From e814179663e5527d8b8a52138fced8919627d85b Mon Sep 17 00:00:00 2001 From: Nikolay Degterinsky Date: Wed, 28 Dec 2022 13:26:23 +0000 Subject: [PATCH 001/406] Add default password type --- programs/client/Client.cpp | 1 + programs/server/config.xml | 5 ++ src/Access/AccessControl.cpp | 26 +++++++ src/Access/AccessControl.h | 9 ++- src/Access/Common/AuthenticationData.cpp | 46 ++++++++++- src/Access/Common/AuthenticationData.h | 2 + src/Client/ClientBase.cpp | 16 +++- src/Client/Connection.cpp | 7 ++ src/Client/Connection.h | 3 + src/Client/IServerConnection.h | 3 + src/Client/LocalConnection.h | 1 + src/Core/ProtocolDefines.h | 4 +- .../Access/InterpreterCreateUserQuery.cpp | 14 +++- src/Parsers/Access/ASTCreateUserQuery.h | 5 +- src/Parsers/Access/ParserCreateUserQuery.cpp | 78 +++++-------------- src/Server/TCPHandler.cpp | 7 ++ 16 files changed, 157 insertions(+), 70 deletions(-) diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp index 9923b8b365a5..9d053275d2e4 100644 --- a/programs/client/Client.cpp +++ b/programs/client/Client.cpp @@ -263,6 +263,7 @@ try /// Set user password complexity rules auto & access_control = global_context->getAccessControl(); access_control.setPasswordComplexityRules(connection->getPasswordComplexityRules()); + access_control.setDefaultPasswordType(connection->getDefaultPasswordType()); if (is_interactive && !delayed_interactive) { diff --git a/programs/server/config.xml b/programs/server/config.xml index 0cbc3d9339e7..8fc766e9d12e 100644 --- a/programs/server/config.xml +++ b/programs/server/config.xml @@ -466,6 +466,11 @@ 1 1 + + sha256 + @@ -139,6 +147,7 @@ 22548578304 1 0 + 100 cache @@ -146,6 +155,7 @@ local_cache_2/ 22548578304 0 + 100 cache @@ -155,6 +165,7 @@ 1 1 0 + 100 @@ -163,6 +174,7 @@ s3_cache_multi/ 22548578304 0 + 100 cache @@ -170,6 +182,7 @@ s3_cache_multi_2/ 22548578304 0 + 100 From e6e71dc832ca1d1b54df741d95a45ae2b3f69355 Mon Sep 17 00:00:00 2001 From: kssenii Date: Sun, 26 Mar 2023 18:37:26 +0200 Subject: [PATCH 083/406] Do not take KeyGuard::Lock under CacheMetadataGuard::Lock to make metadata lock truly lightweight --- src/Interpreters/Cache/FileCache.cpp | 94 +++++++++++++------ src/Interpreters/Cache/FileCache.h | 4 + src/Interpreters/Cache/Metadata.cpp | 84 ++++++++++++----- src/Interpreters/Cache/Metadata.h | 44 +++++---- .../tests/gtest_lru_file_cache.cpp | 80 +++++++++++++++- 5 files changed, 234 insertions(+), 72 deletions(-) diff --git a/src/Interpreters/Cache/FileCache.cpp b/src/Interpreters/Cache/FileCache.cpp index 0db58f2ab37c..8c8959724d3e 100644 --- a/src/Interpreters/Cache/FileCache.cpp +++ b/src/Interpreters/Cache/FileCache.cpp @@ -115,6 +115,7 @@ void FileCache::initialize() } is_initialized = true; + cleanup_task->activate(); cleanup_task->scheduleAfter(delayed_cleanup_interval_ms); } @@ -833,43 +834,46 @@ void FileCache::loadMetadata() } size_t total_size = 0; - for (auto key_prefix_it = fs::directory_iterator{cache_base_path}; key_prefix_it != fs::directory_iterator(); ++key_prefix_it) + for (auto key_prefix_it = fs::directory_iterator{cache_base_path}; key_prefix_it != fs::directory_iterator();) { const fs::path key_prefix_directory = key_prefix_it->path(); + key_prefix_it++; - if (!key_prefix_it->is_directory()) + if (!fs::is_directory(key_prefix_directory)) { if (key_prefix_directory.filename() != "status") { LOG_WARNING( log, "Unexpected file {} (not a directory), will skip it", - key_prefix_it->path().string()); + key_prefix_directory.string()); } continue; } if (fs::is_empty(key_prefix_directory)) { + LOG_DEBUG(log, "Removing empty key prefix directory: {}", key_prefix_directory.string()); fs::remove(key_prefix_directory); continue; } - fs::directory_iterator key_it{}; - for (; key_it != fs::directory_iterator(); ++key_it) + for (fs::directory_iterator key_it{key_prefix_directory}; key_it != fs::directory_iterator();) { const fs::path key_directory = key_it->path(); + ++key_it; - if (!key_it->is_directory()) + if (!fs::is_directory(key_directory)) { LOG_DEBUG( log, "Unexpected file: {} (not a directory). Expected a directory", - key_it->path().string()); + key_directory.string()); continue; } if (fs::is_empty(key_directory)) { + LOG_DEBUG(log, "Removing empty key directory: {}", key_directory.string()); fs::remove(key_directory); continue; } @@ -932,7 +936,7 @@ void FileCache::loadMetadata() log, "Cache capacity changed (max size: {}, used: {}), " "cached file `{}` does not fit in cache anymore (size: {})", - queue.getSizeLimit(), queue.getSize(), key_it->path().string(), size); + queue.getSizeLimit(), queue.getSize(), key_directory.string(), size); fs::remove(offset_it->path()); } @@ -961,46 +965,66 @@ void FileCache::loadMetadata() LockedKeyMetadataPtr FileCache::lockKeyMetadata(const Key & key, KeyNotFoundPolicy key_not_found_policy, bool is_initial_load) { - auto lock = metadata.lock(); - - auto it = metadata.find(key); - if (it == metadata.end()) + KeyMetadataPtr key_metadata; { - if (key_not_found_policy == KeyNotFoundPolicy::THROW) - throw Exception(ErrorCodes::LOGICAL_ERROR, "No such key `{}` in cache", key.toString()); - else if (key_not_found_policy == KeyNotFoundPolicy::RETURN_NULL) - return nullptr; + auto lock = metadata.lock(); - it = metadata.emplace(key, std::make_shared(/* created_base_directory */is_initial_load)).first; - } + auto it = metadata.find(key); + if (it == metadata.end()) + { + if (key_not_found_policy == KeyNotFoundPolicy::THROW) + throw Exception(ErrorCodes::LOGICAL_ERROR, "No such key `{}` in cache", key.toString()); + else if (key_not_found_policy == KeyNotFoundPolicy::RETURN_NULL) + return nullptr; + + it = metadata.emplace( + key, + std::make_shared(/* base_directory_already_exists */is_initial_load, metadata.getCleanupQueue())).first; + } - auto key_metadata = it->second; - auto key_lock = key_metadata->lock(); + key_metadata = it->second; + } - if (key_metadata->inCleanupQueue(key_lock)) { - /// No race is guaranteed because KeyGuard::Lock and CacheMetadataGuard::Lock are hold. - metadata.getCleanupQueue().remove(key); + auto key_lock = key_metadata->lock(); + + const auto cleanup_state = key_metadata->getCleanupState(key_lock); + + if (cleanup_state == KeyMetadata::CleanupState::NOT_SUBMITTED) + { + return std::make_unique(key, key_metadata, std::move(key_lock), getPathInLocalCache(key)); + } if (key_not_found_policy == KeyNotFoundPolicy::THROW) throw Exception(ErrorCodes::LOGICAL_ERROR, "No such key `{}` in cache", key.toString()); else if (key_not_found_policy == KeyNotFoundPolicy::RETURN_NULL) return nullptr; + + if (cleanup_state == KeyMetadata::CleanupState::SUBMITTED_TO_CLEANUP_QUEUE) + { + key_metadata->removeFromCleanupQueue(key, key_lock); + return std::make_unique(key, key_metadata, std::move(key_lock), getPathInLocalCache(key)); + } + + chassert(cleanup_state == KeyMetadata::CleanupState::CLEANED_BY_CLEANUP_THREAD); + chassert(key_not_found_policy == KeyNotFoundPolicy::CREATE_EMPTY); } - return std::make_unique( - key, key_metadata, std::move(key_lock), getPathInLocalCache(key), metadata.getCleanupQueue()); + /// Not we are at a case: + /// cleanup_state == KeyMetadata::CleanupState::CLEANED_BY_CLEANUP_THREAD + /// and KeyNotFoundPolicy == CREATE_EMPTY + /// Retry. + return lockKeyMetadata(key, key_not_found_policy); } LockedKeyMetadataPtr FileCache::lockKeyMetadata(const Key & key, KeyMetadataPtr key_metadata) const { auto key_lock = key_metadata->lock(); - if (key_metadata->inCleanupQueue(key_lock)) + if (key_metadata->getCleanupState(key_lock) != KeyMetadata::CleanupState::NOT_SUBMITTED) throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot lock key: it was removed from cache"); - return std::make_unique( - key, key_metadata, std::move(key_lock), getPathInLocalCache(key), metadata.getCleanupQueue()); + return std::make_unique(key, key_metadata, std::move(key_lock), getPathInLocalCache(key)); } void FileCache::iterateCacheMetadata(const CacheMetadataGuard::Lock &, std::function && func) @@ -1010,18 +1034,28 @@ void FileCache::iterateCacheMetadata(const CacheMetadataGuard::Lock &, std::func { auto key_lock = key_metadata->lock(); - if (key_metadata->inCleanupQueue(key_lock)) + if (key_metadata->getCleanupState(key_lock) != KeyMetadata::CleanupState::NOT_SUBMITTED) continue; func(*key_metadata); } } +FileCache::~FileCache() +{ + cleanup_task->deactivate(); +} + +void FileCache::cleanup() +{ + metadata.doCleanup(); +} + void FileCache::cleanupThreadFunc() { try { - metadata.doCleanup(); + cleanup(); } catch (...) { diff --git a/src/Interpreters/Cache/FileCache.h b/src/Interpreters/Cache/FileCache.h index 2b94ac4fb270..f7a790f0c1a5 100644 --- a/src/Interpreters/Cache/FileCache.h +++ b/src/Interpreters/Cache/FileCache.h @@ -46,6 +46,8 @@ class FileCache : private boost::noncopyable FileCache(const String & cache_base_path_, const FileCacheSettings & cache_settings_); + ~FileCache(); + void initialize(); const String & getBasePath() const { return cache_base_path; } @@ -112,6 +114,8 @@ class FileCache : private boost::noncopyable LockedKeyMetadataPtr lockKeyMetadata(const Key & key, KeyMetadataPtr key_metadata) const; + void cleanup(); + /// For per query cache limit. struct QueryContextHolder : private boost::noncopyable { diff --git a/src/Interpreters/Cache/Metadata.cpp b/src/Interpreters/Cache/Metadata.cpp index 01bab4f35c74..a68d4b3e5ec2 100644 --- a/src/Interpreters/Cache/Metadata.cpp +++ b/src/Interpreters/Cache/Metadata.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include namespace fs = std::filesystem; @@ -103,35 +104,27 @@ std::string KeyMetadata::toString() const return result; } -void CleanupQueue::add(const FileCacheKey & key) +void KeyMetadata::addToCleanupQueue(const FileCacheKey & key, const KeyGuard::Lock &) { - std::lock_guard lock(mutex); - keys.insert(key); + cleanup_queue.add(key); + cleanup_state = CleanupState::SUBMITTED_TO_CLEANUP_QUEUE; } -void CleanupQueue::remove(const FileCacheKey & key) +void KeyMetadata::removeFromCleanupQueue(const FileCacheKey & key, const KeyGuard::Lock &) { - std::lock_guard lock(mutex); - bool erased = keys.erase(key); - if (!erased) - throw Exception(ErrorCodes::LOGICAL_ERROR, "No such key to erase: {}", key.toString()); -} - -bool CleanupQueue::tryPop(FileCacheKey & key) -{ - std::lock_guard lock(mutex); - if (keys.empty()) - return false; - auto it = keys.begin(); - key = *it; - keys.erase(it); - return true; + cleanup_queue.remove(key); + cleanup_state = CleanupState::NOT_SUBMITTED; } void CacheMetadata::doCleanup() { auto lock = guard.lock(); + LOG_INFO( + &Poco::Logger::get("FileCacheCleanupThread"), + "Performing background cleanup (size: {})", + cleanup_queue.getSize()); + /// Let's mention this case. /// This metadata cleanup is delayed so what is we marked key as deleted and /// put it to deletion queue, but then the same key was added to cache before @@ -146,6 +139,15 @@ void CacheMetadata::doCleanup() if (it == end()) throw Exception(ErrorCodes::LOGICAL_ERROR, "No such key {} in metadata", cleanup_key.toString()); + auto key_metadata = it->second; + auto key_lock = key_metadata->lock(); + /// As in lockKeyMetadata we extract key metadata from cache metadata + /// under CacheMetadataGuard::Lock, but take KeyGuard::Lock only after we + /// released cache CacheMetadataGuard::Lock, then we must to take into + /// account it here. + if (key_metadata->getCleanupState(key_lock) == KeyMetadata::CleanupState::NOT_SUBMITTED) + continue; + erase(it); try @@ -170,25 +172,21 @@ LockedKeyMetadata::LockedKeyMetadata( const FileCacheKey & key_, std::shared_ptr key_metadata_, KeyGuard::Lock && lock_, - const std::string & key_path_, - CleanupQueue & cleanup_keys_metadata_queue_) + const std::string & key_path_) : key(key_) , key_path(key_path_) , key_metadata(key_metadata_) , lock(std::move(lock_)) - , cleanup_keys_metadata_queue(cleanup_keys_metadata_queue_) , log(&Poco::Logger::get("LockedKeyMetadata")) { } LockedKeyMetadata::~LockedKeyMetadata() { - /// Someone might still need this directory. if (!key_metadata->empty()) return; - cleanup_keys_metadata_queue.add(key); - key_metadata->in_cleanup_queue = true; + key_metadata->addToCleanupQueue(key, lock); } void LockedKeyMetadata::createKeyDirectoryIfNot() @@ -269,4 +267,40 @@ void LockedKeyMetadata::shrinkFileSegmentToDownloadedSize( assert(file_segment_metadata->size() == entry.size); } +void CleanupQueue::add(const FileCacheKey & key) +{ + std::lock_guard lock(mutex); + auto [_, inserted] = keys.insert(key); + if (!inserted) + { + throw Exception( + ErrorCodes::LOGICAL_ERROR, "Key {} is already in removal queue", key.toString()); + } +} + +void CleanupQueue::remove(const FileCacheKey & key) +{ + std::lock_guard lock(mutex); + bool erased = keys.erase(key); + if (!erased) + throw Exception(ErrorCodes::LOGICAL_ERROR, "No such key {} in removal queue", key.toString()); +} + +bool CleanupQueue::tryPop(FileCacheKey & key) +{ + std::lock_guard lock(mutex); + if (keys.empty()) + return false; + auto it = keys.begin(); + key = *it; + keys.erase(it); + return true; +} + +size_t CleanupQueue::getSize() const +{ + std::lock_guard lock(mutex); + return keys.size(); +} + } diff --git a/src/Interpreters/Cache/Metadata.h b/src/Interpreters/Cache/Metadata.h index d6485165d886..07a719e05efe 100644 --- a/src/Interpreters/Cache/Metadata.h +++ b/src/Interpreters/Cache/Metadata.h @@ -11,6 +11,7 @@ using FileSegmentPtr = std::shared_ptr; struct LockedKeyMetadata; class LockedCachePriority; struct KeysQueue; +struct CleanupQueue; struct FileSegmentMetadata : private boost::noncopyable @@ -36,12 +37,12 @@ struct FileSegmentMetadata : private boost::noncopyable : file_segment(std::move(other.file_segment)), queue_iterator(std::move(other.queue_iterator)) {} }; - struct KeyMetadata : public std::map, private boost::noncopyable { friend struct LockedKeyMetadata; public: - explicit KeyMetadata(bool created_base_directory_) : created_base_directory(created_base_directory_) {} + explicit KeyMetadata(bool created_base_directory_, CleanupQueue & cleanup_queue_) + : created_base_directory(created_base_directory_), cleanup_queue(cleanup_queue_) {} const FileSegmentMetadata * getByOffset(size_t offset) const; FileSegmentMetadata * getByOffset(size_t offset); @@ -55,30 +56,41 @@ struct KeyMetadata : public std::map, private boost bool createdBaseDirectory(const KeyGuard::Lock &) const { return created_base_directory; } - bool inCleanupQueue(const KeyGuard::Lock &) const { return in_cleanup_queue; } + enum class CleanupState + { + NOT_SUBMITTED, + SUBMITTED_TO_CLEANUP_QUEUE, + CLEANED_BY_CLEANUP_THREAD, + }; + + CleanupState getCleanupState(const KeyGuard::Lock &) const { return cleanup_state; } + + void addToCleanupQueue(const FileCacheKey & key, const KeyGuard::Lock &); + + void removeFromCleanupQueue(const FileCacheKey & key, const KeyGuard::Lock &); private: mutable KeyGuard guard; bool created_base_directory = false; - bool in_cleanup_queue = false; + CleanupState cleanup_state = CleanupState::NOT_SUBMITTED; + CleanupQueue & cleanup_queue; }; using KeyMetadataPtr = std::shared_ptr; - struct CleanupQueue { friend struct CacheMetadata; public: void add(const FileCacheKey & key); - void remove(const FileCacheKey & key); + size_t getSize() const; private: bool tryPop(FileCacheKey & key); std::unordered_set keys; - std::mutex mutex; + mutable std::mutex mutex; }; struct CacheMetadata : public std::unordered_map, private boost::noncopyable @@ -88,19 +100,21 @@ struct CacheMetadata : public std::unordered_map, CacheMetadataGuard::Lock lock() { return guard.lock(); } - CleanupQueue & getCleanupQueue() const { return cleanup_queue; } - - void removeFromCleanupQueue(const FileCacheKey & key, const CacheMetadataGuard::Lock &) const; - void doCleanup(); + CleanupQueue & getCleanupQueue() { return cleanup_queue; } + private: + void addToCleanupQueue(const FileCacheKey & key, const KeyGuard::Lock &); + void removeFromCleanupQueue(const FileCacheKey & key, const KeyGuard::Lock &); + const std::string base_directory; CacheMetadataGuard guard; - mutable CleanupQueue cleanup_queue; + CleanupQueue cleanup_queue; }; + /** * `LockedKeyMetadata` is an object which makes sure that as long as it exists the following is true: * 1. the key cannot be removed from cache @@ -119,8 +133,7 @@ struct LockedKeyMetadata : private boost::noncopyable const FileCacheKey & key_, std::shared_ptr key_metadata_, KeyGuard::Lock && key_lock_, - const std::string & key_path_, - CleanupQueue & cleanup_keys_metadata_queue_); + const std::string & key_path_); ~LockedKeyMetadata(); @@ -138,10 +151,9 @@ struct LockedKeyMetadata : private boost::noncopyable private: const FileCacheKey key; - const std::string & key_path; + const std::string key_path; const std::shared_ptr key_metadata; KeyGuard::Lock lock; /// `lock` must be destructed before `key_metadata`. - CleanupQueue & cleanup_keys_metadata_queue; Poco::Logger * log; }; diff --git a/src/Interpreters/tests/gtest_lru_file_cache.cpp b/src/Interpreters/tests/gtest_lru_file_cache.cpp index bfd76d3a4bd4..93df029a592f 100644 --- a/src/Interpreters/tests/gtest_lru_file_cache.cpp +++ b/src/Interpreters/tests/gtest_lru_file_cache.cpp @@ -16,6 +16,9 @@ #include #include #include +#include +#include +#include #include #include @@ -63,6 +66,7 @@ using HolderPtr = FileSegmentsHolderPtr; fs::path caches_dir = fs::current_path() / "lru_cache_test"; std::string cache_base_path = caches_dir / "cache1" / ""; + void assertEqual(const HolderPtr & holder, const Ranges & expected_ranges, const States & expected_states = {}) { std::cerr << "Holder: " << holder->toString() << "\n"; @@ -131,7 +135,6 @@ void download(const HolderPtr & holder) class FileCacheTest : public ::testing::Test { public: - static void setupLogs(const std::string & level) { Poco::AutoPtr channel(new Poco::ConsoleChannel(std::cerr)); @@ -165,6 +168,14 @@ TEST_F(FileCacheTest, get) /// To work with cache need query_id and query context. std::string query_id = "query_id"; + + Poco::XML::DOMParser dom_parser; + std::string xml(R"CONFIG( +)CONFIG"); + Poco::AutoPtr document = dom_parser.parseString(xml); + Poco::AutoPtr config = new Poco::Util::XMLConfiguration(document); + getMutableContext().context->setConfig(config); + auto query_context = DB::Context::createCopy(getContext().context); query_context->makeQueryContext(); query_context->setCurrentQueryId(query_id); @@ -178,23 +189,36 @@ TEST_F(FileCacheTest, get) std::cerr << "Step 1\n"; auto cache = FileCache(cache_base_path, settings); + std::cerr << "Step 1\n"; cache.initialize(); + std::cerr << "Step 1\n"; auto key = cache.createKeyForPath("key1"); + std::cerr << "Step 1\n"; { + std::cerr << "Step 1\n"; auto holder = cache.getOrSet(key, 0, 10, {}); /// Add range [0, 9] + std::cerr << "Step 1\n"; assertEqual(holder, { Range(0, 9) }, { State::EMPTY }); + std::cerr << "Step 1\n"; download(holder->front()); + std::cerr << "Step 1\n"; assertEqual(holder, { Range(0, 9) }, { State::DOWNLOADED }); + std::cerr << "Step 1\n"; } /// Current cache: [__________] /// ^ ^ /// 0 9 + std::cerr << "Step 1\n"; assertEqual(cache.getSnapshot(key), { Range(0, 9) }); + std::cerr << "Step 1\n"; assertEqual(cache.dumpQueue(), { Range(0, 9) }); + std::cerr << "Step 1\n"; ASSERT_EQ(cache.getFileSegmentsNum(), 1); + std::cerr << "Step 1\n"; ASSERT_EQ(cache.getUsedCacheSize(), 10); + std::cerr << "Step 1\n"; std::cerr << "Step 2\n"; @@ -520,6 +544,60 @@ TEST_F(FileCacheTest, get) { State::EMPTY, State::EMPTY, State::EMPTY }); } + std::cerr << "Step 13\n"; + { + /// Test delated cleanup + + auto cache = FileCache(cache_base_path, settings); + cache.initialize(); + cache.cleanup(); + const auto key = cache.createKeyForPath("key10"); + const auto key_path = cache.getPathInLocalCache(key); + + cache.removeAllReleasable(); + ASSERT_EQ(cache.getUsedCacheSize(), 0); + ASSERT_TRUE(!fs::exists(key_path)); + ASSERT_TRUE(!fs::exists(fs::path(key_path).parent_path())); + + download(cache.getOrSet(key, 0, 10, {})); + ASSERT_EQ(cache.getUsedCacheSize(), 10); + ASSERT_TRUE(fs::exists(cache.getPathInLocalCache(key, 0, FileSegmentKind::Regular))); + + cache.removeAllReleasable(); + ASSERT_EQ(cache.getUsedCacheSize(), 0); + ASSERT_TRUE(fs::exists(key_path)); + ASSERT_TRUE(!fs::exists(cache.getPathInLocalCache(key, 0, FileSegmentKind::Regular))); + + cache.cleanup(); + ASSERT_TRUE(!fs::exists(key_path)); + ASSERT_TRUE(!fs::exists(fs::path(key_path).parent_path())); + } + + std::cerr << "Step 14\n"; + { + /// Test background thread delated cleanup + + auto settings2{settings}; + settings2.delayed_cleanup_interval_ms = 0; + auto cache = FileCache(cache_base_path, settings2); + cache.initialize(); + const auto key = cache.createKeyForPath("key10"); + const auto key_path = cache.getPathInLocalCache(key); + + cache.removeAllReleasable(); + ASSERT_EQ(cache.getUsedCacheSize(), 0); + ASSERT_TRUE(!fs::exists(key_path)); + ASSERT_TRUE(!fs::exists(fs::path(key_path).parent_path())); + + download(cache.getOrSet(key, 0, 10, {})); + ASSERT_EQ(cache.getUsedCacheSize(), 10); + ASSERT_TRUE(fs::exists(key_path)); + + cache.removeAllReleasable(); + ASSERT_EQ(cache.getUsedCacheSize(), 0); + sleepForSeconds(2); + ASSERT_TRUE(!fs::exists(key_path)); + } } TEST_F(FileCacheTest, writeBuffer) From 1adef76cbdacee93d0ed2fcc59088c8ad7021046 Mon Sep 17 00:00:00 2001 From: kssenii Date: Sun, 26 Mar 2023 22:10:14 +0200 Subject: [PATCH 084/406] Better comment --- src/Interpreters/Cache/Metadata.cpp | 9 +++++---- src/Interpreters/tests/gtest_lru_file_cache.cpp | 13 ------------- 2 files changed, 5 insertions(+), 17 deletions(-) diff --git a/src/Interpreters/Cache/Metadata.cpp b/src/Interpreters/Cache/Metadata.cpp index a68d4b3e5ec2..b9f7353e6369 100644 --- a/src/Interpreters/Cache/Metadata.cpp +++ b/src/Interpreters/Cache/Metadata.cpp @@ -141,10 +141,11 @@ void CacheMetadata::doCleanup() auto key_metadata = it->second; auto key_lock = key_metadata->lock(); - /// As in lockKeyMetadata we extract key metadata from cache metadata - /// under CacheMetadataGuard::Lock, but take KeyGuard::Lock only after we - /// released cache CacheMetadataGuard::Lock, then we must to take into - /// account it here. + /// As in lockKeyMetadata we extract key metadata from cache metadata under + /// CacheMetadataGuard::Lock, but take KeyGuard::Lock only after we released + /// cache CacheMetadataGuard::Lock (because CacheMetadataGuard::Lock must be lightweight). + /// So it is possible that a key which was submitted to cleanup queue was afterwards added + /// to cache, so here we need to check this case. if (key_metadata->getCleanupState(key_lock) == KeyMetadata::CleanupState::NOT_SUBMITTED) continue; diff --git a/src/Interpreters/tests/gtest_lru_file_cache.cpp b/src/Interpreters/tests/gtest_lru_file_cache.cpp index 93df029a592f..16c8498f04e5 100644 --- a/src/Interpreters/tests/gtest_lru_file_cache.cpp +++ b/src/Interpreters/tests/gtest_lru_file_cache.cpp @@ -189,36 +189,23 @@ TEST_F(FileCacheTest, get) std::cerr << "Step 1\n"; auto cache = FileCache(cache_base_path, settings); - std::cerr << "Step 1\n"; cache.initialize(); - std::cerr << "Step 1\n"; auto key = cache.createKeyForPath("key1"); - std::cerr << "Step 1\n"; { - std::cerr << "Step 1\n"; auto holder = cache.getOrSet(key, 0, 10, {}); /// Add range [0, 9] - std::cerr << "Step 1\n"; assertEqual(holder, { Range(0, 9) }, { State::EMPTY }); - std::cerr << "Step 1\n"; download(holder->front()); - std::cerr << "Step 1\n"; assertEqual(holder, { Range(0, 9) }, { State::DOWNLOADED }); - std::cerr << "Step 1\n"; } /// Current cache: [__________] /// ^ ^ /// 0 9 - std::cerr << "Step 1\n"; assertEqual(cache.getSnapshot(key), { Range(0, 9) }); - std::cerr << "Step 1\n"; assertEqual(cache.dumpQueue(), { Range(0, 9) }); - std::cerr << "Step 1\n"; ASSERT_EQ(cache.getFileSegmentsNum(), 1); - std::cerr << "Step 1\n"; ASSERT_EQ(cache.getUsedCacheSize(), 10); - std::cerr << "Step 1\n"; std::cerr << "Step 2\n"; From f5fa43b79b22f122f4bc5644c2e5d8eb9494b03f Mon Sep 17 00:00:00 2001 From: kssenii Date: Sun, 26 Mar 2023 22:33:59 +0200 Subject: [PATCH 085/406] Remove redundant --- src/Interpreters/Cache/FileCache.cpp | 20 +++++--------------- src/Interpreters/Cache/FileCache.h | 6 ------ 2 files changed, 5 insertions(+), 21 deletions(-) diff --git a/src/Interpreters/Cache/FileCache.cpp b/src/Interpreters/Cache/FileCache.cpp index 8c8959724d3e..cf1a56c8ee77 100644 --- a/src/Interpreters/Cache/FileCache.cpp +++ b/src/Interpreters/Cache/FileCache.cpp @@ -514,22 +514,7 @@ bool FileCache::tryReserve(const Key & key, size_t offset, size_t size, KeyMetad auto locked_key = lockKeyMetadata(key, key_metadata); LOG_TEST(log, "Reserving {} bytes for key {} at offset {}", size, key.toString(), offset); - const bool reserved = tryReserveUnlocked(locked_key, offset, size, lock); - if (reserved) - LOG_TEST(log, "Successfully reserved {} bytes for key {} at offset {}", size, key.toString(), offset); - else - LOG_TEST(log, "Failed to reserve {} bytes for key {} at offset {}", size, key.toString(), offset); - - return reserved; -} - -bool FileCache::tryReserveUnlocked( - LockedKeyMetadataPtr locked_key, - size_t offset, - size_t size, - const CacheGuard::Lock & lock) -{ auto query_context = query_limit ? query_limit->tryGetQueryContext(lock) : nullptr; bool reserved; @@ -545,7 +530,12 @@ bool FileCache::tryReserveUnlocked( } if (reserved) + { + LOG_TEST(log, "Successfully reserved {} bytes for key {} at offset {}", size, key.toString(), offset); locked_key->createKeyDirectoryIfNot(); + } + else + LOG_TEST(log, "Failed to reserve {} bytes for key {} at offset {}", size, key.toString(), offset); return reserved; } diff --git a/src/Interpreters/Cache/FileCache.h b/src/Interpreters/Cache/FileCache.h index f7a790f0c1a5..3f2833486cb7 100644 --- a/src/Interpreters/Cache/FileCache.h +++ b/src/Interpreters/Cache/FileCache.h @@ -213,12 +213,6 @@ class FileCache : private boost::noncopyable FileSegmentPtr file_segment, const CacheGuard::Lock &); - bool tryReserveUnlocked( - LockedKeyMetadataPtr locked_key, - size_t offset, - size_t size, - const CacheGuard::Lock &); - bool tryReserveImpl( IFileCachePriority & priority_queue, LockedKeyMetadataPtr locked_key, From b808c35480571a21adc43d9367c9d9414b622da3 Mon Sep 17 00:00:00 2001 From: kssenii Date: Sun, 26 Mar 2023 23:17:04 +0200 Subject: [PATCH 086/406] Fix style check --- src/Interpreters/Cache/Metadata.h | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Interpreters/Cache/Metadata.h b/src/Interpreters/Cache/Metadata.h index 07a719e05efe..d2903d194efb 100644 --- a/src/Interpreters/Cache/Metadata.h +++ b/src/Interpreters/Cache/Metadata.h @@ -114,7 +114,6 @@ struct CacheMetadata : public std::unordered_map, }; - /** * `LockedKeyMetadata` is an object which makes sure that as long as it exists the following is true: * 1. the key cannot be removed from cache From 569a1508eb5c01e872fcc6c23867ae50864861e1 Mon Sep 17 00:00:00 2001 From: kssenii Date: Mon, 27 Mar 2023 14:05:12 +0200 Subject: [PATCH 087/406] Fix --- src/Interpreters/Cache/FileCache.cpp | 4 ++-- src/Interpreters/Cache/Metadata.cpp | 5 ----- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/src/Interpreters/Cache/FileCache.cpp b/src/Interpreters/Cache/FileCache.cpp index cf1a56c8ee77..9ab43d78661a 100644 --- a/src/Interpreters/Cache/FileCache.cpp +++ b/src/Interpreters/Cache/FileCache.cpp @@ -39,7 +39,6 @@ FileCache::FileCache( query_limit = std::make_unique(); cleanup_task = Context::getGlobalContextInstance()->getSchedulePool().createTask("FileCacheCleanup", [this]{ cleanupThreadFunc(); }); - cleanup_task->deactivate(); } FileCache::Key FileCache::createKeyForPath(const String & path) @@ -1033,7 +1032,8 @@ void FileCache::iterateCacheMetadata(const CacheMetadataGuard::Lock &, std::func FileCache::~FileCache() { - cleanup_task->deactivate(); + if (cleanup_task) + cleanup_task->deactivate(); } void FileCache::cleanup() diff --git a/src/Interpreters/Cache/Metadata.cpp b/src/Interpreters/Cache/Metadata.cpp index b9f7353e6369..ff7fcd96dc02 100644 --- a/src/Interpreters/Cache/Metadata.cpp +++ b/src/Interpreters/Cache/Metadata.cpp @@ -120,11 +120,6 @@ void CacheMetadata::doCleanup() { auto lock = guard.lock(); - LOG_INFO( - &Poco::Logger::get("FileCacheCleanupThread"), - "Performing background cleanup (size: {})", - cleanup_queue.getSize()); - /// Let's mention this case. /// This metadata cleanup is delayed so what is we marked key as deleted and /// put it to deletion queue, but then the same key was added to cache before From daa3d913c03fedc93709d2ddaca87244ca5a0429 Mon Sep 17 00:00:00 2001 From: kssenii Date: Mon, 27 Mar 2023 19:06:49 +0200 Subject: [PATCH 088/406] Fix --- src/Interpreters/Cache/FileCache.cpp | 5 +++++ src/Interpreters/Cache/FileCache.h | 2 ++ src/Interpreters/Context.cpp | 6 ++++++ 3 files changed, 13 insertions(+) diff --git a/src/Interpreters/Cache/FileCache.cpp b/src/Interpreters/Cache/FileCache.cpp index 9ab43d78661a..e506fa8e1715 100644 --- a/src/Interpreters/Cache/FileCache.cpp +++ b/src/Interpreters/Cache/FileCache.cpp @@ -1031,6 +1031,11 @@ void FileCache::iterateCacheMetadata(const CacheMetadataGuard::Lock &, std::func } FileCache::~FileCache() +{ + deactivateBackgroundOperations(); +} + +void FileCache::deactivateBackgroundOperations() { if (cleanup_task) cleanup_task->deactivate(); diff --git a/src/Interpreters/Cache/FileCache.h b/src/Interpreters/Cache/FileCache.h index 3f2833486cb7..050d5d88f375 100644 --- a/src/Interpreters/Cache/FileCache.h +++ b/src/Interpreters/Cache/FileCache.h @@ -116,6 +116,8 @@ class FileCache : private boost::noncopyable void cleanup(); + void deactivateBackgroundOperations(); + /// For per query cache limit. struct QueryContextHolder : private boost::noncopyable { diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index 2cfa55f0d870..bd8a82a5bb80 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -42,6 +42,8 @@ #include #include #include +#include +#include #include #include #include @@ -436,6 +438,10 @@ struct ContextSharedPart : boost::noncopyable } } + const auto & caches = FileCacheFactory::instance().getAll(); + for (const auto & [_, cache] : caches) + cache->cache->deactivateBackgroundOperations(); + try { shutdown(); From 667905f6459bdfe6ad209568a5704322b9627a2f Mon Sep 17 00:00:00 2001 From: kssenii Date: Mon, 27 Mar 2023 19:24:46 +0200 Subject: [PATCH 089/406] Fix duplicate include --- src/Interpreters/Context.cpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index bd8a82a5bb80..06fddaf4cba4 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -107,15 +107,12 @@ #include #include #include -#include #include #include #include #include #include -#include - #if USE_ROCKSDB #include #endif From b541ec485dbaaf49e83eb60b4a7fed0c7ebdddcf Mon Sep 17 00:00:00 2001 From: kssenii Date: Tue, 28 Mar 2023 11:55:55 +0200 Subject: [PATCH 090/406] Fix --- src/Interpreters/Context.cpp | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index 06fddaf4cba4..b12ac394f0e8 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -435,10 +435,6 @@ struct ContextSharedPart : boost::noncopyable } } - const auto & caches = FileCacheFactory::instance().getAll(); - for (const auto & [_, cache] : caches) - cache->cache->deactivateBackgroundOperations(); - try { shutdown(); @@ -517,6 +513,12 @@ struct ContextSharedPart : boost::noncopyable /// take it as well, which will cause deadlock. delete_ddl_worker.reset(); + /// Background operations in cache use background schedule pool. + /// Deactivate them before destructing it. + const auto & caches = FileCacheFactory::instance().getAll(); + for (const auto & [_, cache] : caches) + cache->cache->deactivateBackgroundOperations(); + { auto lock = std::lock_guard(mutex); From c9ba1e7a71d71a19c0e93247e8b1ebbf0fb7447c Mon Sep 17 00:00:00 2001 From: kssenii Date: Tue, 28 Mar 2023 22:14:45 +0200 Subject: [PATCH 091/406] Move checks --- src/Interpreters/Cache/FileCache.cpp | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/Interpreters/Cache/FileCache.cpp b/src/Interpreters/Cache/FileCache.cpp index e506fa8e1715..34d9000d0ec0 100644 --- a/src/Interpreters/Cache/FileCache.cpp +++ b/src/Interpreters/Cache/FileCache.cpp @@ -733,9 +733,6 @@ bool FileCache::tryReserveImpl( void FileCache::removeKeyIfExists(const Key & key) { assertInitialized(); -#ifndef NDEBUG - assertCacheCorrectness(); -#endif auto lock = cache_guard.lock(); auto locked_key = lockKeyMetadata(key, KeyNotFoundPolicy::RETURN_NULL); @@ -767,9 +764,6 @@ void FileCache::removeKeyIfExists(const Key & key) void FileCache::removeAllReleasable() { assertInitialized(); -#ifndef NDEBUG - assertCacheCorrectness(); -#endif using QueueEntry = IFileCachePriority::Entry; using IterationResult = IFileCachePriority::IterationResult; @@ -1048,6 +1042,10 @@ void FileCache::cleanup() void FileCache::cleanupThreadFunc() { +#ifndef NDEBUG + assertCacheCorrectness(); +#endif + try { cleanup(); From ba582dd74c551a5844c7c63d4be4c617dd33a944 Mon Sep 17 00:00:00 2001 From: kssenii Date: Wed, 29 Mar 2023 15:27:00 +0200 Subject: [PATCH 092/406] Move assertions to a better place --- src/Interpreters/Cache/FileCache.cpp | 46 ++----------------- src/Interpreters/Cache/FileSegment.cpp | 23 +++++++++- src/Interpreters/Cache/FileSegment.h | 4 +- src/Interpreters/Cache/IFileCachePriority.h | 5 +- src/Interpreters/Cache/LRUFileCachePriority.h | 6 ++- .../Cache/LockedFileCachePriority.h | 4 +- src/Interpreters/Cache/Metadata.cpp | 2 +- 7 files changed, 38 insertions(+), 52 deletions(-) diff --git a/src/Interpreters/Cache/FileCache.cpp b/src/Interpreters/Cache/FileCache.cpp index 34d9000d0ec0..c007f45d350e 100644 --- a/src/Interpreters/Cache/FileCache.cpp +++ b/src/Interpreters/Cache/FileCache.cpp @@ -1131,48 +1131,12 @@ size_t FileCache::getFileSegmentsNum() const void FileCache::assertCacheCorrectness() { + auto lock = metadata.lock(); + iterateCacheMetadata(lock, [&](KeyMetadata & key_metadata) { - auto lock = metadata.lock(); - iterateCacheMetadata(lock, [&](KeyMetadata & key_metadata) - { - for (auto & [offset, file_segment_metadata] : key_metadata) - { - file_segment_metadata.file_segment->assertCorrectness(); - if (file_segment_metadata.size()) - chassert(file_segment_metadata.queue_iterator); - } - }); - } - - { - auto lock = cache_guard.lock(); - - LockedCachePriority queue(lock, *main_priority); - [[maybe_unused]] size_t total_size = 0; - - using QueueEntry = IFileCachePriority::Entry; - using IterationResult = IFileCachePriority::IterationResult; - - queue.iterate([&](const QueueEntry & entry) -> IterationResult - { - auto locked_key = lockKeyMetadata(entry.key, entry.getKeyMetadata()); - auto * file_segment_metadata = locked_key->getKeyMetadata()->getByOffset(entry.offset); - - if (file_segment_metadata->size() != entry.size) - { - throw Exception( - ErrorCodes::LOGICAL_ERROR, "Expected {} == {} size ({})", - file_segment_metadata->size(), entry.size, file_segment_metadata->file_segment->getInfoForLog()); - } - - total_size += entry.size; - return IterationResult::CONTINUE; - }); - - assert(queue.getSize() == total_size); - assert(queue.getSize() <= queue.getSizeLimit()); - assert(queue.getElementsCount() <= queue.getElementsLimit()); - } + for (auto & [offset, file_segment_metadata] : key_metadata) + file_segment_metadata.file_segment->assertCorrectness(); + }); } FileCache::QueryContextHolder::QueryContextHolder( diff --git a/src/Interpreters/Cache/FileSegment.cpp b/src/Interpreters/Cache/FileSegment.cpp index b18a5c6d7762..580f4cfa670a 100644 --- a/src/Interpreters/Cache/FileSegment.cpp +++ b/src/Interpreters/Cache/FileSegment.cpp @@ -466,6 +466,7 @@ bool FileSegment::reserve(size_t size_to_reserve) /// It is made atomic because of getInfoForLog. reserved_size += size_to_reserve; } + chassert(assertCorrectness()); } return reserved; @@ -689,18 +690,36 @@ String FileSegment::stateToString(FileSegment::State state) UNREACHABLE(); } -void FileSegment::assertCorrectness() const +bool FileSegment::assertCorrectness() const { auto lock = segment_guard.lock(); assertCorrectnessUnlocked(lock); + return true; } -void FileSegment::assertCorrectnessUnlocked(const FileSegmentGuard::Lock & lock) const +bool FileSegment::assertCorrectnessUnlocked(const FileSegmentGuard::Lock & lock) const { auto current_downloader = getDownloaderUnlocked(lock); chassert(current_downloader.empty() == (download_state != FileSegment::State::DOWNLOADING)); chassert(!current_downloader.empty() == (download_state == FileSegment::State::DOWNLOADING)); chassert(download_state != FileSegment::State::DOWNLOADED || std::filesystem::file_size(file_path) > 0); + + auto locked_key = lockKeyMetadata(true); + + const auto & file_segment_metadata = locked_key->getKeyMetadata()->tryGetByOffset(offset()); + chassert(reserved_size == 0 || file_segment_metadata->queue_iterator); + + if (file_segment_metadata->queue_iterator) + { + const auto & entry = *file_segment_metadata->queue_iterator; + if (isCompleted(false)) + chassert(reserved_size == entry.getEntry().size); + else + /// We cannot check == here because reserved_size is not + /// guarded by any mutex, it is just an atomic. + chassert(reserved_size <= entry.getEntry().size); + } + return true; } void FileSegment::throwIfDetachedUnlocked(const FileSegmentGuard::Lock & lock) const diff --git a/src/Interpreters/Cache/FileSegment.h b/src/Interpreters/Cache/FileSegment.h index 139a09f59393..fcba974ef878 100644 --- a/src/Interpreters/Cache/FileSegment.h +++ b/src/Interpreters/Cache/FileSegment.h @@ -222,7 +222,7 @@ friend struct LockedKeyMetadata; /// Completed states: DOWNALODED, DETACHED. bool isCompleted(bool sync = false) const; - void assertCorrectness() const; + bool assertCorrectness() const; /** * ========== Methods for _only_ file segment's `downloader` ================== @@ -282,7 +282,7 @@ friend struct LockedKeyMetadata; void assertNotDetached() const; void assertNotDetachedUnlocked(const FileSegmentGuard::Lock &) const; void assertIsDownloaderUnlocked(const std::string & operation, const FileSegmentGuard::Lock &) const; - void assertCorrectnessUnlocked(const FileSegmentGuard::Lock &) const; + bool assertCorrectnessUnlocked(const FileSegmentGuard::Lock &) const; LockedKeyMetadataPtr lockKeyMetadata(bool assert_exists = true) const; KeyMetadataPtr getKeyMetadata() const; diff --git a/src/Interpreters/Cache/IFileCachePriority.h b/src/Interpreters/Cache/IFileCachePriority.h index e27656d7958a..d0c369f88530 100644 --- a/src/Interpreters/Cache/IFileCachePriority.h +++ b/src/Interpreters/Cache/IFileCachePriority.h @@ -61,9 +61,10 @@ friend class LockedCachePriority; public: virtual ~IIterator() = default; + virtual const Entry & getEntry() const = 0; + protected: - virtual Entry & operator *() = 0; - virtual const Entry & operator *() const = 0; + virtual Entry & getEntry() = 0; virtual size_t use() = 0; diff --git a/src/Interpreters/Cache/LRUFileCachePriority.h b/src/Interpreters/Cache/LRUFileCachePriority.h index 990b76608604..469238d10ab8 100644 --- a/src/Interpreters/Cache/LRUFileCachePriority.h +++ b/src/Interpreters/Cache/LRUFileCachePriority.h @@ -48,8 +48,10 @@ class LRUFileCachePriority::LRUFileCacheIterator : public IFileCachePriority::II LRUFileCachePriority * cache_priority_, LRUFileCachePriority::LRUQueueIterator queue_iter_); - Entry & operator *() override { return *queue_iter; } - const Entry & operator *() const override { return *queue_iter; } + const Entry & getEntry() const override { return *queue_iter; } + +protected: + Entry & getEntry() override { return *queue_iter; } size_t use() override; diff --git a/src/Interpreters/Cache/LockedFileCachePriority.h b/src/Interpreters/Cache/LockedFileCachePriority.h index 0185e8dd9c40..3cc6e2165cbf 100644 --- a/src/Interpreters/Cache/LockedFileCachePriority.h +++ b/src/Interpreters/Cache/LockedFileCachePriority.h @@ -41,8 +41,8 @@ class LockedCachePriorityIterator LockedCachePriorityIterator(const CacheGuard::Lock & lock_, IFileCachePriority::Iterator & iterator_) : lock(lock_), iterator(iterator_) {} - IFileCachePriority::Entry & operator *() { return **iterator; } - const IFileCachePriority::Entry & operator *() const { return **iterator; } + IFileCachePriority::Entry & getEntry() { return iterator->getEntry(); } + const IFileCachePriority::Entry & getEntry() const { return iterator->getEntry(); } size_t use() { return iterator->use(); } diff --git a/src/Interpreters/Cache/Metadata.cpp b/src/Interpreters/Cache/Metadata.cpp index ff7fcd96dc02..4f6af172253c 100644 --- a/src/Interpreters/Cache/Metadata.cpp +++ b/src/Interpreters/Cache/Metadata.cpp @@ -247,7 +247,7 @@ void LockedKeyMetadata::shrinkFileSegmentToDownloadedSize( file_segment->getInfoForLogUnlocked(segment_lock)); } - auto & entry = *LockedCachePriorityIterator(cache_lock, file_segment_metadata->queue_iterator); + auto & entry = LockedCachePriorityIterator(cache_lock, file_segment_metadata->queue_iterator).getEntry(); assert(file_segment->downloaded_size <= file_segment->reserved_size); assert(entry.size == file_segment->reserved_size); assert(entry.size >= file_segment->downloaded_size); From 9acbcb5d5008813af883e2c33168cf502a7895a8 Mon Sep 17 00:00:00 2001 From: kssenii Date: Wed, 29 Mar 2023 15:34:00 +0200 Subject: [PATCH 093/406] Make max_size less in CI (for more pressure on cache) --- tests/config/config.d/storage_conf.xml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/config/config.d/storage_conf.xml b/tests/config/config.d/storage_conf.xml index 0e0ad38430ad..cb5a75f96989 100644 --- a/tests/config/config.d/storage_conf.xml +++ b/tests/config/config.d/storage_conf.xml @@ -55,7 +55,7 @@ cache s3_disk s3_cache/ - 2147483648 + 128Mi 1 0 100 @@ -64,7 +64,7 @@ cache s3_disk_2 s3_cache_2/ - 2Gi + 128Mi 0 100Mi 100 @@ -73,7 +73,7 @@ cache s3_disk_3 s3_disk_3_cache/ - 22548578304 + 128Mi 22548578304 1 1 @@ -84,7 +84,7 @@ cache s3_disk_4 s3_cache_4/ - 22548578304 + 128Mi 1 1 0 @@ -94,7 +94,7 @@ cache s3_disk_5 s3_cache_5/ - 22548578304 + 128Mi 0 100 @@ -102,7 +102,7 @@ cache s3_disk_6 s3_cache_6/ - 22548578304 + 128Mi 0 1 100 @@ -120,7 +120,7 @@ cache s3_disk_6 s3_cache_small_segment_size/ - 22548578304 + 128Mi 10Ki 0 1 From 245d04bb84288d405fdde25f2358707ec331914d Mon Sep 17 00:00:00 2001 From: kssenii Date: Wed, 29 Mar 2023 23:56:32 +0200 Subject: [PATCH 094/406] Fix changes from commit ba582dd, refactor a bit --- .../Cached/CachedObjectStorage.cpp | 7 - .../Cached/CachedObjectStorage.h | 2 - src/Interpreters/Cache/FileCache.cpp | 144 +++--------------- src/Interpreters/Cache/FileCache.h | 18 +-- src/Interpreters/Cache/FileSegment.cpp | 18 +-- src/Interpreters/Cache/FileSegment.h | 4 +- src/Interpreters/Cache/Metadata.cpp | 127 ++++++++++++++- src/Interpreters/Cache/Metadata.h | 32 +++- 8 files changed, 190 insertions(+), 162 deletions(-) diff --git a/src/Disks/ObjectStorages/Cached/CachedObjectStorage.cpp b/src/Disks/ObjectStorages/Cached/CachedObjectStorage.cpp index e1282de37cb4..f783d3bf852c 100644 --- a/src/Disks/ObjectStorages/Cached/CachedObjectStorage.cpp +++ b/src/Disks/ObjectStorages/Cached/CachedObjectStorage.cpp @@ -46,12 +46,6 @@ FileCache::Key CachedObjectStorage::getCacheKey(const std::string & path) const return cache->createKeyForPath(path); } -String CachedObjectStorage::getCachePath(const std::string & path) const -{ - FileCache::Key cache_key = getCacheKey(path); - return cache->getPathInLocalCache(cache_key); -} - std::string CachedObjectStorage::generateBlobNameForPath(const std::string & path) { return object_storage->generateBlobNameForPath(path); @@ -192,7 +186,6 @@ std::unique_ptr CachedObjectStorage::writeObject( /// N if (cache_on_write) { auto key = getCacheKey(path_key_for_cache); - LOG_TEST(log, "Caching file `{}` to `{}` with key {}", object.absolute_path, getCachePath(path_key_for_cache), key.toString()); return std::make_unique( std::move(implementation_buffer), diff --git a/src/Disks/ObjectStorages/Cached/CachedObjectStorage.h b/src/Disks/ObjectStorages/Cached/CachedObjectStorage.h index dbea90803d05..bb0e25eb9a3a 100644 --- a/src/Disks/ObjectStorages/Cached/CachedObjectStorage.h +++ b/src/Disks/ObjectStorages/Cached/CachedObjectStorage.h @@ -120,8 +120,6 @@ class CachedObjectStorage final : public IObjectStorage private: FileCache::Key getCacheKey(const std::string & path) const; - String getCachePath(const std::string & path) const; - ReadSettings patchSettings(const ReadSettings & read_settings) const override; ObjectStoragePtr object_storage; diff --git a/src/Interpreters/Cache/FileCache.cpp b/src/Interpreters/Cache/FileCache.cpp index c007f45d350e..7676b005956f 100644 --- a/src/Interpreters/Cache/FileCache.cpp +++ b/src/Interpreters/Cache/FileCache.cpp @@ -48,36 +48,12 @@ FileCache::Key FileCache::createKeyForPath(const String & path) String FileCache::getPathInLocalCache(const Key & key, size_t offset, FileSegmentKind segment_kind) const { - String file_suffix; - switch (segment_kind) - { - case FileSegmentKind::Persistent: - file_suffix = "_persistent"; - break; - case FileSegmentKind::Temporary: - file_suffix = "_temporary"; - break; - case FileSegmentKind::Regular: - file_suffix = ""; - break; - } - - auto key_str = key.toString(); - return fs::path(cache_base_path) - / key_str.substr(0, 3) - / key_str - / (std::to_string(offset) + file_suffix); + return metadata.getPathInLocalCache(key, offset, segment_kind); } String FileCache::getPathInLocalCache(const Key & key) const { - return getPathInLocalCache(cache_base_path, key); -} - -String FileCache::getPathInLocalCache(const std::string & base_directory, const Key & key) -{ - auto key_str = key.toString(); - return fs::path(base_directory) / key_str.substr(0, 3) / key_str; + return metadata.getPathInLocalCache(key); } void FileCache::assertInitialized() const @@ -360,7 +336,7 @@ FileSegmentsHolderPtr FileCache::set(const Key & key, size_t offset, size_t size { assertInitialized(); - auto locked_key = lockKeyMetadata(key, KeyNotFoundPolicy::CREATE_EMPTY); + auto locked_key = metadata.lockKeyMetadata(key, CacheMetadata::KeyNotFoundPolicy::CREATE_EMPTY); FileSegment::Range range(offset, offset + size - 1); auto file_segments = getImpl(*locked_key, range); @@ -390,7 +366,7 @@ FileSegmentsHolderPtr FileCache::getOrSet( FileSegment::Range range(offset, offset + size - 1); - auto locked_key = lockKeyMetadata(key, KeyNotFoundPolicy::CREATE_EMPTY); + auto locked_key = metadata.lockKeyMetadata(key, CacheMetadata::KeyNotFoundPolicy::CREATE_EMPTY); /// Get all segments which intersect with the given range. auto file_segments = getImpl(*locked_key, range); @@ -411,7 +387,7 @@ FileSegmentsHolderPtr FileCache::get(const Key & key, size_t offset, size_t size { assertInitialized(); - auto locked_key = lockKeyMetadata(key, KeyNotFoundPolicy::RETURN_NULL); + auto locked_key = metadata.lockKeyMetadata(key, CacheMetadata::KeyNotFoundPolicy::RETURN_NULL); if (locked_key) { FileSegment::Range range(offset, offset + size - 1); @@ -510,9 +486,9 @@ bool FileCache::tryReserve(const Key & key, size_t offset, size_t size, KeyMetad { assertInitialized(); auto lock = cache_guard.lock(); - auto locked_key = lockKeyMetadata(key, key_metadata); + auto locked_key = metadata.lockKeyMetadata(key, key_metadata); - LOG_TEST(log, "Reserving {} bytes for key {} at offset {}", size, key.toString(), offset); + LOG_TEST(log, "Reserving {} byts for key {} at offset {}", size, key.toString(), offset); auto query_context = query_limit ? query_limit->tryGetQueryContext(lock) : nullptr; bool reserved; @@ -553,7 +529,7 @@ void FileCache::iterateCacheAndCollectKeyLocks( if (locked) current = locked_it->second; else - current = lockKeyMetadata(entry.key, entry.getKeyMetadata()); + current = metadata.lockKeyMetadata(entry.key, entry.getKeyMetadata()); auto res = func(entry, *current); if (res.lock_key && !locked) @@ -735,7 +711,7 @@ void FileCache::removeKeyIfExists(const Key & key) assertInitialized(); auto lock = cache_guard.lock(); - auto locked_key = lockKeyMetadata(key, KeyNotFoundPolicy::RETURN_NULL); + auto locked_key = metadata.lockKeyMetadata(key, CacheMetadata::KeyNotFoundPolicy::RETURN_NULL); if (!locked_key) return; @@ -777,7 +753,7 @@ void FileCache::removeAllReleasable() LockedCachePriority(lock, *main_priority).iterate([&](const QueueEntry & entry) -> IterationResult { - auto locked_key = lockKeyMetadata(entry.key, entry.getKeyMetadata()); + auto locked_key = metadata.lockKeyMetadata(entry.key, entry.getKeyMetadata()); auto * file_segment_metadata = locked_key->getKeyMetadata()->getByOffset(entry.offset); if (file_segment_metadata->releasable()) @@ -862,7 +838,7 @@ void FileCache::loadMetadata() } const auto key = Key(unhexUInt(key_directory.filename().string().data())); - auto locked_key = lockKeyMetadata(key, KeyNotFoundPolicy::CREATE_EMPTY, /* is_initial_load */true); + auto locked_key = metadata.lockKeyMetadata(key, CacheMetadata::KeyNotFoundPolicy::CREATE_EMPTY, /* is_initial_load */true); for (fs::directory_iterator offset_it{key_directory}; offset_it != fs::directory_iterator(); ++offset_it) { @@ -946,82 +922,9 @@ void FileCache::loadMetadata() } } -LockedKeyMetadataPtr FileCache::lockKeyMetadata(const Key & key, KeyNotFoundPolicy key_not_found_policy, bool is_initial_load) -{ - KeyMetadataPtr key_metadata; - { - auto lock = metadata.lock(); - - auto it = metadata.find(key); - if (it == metadata.end()) - { - if (key_not_found_policy == KeyNotFoundPolicy::THROW) - throw Exception(ErrorCodes::LOGICAL_ERROR, "No such key `{}` in cache", key.toString()); - else if (key_not_found_policy == KeyNotFoundPolicy::RETURN_NULL) - return nullptr; - - it = metadata.emplace( - key, - std::make_shared(/* base_directory_already_exists */is_initial_load, metadata.getCleanupQueue())).first; - } - - key_metadata = it->second; - } - - { - auto key_lock = key_metadata->lock(); - - const auto cleanup_state = key_metadata->getCleanupState(key_lock); - - if (cleanup_state == KeyMetadata::CleanupState::NOT_SUBMITTED) - { - return std::make_unique(key, key_metadata, std::move(key_lock), getPathInLocalCache(key)); - } - - if (key_not_found_policy == KeyNotFoundPolicy::THROW) - throw Exception(ErrorCodes::LOGICAL_ERROR, "No such key `{}` in cache", key.toString()); - else if (key_not_found_policy == KeyNotFoundPolicy::RETURN_NULL) - return nullptr; - - if (cleanup_state == KeyMetadata::CleanupState::SUBMITTED_TO_CLEANUP_QUEUE) - { - key_metadata->removeFromCleanupQueue(key, key_lock); - return std::make_unique(key, key_metadata, std::move(key_lock), getPathInLocalCache(key)); - } - - chassert(cleanup_state == KeyMetadata::CleanupState::CLEANED_BY_CLEANUP_THREAD); - chassert(key_not_found_policy == KeyNotFoundPolicy::CREATE_EMPTY); - } - - /// Not we are at a case: - /// cleanup_state == KeyMetadata::CleanupState::CLEANED_BY_CLEANUP_THREAD - /// and KeyNotFoundPolicy == CREATE_EMPTY - /// Retry. - return lockKeyMetadata(key, key_not_found_policy); -} - -LockedKeyMetadataPtr FileCache::lockKeyMetadata(const Key & key, KeyMetadataPtr key_metadata) const +LockedKeyMetadataPtr FileCache::lockKeyMetadata(const Key & key, KeyMetadataPtr key_metadata, bool return_null_if_in_cleanup_queue) const { - auto key_lock = key_metadata->lock(); - - if (key_metadata->getCleanupState(key_lock) != KeyMetadata::CleanupState::NOT_SUBMITTED) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot lock key: it was removed from cache"); - - return std::make_unique(key, key_metadata, std::move(key_lock), getPathInLocalCache(key)); -} - -void FileCache::iterateCacheMetadata(const CacheMetadataGuard::Lock &, std::function && func) -{ - FileSegments file_segments; - for (const auto & [key, key_metadata] : metadata) - { - auto key_lock = key_metadata->lock(); - - if (key_metadata->getCleanupState(key_lock) != KeyMetadata::CleanupState::NOT_SUBMITTED) - continue; - - func(*key_metadata); - } + return metadata.lockKeyMetadata(key, key_metadata, return_null_if_in_cleanup_queue); } FileCache::~FileCache() @@ -1066,9 +969,9 @@ FileSegmentsHolderPtr FileCache::getSnapshot() #endif FileSegments file_segments; - iterateCacheMetadata(metadata.lock(), [&](KeyMetadata & key_metadata) + metadata.iterate([&](const LockedKeyMetadata & locked_key) { - for (const auto & [_, file_segment_metadata] : key_metadata) + for (const auto & [_, file_segment_metadata] : *locked_key.getKeyMetadata()) file_segments.push_back(FileSegment::getSnapshot(file_segment_metadata.file_segment)); }); return std::make_unique(std::move(file_segments)); @@ -1077,7 +980,7 @@ FileSegmentsHolderPtr FileCache::getSnapshot() FileSegmentsHolderPtr FileCache::getSnapshot(const Key & key) { FileSegments file_segments; - auto locked_key = lockKeyMetadata(key, KeyNotFoundPolicy::THROW); + auto locked_key = metadata.lockKeyMetadata(key, CacheMetadata::KeyNotFoundPolicy::THROW); for (const auto & [_, file_segment_metadata] : *locked_key->getKeyMetadata()) file_segments.push_back(FileSegment::getSnapshot(file_segment_metadata.file_segment)); return std::make_unique(std::move(file_segments)); @@ -1092,7 +995,7 @@ FileSegmentsHolderPtr FileCache::dumpQueue() FileSegments file_segments; LockedCachePriority(cache_guard.lock(), *main_priority).iterate([&](const QueueEntry & entry) { - auto tx = lockKeyMetadata(entry.key, entry.getKeyMetadata()); + auto tx = metadata.lockKeyMetadata(entry.key, entry.getKeyMetadata()); auto * file_segment_metadata = tx->getKeyMetadata()->getByOffset(entry.offset); file_segments.push_back(FileSegment::getSnapshot(file_segment_metadata->file_segment)); return IterationResult::CONTINUE; @@ -1105,7 +1008,7 @@ std::vector FileCache::tryGetCachePaths(const Key & key) { assertInitialized(); - auto locked_key = lockKeyMetadata(key, KeyNotFoundPolicy::RETURN_NULL); + auto locked_key = metadata.lockKeyMetadata(key, CacheMetadata::KeyNotFoundPolicy::RETURN_NULL); if (!locked_key) return {}; @@ -1114,7 +1017,7 @@ std::vector FileCache::tryGetCachePaths(const Key & key) for (const auto & [offset, file_segment_metadata] : *locked_key->getKeyMetadata()) { if (file_segment_metadata.file_segment->state() == FileSegment::State::DOWNLOADED) - cache_paths.push_back(getPathInLocalCache(key, offset, file_segment_metadata.file_segment->getKind())); + cache_paths.push_back(metadata.getPathInLocalCache(key, offset, file_segment_metadata.file_segment->getKind())); } return cache_paths; } @@ -1131,11 +1034,12 @@ size_t FileCache::getFileSegmentsNum() const void FileCache::assertCacheCorrectness() { - auto lock = metadata.lock(); - iterateCacheMetadata(lock, [&](KeyMetadata & key_metadata) + metadata.iterate([&](const LockedKeyMetadata & locked_key) { - for (auto & [offset, file_segment_metadata] : key_metadata) - file_segment_metadata.file_segment->assertCorrectness(); + for (auto & [offset, file_segment_metadata] : *locked_key.getKeyMetadata()) + { + locked_key.assertFileSegmentCorrectness(*file_segment_metadata.file_segment); + } }); } diff --git a/src/Interpreters/Cache/FileCache.h b/src/Interpreters/Cache/FileCache.h index 050d5d88f375..b1d7fa548df7 100644 --- a/src/Interpreters/Cache/FileCache.h +++ b/src/Interpreters/Cache/FileCache.h @@ -90,8 +90,6 @@ class FileCache : private boost::noncopyable String getPathInLocalCache(const Key & key) const; - static String getPathInLocalCache(const std::string & base_directory, const Key & key); - std::vector tryGetCachePaths(const Key & key); size_t getUsedCacheSize() const; @@ -112,12 +110,12 @@ class FileCache : private boost::noncopyable CacheGuard::Lock cacheLock() { return cache_guard.lock(); } - LockedKeyMetadataPtr lockKeyMetadata(const Key & key, KeyMetadataPtr key_metadata) const; - void cleanup(); void deactivateBackgroundOperations(); + LockedKeyMetadataPtr lockKeyMetadata(const Key & key, KeyMetadataPtr key_metadata, bool return_null_if_in_cleanup_queue = true) const; + /// For per query cache limit. struct QueryContextHolder : private boost::noncopyable { @@ -151,15 +149,6 @@ class FileCache : private boost::noncopyable CacheMetadata metadata; - enum class KeyNotFoundPolicy - { - THROW, - CREATE_EMPTY, - RETURN_NULL, - }; - - LockedKeyMetadataPtr lockKeyMetadata(const Key & key, KeyNotFoundPolicy key_not_found_policy, bool is_initial_load = false); - FileCachePriorityPtr main_priority; mutable CacheGuard cache_guard; @@ -235,9 +224,6 @@ class FileCache : private boost::noncopyable IterateAndCollectLocksFunc && func, LockedKeyMetadataMap & locked_map) const; - void iterateCacheMetadata( - const CacheMetadataGuard::Lock &, std::function && func); - void assertCacheCorrectness(); BackgroundSchedulePool::TaskHolder cleanup_task; diff --git a/src/Interpreters/Cache/FileSegment.cpp b/src/Interpreters/Cache/FileSegment.cpp index 580f4cfa670a..5e8c2b62f58e 100644 --- a/src/Interpreters/Cache/FileSegment.cpp +++ b/src/Interpreters/Cache/FileSegment.cpp @@ -692,26 +692,24 @@ String FileSegment::stateToString(FileSegment::State state) bool FileSegment::assertCorrectness() const { - auto lock = segment_guard.lock(); - assertCorrectnessUnlocked(lock); + auto key_lock = lockKeyMetadata(true); + key_lock->assertFileSegmentCorrectness(*this); return true; } -bool FileSegment::assertCorrectnessUnlocked(const FileSegmentGuard::Lock & lock) const +bool FileSegment::assertCorrectnessUnlocked(const FileSegmentMetadata & metadata, const KeyGuard::Lock &) const { + auto lock = segment_guard.lock(); + auto current_downloader = getDownloaderUnlocked(lock); chassert(current_downloader.empty() == (download_state != FileSegment::State::DOWNLOADING)); chassert(!current_downloader.empty() == (download_state == FileSegment::State::DOWNLOADING)); chassert(download_state != FileSegment::State::DOWNLOADED || std::filesystem::file_size(file_path) > 0); - auto locked_key = lockKeyMetadata(true); - - const auto & file_segment_metadata = locked_key->getKeyMetadata()->tryGetByOffset(offset()); - chassert(reserved_size == 0 || file_segment_metadata->queue_iterator); - - if (file_segment_metadata->queue_iterator) + chassert(reserved_size == 0 || metadata.queue_iterator); + if (metadata.queue_iterator) { - const auto & entry = *file_segment_metadata->queue_iterator; + const auto & entry = *metadata.queue_iterator; if (isCompleted(false)) chassert(reserved_size == entry.getEntry().size); else diff --git a/src/Interpreters/Cache/FileSegment.h b/src/Interpreters/Cache/FileSegment.h index fcba974ef878..4ccbf3c10abc 100644 --- a/src/Interpreters/Cache/FileSegment.h +++ b/src/Interpreters/Cache/FileSegment.h @@ -30,6 +30,7 @@ class ReadBufferFromFileBase; class FileSegment; using FileSegmentPtr = std::shared_ptr; using FileSegments = std::list; +struct FileSegmentMetadata; struct LockedKeyMetadata; using LockedKeyMetadataPtr = std::shared_ptr; struct KeyMetadata; @@ -224,6 +225,8 @@ friend struct LockedKeyMetadata; bool assertCorrectness() const; + bool assertCorrectnessUnlocked(const FileSegmentMetadata & metadata, const KeyGuard::Lock &) const; + /** * ========== Methods for _only_ file segment's `downloader` ================== */ @@ -282,7 +285,6 @@ friend struct LockedKeyMetadata; void assertNotDetached() const; void assertNotDetachedUnlocked(const FileSegmentGuard::Lock &) const; void assertIsDownloaderUnlocked(const std::string & operation, const FileSegmentGuard::Lock &) const; - bool assertCorrectnessUnlocked(const FileSegmentGuard::Lock &) const; LockedKeyMetadataPtr lockKeyMetadata(bool assert_exists = true) const; KeyMetadataPtr getKeyMetadata() const; diff --git a/src/Interpreters/Cache/Metadata.cpp b/src/Interpreters/Cache/Metadata.cpp index 4f6af172253c..07f5ff1f1291 100644 --- a/src/Interpreters/Cache/Metadata.cpp +++ b/src/Interpreters/Cache/Metadata.cpp @@ -116,6 +116,120 @@ void KeyMetadata::removeFromCleanupQueue(const FileCacheKey & key, const KeyGuar cleanup_state = CleanupState::NOT_SUBMITTED; } +String CacheMetadata::getPathInLocalCache(const Key & key, size_t offset, FileSegmentKind segment_kind) const +{ + String file_suffix; + switch (segment_kind) + { + case FileSegmentKind::Persistent: + file_suffix = "_persistent"; + break; + case FileSegmentKind::Temporary: + file_suffix = "_temporary"; + break; + case FileSegmentKind::Regular: + file_suffix = ""; + break; + } + + auto key_str = key.toString(); + return fs::path(base_directory) + / key_str.substr(0, 3) + / key_str + / (std::to_string(offset) + file_suffix); +} + +String CacheMetadata::getPathInLocalCache(const Key & key) const +{ + auto key_str = key.toString(); + return fs::path(base_directory) / key_str.substr(0, 3) / key_str; +} + +LockedKeyMetadataPtr CacheMetadata::lockKeyMetadata(const FileCacheKey & key, KeyNotFoundPolicy key_not_found_policy, bool is_initial_load) +{ + KeyMetadataPtr key_metadata; + { + auto lock = guard.lock(); + + auto it = find(key); + if (it == end()) + { + if (key_not_found_policy == KeyNotFoundPolicy::THROW) + throw Exception(ErrorCodes::LOGICAL_ERROR, "No such key `{}` in cache", key.toString()); + else if (key_not_found_policy == KeyNotFoundPolicy::RETURN_NULL) + return nullptr; + + it = emplace( + key, + std::make_shared(/* base_directory_already_exists */is_initial_load, cleanup_queue)).first; + } + + key_metadata = it->second; + } + + { + auto key_lock = key_metadata->lock(); + + const auto cleanup_state = key_metadata->getCleanupState(key_lock); + + if (cleanup_state == KeyMetadata::CleanupState::NOT_SUBMITTED) + { + return std::make_unique(key, key_metadata, std::move(key_lock), getPathInLocalCache(key)); + } + + if (key_not_found_policy == KeyNotFoundPolicy::THROW) + throw Exception(ErrorCodes::LOGICAL_ERROR, "No such key `{}` in cache", key.toString()); + else if (key_not_found_policy == KeyNotFoundPolicy::RETURN_NULL) + return nullptr; + + if (cleanup_state == KeyMetadata::CleanupState::SUBMITTED_TO_CLEANUP_QUEUE) + { + key_metadata->removeFromCleanupQueue(key, key_lock); + return std::make_unique(key, key_metadata, std::move(key_lock), getPathInLocalCache(key)); + } + + chassert(cleanup_state == KeyMetadata::CleanupState::CLEANED_BY_CLEANUP_THREAD); + chassert(key_not_found_policy == KeyNotFoundPolicy::CREATE_EMPTY); + } + + /// Not we are at a case: + /// cleanup_state == KeyMetadata::CleanupState::CLEANED_BY_CLEANUP_THREAD + /// and KeyNotFoundPolicy == CREATE_EMPTY + /// Retry. + return lockKeyMetadata(key, key_not_found_policy); +} + +LockedKeyMetadataPtr CacheMetadata::lockKeyMetadata( + const FileCacheKey & key, KeyMetadataPtr key_metadata, bool return_null_if_in_cleanup_queue) const +{ + auto key_lock = key_metadata->lock(); + + auto cleanup_state = key_metadata->getCleanupState(key_lock); + if (cleanup_state != KeyMetadata::CleanupState::NOT_SUBMITTED) + { + if (return_null_if_in_cleanup_queue + && cleanup_state == KeyMetadata::CleanupState::SUBMITTED_TO_CLEANUP_QUEUE) + return nullptr; + + throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot lock key: it was removed from cache"); + } + + return std::make_unique(key, key_metadata, std::move(key_lock), getPathInLocalCache(key)); +} + +void CacheMetadata::iterate(IterateCacheMetadataFunc && func) +{ + auto lock = guard.lock(); + for (const auto & [key, key_metadata] : *this) + { + auto locked_key = lockKeyMetadata(key, key_metadata, /* return_null_if_in_cleanup_queue */true); + if (!locked_key) + continue; + + func(*locked_key); + } +} + void CacheMetadata::doCleanup() { auto lock = guard.lock(); @@ -136,6 +250,7 @@ void CacheMetadata::doCleanup() auto key_metadata = it->second; auto key_lock = key_metadata->lock(); + /// As in lockKeyMetadata we extract key metadata from cache metadata under /// CacheMetadataGuard::Lock, but take KeyGuard::Lock only after we released /// cache CacheMetadataGuard::Lock (because CacheMetadataGuard::Lock must be lightweight). @@ -144,11 +259,12 @@ void CacheMetadata::doCleanup() if (key_metadata->getCleanupState(key_lock) == KeyMetadata::CleanupState::NOT_SUBMITTED) continue; + key_metadata->cleanup_state = KeyMetadata::CleanupState::CLEANED_BY_CLEANUP_THREAD; erase(it); try { - const fs::path key_directory = FileCache::getPathInLocalCache(base_directory, cleanup_key); + const fs::path key_directory = getPathInLocalCache(cleanup_key); if (fs::exists(key_directory)) fs::remove_all(key_directory); @@ -263,6 +379,15 @@ void LockedKeyMetadata::shrinkFileSegmentToDownloadedSize( assert(file_segment_metadata->size() == entry.size); } +void LockedKeyMetadata::assertFileSegmentCorrectness(const FileSegment & file_segment) const +{ + if (file_segment.key() != key) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Expected {} = {}", file_segment.key().toString(), key.toString()); + + const auto & file_segment_metadata = *key_metadata->getByOffset(file_segment.offset()); + file_segment.assertCorrectnessUnlocked(file_segment_metadata, lock); +} + void CleanupQueue::add(const FileCacheKey & key) { std::lock_guard lock(mutex); diff --git a/src/Interpreters/Cache/Metadata.h b/src/Interpreters/Cache/Metadata.h index d2903d194efb..3e37437c97d4 100644 --- a/src/Interpreters/Cache/Metadata.h +++ b/src/Interpreters/Cache/Metadata.h @@ -3,12 +3,14 @@ #include #include #include +#include namespace DB { class FileSegment; using FileSegmentPtr = std::shared_ptr; struct LockedKeyMetadata; +using LockedKeyMetadataPtr = std::shared_ptr; class LockedCachePriority; struct KeysQueue; struct CleanupQueue; @@ -40,6 +42,7 @@ struct FileSegmentMetadata : private boost::noncopyable struct KeyMetadata : public std::map, private boost::noncopyable { friend struct LockedKeyMetadata; + friend struct CacheMetadata; public: explicit KeyMetadata(bool created_base_directory_, CleanupQueue & cleanup_queue_) : created_base_directory(created_base_directory_), cleanup_queue(cleanup_queue_) {} @@ -96,17 +99,32 @@ struct CleanupQueue struct CacheMetadata : public std::unordered_map, private boost::noncopyable { public: + using Key = FileCacheKey; + explicit CacheMetadata(const std::string & base_directory_) : base_directory(base_directory_) {} - CacheMetadataGuard::Lock lock() { return guard.lock(); } + String getPathInLocalCache(const Key & key, size_t offset, FileSegmentKind segment_kind) const; - void doCleanup(); + String getPathInLocalCache(const Key & key) const; + + enum class KeyNotFoundPolicy + { + THROW, + CREATE_EMPTY, + RETURN_NULL, + }; + LockedKeyMetadataPtr lockKeyMetadata(const Key & key, KeyNotFoundPolicy key_not_found_policy, bool is_initial_load = false); + + LockedKeyMetadataPtr lockKeyMetadata(const Key & key, KeyMetadataPtr key_metadata, bool return_null_if_in_cleanup_queue = false) const; - CleanupQueue & getCleanupQueue() { return cleanup_queue; } + using IterateCacheMetadataFunc = std::function; + void iterate(IterateCacheMetadataFunc && func); + + void doCleanup(); private: - void addToCleanupQueue(const FileCacheKey & key, const KeyGuard::Lock &); - void removeFromCleanupQueue(const FileCacheKey & key, const KeyGuard::Lock &); + void addToCleanupQueue(const Key & key, const KeyGuard::Lock &); + void removeFromCleanupQueue(const Key & key, const KeyGuard::Lock &); const std::string base_directory; CacheMetadataGuard guard; @@ -148,6 +166,10 @@ struct LockedKeyMetadata : private boost::noncopyable bool isLastOwnerOfFileSegment(size_t offset) const; + void assertFileSegmentCorrectness(const FileSegment & file_segment) const; + + KeyMetadata::CleanupState getCleanupState() const { return key_metadata->cleanup_state; } + private: const FileCacheKey key; const std::string key_path; From d94d08c18d51c2971f3bdcea114e663f56d09c8a Mon Sep 17 00:00:00 2001 From: kssenii Date: Thu, 30 Mar 2023 14:21:13 +0200 Subject: [PATCH 095/406] Fix tests --- .../IO/CachedOnDiskReadBufferFromFile.cpp | 22 ++++++++++--------- src/IO/ReadBufferFromS3.cpp | 10 +++++---- src/Interpreters/Cache/FileCache.cpp | 4 ++-- src/Interpreters/Cache/FileCache_fwd.h | 2 +- src/Interpreters/Cache/FileSegment.cpp | 17 +++++++++++--- src/Interpreters/Cache/FileSegment.h | 2 +- src/Interpreters/Cache/Metadata.cpp | 13 ++++------- .../tests/gtest_lru_file_cache.cpp | 4 ++-- .../02344_describe_cache.reference | 4 ++-- 9 files changed, 44 insertions(+), 34 deletions(-) diff --git a/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp b/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp index 8be231cf3155..066d33364865 100644 --- a/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp +++ b/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp @@ -278,7 +278,7 @@ CachedOnDiskReadBufferFromFile::getReadBufferForFileSegment(FileSegment & file_s return getCacheReadBuffer(file_segment); } - download_state = file_segment.wait(); + download_state = file_segment.wait(file_offset_of_buffer_end); continue; } case FileSegment::State::DOWNLOADED: @@ -390,10 +390,11 @@ CachedOnDiskReadBufferFromFile::getImplementationBuffer(FileSegment & file_segme LOG_TEST( log, - "Current file segment: {}, read type: {}, current file offset: {}", - range.toString(), + "Current read type: {}, read offset: {}, impl read range: {}, file segment: {}", toString(read_type), - file_offset_of_buffer_end); + file_offset_of_buffer_end, + read_buffer_for_file_segment->getRemainingReadRange().toString(), + file_segment.getInfoForLog()); read_buffer_for_file_segment->setReadUntilPosition(range.right + 1); /// [..., range.right] @@ -439,7 +440,7 @@ CachedOnDiskReadBufferFromFile::getImplementationBuffer(FileSegment & file_segme if (bytes_to_predownload) { - size_t current_write_offset = file_segment.getCurrentWriteOffset(false); + const size_t current_write_offset = file_segment.getCurrentWriteOffset(false); read_buffer_for_file_segment->seek(current_write_offset, SEEK_SET); } else @@ -450,7 +451,7 @@ CachedOnDiskReadBufferFromFile::getImplementationBuffer(FileSegment & file_segme assert(static_cast(read_buffer_for_file_segment->getFileOffsetOfBufferEnd()) == file_offset_of_buffer_end); } - auto current_write_offset = file_segment.getCurrentWriteOffset(false); + const auto current_write_offset = file_segment.getCurrentWriteOffset(false); if (current_write_offset != static_cast(read_buffer_for_file_segment->getPosition())) { throw Exception( @@ -830,9 +831,9 @@ bool CachedOnDiskReadBufferFromFile::nextImplStep() LOG_TEST( log, - "Current count: {}, position: {}, read range: {}, file segment: {}", - implementation_buffer->count(), - implementation_buffer->getPosition(), + "Current read type: {}, read offset: {}, impl read range: {}, file segment: {}", + toString(read_type), + file_offset_of_buffer_end, implementation_buffer->getRemainingReadRange().toString(), file_segment.getInfoForLog()); @@ -980,7 +981,7 @@ bool CachedOnDiskReadBufferFromFile::nextImplStep() LOG_TEST( log, "Key: {}. Returning with {} bytes, buffer position: {} (offset: {}, predownloaded: {}), " - "buffer available: {}, current range: {}, current offset: {}, file segment state: {}, " + "buffer available: {}, current range: {}, file offset of buffer end: {}, impl offset: {}, file segment state: {}, " "current write offset: {}, read_type: {}, reading until position: {}, started with offset: {}, " "remaining ranges: {}", cache_key.toString(), @@ -991,6 +992,7 @@ bool CachedOnDiskReadBufferFromFile::nextImplStep() available(), current_read_range.toString(), file_offset_of_buffer_end, + implementation_buffer->getFileOffsetOfBufferEnd(), FileSegment::stateToString(file_segment.state()), file_segment.getCurrentWriteOffset(false), toString(read_type), diff --git a/src/IO/ReadBufferFromS3.cpp b/src/IO/ReadBufferFromS3.cpp index 91905330b74e..e1458e3cb30b 100644 --- a/src/IO/ReadBufferFromS3.cpp +++ b/src/IO/ReadBufferFromS3.cpp @@ -201,11 +201,13 @@ off_t ReadBufferFromS3::seek(off_t offset_, int whence) return offset; if (impl && restricted_seek) + { throw Exception( - ErrorCodes::CANNOT_SEEK_THROUGH_FILE, - "Seek is allowed only before first read attempt from the buffer (current offset: " - "{}, new offset: {}, reading until position: {}, available: {})", - offset, offset_, read_until_position, available()); + ErrorCodes::LOGICAL_ERROR, + "Seek is allowed only before first read attempt from the buffer (current offset: " + "{}, new offset: {}, reading until position: {}, available: {})", + offset, offset_, read_until_position, available()); + } if (whence != SEEK_SET) throw Exception(ErrorCodes::CANNOT_SEEK_THROUGH_FILE, "Only SEEK_SET mode is allowed."); diff --git a/src/Interpreters/Cache/FileCache.cpp b/src/Interpreters/Cache/FileCache.cpp index 7676b005956f..5441bdc0d0a9 100644 --- a/src/Interpreters/Cache/FileCache.cpp +++ b/src/Interpreters/Cache/FileCache.cpp @@ -602,12 +602,12 @@ bool FileCache::tryReserveImpl( auto * file_segment_metadata = current_locked_key.getKeyMetadata()->getByOffset(entry.offset); chassert(file_segment_metadata->queue_iterator); - chassert(entry.size == file_segment_metadata->size()); + chassert((entry.size == file_segment_metadata->size()) || (file_segment_metadata->file_segment->state() == FileSegment::State::DOWNLOADING)); /// It is guaranteed that file_segment_metadata is not removed from cache as long as /// pointer to corresponding file segment is hold by any other thread. - const size_t file_segment_size = file_segment_metadata->size(); + const size_t file_segment_size = entry.size; bool remove_current_it = false; bool save_locked_key = false; diff --git a/src/Interpreters/Cache/FileCache_fwd.h b/src/Interpreters/Cache/FileCache_fwd.h index d14f042f93d1..47166ffd602c 100644 --- a/src/Interpreters/Cache/FileCache_fwd.h +++ b/src/Interpreters/Cache/FileCache_fwd.h @@ -8,7 +8,7 @@ static constexpr int FILECACHE_DEFAULT_MAX_FILE_SEGMENT_SIZE = 100 * 1024 * 1024 static constexpr int FILECACHE_DEFAULT_MAX_ELEMENTS = 1024 * 1024; static constexpr int FILECACHE_DEFAULT_HITS_THRESHOLD = 0; static constexpr size_t FILECACHE_BYPASS_THRESHOLD = 256 * 1024 * 1024; -static constexpr size_t FILECACHE_DELAYED_CLEANUP_INTERVAL_MS = 1000 * 60 * 10; /// 10 min +static constexpr size_t FILECACHE_DELAYED_CLEANUP_INTERVAL_MS = 1000 * 60 * 4; /// 4 min class FileCache; using FileCachePtr = std::shared_ptr; diff --git a/src/Interpreters/Cache/FileSegment.cpp b/src/Interpreters/Cache/FileSegment.cpp index 5e8c2b62f58e..a5ff0d358d7f 100644 --- a/src/Interpreters/Cache/FileSegment.cpp +++ b/src/Interpreters/Cache/FileSegment.cpp @@ -203,6 +203,7 @@ void FileSegment::resetDownloaderUnlocked(const FileSegmentGuard::Lock &) { LOG_TEST(log, "Resetting downloader from {}", downloader_id); downloader_id.clear(); + cv.notify_all(); } void FileSegment::assertIsDownloaderUnlocked(const std::string & operation, const FileSegmentGuard::Lock & lock) const @@ -356,7 +357,7 @@ void FileSegment::write(const char * from, size_t size, size_t offset) chassert(getFirstNonDownloadedOffset(false) == offset + size); } -FileSegment::State FileSegment::wait() +FileSegment::State FileSegment::wait(size_t offset) { auto lock = segment_guard.lock(); @@ -373,7 +374,13 @@ FileSegment::State FileSegment::wait() chassert(!getDownloaderUnlocked(lock).empty()); chassert(!isDownloaderUnlocked(lock)); - cv.wait_for(lock, std::chrono::seconds(60), [this]() { return download_state != State::DOWNLOADING; }); + [[maybe_unused]] const bool ok = cv.wait_for(lock, std::chrono::seconds(60), [&, this]() + { + return download_state != State::DOWNLOADING || offset < getCurrentWriteOffset(true); + }); + /// If we exited by timeout, it means we are missing notify somewhere. + /// Make sure this case is caught by stress tests. + chassert(ok); } return download_state; @@ -519,15 +526,19 @@ void FileSegment::completePartAndResetDownloaderUnlocked(const FileSegmentGuard: resetDownloaderUnlocked(lock); LOG_TEST(log, "Complete batch. ({})", getInfoForLogUnlocked(lock)); - cv.notify_all(); } void FileSegment::setBroken() { auto lock = segment_guard.lock(); + assertNotDetachedUnlocked(lock); assertIsDownloaderUnlocked("setBroken", lock); + resetDownloadingStateUnlocked(lock); + if (download_state != State::DOWNLOADED) + download_state = State::PARTIALLY_DOWNLOADED_NO_CONTINUATION; + resetDownloaderUnlocked(lock); } diff --git a/src/Interpreters/Cache/FileSegment.h b/src/Interpreters/Cache/FileSegment.h index 4ccbf3c10abc..6e6ac12fdb66 100644 --- a/src/Interpreters/Cache/FileSegment.h +++ b/src/Interpreters/Cache/FileSegment.h @@ -183,7 +183,7 @@ friend struct LockedKeyMetadata; DownloaderId getDownloader() const; /// Wait for the change of state from DOWNLOADING to any other. - State wait(); + State wait(size_t offset); bool isDownloaded() const; diff --git a/src/Interpreters/Cache/Metadata.cpp b/src/Interpreters/Cache/Metadata.cpp index 07f5ff1f1291..346eaf2641cd 100644 --- a/src/Interpreters/Cache/Metadata.cpp +++ b/src/Interpreters/Cache/Metadata.cpp @@ -110,9 +110,9 @@ void KeyMetadata::addToCleanupQueue(const FileCacheKey & key, const KeyGuard::Lo cleanup_state = CleanupState::SUBMITTED_TO_CLEANUP_QUEUE; } -void KeyMetadata::removeFromCleanupQueue(const FileCacheKey & key, const KeyGuard::Lock &) +void KeyMetadata::removeFromCleanupQueue(const FileCacheKey &, const KeyGuard::Lock &) { - cleanup_queue.remove(key); + /// Just mark cleanup_state as "not to be removed", the cleanup thread will check it and skip the key. cleanup_state = CleanupState::NOT_SUBMITTED; } @@ -246,7 +246,7 @@ void CacheMetadata::doCleanup() { auto it = find(cleanup_key); if (it == end()) - throw Exception(ErrorCodes::LOGICAL_ERROR, "No such key {} in metadata", cleanup_key.toString()); + continue; auto key_metadata = it->second; auto key_lock = key_metadata->lock(); @@ -391,12 +391,7 @@ void LockedKeyMetadata::assertFileSegmentCorrectness(const FileSegment & file_se void CleanupQueue::add(const FileCacheKey & key) { std::lock_guard lock(mutex); - auto [_, inserted] = keys.insert(key); - if (!inserted) - { - throw Exception( - ErrorCodes::LOGICAL_ERROR, "Key {} is already in removal queue", key.toString()); - } + keys.insert(key); } void CleanupQueue::remove(const FileCacheKey & key) diff --git a/src/Interpreters/tests/gtest_lru_file_cache.cpp b/src/Interpreters/tests/gtest_lru_file_cache.cpp index 086c7a57ef90..03a5854f904f 100644 --- a/src/Interpreters/tests/gtest_lru_file_cache.cpp +++ b/src/Interpreters/tests/gtest_lru_file_cache.cpp @@ -416,7 +416,7 @@ TEST_F(FileCacheTest, get) } cv.notify_one(); - file_segment2.wait(); + file_segment2.wait(file_segment2.range().left); ASSERT_TRUE(file_segment2.state() == State::DOWNLOADED); }); @@ -479,7 +479,7 @@ TEST_F(FileCacheTest, get) } cv.notify_one(); - file_segment2.wait(); + file_segment2.wait(file_segment2.range().left); ASSERT_TRUE(file_segment2.state() == DB::FileSegment::State::PARTIALLY_DOWNLOADED); ASSERT_TRUE(file_segment2.getOrSetDownloader() == DB::FileSegment::getCallerId()); download(file_segment2); diff --git a/tests/queries/0_stateless/02344_describe_cache.reference b/tests/queries/0_stateless/02344_describe_cache.reference index d3bb37af5cfa..7cda2af34670 100644 --- a/tests/queries/0_stateless/02344_describe_cache.reference +++ b/tests/queries/0_stateless/02344_describe_cache.reference @@ -1,2 +1,2 @@ -2147483648 1048576 104857600 1 0 0 0 s3_cache/ 0 -2147483648 1048576 104857600 0 0 0 0 s3_cache_2/ 0 +134217728 1048576 104857600 1 0 0 0 s3_cache/ 0 +134217728 1048576 104857600 0 0 0 0 s3_cache_2/ 0 From 3b8869a8d09f8ea08d64c96cccb76291e1c65d76 Mon Sep 17 00:00:00 2001 From: kssenii Date: Thu, 30 Mar 2023 17:32:14 +0200 Subject: [PATCH 096/406] Ping CI From 54df42136ba2945c827e4bb98d709e59f29f88e4 Mon Sep 17 00:00:00 2001 From: kssenii Date: Sat, 1 Apr 2023 17:53:03 +0200 Subject: [PATCH 097/406] Rewrite space reservation so that at most one key lock is taken at a time --- src/Interpreters/Cache/FileCache.cpp | 257 ++++++++++--------------- src/Interpreters/Cache/FileCache.h | 14 +- src/Interpreters/Cache/FileSegment.cpp | 46 ++--- src/Interpreters/Cache/FileSegment.h | 12 +- src/Interpreters/Cache/Metadata.cpp | 227 ++++++++++------------ src/Interpreters/Cache/Metadata.h | 89 +++++---- 6 files changed, 293 insertions(+), 352 deletions(-) diff --git a/src/Interpreters/Cache/FileCache.cpp b/src/Interpreters/Cache/FileCache.cpp index 5441bdc0d0a9..9182e3887bfa 100644 --- a/src/Interpreters/Cache/FileCache.cpp +++ b/src/Interpreters/Cache/FileCache.cpp @@ -102,7 +102,7 @@ FileSegments FileCache::getImpl(const LockedKeyMetadata & locked_key, const File if (bypass_cache_threshold && range.size() > bypass_cache_threshold) { auto file_segment = std::make_shared( - range.left, range.size(), locked_key.getKey(), std::weak_ptr(), this, + locked_key.getKey(), range.left, range.size(), std::weak_ptr(), nullptr, this, FileSegment::State::DETACHED, CreateFileSegmentSettings{}); return { file_segment }; } @@ -112,16 +112,16 @@ FileSegments FileCache::getImpl(const LockedKeyMetadata & locked_key, const File return {}; FileSegments result; - auto add_to_result = [&](const FileSegmentMetadata & file_segment_metadata) + auto add_to_result = [&](const FileSegmentPtr & file_segment) { - if (file_segment_metadata.file_segment->isDownloaded()) + if (file_segment->isDownloaded()) { - if (file_segment_metadata.file_segment->getDownloadedSize(true) == 0) + if (file_segment->getDownloadedSize(true) == 0) { throw Exception( ErrorCodes::LOGICAL_ERROR, "Cannot have zero size downloaded file segments. {}", - file_segment_metadata.file_segment->getInfoForLog()); + file_segment->getInfoForLog()); } #ifndef NDEBUG @@ -131,13 +131,13 @@ FileSegments FileCache::getImpl(const LockedKeyMetadata & locked_key, const File * expensive compared to overall query execution time. */ - fs::path path = file_segment_metadata.file_segment->getPathInLocalCache(); + fs::path path = file_segment->getPathInLocalCache(); if (!fs::exists(path)) { throw Exception( ErrorCodes::LOGICAL_ERROR, "File path does not exist, but file has DOWNLOADED state. {}", - file_segment_metadata.file_segment->getInfoForLog()); + file_segment->getInfoForLog()); } if (fs::file_size(path) == 0) @@ -145,12 +145,12 @@ FileSegments FileCache::getImpl(const LockedKeyMetadata & locked_key, const File throw Exception( ErrorCodes::LOGICAL_ERROR, "Cannot have zero size downloaded file segments. {}", - file_segment_metadata.file_segment->getInfoForLog()); + file_segment->getInfoForLog()); } #endif } - result.push_back(file_segment_metadata.file_segment); + result.push_back(file_segment); }; auto segment_it = file_segments.lower_bound(range.left); @@ -167,7 +167,7 @@ FileSegments FileCache::getImpl(const LockedKeyMetadata & locked_key, const File if (file_segment_metadata.file_segment->range().right < range.left) return {}; - add_to_result(file_segment_metadata); + add_to_result(file_segment_metadata.file_segment); } else /// segment_it <-- segmment{k} { @@ -183,7 +183,7 @@ FileSegments FileCache::getImpl(const LockedKeyMetadata & locked_key, const File /// [___________ /// ^ /// range.left - add_to_result(prev_file_segment_metadata); + add_to_result(prev_file_segment_metadata.file_segment); } } @@ -199,7 +199,7 @@ FileSegments FileCache::getImpl(const LockedKeyMetadata & locked_key, const File if (range.right < file_segment_metadata.file_segment->range().left) break; - add_to_result(file_segment_metadata); + add_to_result(file_segment_metadata.file_segment); ++segment_it; } } @@ -292,8 +292,8 @@ void FileCache::fillHolesWithEmptyFileSegments( if (fill_with_detached_file_segments) { auto file_segment = std::make_shared( - current_pos, hole_size, locked_key.getKey(), std::weak_ptr(), - this, FileSegment::State::DETACHED, settings); + locked_key.getKey(), current_pos, hole_size, std::weak_ptr(), + nullptr, this, FileSegment::State::DETACHED, settings); file_segments.insert(it, file_segment); } @@ -319,7 +319,8 @@ void FileCache::fillHolesWithEmptyFileSegments( if (fill_with_detached_file_segments) { auto file_segment = std::make_shared( - current_pos, hole_size, locked_key.getKey(), std::weak_ptr() , this, FileSegment::State::DETACHED, settings); + locked_key.getKey(), current_pos, hole_size, std::weak_ptr(), + nullptr, this, FileSegment::State::DETACHED, settings); file_segments.insert(file_segments.end(), file_segment); } @@ -404,7 +405,8 @@ FileSegmentsHolderPtr FileCache::get(const Key & key, size_t offset, size_t size } auto file_segment = std::make_shared( - offset, size, key, std::weak_ptr(), this, FileSegment::State::DETACHED, CreateFileSegmentSettings{}); + key, offset, size, std::weak_ptr(), nullptr, + this, FileSegment::State::DETACHED, CreateFileSegmentSettings{}); return std::make_unique(FileSegments{file_segment}); } @@ -422,10 +424,7 @@ KeyMetadata::iterator FileCache::addFileSegment( chassert(size > 0); /// Empty file segments in cache are not allowed. const auto & key = locked_key.getKey(); - auto key_metadata = locked_key.getKeyMetadata(); - - auto it = key_metadata->find(offset); - if (it != key_metadata->end()) + if (locked_key.tryGetByOffset(offset)) { throw Exception( ErrorCodes::LOGICAL_ERROR, @@ -450,7 +449,7 @@ KeyMetadata::iterator FileCache::addFileSegment( auto stash_queue = LockedCachePriority(*lock, *stash->queue); auto & stash_records = stash->records; - stash_records.emplace(stash_key, stash_queue.add(key, offset, 0, key_metadata)); + stash_records.emplace(stash_key, stash_queue.add(key, offset, 0, locked_key.getKeyMetadata())); if (stash_queue.getElementsCount() > stash->queue->getElementsLimit()) stash_queue.pop(); @@ -469,26 +468,36 @@ KeyMetadata::iterator FileCache::addFileSegment( result_state = state; } - auto file_segment = std::make_shared(offset, size, key, key_metadata, this, result_state, settings); + auto file_segment = std::make_shared(key, offset, size, locked_key.getKeyMetadata(), nullptr, this, result_state, settings); std::optional locked_queue( lock ? LockedCachePriority(*lock, *main_priority) : std::optional{}); - FileSegmentMetadata file_segment_metadata(std::move(file_segment), locked_key, locked_queue ? &*locked_queue : nullptr); + auto [file_segment_metadata_it, inserted] = locked_key.getKeyMetadata()->emplace( + std::piecewise_construct, + std::forward_as_tuple(offset), + std::forward_as_tuple(std::move(file_segment), locked_key, locked_queue ? &*locked_queue : nullptr)); - auto [file_segment_metadata_it, inserted] = key_metadata->emplace(offset, std::move(file_segment_metadata)); assert(inserted); return file_segment_metadata_it; } -bool FileCache::tryReserve(const Key & key, size_t offset, size_t size, KeyMetadataPtr key_metadata) +void FileCache::removeFileSegment(LockedKeyMetadata & locked_key, FileSegmentPtr file_segment, const CacheGuard::Lock & cache_lock) +{ + /// We must hold pointer to file segment while removing it + /// (because we remove file segment under file segment lock). + + chassert(file_segment->key() == locked_key.getKey()); + locked_key.removeFileSegment(file_segment->offset(), file_segment->lock(), cache_lock); +} + +bool FileCache::tryReserve(const FileSegment & file_segment, size_t size) { assertInitialized(); auto lock = cache_guard.lock(); - auto locked_key = metadata.lockKeyMetadata(key, key_metadata); - LOG_TEST(log, "Reserving {} byts for key {} at offset {}", size, key.toString(), offset); + LOG_TEST(log, "Reserving {} byts for key {} at offset {}", size, file_segment.key().toString(), file_segment.offset()); auto query_context = query_limit ? query_limit->tryGetQueryContext(lock) : nullptr; bool reserved; @@ -497,62 +506,27 @@ bool FileCache::tryReserve(const Key & key, size_t offset, size_t size, KeyMetad { const bool query_limit_exceeded = query_context->getSize() + size > query_context->getSizeLimit(); reserved = (!query_limit_exceeded || query_context->recacheOnFileCacheQueryLimitExceeded()) - && tryReserveImpl(query_context->getPriority(), locked_key, offset, size, query_context.get(), lock); + && tryReserveImpl(file_segment, size, query_context->getPriority(), query_context.get(), lock); } else { - reserved = tryReserveImpl(*main_priority, locked_key, offset, size, nullptr, lock); + reserved = tryReserveImpl(file_segment, size, *main_priority, nullptr, lock); } - if (reserved) + const auto & key_metadata = file_segment.getKeyMetadata(); + if (reserved && !key_metadata->created_base_directory) { - LOG_TEST(log, "Successfully reserved {} bytes for key {} at offset {}", size, key.toString(), offset); - locked_key->createKeyDirectoryIfNot(); + fs::create_directories(metadata.getPathInLocalCache(file_segment.key())); + key_metadata->created_base_directory = true; } - else - LOG_TEST(log, "Failed to reserve {} bytes for key {} at offset {}", size, key.toString(), offset); return reserved; } -void FileCache::iterateCacheAndCollectKeyLocks( - LockedCachePriority & priority, - IterateAndCollectLocksFunc && func, - LockedKeyMetadataMap & locked_map) const -{ - priority.iterate([&, func = std::move(func)](const IFileCachePriority::Entry & entry) - { - LockedKeyMetadataPtr current; - - auto locked_it = locked_map.find(entry.key); - const bool locked = locked_it != locked_map.end(); - if (locked) - current = locked_it->second; - else - current = metadata.lockKeyMetadata(entry.key, entry.getKeyMetadata()); - - auto res = func(entry, *current); - if (res.lock_key && !locked) - locked_map.emplace(entry.key, current); - - return res.iteration_result; - }); -} - -void FileCache::removeFileSegment(LockedKeyMetadata & locked_key, FileSegmentPtr file_segment, const CacheGuard::Lock & cache_lock) -{ - /// We must hold pointer to file segment while removing it - /// (because we remove file segment under file segment lock). - - chassert(file_segment->key() == locked_key.getKey()); - locked_key.removeFileSegment(file_segment->offset(), file_segment->lock(), cache_lock); -} - bool FileCache::tryReserveImpl( - IFileCachePriority & priority_queue, - LockedKeyMetadataPtr locked_key, - size_t offset, + const FileSegment & file_segment, size_t size, + IFileCachePriority & priority_queue, FileCacheQueryLimit::LockedQueryContext * query_context, const CacheGuard::Lock & cache_lock) { @@ -560,8 +534,7 @@ bool FileCache::tryReserveImpl( /// We add/remove entries from both (global and local) priority queues, /// but iterate only local, though check the limits in both. - const auto & key = locked_key->getKey(); - LOG_TEST(log, "Reserving space {} for {}:{}", size, key.toString(), offset); + LOG_TEST(log, "Reserving space {} for {}:{}", size, file_segment.key().toString(), file_segment.offset()); LockedCachePriority locked_priority_queue(cache_lock, priority_queue); LockedCachePriority locked_main_priority(cache_lock, *main_priority); @@ -570,8 +543,10 @@ bool FileCache::tryReserveImpl( chassert(queue_size <= locked_priority_queue.getElementsLimit()); /// A file_segment_metadata acquires a LRUQueue iterator on first successful space reservation attempt. - auto * file_segment_for_reserve = locked_key->getKeyMetadata()->getByOffset(offset); - if (!file_segment_for_reserve->queue_iterator) + auto & queue_iterator = file_segment.getQueueIterator(); + if (queue_iterator) + chassert(file_segment.getReservedSize() > 0); + else queue_size += 1; size_t removed_size = 0; @@ -584,119 +559,101 @@ bool FileCache::tryReserveImpl( || (query_context && query_context->getSize() + size - removed_size > query_context->getSizeLimit()); }; - LockedKeyMetadataMap locked; - locked[key] = locked_key; + struct DeletionInfo + { + KeyMetadataPtr key_metadata; + std::vector> offsets; + }; + std::unordered_map to_delete; - using QueueEntry = IFileCachePriority::Entry; using IterationResult = IFileCachePriority::IterationResult; - - std::unordered_map> offsets_per_key_to_delete; - - iterateCacheAndCollectKeyLocks( - locked_priority_queue, - [&](const QueueEntry & entry, LockedKeyMetadata & current_locked_key) -> IterateAndLockResult + auto iterate = [this, &removed_size, &queue_size, &cache_lock, &to_delete](const IFileCachePriority::Entry & entry) { - if (!is_overflow()) - return { IterationResult::BREAK, false }; - - auto * file_segment_metadata = current_locked_key.getKeyMetadata()->getByOffset(entry.offset); + auto current_locked_key = metadata.lockKeyMetadata(entry.key, entry.getKeyMetadata()); + auto & file_segment_metadata = *current_locked_key->getByOffset(entry.offset); - chassert(file_segment_metadata->queue_iterator); - chassert((entry.size == file_segment_metadata->size()) || (file_segment_metadata->file_segment->state() == FileSegment::State::DOWNLOADING)); + chassert(file_segment_metadata.getQueueIterator()); + chassert((entry.size == file_segment_metadata.size()) || (file_segment_metadata.file_segment->state() == FileSegment::State::DOWNLOADING)); + chassert(entry.offset == file_segment_metadata.file_segment->offset()); - /// It is guaranteed that file_segment_metadata is not removed from cache as long as - /// pointer to corresponding file segment is hold by any other thread. + auto iteration_result = IterationResult::CONTINUE; - const size_t file_segment_size = entry.size; + const bool is_persistent = allow_persistent_files && file_segment_metadata.file_segment->isPersistent(); + const bool releasable = file_segment_metadata.releasable() && !is_persistent; - bool remove_current_it = false; - bool save_locked_key = false; - - if (file_segment_metadata->releasable()) + if (releasable) { - auto file_segment = file_segment_metadata->file_segment; + const size_t file_segment_size = entry.size; + const auto & current_file_segment = file_segment_metadata.file_segment; - chassert(entry.offset == file_segment->offset()); - if (file_segment->isPersistent() && allow_persistent_files) + if (current_file_segment->state() == FileSegment::State::DOWNLOADED) { - return { IterationResult::CONTINUE, false }; + /// file_segment_metadata will actually be removed only if we managed to reserve enough space. + auto & deletion_info = to_delete[current_file_segment->key()]; + deletion_info.offsets.emplace_back(file_segment_metadata.getEvictHolder(), current_file_segment->offset()); + deletion_info.key_metadata = current_locked_key->getKeyMetadata(); } - - switch (file_segment->state()) + else { - case FileSegment::State::DOWNLOADED: - { - /// file_segment_metadata will actually be removed only if we managed to reserve enough space. - - offsets_per_key_to_delete[file_segment->key()].push_back(file_segment->offset()); - save_locked_key = true; - break; - } - default: - { - remove_current_it = true; - file_segment_metadata->queue_iterator = {}; - removeFileSegment(current_locked_key, file_segment, cache_lock); - break; - } + iteration_result = IterationResult::REMOVE_AND_CONTINUE; + file_segment_metadata.getQueueIterator() = {}; + removeFileSegment(*current_locked_key, current_file_segment, cache_lock); } removed_size += file_segment_size; --queue_size; } + return iteration_result; + }; - if (remove_current_it) - return { IterationResult::REMOVE_AND_CONTINUE, save_locked_key }; + locked_priority_queue.iterate([&](const IFileCachePriority::Entry & entry) + { + if (!is_overflow()) + return IterationResult::BREAK; - return { IterationResult::CONTINUE, save_locked_key }; - }, locked); + return iterate(entry); + }); if (is_overflow()) return false; - for (auto it = locked.begin(); it != locked.end();) + for (const auto & [current_key, deletion_info] : to_delete) { - auto & current_locked_key = it->second; - auto & offsets_to_delete = offsets_per_key_to_delete[current_locked_key->getKey()]; - /// TODO: Add assertion. - for (const auto & offset_to_delete : offsets_to_delete) + auto current_locked_metadata = metadata.lockKeyMetadata(current_key, deletion_info.key_metadata); + for (const auto & offset_to_delete : deletion_info.offsets) { - auto * file_segment_metadata = current_locked_key->getKeyMetadata()->getByOffset(offset_to_delete); - removeFileSegment(*current_locked_key, file_segment_metadata->file_segment, cache_lock); + auto * file_segment_metadata = current_locked_metadata->getByOffset(offset_to_delete.second); + removeFileSegment(*current_locked_metadata, file_segment_metadata->file_segment, cache_lock); if (query_context) - query_context->remove(key, offset); + query_context->remove(current_key, offset_to_delete.second); } - offsets_to_delete.clear(); - - /// Do not hold the key lock longer than required. - it = locked.erase(it); } /// queue_iteratir is std::nullopt here if no space has been reserved yet, a file_segment_metadata /// acquires queue iterator on first successful space reservation attempt. /// If queue iterator already exists, we need to update the size after each space reservation. - if (file_segment_for_reserve->queue_iterator) - LockedCachePriorityIterator(cache_lock, file_segment_for_reserve->queue_iterator).incrementSize(size); + if (queue_iterator) + LockedCachePriorityIterator(cache_lock, queue_iterator).incrementSize(size); else { /// Space reservation is incremental, so file_segment_metadata is created first (with state empty), - /// and queue_iterator is assigned on first space reservation attempt. - file_segment_for_reserve->queue_iterator = locked_main_priority.add(key, offset, size, locked_key->getKeyMetadata()); + /// and getQueueIterator() is assigned on first space reservation attempt. + queue_iterator = locked_main_priority.add(file_segment.key(), file_segment.offset(), size, file_segment.getKeyMetadata()); } if (query_context) { - auto queue_iterator = query_context->tryGet(key, offset); - if (queue_iterator) + auto query_queue_it = query_context->tryGet(file_segment.key(), file_segment.offset()); + if (query_queue_it) { - LockedCachePriorityIterator(cache_lock, queue_iterator).incrementSize(size); + LockedCachePriorityIterator(cache_lock, query_queue_it).incrementSize(size); } else { auto it = LockedCachePriority( - cache_lock, query_context->getPriority()).add(key, offset, size, locked_key->getKeyMetadata()); + cache_lock, query_context->getPriority()).add(file_segment.key(), file_segment.offset(), size, file_segment.getKeyMetadata()); - query_context->add(key, offset, it); + query_context->add(file_segment.key(), file_segment.offset(), it); } } @@ -754,11 +711,11 @@ void FileCache::removeAllReleasable() LockedCachePriority(lock, *main_priority).iterate([&](const QueueEntry & entry) -> IterationResult { auto locked_key = metadata.lockKeyMetadata(entry.key, entry.getKeyMetadata()); - auto * file_segment_metadata = locked_key->getKeyMetadata()->getByOffset(entry.offset); + auto * file_segment_metadata = locked_key->getByOffset(entry.offset); if (file_segment_metadata->releasable()) { - file_segment_metadata->queue_iterator = {}; + file_segment_metadata->getQueueIterator() = {}; removeFileSegment(*locked_key, file_segment_metadata->file_segment, lock); return IterationResult::REMOVE_AND_CONTINUE; } @@ -882,12 +839,12 @@ void FileCache::loadMetadata() auto file_segment_metadata_it = addFileSegment( *locked_key, offset, size, FileSegment::State::DOWNLOADED, CreateFileSegmentSettings(segment_kind), &lock); - chassert(file_segment_metadata_it->second.queue_iterator); + chassert(file_segment_metadata_it->second.getQueueIterator()); chassert(file_segment_metadata_it->second.size() == size); total_size += size; queue_entries.emplace_back( - file_segment_metadata_it->second.queue_iterator, file_segment_metadata_it->second.file_segment); + file_segment_metadata_it->second.getQueueIterator(), file_segment_metadata_it->second.file_segment); } else { @@ -971,7 +928,7 @@ FileSegmentsHolderPtr FileCache::getSnapshot() FileSegments file_segments; metadata.iterate([&](const LockedKeyMetadata & locked_key) { - for (const auto & [_, file_segment_metadata] : *locked_key.getKeyMetadata()) + for (const auto & [_, file_segment_metadata] : locked_key) file_segments.push_back(FileSegment::getSnapshot(file_segment_metadata.file_segment)); }); return std::make_unique(std::move(file_segments)); @@ -995,8 +952,8 @@ FileSegmentsHolderPtr FileCache::dumpQueue() FileSegments file_segments; LockedCachePriority(cache_guard.lock(), *main_priority).iterate([&](const QueueEntry & entry) { - auto tx = metadata.lockKeyMetadata(entry.key, entry.getKeyMetadata()); - auto * file_segment_metadata = tx->getKeyMetadata()->getByOffset(entry.offset); + auto locked_metadata = metadata.lockKeyMetadata(entry.key, entry.getKeyMetadata()); + auto * file_segment_metadata = locked_metadata->getByOffset(entry.offset); file_segments.push_back(FileSegment::getSnapshot(file_segment_metadata->file_segment)); return IterationResult::CONTINUE; }); @@ -1036,7 +993,7 @@ void FileCache::assertCacheCorrectness() { metadata.iterate([&](const LockedKeyMetadata & locked_key) { - for (auto & [offset, file_segment_metadata] : *locked_key.getKeyMetadata()) + for (const auto & [offset, file_segment_metadata] : locked_key) { locked_key.assertFileSegmentCorrectness(*file_segment_metadata.file_segment); } diff --git a/src/Interpreters/Cache/FileCache.h b/src/Interpreters/Cache/FileCache.h index b1d7fa548df7..6490ed8f99ae 100644 --- a/src/Interpreters/Cache/FileCache.h +++ b/src/Interpreters/Cache/FileCache.h @@ -29,7 +29,6 @@ namespace DB struct LockedKeyMetadata; using LockedKeyMetadataPtr = std::shared_ptr; -using LockedKeyMetadataMap = std::unordered_map; namespace ErrorCodes { @@ -98,7 +97,7 @@ class FileCache : private boost::noncopyable size_t getMaxFileSegmentSize() const { return max_file_segment_size; } - bool tryReserve(const Key & key, size_t offset, size_t size, KeyMetadataPtr key_metadata); + bool tryReserve(const FileSegment & file_segment, size_t size); FileSegmentsHolderPtr getSnapshot(); @@ -205,10 +204,9 @@ class FileCache : private boost::noncopyable const CacheGuard::Lock &); bool tryReserveImpl( - IFileCachePriority & priority_queue, - LockedKeyMetadataPtr locked_key, - size_t offset, + const FileSegment & file_segment, size_t size, + IFileCachePriority & priority_queue, QueryLimit::LockedQueryContext * query_context, const CacheGuard::Lock &); @@ -218,12 +216,6 @@ class FileCache : private boost::noncopyable bool lock_key = false; }; - using IterateAndCollectLocksFunc = std::function; - void iterateCacheAndCollectKeyLocks( - LockedCachePriority & priority, - IterateAndCollectLocksFunc && func, - LockedKeyMetadataMap & locked_map) const; - void assertCacheCorrectness(); BackgroundSchedulePool::TaskHolder cleanup_task; diff --git a/src/Interpreters/Cache/FileSegment.cpp b/src/Interpreters/Cache/FileSegment.cpp index a5ff0d358d7f..93bb76281b77 100644 --- a/src/Interpreters/Cache/FileSegment.cpp +++ b/src/Interpreters/Cache/FileSegment.cpp @@ -25,14 +25,14 @@ String toString(FileSegmentKind kind) } FileSegment::FileSegment( - size_t offset_, - size_t size_, - const Key & key_, + const Key & key_, size_t offset_, size_t size_, std::weak_ptr key_metadata_, + IFileCachePriority::Iterator queue_iterator_, FileCache * cache_, State download_state_, const CreateFileSegmentSettings & settings) : segment_range(offset_, offset_ + size_ - 1) + , queue_iterator(queue_iterator_) , download_state(download_state_) , key_metadata(key_metadata_) , file_key(key_) @@ -464,7 +464,7 @@ bool FileSegment::reserve(size_t size_to_reserve) if (is_unbound && is_file_segment_size_exceeded) segment_range.right = range().left + expected_downloaded_size + size_to_reserve; - reserved = cache->tryReserve(key(), offset(), size_to_reserve, getKeyMetadata()); + reserved = cache->tryReserve(*this, size_to_reserve); if (reserved) { /// No lock is required because reserved size is always @@ -702,13 +702,6 @@ String FileSegment::stateToString(FileSegment::State state) } bool FileSegment::assertCorrectness() const -{ - auto key_lock = lockKeyMetadata(true); - key_lock->assertFileSegmentCorrectness(*this); - return true; -} - -bool FileSegment::assertCorrectnessUnlocked(const FileSegmentMetadata & metadata, const KeyGuard::Lock &) const { auto lock = segment_guard.lock(); @@ -717,10 +710,10 @@ bool FileSegment::assertCorrectnessUnlocked(const FileSegmentMetadata & metadata chassert(!current_downloader.empty() == (download_state == FileSegment::State::DOWNLOADING)); chassert(download_state != FileSegment::State::DOWNLOADED || std::filesystem::file_size(file_path) > 0); - chassert(reserved_size == 0 || metadata.queue_iterator); - if (metadata.queue_iterator) + chassert(reserved_size == 0 || queue_iterator); + if (queue_iterator) { - const auto & entry = *metadata.queue_iterator; + const auto & entry = *queue_iterator; if (isCompleted(false)) chassert(reserved_size == entry.getEntry().size); else @@ -757,10 +750,11 @@ FileSegmentPtr FileSegment::getSnapshot(const FileSegmentPtr & file_segment) auto lock = file_segment->segment_guard.lock(); auto snapshot = std::make_shared( + file_segment->key(), file_segment->offset(), file_segment->range().size(), - file_segment->key(), std::weak_ptr(), + nullptr, file_segment->cache, State::DETACHED, CreateFileSegmentSettings(file_segment->getKind())); @@ -817,25 +811,21 @@ FileSegments::iterator FileSegmentsHolder::completeAndPopFrontImpl() { auto & file_segment = front(); if (file_segment.isDetached()) - { return file_segments.erase(file_segments.begin()); - } auto cache_lock = file_segment.cache->cacheLock(); - /// File segment pointer must be reset right after calling complete() and - /// under the same mutex, because complete() checks for segment pointers. - auto locked_key = file_segment.lockKeyMetadata(/* assert_exists */false); - if (locked_key) - { - auto queue_iter = locked_key->getKeyMetadata()->tryGetByOffset(file_segment.offset())->queue_iterator; - if (queue_iter) - LockedCachePriorityIterator(cache_lock, queue_iter).use(); + auto queue_iter = file_segment.getQueueIterator(); + if (queue_iter) + LockedCachePriorityIterator(cache_lock, queue_iter).use(); - if (!file_segment.isCompleted()) - { + if (!file_segment.isCompleted()) + { + /// File segment pointer must be reset right after calling complete() and + /// under the same mutex, because complete() checks for segment pointers. + auto locked_key = file_segment.lockKeyMetadata(/* assert_exists */false); + if (locked_key) file_segment.completeUnlocked(*locked_key, cache_lock); - } } return file_segments.erase(file_segments.begin()); diff --git a/src/Interpreters/Cache/FileSegment.h b/src/Interpreters/Cache/FileSegment.h index 6e6ac12fdb66..c5dd9e1c4446 100644 --- a/src/Interpreters/Cache/FileSegment.h +++ b/src/Interpreters/Cache/FileSegment.h @@ -10,6 +10,7 @@ #include #include #include +#include #include #include @@ -78,6 +79,7 @@ friend struct FileSegmentsHolder; friend class FileSegmentRangeWriter; friend class StorageSystemFilesystemCache; friend struct LockedKeyMetadata; +friend struct FileSegmentMetadata; public: using Key = FileCacheKey; @@ -121,10 +123,11 @@ friend struct LockedKeyMetadata; }; FileSegment( + const Key & key_, size_t offset_, size_t size_, - const Key & key_, std::weak_ptr key_metadata, + IFileCachePriority::Iterator queue_iterator_, FileCache * cache_, State download_state_, const CreateFileSegmentSettings & create_settings); @@ -225,8 +228,6 @@ friend struct LockedKeyMetadata; bool assertCorrectness() const; - bool assertCorrectnessUnlocked(const FileSegmentMetadata & metadata, const KeyGuard::Lock &) const; - /** * ========== Methods for _only_ file segment's `downloader` ================== */ @@ -266,6 +267,8 @@ friend struct LockedKeyMetadata; size_t getReservedSize() const; + IFileCachePriority::Iterator & getQueueIterator() const { return queue_iterator; } + private: String getInfoForLogUnlocked(const FileSegmentGuard::Lock &) const; String getDownloaderUnlocked(const FileSegmentGuard::Lock &) const; @@ -303,6 +306,9 @@ friend struct LockedKeyMetadata; Range segment_range; + /// Iterator is put here on first reservation attempt, if successful. + mutable IFileCachePriority::Iterator queue_iterator; + std::atomic download_state; /// The one who prepares the download diff --git a/src/Interpreters/Cache/Metadata.cpp b/src/Interpreters/Cache/Metadata.cpp index 346eaf2641cd..c092d996f59b 100644 --- a/src/Interpreters/Cache/Metadata.cpp +++ b/src/Interpreters/Cache/Metadata.cpp @@ -16,17 +16,9 @@ namespace ErrorCodes } FileSegmentMetadata::FileSegmentMetadata( - FileSegmentPtr file_segment_, - LockedKeyMetadata & locked_key, - LockedCachePriority * locked_queue) + FileSegmentPtr file_segment_, LockedKeyMetadata & locked_key, LockedCachePriority * locked_queue) : file_segment(file_segment_) { - /** - * File segment can be created with either DOWNLOADED or EMPTY file segment's state. - * File segment acquires DOWNLOADING state and creates LRUQueue iterator on first - * successful getOrSetDownaloder call. - */ - switch (file_segment->state()) { case FileSegment::State::DOWNLOADED: @@ -37,7 +29,7 @@ FileSegmentMetadata::FileSegmentMetadata( ErrorCodes::LOGICAL_ERROR, "Adding file segment with state DOWNLOADED requires locked queue lock"); } - queue_iterator = locked_queue->add( + file_segment->getQueueIterator() = locked_queue->add( file_segment->key(), file_segment->offset(), file_segment->range().size(), locked_key.getKeyMetadata()); break; @@ -60,60 +52,9 @@ size_t FileSegmentMetadata::size() const return file_segment->getReservedSize(); } -const FileSegmentMetadata * KeyMetadata::getByOffset(size_t offset) const +IFileCachePriority::Iterator & FileSegmentMetadata::getQueueIterator() const { - auto it = find(offset); - if (it == end()) - throw Exception(ErrorCodes::LOGICAL_ERROR, "There is not offset {}", offset); - return &(it->second); -} - -FileSegmentMetadata * KeyMetadata::getByOffset(size_t offset) -{ - auto it = find(offset); - if (it == end()) - throw Exception(ErrorCodes::LOGICAL_ERROR, "There is not offset {}", offset); - return &(it->second); -} - -const FileSegmentMetadata * KeyMetadata::tryGetByOffset(size_t offset) const -{ - auto it = find(offset); - if (it == end()) - return nullptr; - return &(it->second); -} - -FileSegmentMetadata * KeyMetadata::tryGetByOffset(size_t offset) -{ - auto it = find(offset); - if (it == end()) - return nullptr; - return &(it->second); -} - -std::string KeyMetadata::toString() const -{ - std::string result; - for (auto it = begin(); it != end(); ++it) - { - if (it != begin()) - result += ", "; - result += std::to_string(it->first); - } - return result; -} - -void KeyMetadata::addToCleanupQueue(const FileCacheKey & key, const KeyGuard::Lock &) -{ - cleanup_queue.add(key); - cleanup_state = CleanupState::SUBMITTED_TO_CLEANUP_QUEUE; -} - -void KeyMetadata::removeFromCleanupQueue(const FileCacheKey &, const KeyGuard::Lock &) -{ - /// Just mark cleanup_state as "not to be removed", the cleanup thread will check it and skip the key. - cleanup_state = CleanupState::NOT_SUBMITTED; + return file_segment->getQueueIterator(); } String CacheMetadata::getPathInLocalCache(const Key & key, size_t offset, FileSegmentKind segment_kind) const @@ -132,16 +73,13 @@ String CacheMetadata::getPathInLocalCache(const Key & key, size_t offset, FileSe break; } - auto key_str = key.toString(); - return fs::path(base_directory) - / key_str.substr(0, 3) - / key_str - / (std::to_string(offset) + file_suffix); + const auto key_str = key.toString(); + return fs::path(base_directory) / key_str.substr(0, 3) / key_str / (std::to_string(offset) + file_suffix); } String CacheMetadata::getPathInLocalCache(const Key & key) const { - auto key_str = key.toString(); + const auto key_str = key.toString(); return fs::path(base_directory) / key_str.substr(0, 3) / key_str; } @@ -168,53 +106,49 @@ LockedKeyMetadataPtr CacheMetadata::lockKeyMetadata(const FileCacheKey & key, Ke } { - auto key_lock = key_metadata->lock(); - - const auto cleanup_state = key_metadata->getCleanupState(key_lock); + auto locked_metadata = std::make_unique(key, key_metadata, getPathInLocalCache(key)); + const auto key_state = locked_metadata->getKeyState(); - if (cleanup_state == KeyMetadata::CleanupState::NOT_SUBMITTED) - { - return std::make_unique(key, key_metadata, std::move(key_lock), getPathInLocalCache(key)); - } + if (key_state == KeyMetadata::KeyState::ACTIVE) + return locked_metadata; if (key_not_found_policy == KeyNotFoundPolicy::THROW) throw Exception(ErrorCodes::LOGICAL_ERROR, "No such key `{}` in cache", key.toString()); - else if (key_not_found_policy == KeyNotFoundPolicy::RETURN_NULL) + + if (key_not_found_policy == KeyNotFoundPolicy::RETURN_NULL) return nullptr; - if (cleanup_state == KeyMetadata::CleanupState::SUBMITTED_TO_CLEANUP_QUEUE) + if (key_state == KeyMetadata::KeyState::REMOVING) { - key_metadata->removeFromCleanupQueue(key, key_lock); - return std::make_unique(key, key_metadata, std::move(key_lock), getPathInLocalCache(key)); + locked_metadata->removeFromCleanupQueue(); + return locked_metadata; } - chassert(cleanup_state == KeyMetadata::CleanupState::CLEANED_BY_CLEANUP_THREAD); + chassert(key_state == KeyMetadata::KeyState::REMOVED); chassert(key_not_found_policy == KeyNotFoundPolicy::CREATE_EMPTY); } /// Not we are at a case: - /// cleanup_state == KeyMetadata::CleanupState::CLEANED_BY_CLEANUP_THREAD + /// key_state == KeyMetadata::KeyState::REMOVED /// and KeyNotFoundPolicy == CREATE_EMPTY /// Retry. return lockKeyMetadata(key, key_not_found_policy); } LockedKeyMetadataPtr CacheMetadata::lockKeyMetadata( - const FileCacheKey & key, KeyMetadataPtr key_metadata, bool return_null_if_in_cleanup_queue) const + const FileCacheKey & key, KeyMetadataPtr key_metadata, bool skip_if_in_cleanup_queue) const { - auto key_lock = key_metadata->lock(); + auto locked_metadata = std::make_unique(key, key_metadata, getPathInLocalCache(key)); + const auto key_state = locked_metadata->getKeyState(); - auto cleanup_state = key_metadata->getCleanupState(key_lock); - if (cleanup_state != KeyMetadata::CleanupState::NOT_SUBMITTED) - { - if (return_null_if_in_cleanup_queue - && cleanup_state == KeyMetadata::CleanupState::SUBMITTED_TO_CLEANUP_QUEUE) - return nullptr; + if (key_state == KeyMetadata::KeyState::ACTIVE) + return locked_metadata; - throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot lock key: it was removed from cache"); - } + if (skip_if_in_cleanup_queue + && key_state == KeyMetadata::KeyState::REMOVING) + return nullptr; - return std::make_unique(key, key_metadata, std::move(key_lock), getPathInLocalCache(key)); + throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot lock key {}: it was removed from cache", key.toString()); } void CacheMetadata::iterate(IterateCacheMetadataFunc && func) @@ -222,7 +156,7 @@ void CacheMetadata::iterate(IterateCacheMetadataFunc && func) auto lock = guard.lock(); for (const auto & [key, key_metadata] : *this) { - auto locked_key = lockKeyMetadata(key, key_metadata, /* return_null_if_in_cleanup_queue */true); + auto locked_key = lockKeyMetadata(key, key_metadata, /* skip_if_in_cleanup_queue */true); if (!locked_key) continue; @@ -248,18 +182,16 @@ void CacheMetadata::doCleanup() if (it == end()) continue; - auto key_metadata = it->second; - auto key_lock = key_metadata->lock(); + auto locked_metadata = std::make_unique(it->first, it->second, getPathInLocalCache(it->first)); + const auto key_state = locked_metadata->getKeyState(); - /// As in lockKeyMetadata we extract key metadata from cache metadata under - /// CacheMetadataGuard::Lock, but take KeyGuard::Lock only after we released - /// cache CacheMetadataGuard::Lock (because CacheMetadataGuard::Lock must be lightweight). - /// So it is possible that a key which was submitted to cleanup queue was afterwards added - /// to cache, so here we need to check this case. - if (key_metadata->getCleanupState(key_lock) == KeyMetadata::CleanupState::NOT_SUBMITTED) + if (key_state == KeyMetadata::KeyState::ACTIVE) + { + /// Key was added back to cache after we submitted it to removal queue. continue; + } - key_metadata->cleanup_state = KeyMetadata::CleanupState::CLEANED_BY_CLEANUP_THREAD; + locked_metadata->markAsRemoved(); erase(it); try @@ -283,12 +215,11 @@ void CacheMetadata::doCleanup() LockedKeyMetadata::LockedKeyMetadata( const FileCacheKey & key_, std::shared_ptr key_metadata_, - KeyGuard::Lock && lock_, const std::string & key_path_) : key(key_) , key_path(key_path_) , key_metadata(key_metadata_) - , lock(std::move(lock_)) + , lock(key_metadata->lock()) , log(&Poco::Logger::get("LockedKeyMetadata")) { } @@ -298,21 +229,33 @@ LockedKeyMetadata::~LockedKeyMetadata() if (!key_metadata->empty()) return; - key_metadata->addToCleanupQueue(key, lock); + key_metadata->key_state = KeyMetadata::KeyState::REMOVING; + key_metadata->cleanup_queue.add(key); } -void LockedKeyMetadata::createKeyDirectoryIfNot() +void LockedKeyMetadata::removeFromCleanupQueue() { - if (key_metadata->createdBaseDirectory(lock)) - return; + if (key_metadata->key_state != KeyMetadata::KeyState::REMOVING) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot remove non-removing"); - fs::create_directories(key_path); - key_metadata->created_base_directory = true; + /// Just mark key_state as "not to be removed", the cleanup thread will check it and skip the key. + key_metadata->key_state = KeyMetadata::KeyState::ACTIVE; +} + +bool LockedKeyMetadata::markAsRemoved() +{ + chassert(key_metadata->key_state != KeyMetadata::KeyState::REMOVED); + + if (key_metadata->key_state == KeyMetadata::KeyState::ACTIVE) + return false; + + key_metadata->key_state = KeyMetadata::KeyState::REMOVED; + return true; } bool LockedKeyMetadata::isLastOwnerOfFileSegment(size_t offset) const { - const auto * file_segment_metadata = key_metadata->getByOffset(offset); + const auto * file_segment_metadata = getByOffset(offset); return file_segment_metadata->file_segment.use_count() == 2; } @@ -325,10 +268,10 @@ void LockedKeyMetadata::removeFileSegment( log, "Remove from cache. Key: {}, offset: {}", key.toString(), offset); - auto * file_segment_metadata = key_metadata->getByOffset(offset); + auto * file_segment_metadata = getByOffset(offset); - if (file_segment_metadata->queue_iterator) - LockedCachePriorityIterator(cache_lock, file_segment_metadata->queue_iterator).remove(); + if (file_segment_metadata->getQueueIterator()) + LockedCachePriorityIterator(cache_lock, file_segment_metadata->getQueueIterator()).remove(); const auto cache_file_path = file_segment_metadata->file_segment->getPathInLocalCache(); file_segment_metadata->file_segment->detach(segment_lock, *this); @@ -349,7 +292,7 @@ void LockedKeyMetadata::shrinkFileSegmentToDownloadedSize( * because of no space left in cache, we need to be able to cut file segment's size to downloaded_size. */ - auto * file_segment_metadata = key_metadata->getByOffset(offset); + auto * file_segment_metadata = getByOffset(offset); const auto & file_segment = file_segment_metadata->file_segment; size_t downloaded_size = file_segment->downloaded_size; @@ -363,14 +306,15 @@ void LockedKeyMetadata::shrinkFileSegmentToDownloadedSize( file_segment->getInfoForLogUnlocked(segment_lock)); } - auto & entry = LockedCachePriorityIterator(cache_lock, file_segment_metadata->queue_iterator).getEntry(); + auto & entry = LockedCachePriorityIterator(cache_lock, file_segment_metadata->getQueueIterator()).getEntry(); assert(file_segment->downloaded_size <= file_segment->reserved_size); assert(entry.size == file_segment->reserved_size); assert(entry.size >= file_segment->downloaded_size); CreateFileSegmentSettings create_settings(file_segment->getKind()); file_segment_metadata->file_segment = std::make_shared( - offset, downloaded_size, key, key_metadata, file_segment->cache, FileSegment::State::DOWNLOADED, create_settings); + key, offset, downloaded_size, key_metadata, file_segment->getQueueIterator(), + file_segment->cache, FileSegment::State::DOWNLOADED, create_settings); if (file_segment->reserved_size > file_segment->downloaded_size) entry.size = downloaded_size; @@ -384,8 +328,51 @@ void LockedKeyMetadata::assertFileSegmentCorrectness(const FileSegment & file_se if (file_segment.key() != key) throw Exception(ErrorCodes::LOGICAL_ERROR, "Expected {} = {}", file_segment.key().toString(), key.toString()); - const auto & file_segment_metadata = *key_metadata->getByOffset(file_segment.offset()); - file_segment.assertCorrectnessUnlocked(file_segment_metadata, lock); + file_segment.assertCorrectness(); +} + +const FileSegmentMetadata * LockedKeyMetadata::getByOffset(size_t offset) const +{ + auto it = key_metadata->find(offset); + if (it == key_metadata->end()) + throw Exception(ErrorCodes::LOGICAL_ERROR, "There is not offset {}", offset); + return &(it->second); +} + +FileSegmentMetadata * LockedKeyMetadata::getByOffset(size_t offset) +{ + auto it = key_metadata->find(offset); + if (it == key_metadata->end()) + throw Exception(ErrorCodes::LOGICAL_ERROR, "There is not offset {}", offset); + return &(it->second); +} + +const FileSegmentMetadata * LockedKeyMetadata::tryGetByOffset(size_t offset) const +{ + auto it = key_metadata->find(offset); + if (it == key_metadata->end()) + return nullptr; + return &(it->second); +} + +FileSegmentMetadata * LockedKeyMetadata::tryGetByOffset(size_t offset) +{ + auto it = key_metadata->find(offset); + if (it == key_metadata->end()) + return nullptr; + return &(it->second); +} + +std::string LockedKeyMetadata::toString() const +{ + std::string result; + for (auto it = key_metadata->begin(); it != key_metadata->end(); ++it) + { + if (it != key_metadata->begin()) + result += ", "; + result += std::to_string(it->first); + } + return result; } void CleanupQueue::add(const FileCacheKey & key) diff --git a/src/Interpreters/Cache/Metadata.h b/src/Interpreters/Cache/Metadata.h index 3e37437c97d4..4fc0fed99d83 100644 --- a/src/Interpreters/Cache/Metadata.h +++ b/src/Interpreters/Cache/Metadata.h @@ -18,64 +18,59 @@ struct CleanupQueue; struct FileSegmentMetadata : private boost::noncopyable { - FileSegmentPtr file_segment; - - /// Iterator is put here on first reservation attempt, if successful. - IFileCachePriority::Iterator queue_iterator; + FileSegmentMetadata( + FileSegmentPtr file_segment_, + LockedKeyMetadata & locked_key, + LockedCachePriority * locked_queue); /// Pointer to file segment is always hold by the cache itself. /// Apart from pointer in cache, it can be hold by cache users, when they call /// getorSet(), but cache users always hold it via FileSegmentsHolder. bool releasable() const { return file_segment.unique(); } + IFileCachePriority::Iterator & getQueueIterator() const; + + bool valid() const { return *evict_flag == false; } + size_t size() const; - FileSegmentMetadata( - FileSegmentPtr file_segment_, - LockedKeyMetadata & locked_key, - LockedCachePriority * locked_queue); + FileSegmentPtr file_segment; + + using EvictFlag = std::shared_ptr>; + EvictFlag evict_flag = std::make_shared>(false); + struct EvictHolder : boost::noncopyable + { + explicit EvictHolder(EvictFlag evict_flag_) : evict_flag(evict_flag_) { *evict_flag = true; } + ~EvictHolder() { *evict_flag = false; } + EvictFlag evict_flag; + }; + using EvictHolderPtr = std::unique_ptr; - FileSegmentMetadata(FileSegmentMetadata && other) noexcept - : file_segment(std::move(other.file_segment)), queue_iterator(std::move(other.queue_iterator)) {} + EvictHolderPtr getEvictHolder() { return std::make_unique(evict_flag); } }; struct KeyMetadata : public std::map, private boost::noncopyable { friend struct LockedKeyMetadata; - friend struct CacheMetadata; public: explicit KeyMetadata(bool created_base_directory_, CleanupQueue & cleanup_queue_) : created_base_directory(created_base_directory_), cleanup_queue(cleanup_queue_) {} - const FileSegmentMetadata * getByOffset(size_t offset) const; - FileSegmentMetadata * getByOffset(size_t offset); - - const FileSegmentMetadata * tryGetByOffset(size_t offset) const; - FileSegmentMetadata * tryGetByOffset(size_t offset); - - std::string toString() const; - - KeyGuard::Lock lock() const { return guard.lock(); } - - bool createdBaseDirectory(const KeyGuard::Lock &) const { return created_base_directory; } - - enum class CleanupState + enum class KeyState { - NOT_SUBMITTED, - SUBMITTED_TO_CLEANUP_QUEUE, - CLEANED_BY_CLEANUP_THREAD, + ACTIVE, + REMOVING, + REMOVED, }; - CleanupState getCleanupState(const KeyGuard::Lock &) const { return cleanup_state; } + bool created_base_directory = false; - void addToCleanupQueue(const FileCacheKey & key, const KeyGuard::Lock &); +private: + KeyGuard::Lock lock() const { return guard.lock(); } - void removeFromCleanupQueue(const FileCacheKey & key, const KeyGuard::Lock &); + KeyState key_state = KeyState::ACTIVE; -private: mutable KeyGuard guard; - bool created_base_directory = false; - CleanupState cleanup_state = CleanupState::NOT_SUBMITTED; CleanupQueue & cleanup_queue; }; @@ -115,7 +110,7 @@ struct CacheMetadata : public std::unordered_map, }; LockedKeyMetadataPtr lockKeyMetadata(const Key & key, KeyNotFoundPolicy key_not_found_policy, bool is_initial_load = false); - LockedKeyMetadataPtr lockKeyMetadata(const Key & key, KeyMetadataPtr key_metadata, bool return_null_if_in_cleanup_queue = false) const; + LockedKeyMetadataPtr lockKeyMetadata(const Key & key, KeyMetadataPtr key_metadata, bool skip_if_in_cleanup_queue = false) const; using IterateCacheMetadataFunc = std::function; void iterate(IterateCacheMetadataFunc && func); @@ -123,9 +118,6 @@ struct CacheMetadata : public std::unordered_map, void doCleanup(); private: - void addToCleanupQueue(const Key & key, const KeyGuard::Lock &); - void removeFromCleanupQueue(const Key & key, const KeyGuard::Lock &); - const std::string base_directory; CacheMetadataGuard guard; CleanupQueue cleanup_queue; @@ -149,16 +141,25 @@ struct LockedKeyMetadata : private boost::noncopyable LockedKeyMetadata( const FileCacheKey & key_, std::shared_ptr key_metadata_, - KeyGuard::Lock && key_lock_, const std::string & key_path_); ~LockedKeyMetadata(); const FileCacheKey & getKey() const { return key; } - void createKeyDirectoryIfNot(); + auto begin() const { return key_metadata->begin(); } + auto end() const { return key_metadata->end(); } + + const FileSegmentMetadata * getByOffset(size_t offset) const; + FileSegmentMetadata * getByOffset(size_t offset); + + const FileSegmentMetadata * tryGetByOffset(size_t offset) const; + FileSegmentMetadata * tryGetByOffset(size_t offset); + + KeyMetadata::KeyState getKeyState() const { return key_metadata->key_state; } KeyMetadataPtr getKeyMetadata() const { return key_metadata; } + KeyMetadataPtr getKeyMetadata() { return key_metadata; } void removeFileSegment(size_t offset, const FileSegmentGuard::Lock &, const CacheGuard::Lock &); @@ -168,7 +169,15 @@ struct LockedKeyMetadata : private boost::noncopyable void assertFileSegmentCorrectness(const FileSegment & file_segment) const; - KeyMetadata::CleanupState getCleanupState() const { return key_metadata->cleanup_state; } + bool isRemovalCandidate() const; + + bool markAsRemovalCandidate(size_t offset); + + void removeFromCleanupQueue(); + + bool markAsRemoved(); + + std::string toString() const; private: const FileCacheKey key; From e2e32fa34c92ff5009a64bd6ff79fdf0168d9366 Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Thu, 6 Apr 2023 10:25:37 +0000 Subject: [PATCH 098/406] Add config --- utils/keeper-bench/CMakeLists.txt | 2 +- utils/keeper-bench/Generator.cpp | 8 ++++++++ utils/keeper-bench/Generator.h | 2 ++ utils/keeper-bench/Runner.h | 17 ++++++++++++++++- utils/keeper-bench/main.cpp | 16 +++++++++------- 5 files changed, 36 insertions(+), 9 deletions(-) diff --git a/utils/keeper-bench/CMakeLists.txt b/utils/keeper-bench/CMakeLists.txt index 2596be4adddc..97d30117d69c 100644 --- a/utils/keeper-bench/CMakeLists.txt +++ b/utils/keeper-bench/CMakeLists.txt @@ -1,2 +1,2 @@ clickhouse_add_executable(keeper-bench Generator.cpp Runner.cpp Stats.cpp main.cpp) -target_link_libraries(keeper-bench PRIVATE clickhouse_common_zookeeper_no_log) +target_link_libraries(keeper-bench PRIVATE clickhouse_common_config_no_zookeeper_log) diff --git a/utils/keeper-bench/Generator.cpp b/utils/keeper-bench/Generator.cpp index b6d8223862c1..3b2378c43966 100644 --- a/utils/keeper-bench/Generator.cpp +++ b/utils/keeper-bench/Generator.cpp @@ -1,4 +1,5 @@ #include "Generator.h" +#include #include #include @@ -338,3 +339,10 @@ std::unique_ptr getGenerator(const std::string & name) throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "Unknown generator {}", name); } + +std::unique_ptr constructGeneratorFromConfig(const std::string & config_path) +{ + ConfigProcessor config_processor(config_path, true, false); + auto loaded_config = config_processor.loadConfig(); + return nullptr; +} diff --git a/utils/keeper-bench/Generator.h b/utils/keeper-bench/Generator.h index e2c546e4bce0..25e4d96caeff 100644 --- a/utils/keeper-bench/Generator.h +++ b/utils/keeper-bench/Generator.h @@ -137,4 +137,6 @@ class MixedRequestGenerator final : public IGenerator }; +std::unique_ptr constructGeneratorFromConfig(const std::string & config_path); + std::unique_ptr getGenerator(const std::string & name); diff --git a/utils/keeper-bench/Runner.h b/utils/keeper-bench/Runner.h index a00b7b43effb..d3f2d1800d90 100644 --- a/utils/keeper-bench/Runner.h +++ b/utils/keeper-bench/Runner.h @@ -22,12 +22,18 @@ namespace CurrentMetrics extern const Metric LocalThreadActive; } +namespace DB::ErrorCodes +{ + extern const int BAD_ARGUMENTS; +} + class Runner { public: Runner( size_t concurrency_, const std::string & generator_name, + const std::string & config_path, const Strings & hosts_strings_, double max_time_, double delay_, @@ -36,7 +42,6 @@ class Runner : concurrency(concurrency_) , pool(CurrentMetrics::LocalThread, CurrentMetrics::LocalThreadActive, concurrency) , hosts_strings(hosts_strings_) - , generator(getGenerator(generator_name)) , max_time(max_time_) , delay(delay_) , continue_on_error(continue_on_error_) @@ -44,6 +49,16 @@ class Runner , info(std::make_shared()) , queue(concurrency) { + if (!generator_name.empty() && !config_path.empty()) + throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, "Both generator name and generator config path are defined. Please define only one of them"); + + if (generator_name.empty() && config_path.empty()) + throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, "Both generator name and generator config path are empty. Please define one of them"); + + if (!generator_name.empty()) + generator = getGenerator(generator_name); + else + generator = constructGeneratorFromConfig(config_path); } void thread(std::vector> zookeepers); diff --git a/utils/keeper-bench/main.cpp b/utils/keeper-bench/main.cpp index 39af28e7580a..83303fb4029c 100644 --- a/utils/keeper-bench/main.cpp +++ b/utils/keeper-bench/main.cpp @@ -19,13 +19,14 @@ int main(int argc, char *argv[]) boost::program_options::options_description desc = createOptionsDescription("Allowed options", getTerminalWidth()); desc.add_options() - ("help", "produce help message") - ("generator", value()->default_value("set_small_data"), "query to execute") - ("concurrency,c", value()->default_value(1), "number of parallel queries") - ("delay,d", value()->default_value(1), "delay between intermediate reports in seconds (set 0 to disable reports)") - ("iterations,i", value()->default_value(0), "amount of queries to be executed") - ("timelimit,t", value()->default_value(0.), "stop launch of queries after specified time limit") - ("hosts,h", value()->multitoken(), "") + ("help", "produce help message") + ("generator", value()->default_value(""), "query to execute") + ("config", value()->default_value(""), "xml file containing generator configuration") + ("concurrency,c", value()->default_value(1), "number of parallel queries") + ("delay,d", value()->default_value(1), "delay between intermediate reports in seconds (set 0 to disable reports)") + ("iterations,i", value()->default_value(0), "amount of queries to be executed") + ("timelimit,t", value()->default_value(0.), "stop launch of queries after specified time limit") + ("hosts,h", value()->multitoken(), "") ("continue_on_errors", "continue testing even if a query fails") ("reconnect", "establish new connection for every query") ; @@ -43,6 +44,7 @@ int main(int argc, char *argv[]) Runner runner(options["concurrency"].as(), options["generator"].as(), + options["config"].as(), options["hosts"].as(), options["timelimit"].as(), options["delay"].as(), From d74ff75d837d05df76f38ec732dd722b2cb53f98 Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Thu, 6 Apr 2023 15:10:58 +0000 Subject: [PATCH 099/406] Add support for create request test --- utils/keeper-bench/Generator.cpp | 654 +++++++++++++++++++++---------- utils/keeper-bench/Generator.h | 262 ++++++++----- utils/keeper-bench/Runner.cpp | 43 +- utils/keeper-bench/Runner.h | 26 +- 4 files changed, 665 insertions(+), 320 deletions(-) diff --git a/utils/keeper-bench/Generator.cpp b/utils/keeper-bench/Generator.cpp index 3b2378c43966..e2c276a274de 100644 --- a/utils/keeper-bench/Generator.cpp +++ b/utils/keeper-bench/Generator.cpp @@ -1,4 +1,5 @@ #include "Generator.h" +#include "Common/Exception.h" #include #include #include @@ -6,13 +7,10 @@ using namespace Coordination; using namespace zkutil; -namespace DB -{ -namespace ErrorCodes +namespace DB::ErrorCodes { extern const int LOGICAL_ERROR; } -} namespace { @@ -38,16 +36,16 @@ std::string generateRandomString(size_t length) return s; } } - -std::string generateRandomPath(const std::string & prefix, size_t length) -{ - return std::filesystem::path(prefix) / generateRandomString(length); -} - -std::string generateRandomData(size_t size) -{ - return generateRandomString(size); -} +// +//std::string generateRandomPath(const std::string & prefix, size_t length) +//{ +// return std::filesystem::path(prefix) / generateRandomString(length); +//} +// +//std::string generateRandomData(size_t size) +//{ +// return generateRandomString(size); +//} void removeRecursive(Coordination::ZooKeeper & zookeeper, const std::string & path) { @@ -98,138 +96,474 @@ void removeRecursive(Coordination::ZooKeeper & zookeeper, const std::string & pa } -void CreateRequestGenerator::startup(Coordination::ZooKeeper & zookeeper) +//void SetRequestGenerator::startup(Coordination::ZooKeeper & zookeeper) +//{ +// removeRecursive(zookeeper, path_prefix); +// +// auto promise = std::make_shared>(); +// auto future = promise->get_future(); +// auto create_callback = [promise] (const CreateResponse & response) +// { +// if (response.error != Coordination::Error::ZOK) +// promise->set_exception(std::make_exception_ptr(zkutil::KeeperException(response.error))); +// else +// promise->set_value(); +// }; +// zookeeper.create(path_prefix, "", false, false, default_acls, create_callback); +// future.get(); +//} +// +//ZooKeeperRequestPtr SetRequestGenerator::generate() +//{ +// auto request = std::make_shared(); +// request->path = path_prefix; +// request->data = generateRandomData(data_size); +// +// return request; +//} +// +//void MixedRequestGenerator::startup(Coordination::ZooKeeper & zookeeper) +//{ +// for (auto & generator : generators) +// generator->startup(zookeeper); +//} +// +//ZooKeeperRequestPtr MixedRequestGenerator::generate() +//{ +// pcg64 rng(randomSeed()); +// std::uniform_int_distribution distribution(0, generators.size() - 1); +// +// return generators[distribution(rng)]->generate(); +//} +// +//void GetRequestGenerator::startup(Coordination::ZooKeeper & zookeeper) +//{ +// auto promise = std::make_shared>(); +// auto future = promise->get_future(); +// auto create_callback = [promise] (const CreateResponse & response) +// { +// if (response.error != Coordination::Error::ZOK) +// promise->set_exception(std::make_exception_ptr(zkutil::KeeperException(response.error))); +// else +// promise->set_value(); +// }; +// zookeeper.create(path_prefix, "", false, false, default_acls, create_callback); +// future.get(); +// size_t total_nodes = 1; +// if (num_nodes) +// total_nodes = *num_nodes; +// +// for (size_t i = 0; i < total_nodes; ++i) +// { +// auto path = generateRandomPath(path_prefix, 5); +// while (std::find(paths_to_get.begin(), paths_to_get.end(), path) != paths_to_get.end()) +// path = generateRandomPath(path_prefix, 5); +// +// auto create_promise = std::make_shared>(); +// auto create_future = create_promise->get_future(); +// auto callback = [create_promise] (const CreateResponse & response) +// { +// if (response.error != Coordination::Error::ZOK) +// create_promise->set_exception(std::make_exception_ptr(zkutil::KeeperException(response.error))); +// else +// create_promise->set_value(); +// }; +// std::string data; +// if (nodes_data_size) +// data = generateRandomString(*nodes_data_size); +// +// zookeeper.create(path, data, false, false, default_acls, callback); +// create_future.get(); +// paths_to_get.push_back(path); +// } +//} +// +//Coordination::ZooKeeperRequestPtr GetRequestGenerator::generate() +//{ +// auto request = std::make_shared(); +// +// size_t path_index = distribution(rng); +// request->path = paths_to_get[path_index]; +// return request; +//} +// +//void ListRequestGenerator::startup(Coordination::ZooKeeper & zookeeper) +//{ +// auto promise = std::make_shared>(); +// auto future = promise->get_future(); +// auto create_callback = [promise] (const CreateResponse & response) +// { +// if (response.error != Coordination::Error::ZOK) +// promise->set_exception(std::make_exception_ptr(zkutil::KeeperException(response.error))); +// else +// promise->set_value(); +// }; +// zookeeper.create(path_prefix, "", false, false, default_acls, create_callback); +// future.get(); +// +// size_t total_nodes = 1; +// if (num_nodes) +// total_nodes = *num_nodes; +// +// size_t path_length = 5; +// if (paths_length) +// path_length = *paths_length; +// +// for (size_t i = 0; i < total_nodes; ++i) +// { +// auto path = generateRandomPath(path_prefix, path_length); +// +// auto create_promise = std::make_shared>(); +// auto create_future = create_promise->get_future(); +// auto callback = [create_promise] (const CreateResponse & response) +// { +// if (response.error != Coordination::Error::ZOK) +// create_promise->set_exception(std::make_exception_ptr(zkutil::KeeperException(response.error))); +// else +// create_promise->set_value(); +// }; +// zookeeper.create(path, "", false, false, default_acls, callback); +// create_future.get(); +// } +//} +// +//Coordination::ZooKeeperRequestPtr ListRequestGenerator::generate() +//{ +// auto request = std::make_shared(); +// request->path = path_prefix; +// return request; +//} + +std::unique_ptr getGenerator(const std::string & name) { - removeRecursive(zookeeper, path_prefix); + //if (name == "create_no_data") + //{ + // return std::make_unique(); + //} + //else if (name == "create_small_data") + //{ + // return std::make_unique("/create_generator", 5, 32); + //} + //else if (name == "create_medium_data") + //{ + // return std::make_unique("/create_generator", 5, 1024); + //} + //else if (name == "create_big_data") + //{ + // return std::make_unique("/create_generator", 5, 512 * 1024); + //} + //else if (name == "get_no_data") + //{ + // return std::make_unique("/get_generator", 10, 0); + //} + //else if (name == "get_small_data") + //{ + // return std::make_unique("/get_generator", 10, 32); + //} + //else if (name == "get_medium_data") + //{ + // return std::make_unique("/get_generator", 10, 1024); + //} + //else if (name == "get_big_data") + //{ + // return std::make_unique("/get_generator", 10, 512 * 1024); + //} + //else if (name == "list_no_nodes") + //{ + // return std::make_unique("/list_generator", 0, 1); + //} + //else if (name == "list_few_nodes") + //{ + // return std::make_unique("/list_generator", 10, 5); + //} + //else if (name == "list_medium_nodes") + //{ + // return std::make_unique("/list_generator", 1000, 5); + //} + //else if (name == "list_a_lot_nodes") + //{ + // return std::make_unique("/list_generator", 100000, 5); + //} + //else if (name == "set_small_data") + //{ + // return std::make_unique("/set_generator", 5); + //} + //else if (name == "mixed_small_data") + //{ + // std::vector> generators; + // generators.push_back(std::make_unique("/set_generator", 5)); + // generators.push_back(std::make_unique("/get_generator", 10, 32)); + // return std::make_unique(std::move(generators)); + //} - auto promise = std::make_shared>(); - auto future = promise->get_future(); - auto create_callback = [promise] (const CreateResponse & response) + throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "Unknown generator {}", name); +} + +NumberGetter +NumberGetter::fromConfig(const std::string & key, const Poco::Util::AbstractConfiguration & config, std::optional default_value) +{ + NumberGetter number_getter; + + if (!config.has(key) && default_value.has_value()) { - if (response.error != Coordination::Error::ZOK) - promise->set_exception(std::make_exception_ptr(zkutil::KeeperException(response.error))); - else - promise->set_value(); - }; - zookeeper.create(path_prefix, "", false, false, default_acls, create_callback); - future.get(); + number_getter.value = *default_value; + } + else if (config.has(key + ".min_value") && config.has(key + ".max_value")) + { + NumberRange range{.min_value = config.getUInt64(key + ".min_value"), .max_value = config.getUInt64(key + ".max_value")}; + if (range.max_value <= range.min_value) + throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, "Range is invalid for key {}: [{}, {}]", key, range.min_value, range.max_value); + number_getter.value = range; + } + else + { + number_getter.value = config.getUInt64(key); + } + + return number_getter; } -ZooKeeperRequestPtr CreateRequestGenerator::generate() +std::string NumberGetter::description() const { - auto request = std::make_shared(); - request->acls = default_acls; - size_t plength = 5; - if (path_length) - plength = *path_length; - auto path_candidate = generateRandomPath(path_prefix, plength); + if (const auto * number = std::get_if(&value)) + return std::to_string(*number); - while (paths_created.contains(path_candidate)) - path_candidate = generateRandomPath(path_prefix, plength); + const auto & range = std::get(value); + return fmt::format("random value from range [{}, {}]", range.min_value, range.max_value); +} - paths_created.insert(path_candidate); +uint64_t NumberGetter::getNumber() const +{ + if (const auto * number = std::get_if(&value)) + return *number; - request->path = path_candidate; - if (data_size) - request->data = generateRandomData(*data_size); + const auto & range = std::get(value); + static pcg64 rng(randomSeed()); + return std::uniform_int_distribution(range.min_value, range.max_value)(rng); +} - return request; +StringGetter StringGetter::fromConfig(const std::string & key, const Poco::Util::AbstractConfiguration & config) +{ + StringGetter string_getter; + if (config.has(key + ".random_string")) + string_getter.value + = NumberGetter::fromConfig(key + ".random_string.size", config); + else + string_getter.value = config.getString(key); + + return string_getter; } +void StringGetter::setString(std::string name) +{ + value = std::move(name); +} -void SetRequestGenerator::startup(Coordination::ZooKeeper & zookeeper) +std::string StringGetter::getString() const { - removeRecursive(zookeeper, path_prefix); + if (const auto * string = std::get_if(&value)) + return *string; - auto promise = std::make_shared>(); - auto future = promise->get_future(); - auto create_callback = [promise] (const CreateResponse & response) - { - if (response.error != Coordination::Error::ZOK) - promise->set_exception(std::make_exception_ptr(zkutil::KeeperException(response.error))); - else - promise->set_value(); - }; - zookeeper.create(path_prefix, "", false, false, default_acls, create_callback); - future.get(); + const auto number_getter = std::get(value); + return generateRandomString(number_getter.getNumber()); } -ZooKeeperRequestPtr SetRequestGenerator::generate() +std::string StringGetter::description() const { - auto request = std::make_shared(); - request->path = path_prefix; - request->data = generateRandomData(data_size); + if (const auto * string = std::get_if(&value)) + return *string; - return request; + const auto number_getter = std::get(value); + return fmt::format("random string with size of {}", number_getter.description()); } -void MixedRequestGenerator::startup(Coordination::ZooKeeper & zookeeper) +bool StringGetter::isRandom() const { - for (auto & generator : generators) - generator->startup(zookeeper); + return std::holds_alternative(value); } -ZooKeeperRequestPtr MixedRequestGenerator::generate() +void RequestGenerator::getFromConfig(const std::string & key, const Poco::Util::AbstractConfiguration & config) { - pcg64 rng(randomSeed()); - std::uniform_int_distribution distribution(0, generators.size() - 1); + getFromConfigImpl(key, config); +} - return generators[distribution(rng)]->generate(); +std::string RequestGenerator::description() +{ + return descriptionImpl(); } -void GetRequestGenerator::startup(Coordination::ZooKeeper & zookeeper) +Coordination::ZooKeeperRequestPtr RequestGenerator::generate(const Coordination::ACLs & acls) { - auto promise = std::make_shared>(); - auto future = promise->get_future(); - auto create_callback = [promise] (const CreateResponse & response) + return generateImpl(acls); +} + +CreateRequestGenerator::CreateRequestGenerator() + : rng(randomSeed()) + , remove_picker(0, 1.0) +{} + +void CreateRequestGenerator::getFromConfigImpl(const std::string & key, const Poco::Util::AbstractConfiguration & config) +{ + path_prefix = config.getString(key + ".path_prefix"); + + if (path_prefix.empty() || path_prefix[0] != '/') + throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, "Invalid path_prefix for Create request generator: '{}'", path_prefix); + + name = StringGetter(NumberGetter::fromConfig(key + ".path_length", config, 5)); + + if (config.has(key + ".data")) + data = StringGetter::fromConfig(key + ".data", config); + + remove_factor = config.getDouble(key + ".remove_factor", 0.0); +} + +std::string CreateRequestGenerator::descriptionImpl() +{ + std::string data_string + = data.has_value() ? fmt::format("data for created nodes: {}", data->description()) : "no data for created nodes"; + return fmt::format( + "Create Request Generator\n" + "- path prefix for created nodes: {}\n" + "- name for created nodes: {}\n" + "- {}\n" + "- remove factor: {}", + path_prefix, + name.description(), + data_string, + remove_factor); +} + +Coordination::ZooKeeperRequestPtr CreateRequestGenerator::generateImpl(const Coordination::ACLs & acls) +{ + if (!paths_created.empty() && remove_picker(rng) < remove_factor) { - if (response.error != Coordination::Error::ZOK) - promise->set_exception(std::make_exception_ptr(zkutil::KeeperException(response.error))); - else - promise->set_value(); - }; - zookeeper.create(path_prefix, "", false, false, default_acls, create_callback); - future.get(); - size_t total_nodes = 1; - if (num_nodes) - total_nodes = *num_nodes; + auto request = std::make_shared(); + auto it = paths_created.begin(); + request->path = *it; + paths_created.erase(it); + return request; + } + + auto request = std::make_shared(); + request->acls = acls; + + std::string path_candidate = std::filesystem::path(path_prefix) / name.getString(); + + while (paths_created.contains(path_candidate)) + path_candidate = std::filesystem::path(path_prefix) / name.getString(); + + paths_created.insert(path_candidate); + + request->path = std::move(path_candidate); + + if (data) + request->data = data->getString(); + + return request; +} + +Generator::Generator(const Poco::Util::AbstractConfiguration & config) +{ + Coordination::ACL acl; + acl.permissions = Coordination::ACL::All; + acl.scheme = "world"; + acl.id = "anyone"; + default_acls.emplace_back(std::move(acl)); - for (size_t i = 0; i < total_nodes; ++i) + static const std::string generator_key = "generator"; + + { + static const std::string setup_key = generator_key + ".setup"; + Poco::Util::AbstractConfiguration::Keys keys; + config.keys(setup_key, keys); + for (const auto & key : keys) + { + if (key.starts_with("node")) + { + const auto & node = root_nodes.emplace_back(parseNode(setup_key + "." + key, config)); + + std::cout << "---- Will create tree ----" << std::endl; + node->dumpTree(); + } + } + } { - auto path = generateRandomPath(path_prefix, 5); - while (std::find(paths_to_get.begin(), paths_to_get.end(), path) != paths_to_get.end()) - path = generateRandomPath(path_prefix, 5); + static const std::string requests_key = generator_key + ".requests"; + Poco::Util::AbstractConfiguration::Keys keys; + config.keys(requests_key, keys); - auto create_promise = std::make_shared>(); - auto create_future = create_promise->get_future(); - auto callback = [create_promise] (const CreateResponse & response) + std::cout << "\n---- Collecting request generators ----" << std::endl; + for (const auto & key : keys) { - if (response.error != Coordination::Error::ZOK) - create_promise->set_exception(std::make_exception_ptr(zkutil::KeeperException(response.error))); - else - create_promise->set_value(); - }; - std::string data; - if (nodes_data_size) - data = generateRandomString(*nodes_data_size); + RequestGeneratorPtr request_generator; + + if (key.starts_with("create")) + request_generator = std::make_unique(); + + if (!request_generator) + throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "Unknown generator {}", key); + + request_generator->getFromConfig(requests_key + "." + key, config); - zookeeper.create(path, data, false, false, default_acls, callback); - create_future.get(); - paths_to_get.push_back(path); + std::cout << fmt::format("\n{}\n", request_generator->description()) << std::endl; + request_generators.push_back(std::move(request_generator)); + } + + if (request_generators.empty()) + throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, "No request generators found in config"); + std::cout << "---- Done collecting request generators ----" << std::endl; } + + request_picker = std::uniform_int_distribution(0, request_generators.size() - 1); } -Coordination::ZooKeeperRequestPtr GetRequestGenerator::generate() +std::shared_ptr Generator::parseNode(const std::string & key, const Poco::Util::AbstractConfiguration & config) { - auto request = std::make_shared(); + auto node = std::make_shared(); + node->name = StringGetter::fromConfig(key + ".name", config); - size_t path_index = distribution(rng); - request->path = paths_to_get[path_index]; - return request; + if (config.has(key + ".data")) + node->data = StringGetter::fromConfig(key + ".data", config); + + Poco::Util::AbstractConfiguration::Keys node_keys; + config.keys(key, node_keys); + + for (const auto & node_key : node_keys) + { + if (!node_key.starts_with("node")) + continue; + + const auto node_key_string = key + "." + node_key; + auto child_node = parseNode(node_key_string, config); + node->children.push_back(child_node); + + if (config.has(node_key_string + ".repeat")) + { + if (!child_node->name.isRandom()) + throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, "Repeating node creation for key {}, but name is not randomly generated", node_key_string); + + auto repeat_count = config.getUInt64(node_key_string + ".repeat"); + for (size_t i = 1; i < repeat_count; ++i) + node->children.push_back(child_node); + } + } + + return node; +} + +void Generator::Node::dumpTree(int level) const +{ + std::string data_string + = data.has_value() ? fmt::format("{}", data->description()) : "no data"; + std::cout << fmt::format("{}name: {}, data: {}", std::string(level, '\t'), name.description(), data_string) << std::endl; + + for (const auto & child : children) + child->dumpTree(level + 1); } -void ListRequestGenerator::startup(Coordination::ZooKeeper & zookeeper) +void Generator::Node::createNode(Coordination::ZooKeeper & zookeeper, const std::string & parent_path, const Coordination::ACLs & acls) const { + auto path = std::filesystem::path(parent_path) / name.getString(); auto promise = std::make_shared>(); auto future = promise->get_future(); auto create_callback = [promise] (const CreateResponse & response) @@ -239,110 +573,32 @@ void ListRequestGenerator::startup(Coordination::ZooKeeper & zookeeper) else promise->set_value(); }; - zookeeper.create(path_prefix, "", false, false, default_acls, create_callback); + zookeeper.create(path, data ? data->getString() : "", false, false, acls, create_callback); future.get(); - size_t total_nodes = 1; - if (num_nodes) - total_nodes = *num_nodes; - - size_t path_length = 5; - if (paths_length) - path_length = *paths_length; - - for (size_t i = 0; i < total_nodes; ++i) - { - auto path = generateRandomPath(path_prefix, path_length); - - auto create_promise = std::make_shared>(); - auto create_future = create_promise->get_future(); - auto callback = [create_promise] (const CreateResponse & response) - { - if (response.error != Coordination::Error::ZOK) - create_promise->set_exception(std::make_exception_ptr(zkutil::KeeperException(response.error))); - else - create_promise->set_value(); - }; - zookeeper.create(path, "", false, false, default_acls, callback); - create_future.get(); - } + for (const auto & child : children) + child->createNode(zookeeper, path, acls); } -Coordination::ZooKeeperRequestPtr ListRequestGenerator::generate() +void Generator::startup(Coordination::ZooKeeper & zookeeper) { - auto request = std::make_shared(); - request->path = path_prefix; - return request; -} - -std::unique_ptr getGenerator(const std::string & name) -{ - if (name == "create_no_data") - { - return std::make_unique(); - } - else if (name == "create_small_data") - { - return std::make_unique("/create_generator", 5, 32); - } - else if (name == "create_medium_data") - { - return std::make_unique("/create_generator", 5, 1024); - } - else if (name == "create_big_data") - { - return std::make_unique("/create_generator", 5, 512 * 1024); - } - else if (name == "get_no_data") - { - return std::make_unique("/get_generator", 10, 0); - } - else if (name == "get_small_data") - { - return std::make_unique("/get_generator", 10, 32); - } - else if (name == "get_medium_data") - { - return std::make_unique("/get_generator", 10, 1024); - } - else if (name == "get_big_data") - { - return std::make_unique("/get_generator", 10, 512 * 1024); - } - else if (name == "list_no_nodes") - { - return std::make_unique("/list_generator", 0, 1); - } - else if (name == "list_few_nodes") - { - return std::make_unique("/list_generator", 10, 5); - } - else if (name == "list_medium_nodes") - { - return std::make_unique("/list_generator", 1000, 5); - } - else if (name == "list_a_lot_nodes") - { - return std::make_unique("/list_generator", 100000, 5); - } - else if (name == "set_small_data") - { - return std::make_unique("/set_generator", 5); - } - else if (name == "mixed_small_data") + std::cout << "\n---- Creating test data ----" << std::endl; + for (const auto & node : root_nodes) { - std::vector> generators; - generators.push_back(std::make_unique("/set_generator", 5)); - generators.push_back(std::make_unique("/get_generator", 10, 32)); - return std::make_unique(std::move(generators)); - } + auto node_name = node->name.getString(); + node->name.setString(node_name); - throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "Unknown generator {}", name); + std::string root_path = std::filesystem::path("/") / node_name; + std::cout << "Cleaning up " << root_path << std::endl; + removeRecursive(zookeeper, root_path); + + node->createNode(zookeeper, "/", default_acls); + } + std::cout << "---- Created test data ----" << std::endl; } -std::unique_ptr constructGeneratorFromConfig(const std::string & config_path) +Coordination::ZooKeeperRequestPtr Generator::generate() { - ConfigProcessor config_processor(config_path, true, false); - auto loaded_config = config_processor.loadConfig(); - return nullptr; + static pcg64 rng(randomSeed()); + return request_generators[request_picker(rng)]->generate(default_acls); } diff --git a/utils/keeper-bench/Generator.h b/utils/keeper-bench/Generator.h index 25e4d96caeff..123e45471205 100644 --- a/utils/keeper-bench/Generator.h +++ b/utils/keeper-bench/Generator.h @@ -6,6 +6,7 @@ #include #include #include +#include #include @@ -13,130 +14,203 @@ std::string generateRandomPath(const std::string & prefix, size_t length = 5); std::string generateRandomData(size_t size); -class IGenerator +// +//class CreateRequestGenerator final : public IGenerator +//{ +//public: +// explicit CreateRequestGenerator( +// std::string path_prefix_ = "/create_generator", +// std::optional path_length_ = std::nullopt, +// std::optional data_size_ = std::nullopt) +// : path_prefix(path_prefix_) +// , path_length(path_length_) +// , data_size(data_size_) +// {} +// +// void startup(Coordination::ZooKeeper & zookeeper) override; +// Coordination::ZooKeeperRequestPtr generate() override; +// +//private: +// std::string path_prefix; +// std::optional path_length; +// std::optional data_size; +// std::unordered_set paths_created; +//}; +// +// +//class GetRequestGenerator final : public IGenerator +//{ +//public: +// explicit GetRequestGenerator( +// std::string path_prefix_ = "/get_generator", +// std::optional num_nodes_ = std::nullopt, +// std::optional nodes_data_size_ = std::nullopt) +// : path_prefix(path_prefix_) +// , num_nodes(num_nodes_) +// , nodes_data_size(nodes_data_size_) +// , rng(randomSeed()) +// , distribution(0, num_nodes ? *num_nodes - 1 : 0) +// {} +// +// void startup(Coordination::ZooKeeper & zookeeper) override; +// Coordination::ZooKeeperRequestPtr generate() override; +// +//private: +// std::string path_prefix; +// std::optional num_nodes; +// std::optional nodes_data_size; +// std::vector paths_to_get; +// +// pcg64 rng; +// std::uniform_int_distribution distribution; +//}; +// +//class ListRequestGenerator final : public IGenerator +//{ +//public: +// explicit ListRequestGenerator( +// std::string path_prefix_ = "/list_generator", +// std::optional num_nodes_ = std::nullopt, +// std::optional paths_length_ = std::nullopt) +// : path_prefix(path_prefix_) +// , num_nodes(num_nodes_) +// , paths_length(paths_length_) +// {} +// +// void startup(Coordination::ZooKeeper & zookeeper) override; +// Coordination::ZooKeeperRequestPtr generate() override; +// +//private: +// std::string path_prefix; +// std::optional num_nodes; +// std::optional paths_length; +//}; +// +//class SetRequestGenerator final : public IGenerator +//{ +//public: +// explicit SetRequestGenerator( +// std::string path_prefix_ = "/set_generator", +// uint64_t data_size_ = 5) +// : path_prefix(path_prefix_) +// , data_size(data_size_) +// {} +// +// void startup(Coordination::ZooKeeper & zookeeper) override; +// Coordination::ZooKeeperRequestPtr generate() override; +// +//private: +// std::string path_prefix; +// uint64_t data_size; +//}; +// +//class MixedRequestGenerator final : public IGenerator +//{ +//public: +// explicit MixedRequestGenerator(std::vector> generators_) +// : generators(std::move(generators_)) +// {} +// +// void startup(Coordination::ZooKeeper & zookeeper) override; +// Coordination::ZooKeeperRequestPtr generate() override; +// +//private: +// std::vector> generators; +//}; + +struct NumberGetter { -public: - IGenerator() + static NumberGetter fromConfig(const std::string & key, const Poco::Util::AbstractConfiguration & config, std::optional default_value = std::nullopt); + uint64_t getNumber() const; + std::string description() const; +private: + struct NumberRange { - Coordination::ACL acl; - acl.permissions = Coordination::ACL::All; - acl.scheme = "world"; - acl.id = "anyone"; - default_acls.emplace_back(std::move(acl)); - } - virtual void startup(Coordination::ZooKeeper & /*zookeeper*/) {} - virtual Coordination::ZooKeeperRequestPtr generate() = 0; - - virtual ~IGenerator() = default; - - Coordination::ACLs default_acls; + uint64_t min_value; + uint64_t max_value; + }; + std::variant value; }; -class CreateRequestGenerator final : public IGenerator +struct StringGetter { -public: - explicit CreateRequestGenerator( - std::string path_prefix_ = "/create_generator", - std::optional path_length_ = std::nullopt, - std::optional data_size_ = std::nullopt) - : path_prefix(path_prefix_) - , path_length(path_length_) - , data_size(data_size_) + explicit StringGetter(NumberGetter number_getter) + : value(std::move(number_getter)) {} - void startup(Coordination::ZooKeeper & zookeeper) override; - Coordination::ZooKeeperRequestPtr generate() override; + StringGetter() = default; + static StringGetter fromConfig(const std::string & key, const Poco::Util::AbstractConfiguration & config); + void setString(std::string name); + std::string getString() const; + std::string description() const; + bool isRandom() const; private: - std::string path_prefix; - std::optional path_length; - std::optional data_size; - std::unordered_set paths_created; + std::variant value; }; - -class GetRequestGenerator final : public IGenerator +struct RequestGenerator { -public: - explicit GetRequestGenerator( - std::string path_prefix_ = "/get_generator", - std::optional num_nodes_ = std::nullopt, - std::optional nodes_data_size_ = std::nullopt) - : path_prefix(path_prefix_) - , num_nodes(num_nodes_) - , nodes_data_size(nodes_data_size_) - , rng(randomSeed()) - , distribution(0, num_nodes ? *num_nodes - 1 : 0) - {} + virtual ~RequestGenerator() = default; - void startup(Coordination::ZooKeeper & zookeeper) override; - Coordination::ZooKeeperRequestPtr generate() override; + void getFromConfig(const std::string & key, const Poco::Util::AbstractConfiguration & config); -private: - std::string path_prefix; - std::optional num_nodes; - std::optional nodes_data_size; - std::vector paths_to_get; + Coordination::ZooKeeperRequestPtr generate(const Coordination::ACLs & acls); - pcg64 rng; - std::uniform_int_distribution distribution; + std::string description(); +private: + virtual void getFromConfigImpl(const std::string & key, const Poco::Util::AbstractConfiguration & config) = 0; + virtual std::string descriptionImpl() = 0; + virtual Coordination::ZooKeeperRequestPtr generateImpl(const Coordination::ACLs & acls) = 0; }; -class ListRequestGenerator final : public IGenerator -{ -public: - explicit ListRequestGenerator( - std::string path_prefix_ = "/list_generator", - std::optional num_nodes_ = std::nullopt, - std::optional paths_length_ = std::nullopt) - : path_prefix(path_prefix_) - , num_nodes(num_nodes_) - , paths_length(paths_length_) - {} - - void startup(Coordination::ZooKeeper & zookeeper) override; - Coordination::ZooKeeperRequestPtr generate() override; +using RequestGeneratorPtr = std::unique_ptr; +struct CreateRequestGenerator final : public RequestGenerator +{ + CreateRequestGenerator(); private: - std::string path_prefix; - std::optional num_nodes; - std::optional paths_length; -}; + void getFromConfigImpl(const std::string & key, const Poco::Util::AbstractConfiguration & config) override; + std::string descriptionImpl() override; + Coordination::ZooKeeperRequestPtr generateImpl(const Coordination::ACLs & acls) override; -class SetRequestGenerator final : public IGenerator -{ -public: - explicit SetRequestGenerator( - std::string path_prefix_ = "/set_generator", - uint64_t data_size_ = 5) - : path_prefix(path_prefix_) - , data_size(data_size_) - {} + std::string path_prefix; + StringGetter name; + std::optional data; - void startup(Coordination::ZooKeeper & zookeeper) override; - Coordination::ZooKeeperRequestPtr generate() override; + double remove_factor; + pcg64 rng; + std::uniform_real_distribution remove_picker; -private: - std::string path_prefix; - uint64_t data_size; + std::unordered_set paths_created; }; -class MixedRequestGenerator final : public IGenerator +class Generator { public: - explicit MixedRequestGenerator(std::vector> generators_) - : generators(std::move(generators_)) - {} + explicit Generator(const Poco::Util::AbstractConfiguration & config); - void startup(Coordination::ZooKeeper & zookeeper) override; - Coordination::ZooKeeperRequestPtr generate() override; + void startup(Coordination::ZooKeeper & zookeeper); + Coordination::ZooKeeperRequestPtr generate(); private: - std::vector> generators; -}; + struct Node + { + StringGetter name; + std::optional data; + std::vector> children; + + void createNode(Coordination::ZooKeeper & zookeeper, const std::string & parent_path, const Coordination::ACLs & acls) const; + void dumpTree(int level = 0) const; + }; + static std::shared_ptr parseNode(const std::string & key, const Poco::Util::AbstractConfiguration & config); -std::unique_ptr constructGeneratorFromConfig(const std::string & config_path); + std::uniform_int_distribution request_picker; + std::vector> root_nodes; + std::vector request_generators; + Coordination::ACLs default_acls; +}; -std::unique_ptr getGenerator(const std::string & name); +std::unique_ptr getGenerator(const std::string & name); diff --git a/utils/keeper-bench/Runner.cpp b/utils/keeper-bench/Runner.cpp index c858b476483f..387b7baebdd4 100644 --- a/utils/keeper-bench/Runner.cpp +++ b/utils/keeper-bench/Runner.cpp @@ -1,15 +1,50 @@ #include "Runner.h" -namespace DB -{ +#include -namespace ErrorCodes +namespace DB::ErrorCodes { extern const int CANNOT_BLOCK_SIGNAL; } -} +Runner::Runner( + size_t concurrency_, + const std::string & generator_name, + const std::string & config_path, + const Strings & hosts_strings_, + double max_time_, + double delay_, + bool continue_on_error_, + size_t max_iterations_) + : concurrency(concurrency_) + , pool(CurrentMetrics::LocalThread, CurrentMetrics::LocalThreadActive, concurrency) + , hosts_strings(hosts_strings_) + , max_time(max_time_) + , delay(delay_) + , continue_on_error(continue_on_error_) + , max_iterations(max_iterations_) + , info(std::make_shared()) + , queue(concurrency) + { + if (!generator_name.empty() && !config_path.empty()) + throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, "Both generator name and generator config path are defined. Please define only one of them"); + if (generator_name.empty() && config_path.empty()) + throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, "Both generator name and generator config path are empty. Please define one of them"); + + if (!generator_name.empty()) + generator = getGenerator(generator_name); + else + { + DB::ConfigProcessor config_processor(config_path, true, false); + auto loaded_config = config_processor.loadConfig(); + + generator = std::make_unique(*loaded_config.configuration); + } + + if (!generator) + throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, "Failed to create generator"); + } void Runner::thread(std::vector> zookeepers) { diff --git a/utils/keeper-bench/Runner.h b/utils/keeper-bench/Runner.h index d3f2d1800d90..f8280ac5f376 100644 --- a/utils/keeper-bench/Runner.h +++ b/utils/keeper-bench/Runner.h @@ -2,6 +2,7 @@ #include #include "Generator.h" #include +#include #include #include #include @@ -38,28 +39,7 @@ class Runner double max_time_, double delay_, bool continue_on_error_, - size_t max_iterations_) - : concurrency(concurrency_) - , pool(CurrentMetrics::LocalThread, CurrentMetrics::LocalThreadActive, concurrency) - , hosts_strings(hosts_strings_) - , max_time(max_time_) - , delay(delay_) - , continue_on_error(continue_on_error_) - , max_iterations(max_iterations_) - , info(std::make_shared()) - , queue(concurrency) - { - if (!generator_name.empty() && !config_path.empty()) - throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, "Both generator name and generator config path are defined. Please define only one of them"); - - if (generator_name.empty() && config_path.empty()) - throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, "Both generator name and generator config path are empty. Please define one of them"); - - if (!generator_name.empty()) - generator = getGenerator(generator_name); - else - generator = constructGeneratorFromConfig(config_path); - } + size_t max_iterations_); void thread(std::vector> zookeepers); @@ -79,7 +59,7 @@ class Runner ThreadPool pool; Strings hosts_strings; - std::unique_ptr generator; + std::unique_ptr generator; double max_time = 0; double delay = 1; bool continue_on_error = false; From 7b33744618994151348c37cc8bacde9cf92c7bb5 Mon Sep 17 00:00:00 2001 From: Smita Kulkarni Date: Thu, 6 Apr 2023 20:11:40 +0200 Subject: [PATCH 100/406] Added a check to clear and not load marks asynchronously from outdated parts --- src/Storages/MergeTree/IDataPartStorage.h | 2 ++ src/Storages/MergeTree/MergeTreeData.cpp | 3 +++ src/Storages/MergeTree/MergeTreeMarksLoader.cpp | 15 +++++++++++++++ 3 files changed, 20 insertions(+) diff --git a/src/Storages/MergeTree/IDataPartStorage.h b/src/Storages/MergeTree/IDataPartStorage.h index f92784cb0dad..98c14bd377c6 100644 --- a/src/Storages/MergeTree/IDataPartStorage.h +++ b/src/Storages/MergeTree/IDataPartStorage.h @@ -284,6 +284,8 @@ class IDataPartStorage : public boost::noncopyable /// It may be flush of buffered data or similar. virtual void precommitTransaction() = 0; virtual bool hasActiveTransaction() const = 0; + + mutable std::atomic is_part_outdated = false; }; using DataPartStoragePtr = std::shared_ptr; diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 66c52e6e24ca..057dba29ea81 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -3670,6 +3670,8 @@ void MergeTreeData::removePartsFromWorkingSet(MergeTreeTransaction * txn, const if (isInMemoryPart(part) && getSettings()->in_memory_parts_enable_wal) getWriteAheadLog()->dropPart(part->name); + + part->getDataPartStorage().is_part_outdated = true; } if (removed_active_part) @@ -3834,6 +3836,7 @@ void MergeTreeData::restoreAndActivatePart(const DataPartPtr & part, DataPartsLo addPartContributionToColumnAndSecondaryIndexSizes(part); addPartContributionToDataVolume(part); modifyPartState(part, DataPartState::Active); + part->getDataPartStorage().is_part_outdated = false; } diff --git a/src/Storages/MergeTree/MergeTreeMarksLoader.cpp b/src/Storages/MergeTree/MergeTreeMarksLoader.cpp index ed8866b0044b..b870421993f4 100644 --- a/src/Storages/MergeTree/MergeTreeMarksLoader.cpp +++ b/src/Storages/MergeTree/MergeTreeMarksLoader.cpp @@ -68,6 +68,11 @@ MarkInCompressedFile MergeTreeMarksLoader::getMark(size_t row_index, size_t colu { if (!marks) { + if (this->data_part_storage->is_part_outdated) + { + throw Exception(ErrorCodes::LOGICAL_ERROR, "Attempting to read from outdated part. path : {}", data_part_storage->getFullPath()); + } + Stopwatch watch(CLOCK_MONOTONIC); if (future.valid()) @@ -196,6 +201,16 @@ std::future MergeTreeMarksLoader::loadMarksAsync() [this]() -> MarkCache::MappedPtr { ProfileEvents::increment(ProfileEvents::BackgroundLoadingMarksTasks); + if (this->data_part_storage->is_part_outdated) + { + if (mark_cache) + { + auto key = mark_cache->hash(fs::path(data_part_storage->getFullPath()) / mrk_path); + marks.reset(); + mark_cache->remove(key); + } + return nullptr; + } return loadMarks(); }, *load_marks_threadpool, From 994b6dd71ce1396b9fd684c06ba0cf1f78c2be9e Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Thu, 6 Apr 2023 19:17:35 +0000 Subject: [PATCH 101/406] Add other requests --- utils/keeper-bench/Generator.cpp | 367 +++++++++++++++++-------------- utils/keeper-bench/Generator.h | 58 ++++- 2 files changed, 258 insertions(+), 167 deletions(-) diff --git a/utils/keeper-bench/Generator.cpp b/utils/keeper-bench/Generator.cpp index e2c276a274de..eba6b7d97477 100644 --- a/utils/keeper-bench/Generator.cpp +++ b/utils/keeper-bench/Generator.cpp @@ -3,6 +3,7 @@ #include #include #include +#include using namespace Coordination; using namespace zkutil; @@ -36,16 +37,6 @@ std::string generateRandomString(size_t length) return s; } } -// -//std::string generateRandomPath(const std::string & prefix, size_t length) -//{ -// return std::filesystem::path(prefix) / generateRandomString(length); -//} -// -//std::string generateRandomData(size_t size) -//{ -// return generateRandomString(size); -//} void removeRecursive(Coordination::ZooKeeper & zookeeper, const std::string & path) { @@ -95,145 +86,6 @@ void removeRecursive(Coordination::ZooKeeper & zookeeper, const std::string & pa remove_future.get(); } - -//void SetRequestGenerator::startup(Coordination::ZooKeeper & zookeeper) -//{ -// removeRecursive(zookeeper, path_prefix); -// -// auto promise = std::make_shared>(); -// auto future = promise->get_future(); -// auto create_callback = [promise] (const CreateResponse & response) -// { -// if (response.error != Coordination::Error::ZOK) -// promise->set_exception(std::make_exception_ptr(zkutil::KeeperException(response.error))); -// else -// promise->set_value(); -// }; -// zookeeper.create(path_prefix, "", false, false, default_acls, create_callback); -// future.get(); -//} -// -//ZooKeeperRequestPtr SetRequestGenerator::generate() -//{ -// auto request = std::make_shared(); -// request->path = path_prefix; -// request->data = generateRandomData(data_size); -// -// return request; -//} -// -//void MixedRequestGenerator::startup(Coordination::ZooKeeper & zookeeper) -//{ -// for (auto & generator : generators) -// generator->startup(zookeeper); -//} -// -//ZooKeeperRequestPtr MixedRequestGenerator::generate() -//{ -// pcg64 rng(randomSeed()); -// std::uniform_int_distribution distribution(0, generators.size() - 1); -// -// return generators[distribution(rng)]->generate(); -//} -// -//void GetRequestGenerator::startup(Coordination::ZooKeeper & zookeeper) -//{ -// auto promise = std::make_shared>(); -// auto future = promise->get_future(); -// auto create_callback = [promise] (const CreateResponse & response) -// { -// if (response.error != Coordination::Error::ZOK) -// promise->set_exception(std::make_exception_ptr(zkutil::KeeperException(response.error))); -// else -// promise->set_value(); -// }; -// zookeeper.create(path_prefix, "", false, false, default_acls, create_callback); -// future.get(); -// size_t total_nodes = 1; -// if (num_nodes) -// total_nodes = *num_nodes; -// -// for (size_t i = 0; i < total_nodes; ++i) -// { -// auto path = generateRandomPath(path_prefix, 5); -// while (std::find(paths_to_get.begin(), paths_to_get.end(), path) != paths_to_get.end()) -// path = generateRandomPath(path_prefix, 5); -// -// auto create_promise = std::make_shared>(); -// auto create_future = create_promise->get_future(); -// auto callback = [create_promise] (const CreateResponse & response) -// { -// if (response.error != Coordination::Error::ZOK) -// create_promise->set_exception(std::make_exception_ptr(zkutil::KeeperException(response.error))); -// else -// create_promise->set_value(); -// }; -// std::string data; -// if (nodes_data_size) -// data = generateRandomString(*nodes_data_size); -// -// zookeeper.create(path, data, false, false, default_acls, callback); -// create_future.get(); -// paths_to_get.push_back(path); -// } -//} -// -//Coordination::ZooKeeperRequestPtr GetRequestGenerator::generate() -//{ -// auto request = std::make_shared(); -// -// size_t path_index = distribution(rng); -// request->path = paths_to_get[path_index]; -// return request; -//} -// -//void ListRequestGenerator::startup(Coordination::ZooKeeper & zookeeper) -//{ -// auto promise = std::make_shared>(); -// auto future = promise->get_future(); -// auto create_callback = [promise] (const CreateResponse & response) -// { -// if (response.error != Coordination::Error::ZOK) -// promise->set_exception(std::make_exception_ptr(zkutil::KeeperException(response.error))); -// else -// promise->set_value(); -// }; -// zookeeper.create(path_prefix, "", false, false, default_acls, create_callback); -// future.get(); -// -// size_t total_nodes = 1; -// if (num_nodes) -// total_nodes = *num_nodes; -// -// size_t path_length = 5; -// if (paths_length) -// path_length = *paths_length; -// -// for (size_t i = 0; i < total_nodes; ++i) -// { -// auto path = generateRandomPath(path_prefix, path_length); -// -// auto create_promise = std::make_shared>(); -// auto create_future = create_promise->get_future(); -// auto callback = [create_promise] (const CreateResponse & response) -// { -// if (response.error != Coordination::Error::ZOK) -// create_promise->set_exception(std::make_exception_ptr(zkutil::KeeperException(response.error))); -// else -// create_promise->set_value(); -// }; -// zookeeper.create(path, "", false, false, default_acls, callback); -// create_future.get(); -// } -//} -// -//Coordination::ZooKeeperRequestPtr ListRequestGenerator::generate() -//{ -// auto request = std::make_shared(); -// request->path = path_prefix; -// return request; -//} - std::unique_ptr getGenerator(const std::string & name) { //if (name == "create_no_data") @@ -288,13 +140,6 @@ std::unique_ptr getGenerator(const std::string & name) //{ // return std::make_unique("/set_generator", 5); //} - //else if (name == "mixed_small_data") - //{ - // std::vector> generators; - // generators.push_back(std::make_unique("/set_generator", 5)); - // generators.push_back(std::make_unique("/get_generator", 10, 32)); - // return std::make_unique(std::move(generators)); - //} throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "Unknown generator {}", name); } @@ -382,6 +227,99 @@ bool StringGetter::isRandom() const return std::holds_alternative(value); } +PathGetter PathGetter::fromConfig(const std::string & key, const Poco::Util::AbstractConfiguration & config) +{ + static constexpr std::string_view path_key_string = "path"; + + PathGetter path_getter; + Poco::Util::AbstractConfiguration::Keys path_keys; + config.keys(key, path_keys); + + for (const auto & path_key : path_keys) + { + if (!path_key.starts_with(path_key_string)) + continue; + + const auto current_path_key_string = key + "." + path_key; + const auto children_of_key = current_path_key_string + ".children_of"; + if (config.has(children_of_key)) + { + auto parent_node = config.getString(children_of_key); + if (parent_node.empty() || parent_node[0] != '/') + throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, "Invalid path for request generator: '{}'", parent_node); + path_getter.parent_paths.push_back(std::move(parent_node)); + } + else + { + auto path = config.getString(key + "." + path_key); + + if (path.empty() || path[0] != '/') + throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, "Invalid path for request generator: '{}'", path); + + path_getter.paths.push_back(std::move(path)); + } + } + + path_getter.path_picker = std::uniform_int_distribution(0, path_getter.paths.size() - 1); + return path_getter; +} + +void PathGetter::initialize(Coordination::ZooKeeper & zookeeper) +{ + for (const auto & parent_path : parent_paths) + { + auto list_promise = std::make_shared>(); + auto list_future = list_promise->get_future(); + auto callback = [list_promise] (const ListResponse & response) + { + if (response.error != Coordination::Error::ZOK) + list_promise->set_exception(std::make_exception_ptr(zkutil::KeeperException(response.error))); + else + list_promise->set_value(response); + }; + zookeeper.list(parent_path, ListRequestType::ALL, std::move(callback), {}); + auto list_response = list_future.get(); + + for (const auto & child : list_response.names) + paths.push_back(std::filesystem::path(parent_path) / child); + } + + path_picker = std::uniform_int_distribution(0, paths.size() - 1); + initialized = true; +} + +std::string PathGetter::getPath() const +{ + if (!initialized) + throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "PathGetter is not initialized"); + + if (paths.size() == 1) + return paths[0]; + + static pcg64 rng(randomSeed()); + return paths[path_picker(rng)]; +} + +std::string PathGetter::description() const +{ + std::string description; + for (const auto & path : parent_paths) + { + if (!description.empty()) + description += ", "; + description += fmt::format("children of {}", path); + } + + for (const auto & path : paths) + { + if (!description.empty()) + description += ", "; + description += path; + } + + return description; +} + void RequestGenerator::getFromConfig(const std::string & key, const Poco::Util::AbstractConfiguration & config) { getFromConfigImpl(key, config); @@ -397,6 +335,11 @@ Coordination::ZooKeeperRequestPtr RequestGenerator::generate(const Coordination: return generateImpl(acls); } +void RequestGenerator::startup(Coordination::ZooKeeper & zookeeper) +{ + startupImpl(zookeeper); +} + CreateRequestGenerator::CreateRequestGenerator() : rng(randomSeed()) , remove_picker(0, 1.0) @@ -404,12 +347,9 @@ CreateRequestGenerator::CreateRequestGenerator() void CreateRequestGenerator::getFromConfigImpl(const std::string & key, const Poco::Util::AbstractConfiguration & config) { - path_prefix = config.getString(key + ".path_prefix"); - - if (path_prefix.empty() || path_prefix[0] != '/') - throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, "Invalid path_prefix for Create request generator: '{}'", path_prefix); + parent_path = PathGetter::fromConfig(key, config); - name = StringGetter(NumberGetter::fromConfig(key + ".path_length", config, 5)); + name = StringGetter(NumberGetter::fromConfig(key + ".name_length", config, 5)); if (config.has(key + ".data")) data = StringGetter::fromConfig(key + ".data", config); @@ -423,16 +363,21 @@ std::string CreateRequestGenerator::descriptionImpl() = data.has_value() ? fmt::format("data for created nodes: {}", data->description()) : "no data for created nodes"; return fmt::format( "Create Request Generator\n" - "- path prefix for created nodes: {}\n" + "- parent path(s) for created nodes: {}\n" "- name for created nodes: {}\n" "- {}\n" "- remove factor: {}", - path_prefix, + parent_path.description(), name.description(), data_string, remove_factor); } +void CreateRequestGenerator::startupImpl(Coordination::ZooKeeper & zookeeper) +{ + parent_path.initialize(zookeeper); +} + Coordination::ZooKeeperRequestPtr CreateRequestGenerator::generateImpl(const Coordination::ACLs & acls) { if (!paths_created.empty() && remove_picker(rng) < remove_factor) @@ -447,10 +392,10 @@ Coordination::ZooKeeperRequestPtr CreateRequestGenerator::generateImpl(const Coo auto request = std::make_shared(); request->acls = acls; - std::string path_candidate = std::filesystem::path(path_prefix) / name.getString(); + std::string path_candidate = std::filesystem::path(parent_path.getPath()) / name.getString(); while (paths_created.contains(path_candidate)) - path_candidate = std::filesystem::path(path_prefix) / name.getString(); + path_candidate = std::filesystem::path(parent_path.getPath()) / name.getString(); paths_created.insert(path_candidate); @@ -462,6 +407,86 @@ Coordination::ZooKeeperRequestPtr CreateRequestGenerator::generateImpl(const Coo return request; } +void SetRequestGenerator::getFromConfigImpl(const std::string & key, const Poco::Util::AbstractConfiguration & config) +{ + path = PathGetter::fromConfig(key, config); + + data = StringGetter::fromConfig(key + ".data", config); +} + +std::string SetRequestGenerator::descriptionImpl() +{ + return fmt::format( + "Set Request Generator\n" + "- path(s) to set: {}\n" + "- data to set: {}", + path.description(), + data.description()); +} + +Coordination::ZooKeeperRequestPtr SetRequestGenerator::generateImpl(const Coordination::ACLs & /*acls*/) +{ + auto request = std::make_shared(); + request->path = path.getPath(); + request->data = data.getString(); + return request; +} + +void SetRequestGenerator::startupImpl(Coordination::ZooKeeper & zookeeper) +{ + path.initialize(zookeeper); +} + +void GetRequestGenerator::getFromConfigImpl(const std::string & key, const Poco::Util::AbstractConfiguration & config) +{ + path = PathGetter::fromConfig(key, config); +} + +std::string GetRequestGenerator::descriptionImpl() +{ + return fmt::format( + "Get Request Generator\n" + "- path(s) to get: {}", + path.description()); +} + +Coordination::ZooKeeperRequestPtr GetRequestGenerator::generateImpl(const Coordination::ACLs & /*acls*/) +{ + auto request = std::make_shared(); + request->path = path.getPath(); + return request; +} + +void GetRequestGenerator::startupImpl(Coordination::ZooKeeper & zookeeper) +{ + path.initialize(zookeeper); +} + +void ListRequestGenerator::getFromConfigImpl(const std::string & key, const Poco::Util::AbstractConfiguration & config) +{ + path = PathGetter::fromConfig(key, config); +} + +std::string ListRequestGenerator::descriptionImpl() +{ + return fmt::format( + "List Request Generator\n" + "- path(s) to get: {}", + path.description()); +} + +Coordination::ZooKeeperRequestPtr ListRequestGenerator::generateImpl(const Coordination::ACLs & /*acls*/) +{ + auto request = std::make_shared(); + request->path = path.getPath(); + return request; +} + +void ListRequestGenerator::startupImpl(Coordination::ZooKeeper & zookeeper) +{ + path.initialize(zookeeper); +} + Generator::Generator(const Poco::Util::AbstractConfiguration & config) { Coordination::ACL acl; @@ -499,6 +524,12 @@ Generator::Generator(const Poco::Util::AbstractConfiguration & config) if (key.starts_with("create")) request_generator = std::make_unique(); + else if (key.starts_with("set")) + request_generator = std::make_unique(); + else if (key.starts_with("get")) + request_generator = std::make_unique(); + else if (key.starts_with("list")) + request_generator = std::make_unique(); if (!request_generator) throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "Unknown generator {}", key); @@ -595,6 +626,12 @@ void Generator::startup(Coordination::ZooKeeper & zookeeper) node->createNode(zookeeper, "/", default_acls); } std::cout << "---- Created test data ----" << std::endl; + + + std::cout << "---- Initializing generators ----" << std::endl; + + for (const auto & generator : request_generators) + generator->startup(zookeeper); } Coordination::ZooKeeperRequestPtr Generator::generate() diff --git a/utils/keeper-bench/Generator.h b/utils/keeper-bench/Generator.h index 123e45471205..af186ea6624a 100644 --- a/utils/keeper-bench/Generator.h +++ b/utils/keeper-bench/Generator.h @@ -150,6 +150,23 @@ struct StringGetter std::variant value; }; +struct PathGetter +{ + static PathGetter fromConfig(const std::string & key, const Poco::Util::AbstractConfiguration & config); + + std::string getPath() const; + std::string description() const; + + void initialize(Coordination::ZooKeeper & zookeeper); +private: + std::vector parent_paths; + + bool initialized = false; + + std::vector paths; + mutable std::uniform_int_distribution path_picker; +}; + struct RequestGenerator { virtual ~RequestGenerator() = default; @@ -159,10 +176,13 @@ struct RequestGenerator Coordination::ZooKeeperRequestPtr generate(const Coordination::ACLs & acls); std::string description(); + + void startup(Coordination::ZooKeeper & zookeeper); private: virtual void getFromConfigImpl(const std::string & key, const Poco::Util::AbstractConfiguration & config) = 0; virtual std::string descriptionImpl() = 0; virtual Coordination::ZooKeeperRequestPtr generateImpl(const Coordination::ACLs & acls) = 0; + virtual void startupImpl(Coordination::ZooKeeper &) {} }; using RequestGeneratorPtr = std::unique_ptr; @@ -174,8 +194,9 @@ struct CreateRequestGenerator final : public RequestGenerator void getFromConfigImpl(const std::string & key, const Poco::Util::AbstractConfiguration & config) override; std::string descriptionImpl() override; Coordination::ZooKeeperRequestPtr generateImpl(const Coordination::ACLs & acls) override; + void startupImpl(Coordination::ZooKeeper & zookeeper) override; - std::string path_prefix; + PathGetter parent_path; StringGetter name; std::optional data; @@ -186,6 +207,40 @@ struct CreateRequestGenerator final : public RequestGenerator std::unordered_set paths_created; }; +struct SetRequestGenerator final : public RequestGenerator +{ +private: + void getFromConfigImpl(const std::string & key, const Poco::Util::AbstractConfiguration & config) override; + std::string descriptionImpl() override; + Coordination::ZooKeeperRequestPtr generateImpl(const Coordination::ACLs & acls) override; + void startupImpl(Coordination::ZooKeeper & zookeeper) override; + + PathGetter path; + StringGetter data; +}; + +struct GetRequestGenerator final : public RequestGenerator +{ +private: + void getFromConfigImpl(const std::string & key, const Poco::Util::AbstractConfiguration & config) override; + std::string descriptionImpl() override; + Coordination::ZooKeeperRequestPtr generateImpl(const Coordination::ACLs & acls) override; + void startupImpl(Coordination::ZooKeeper & zookeeper) override; + + PathGetter path; +}; + +struct ListRequestGenerator final : public RequestGenerator +{ +private: + void getFromConfigImpl(const std::string & key, const Poco::Util::AbstractConfiguration & config) override; + std::string descriptionImpl() override; + Coordination::ZooKeeperRequestPtr generateImpl(const Coordination::ACLs & acls) override; + void startupImpl(Coordination::ZooKeeper & zookeeper) override; + + PathGetter path; +}; + class Generator { public: @@ -193,7 +248,6 @@ class Generator void startup(Coordination::ZooKeeper & zookeeper); Coordination::ZooKeeperRequestPtr generate(); - private: struct Node { From 22a4d7d1e1c5d69542aae79fbd6b63f6ae1b2759 Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Fri, 7 Apr 2023 13:02:42 +0000 Subject: [PATCH 102/406] Support connection definition --- utils/keeper-bench/Generator.cpp | 223 ++++++++++++++++++++++++------- utils/keeper-bench/Generator.h | 34 ++++- utils/keeper-bench/Runner.cpp | 182 ++++++++++++++++++++----- utils/keeper-bench/Runner.h | 26 +++- utils/keeper-bench/main.cpp | 16 +-- 5 files changed, 388 insertions(+), 93 deletions(-) diff --git a/utils/keeper-bench/Generator.cpp b/utils/keeper-bench/Generator.cpp index eba6b7d97477..b050fe6c6392 100644 --- a/utils/keeper-bench/Generator.cpp +++ b/utils/keeper-bench/Generator.cpp @@ -1,5 +1,6 @@ #include "Generator.h" #include "Common/Exception.h" +#include "Common/ZooKeeper/ZooKeeperCommon.h" #include #include #include @@ -320,14 +321,122 @@ std::string PathGetter::description() const return description; } +RequestGetter RequestGetter::fromConfig(const std::string & key, const Poco::Util::AbstractConfiguration & config, bool for_multi) +{ + RequestGetter request_getter; + + Poco::Util::AbstractConfiguration::Keys generator_keys; + config.keys(key, generator_keys); + + bool use_weights = false; + size_t weight_sum = 0; + auto & generators = request_getter.request_generators; + for (const auto & generator_key : generator_keys) + { + RequestGeneratorPtr request_generator; + + if (generator_key.starts_with("create")) + request_generator = std::make_unique(); + else if (generator_key.starts_with("set")) + request_generator = std::make_unique(); + else if (generator_key.starts_with("get")) + request_generator = std::make_unique(); + else if (generator_key.starts_with("list")) + request_generator = std::make_unique(); + else if (generator_key.starts_with("multi")) + { + if (for_multi) + throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, "Nested multi requests are not allowed"); + request_generator = std::make_unique(); + } + else + { + if (for_multi) + continue; + + throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, "Unknown generator {}", key + "." + generator_key); + } + + request_generator->getFromConfig(key + "." + generator_key, config); + + auto weight = request_generator->getWeight(); + use_weights |= weight != 1; + weight_sum += weight; + + generators.push_back(std::move(request_generator)); + } + + if (generators.empty()) + throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, "No request generators found in config for key '{}'", key); + + + size_t max_value = use_weights ? weight_sum - 1 : generators.size() - 1; + request_getter.request_generator_picker = std::uniform_int_distribution(0, max_value); + + /// construct weight vector + if (use_weights) + { + auto & weights = request_getter.weights; + weights.reserve(generators.size()); + weights.push_back(generators[0]->getWeight() - 1); + + for (size_t i = 1; i < generators.size(); ++i) + weights.push_back(weights.back() + generators[i]->getWeight()); + } + + return request_getter; +} + +RequestGeneratorPtr RequestGetter::getRequestGenerator() const +{ + static pcg64 rng(randomSeed()); + + auto random_number = request_generator_picker(rng); + + if (weights.empty()) + return request_generators[random_number]; + + for (size_t i = 0; i < request_generators.size(); ++i) + { + if (random_number <= weights[i]) + return request_generators[i]; + } + + throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "Invalid number generated: {}", random_number); +} + +std::string RequestGetter::description() const +{ + std::string guard(30, '-'); + std::string description = guard; + + for (const auto & request_generator : request_generators) + description += fmt::format("\n{}\n", request_generator->description()); + return description + guard; +} + +void RequestGetter::startup(Coordination::ZooKeeper & zookeeper) +{ + for (const auto & request_generator : request_generators) + request_generator->startup(zookeeper); +} + +const std::vector & RequestGetter::requestGenerators() const +{ + return request_generators; +} + void RequestGenerator::getFromConfig(const std::string & key, const Poco::Util::AbstractConfiguration & config) { + if (config.has(key + ".weight")) + weight = config.getUInt64(key + ".weight"); getFromConfigImpl(key, config); } std::string RequestGenerator::description() { - return descriptionImpl(); + std::string weight_string = weight == 1 ? "" : fmt::format("\n- weight: {}", weight); + return fmt::format("{}{}", descriptionImpl(), weight_string); } Coordination::ZooKeeperRequestPtr RequestGenerator::generate(const Coordination::ACLs & acls) @@ -340,6 +449,11 @@ void RequestGenerator::startup(Coordination::ZooKeeper & zookeeper) startupImpl(zookeeper); } +size_t RequestGenerator::getWeight() const +{ + return weight; +} + CreateRequestGenerator::CreateRequestGenerator() : rng(randomSeed()) , remove_picker(0, 1.0) @@ -487,6 +601,50 @@ void ListRequestGenerator::startupImpl(Coordination::ZooKeeper & zookeeper) path.initialize(zookeeper); } +void MultiRequestGenerator::getFromConfigImpl(const std::string & key, const Poco::Util::AbstractConfiguration & config) +{ + if (config.has(key + ".size")) + size = NumberGetter::fromConfig(key + ".size", config); + + request_getter = RequestGetter::fromConfig(key, config, /*for_multi*/ true); +}; + +std::string MultiRequestGenerator::descriptionImpl() +{ + std::string size_string = size.has_value() ? fmt::format("- number of requests: {}\n", size->description()) : ""; + return fmt::format( + "Multi Request Generator\n" + "{}" + "- requests:\n{}", + size_string, + request_getter.description()); +} + +Coordination::ZooKeeperRequestPtr MultiRequestGenerator::generateImpl(const Coordination::ACLs & acls) +{ + Coordination::Requests ops; + + if (size) + { + auto request_count = size->getNumber(); + + for (size_t i = 0; i < request_count; ++i) + ops.push_back(request_getter.getRequestGenerator()->generate(acls)); + } + else + { + for (const auto & request_generator : request_getter.requestGenerators()) + ops.push_back(request_generator->generate(acls)); + } + + return std::make_shared(ops, acls); +} + +void MultiRequestGenerator::startupImpl(Coordination::ZooKeeper & zookeeper) +{ + request_getter.startup(zookeeper); +} + Generator::Generator(const Poco::Util::AbstractConfiguration & config) { Coordination::ACL acl; @@ -497,55 +655,25 @@ Generator::Generator(const Poco::Util::AbstractConfiguration & config) static const std::string generator_key = "generator"; + static const std::string setup_key = generator_key + ".setup"; + Poco::Util::AbstractConfiguration::Keys keys; + config.keys(setup_key, keys); + for (const auto & key : keys) { - static const std::string setup_key = generator_key + ".setup"; - Poco::Util::AbstractConfiguration::Keys keys; - config.keys(setup_key, keys); - for (const auto & key : keys) + if (key.starts_with("node")) { - if (key.starts_with("node")) - { - const auto & node = root_nodes.emplace_back(parseNode(setup_key + "." + key, config)); + const auto & node = root_nodes.emplace_back(parseNode(setup_key + "." + key, config)); - std::cout << "---- Will create tree ----" << std::endl; - node->dumpTree(); - } + std::cout << "---- Will create tree ----" << std::endl; + node->dumpTree(); } } - { - static const std::string requests_key = generator_key + ".requests"; - Poco::Util::AbstractConfiguration::Keys keys; - config.keys(requests_key, keys); - - std::cout << "\n---- Collecting request generators ----" << std::endl; - for (const auto & key : keys) - { - RequestGeneratorPtr request_generator; - if (key.starts_with("create")) - request_generator = std::make_unique(); - else if (key.starts_with("set")) - request_generator = std::make_unique(); - else if (key.starts_with("get")) - request_generator = std::make_unique(); - else if (key.starts_with("list")) - request_generator = std::make_unique(); - - if (!request_generator) - throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "Unknown generator {}", key); - - request_generator->getFromConfig(requests_key + "." + key, config); - - std::cout << fmt::format("\n{}\n", request_generator->description()) << std::endl; - request_generators.push_back(std::move(request_generator)); - } - - if (request_generators.empty()) - throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, "No request generators found in config"); - std::cout << "---- Done collecting request generators ----" << std::endl; - } - - request_picker = std::uniform_int_distribution(0, request_generators.size() - 1); + std::cout << "\n---- Collecting request generators ----" << std::endl; + static const std::string requests_key = generator_key + ".requests"; + request_getter = RequestGetter::fromConfig(requests_key, config); + std::cout << request_getter.description() << std::endl; + std::cout << "---- Done collecting request generators ----" << std::endl; } std::shared_ptr Generator::parseNode(const std::string & key, const Poco::Util::AbstractConfiguration & config) @@ -627,15 +755,12 @@ void Generator::startup(Coordination::ZooKeeper & zookeeper) } std::cout << "---- Created test data ----" << std::endl; - std::cout << "---- Initializing generators ----" << std::endl; - for (const auto & generator : request_generators) - generator->startup(zookeeper); + request_getter.startup(zookeeper); } Coordination::ZooKeeperRequestPtr Generator::generate() { - static pcg64 rng(randomSeed()); - return request_generators[request_picker(rng)]->generate(default_acls); + return request_getter.getRequestGenerator()->generate(default_acls); } diff --git a/utils/keeper-bench/Generator.h b/utils/keeper-bench/Generator.h index af186ea6624a..d2925e8650a2 100644 --- a/utils/keeper-bench/Generator.h +++ b/utils/keeper-bench/Generator.h @@ -178,14 +178,18 @@ struct RequestGenerator std::string description(); void startup(Coordination::ZooKeeper & zookeeper); + + size_t getWeight() const; private: virtual void getFromConfigImpl(const std::string & key, const Poco::Util::AbstractConfiguration & config) = 0; virtual std::string descriptionImpl() = 0; virtual Coordination::ZooKeeperRequestPtr generateImpl(const Coordination::ACLs & acls) = 0; virtual void startupImpl(Coordination::ZooKeeper &) {} + + size_t weight = 1; }; -using RequestGeneratorPtr = std::unique_ptr; +using RequestGeneratorPtr = std::shared_ptr; struct CreateRequestGenerator final : public RequestGenerator { @@ -241,6 +245,32 @@ struct ListRequestGenerator final : public RequestGenerator PathGetter path; }; +struct RequestGetter +{ + static RequestGetter fromConfig(const std::string & key, const Poco::Util::AbstractConfiguration & config, bool for_multi = false); + + RequestGeneratorPtr getRequestGenerator() const; + std::string description() const; + void startup(Coordination::ZooKeeper & zookeeper); + const std::vector & requestGenerators() const; +private: + std::vector request_generators; + std::vector weights; + mutable std::uniform_int_distribution request_generator_picker; +}; + +struct MultiRequestGenerator final : public RequestGenerator +{ +private: + void getFromConfigImpl(const std::string & key, const Poco::Util::AbstractConfiguration & config) override; + std::string descriptionImpl() override; + Coordination::ZooKeeperRequestPtr generateImpl(const Coordination::ACLs & acls) override; + void startupImpl(Coordination::ZooKeeper & zookeeper) override; + + std::optional size; + RequestGetter request_getter; +}; + class Generator { public: @@ -263,7 +293,7 @@ class Generator std::uniform_int_distribution request_picker; std::vector> root_nodes; - std::vector request_generators; + RequestGetter request_getter; Coordination::ACLs default_acls; }; diff --git a/utils/keeper-bench/Runner.cpp b/utils/keeper-bench/Runner.cpp index 387b7baebdd4..3076bf425583 100644 --- a/utils/keeper-bench/Runner.cpp +++ b/utils/keeper-bench/Runner.cpp @@ -1,5 +1,8 @@ #include "Runner.h" +#include +#include "Common/ZooKeeper/ZooKeeperCommon.h" +#include "Common/ZooKeeper/ZooKeeperConstants.h" #include namespace DB::ErrorCodes @@ -18,33 +21,96 @@ Runner::Runner( size_t max_iterations_) : concurrency(concurrency_) , pool(CurrentMetrics::LocalThread, CurrentMetrics::LocalThreadActive, concurrency) - , hosts_strings(hosts_strings_) , max_time(max_time_) , delay(delay_) , continue_on_error(continue_on_error_) , max_iterations(max_iterations_) , info(std::make_shared()) , queue(concurrency) - { - if (!generator_name.empty() && !config_path.empty()) - throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, "Both generator name and generator config path are defined. Please define only one of them"); +{ - if (generator_name.empty() && config_path.empty()) - throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, "Both generator name and generator config path are empty. Please define one of them"); + DB::ConfigurationPtr config = nullptr; - if (!generator_name.empty()) - generator = getGenerator(generator_name); - else - { - DB::ConfigProcessor config_processor(config_path, true, false); - auto loaded_config = config_processor.loadConfig(); + if (!config_path.empty()) + { + DB::ConfigProcessor config_processor(config_path, true, false); + config = config_processor.loadConfig().configuration; + } - generator = std::make_unique(*loaded_config.configuration); - } + if (!generator_name.empty()) + { + generator = getGenerator(generator_name); if (!generator) throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, "Failed to create generator"); } + else + { + if (!config) + throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, "No config file or generator name defined"); + + generator = std::make_unique(*config); + } + + if (!hosts_strings_.empty()) + { + for (const auto & host : hosts_strings_) + connection_infos.push_back({.host = host}); + } + else + { + if (!config) + throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, "No config file or hosts defined"); + + parseHostsFromConfig(*config); + } +} + +void Runner::parseHostsFromConfig(const Poco::Util::AbstractConfiguration & config) +{ + ConnectionInfo default_connection_info; + + const auto fill_connection_details = [&](const std::string & key, auto & connection_info) + { + if (config.has(key + ".secure")) + connection_info.secure = config.getBool(key + ".secure"); + + if (config.has(key + ".session_timeout_ms")) + connection_info.session_timeout_ms = config.getInt(key + ".session_timeout_ms"); + + if (config.has(key + ".operation_timeout_ms")) + connection_info.operation_timeout_ms = config.getInt(key + ".operation_timeout_ms"); + + if (config.has(key + ".connection_timeout_ms")) + connection_info.connection_timeout_ms = config.getInt(key + ".connection_timeout_ms"); + }; + + fill_connection_details("connections", default_connection_info); + + Poco::Util::AbstractConfiguration::Keys connections_keys; + config.keys("connections", connections_keys); + + for (const auto & key : connections_keys) + { + std::string connection_key = "connections." + key; + auto connection_info = default_connection_info; + if (key.starts_with("host")) + { + connection_info.host = config.getString(connection_key); + connection_infos.push_back(std::move(connection_info)); + } + else if (key.starts_with("connection") && key != "connection_timeout_ms") + { + connection_info.host = config.getString(connection_key + ".host"); + if (config.has(connection_key + ".sessions")) + connection_info.sessions = config.getUInt64(connection_key + ".sessions"); + + fill_connection_details(connection_key, connection_info); + + connection_infos.push_back(std::move(connection_info)); + } + } +} void Runner::thread(std::vector> zookeepers) { @@ -130,7 +196,7 @@ void Runner::thread(std::vector> zookee { try { - zookeepers = getConnections(); + zookeepers = refreshConnections(); break; } catch (...) @@ -147,6 +213,24 @@ void Runner::thread(std::vector> zookee bool Runner::tryPushRequestInteractively(const Coordination::ZooKeeperRequestPtr & request, DB::InterruptListener & interrupt_listener) { + static std::unordered_map counts; + static size_t i = 0; + + counts[request->getOpNum()]++; + + //if (request->getOpNum() == Coordination::OpNum::Multi) + //{ + // for (const auto & multi_request : dynamic_cast(*request).requests) + // counts[dynamic_cast(*multi_request).getOpNum()]++; + //} + + ++i; + if (i % 10000 == 0) + { + for (const auto & [op_num, count] : counts) + std::cout << fmt::format("{}: {}", op_num, count) << std::endl; + } + bool inserted = false; while (!inserted) @@ -187,17 +271,17 @@ bool Runner::tryPushRequestInteractively(const Coordination::ZooKeeperRequestPtr void Runner::runBenchmark() { - auto aux_connections = getConnections(); + createConnections(); std::cerr << "Preparing to run\n"; - generator->startup(*aux_connections[0]); + generator->startup(*connections[0]); std::cerr << "Prepared\n"; try { - auto connections = getConnections(); for (size_t i = 0; i < concurrency; ++i) { - pool.scheduleOrThrowOnError([this, connections]() mutable { thread(connections); }); + auto thread_connections = connections; + pool.scheduleOrThrowOnError([this, connections = std::move(thread_connections)]() mutable { thread(connections); }); } } catch (...) @@ -230,21 +314,55 @@ void Runner::runBenchmark() } -std::vector> Runner::getConnections() +void Runner::createConnections() { - std::vector> zookeepers; - for (const auto & host_string : hosts_strings) - { - Coordination::ZooKeeper::Node node{Poco::Net::SocketAddress{host_string}, false}; - std::vector nodes; - nodes.push_back(node); - zkutil::ZooKeeperArgs args; - args.session_timeout_ms = 30000; - args.connection_timeout_ms = 1000; - args.operation_timeout_ms = 10000; - zookeepers.emplace_back(std::make_shared(nodes, args, nullptr)); + for (size_t connection_info_idx = 0; connection_info_idx < connection_infos.size(); ++connection_info_idx) + { + const auto & connection_info = connection_infos[connection_info_idx]; + std::cout << fmt::format("Creating {} session(s) for:\n" + "- host: {}\n" + "- secure: {}\n" + "- session timeout: {}ms\n" + "- operation timeout: {}ms\n" + "- connection timeout: {}ms", + connection_info.sessions, + connection_info.host, + connection_info.secure, + connection_info.session_timeout_ms, + connection_info.operation_timeout_ms, + connection_info.connection_timeout_ms) << std::endl; + + for (size_t session = 0; session < connection_info.sessions; ++session) + { + connections.emplace_back(getConnection(connection_info)); + connections_to_info_map[connections.size() - 1] = connection_info_idx; + } } +} +std::shared_ptr Runner::getConnection(const ConnectionInfo & connection_info) +{ + Coordination::ZooKeeper::Node node{Poco::Net::SocketAddress{connection_info.host}, connection_info.secure}; + std::vector nodes; + nodes.push_back(node); + zkutil::ZooKeeperArgs args; + args.session_timeout_ms = connection_info.session_timeout_ms; + args.connection_timeout_ms = connection_info.operation_timeout_ms; + args.operation_timeout_ms = connection_info.connection_timeout_ms; + return std::make_shared(nodes, args, nullptr); +} - return zookeepers; +std::vector> Runner::refreshConnections() +{ + std::lock_guard lock(connection_mutex); + for (size_t connection_idx = 0; connection_idx < connections.size(); ++connection_idx) + { + auto & connection = connections[connection_idx]; + if (connection->isExpired()) + { + const auto & connection_info = connection_infos[connections_to_info_map[connection_idx]]; + connection = getConnection(connection_info); + } + } + return connections; } diff --git a/utils/keeper-bench/Runner.h b/utils/keeper-bench/Runner.h index f8280ac5f376..e36089d55196 100644 --- a/utils/keeper-bench/Runner.h +++ b/utils/keeper-bench/Runner.h @@ -1,4 +1,5 @@ #pragma once +#include "Common/ZooKeeper/ZooKeeperConstants.h" #include #include "Generator.h" #include @@ -12,6 +13,7 @@ #include #include +#include #include "Stats.h" using Ports = std::vector; @@ -54,11 +56,12 @@ class Runner private: + void parseHostsFromConfig(const Poco::Util::AbstractConfiguration & config); size_t concurrency = 1; ThreadPool pool; - Strings hosts_strings; + std::unique_ptr generator; double max_time = 0; double delay = 1; @@ -77,5 +80,24 @@ class Runner using Queue = ConcurrentBoundedQueue; Queue queue; - std::vector> getConnections(); + struct ConnectionInfo + { + std::string host; + + bool secure = false; + int32_t session_timeout_ms = Coordination::DEFAULT_SESSION_TIMEOUT_MS; + int32_t connection_timeout_ms = Coordination::DEFAULT_CONNECTION_TIMEOUT_MS; + int32_t operation_timeout_ms = Coordination::DEFAULT_OPERATION_TIMEOUT_MS; + + size_t sessions = 1; + }; + + std::mutex connection_mutex; + std::vector connection_infos; + std::vector> connections; + std::unordered_map connections_to_info_map; + + void createConnections(); + std::shared_ptr getConnection(const ConnectionInfo & connection_info); + std::vector> refreshConnections(); }; diff --git a/utils/keeper-bench/main.cpp b/utils/keeper-bench/main.cpp index 83303fb4029c..bec91dc6ad17 100644 --- a/utils/keeper-bench/main.cpp +++ b/utils/keeper-bench/main.cpp @@ -19,14 +19,14 @@ int main(int argc, char *argv[]) boost::program_options::options_description desc = createOptionsDescription("Allowed options", getTerminalWidth()); desc.add_options() - ("help", "produce help message") - ("generator", value()->default_value(""), "query to execute") - ("config", value()->default_value(""), "xml file containing generator configuration") - ("concurrency,c", value()->default_value(1), "number of parallel queries") - ("delay,d", value()->default_value(1), "delay between intermediate reports in seconds (set 0 to disable reports)") - ("iterations,i", value()->default_value(0), "amount of queries to be executed") - ("timelimit,t", value()->default_value(0.), "stop launch of queries after specified time limit") - ("hosts,h", value()->multitoken(), "") + ("help", "produce help message") + ("generator", value()->default_value(""), "query to execute") + ("config", value()->default_value(""), "xml file containing generator configuration") + ("concurrency,c", value()->default_value(1), "number of parallel queries") + ("delay,d", value()->default_value(1), "delay between intermediate reports in seconds (set 0 to disable reports)") + ("iterations,i", value()->default_value(0), "amount of queries to be executed") + ("timelimit,t", value()->default_value(0.), "stop launch of queries after specified time limit") + ("hosts,h", value()->multitoken()->default_value(Strings{}, ""), "") ("continue_on_errors", "continue testing even if a query fails") ("reconnect", "establish new connection for every query") ; From 9df7a673062d63053903a5a3d3268a8756813248 Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Fri, 7 Apr 2023 14:14:39 +0000 Subject: [PATCH 103/406] Add more configurations --- utils/keeper-bench/Generator.cpp | 15 +++-- utils/keeper-bench/Runner.cpp | 102 +++++++++++++++++++++---------- utils/keeper-bench/Runner.h | 20 +++--- utils/keeper-bench/main.cpp | 58 ++++++++++++------ 4 files changed, 129 insertions(+), 66 deletions(-) diff --git a/utils/keeper-bench/Generator.cpp b/utils/keeper-bench/Generator.cpp index b050fe6c6392..f9f684e49efc 100644 --- a/utils/keeper-bench/Generator.cpp +++ b/utils/keeper-bench/Generator.cpp @@ -591,7 +591,7 @@ std::string ListRequestGenerator::descriptionImpl() Coordination::ZooKeeperRequestPtr ListRequestGenerator::generateImpl(const Coordination::ACLs & /*acls*/) { - auto request = std::make_shared(); + auto request = std::make_shared(); request->path = path.getPath(); return request; } @@ -655,6 +655,7 @@ Generator::Generator(const Poco::Util::AbstractConfiguration & config) static const std::string generator_key = "generator"; + std::cout << "---- Parsing setup ---- " << std::endl; static const std::string setup_key = generator_key + ".setup"; Poco::Util::AbstractConfiguration::Keys keys; config.keys(setup_key, keys); @@ -664,16 +665,18 @@ Generator::Generator(const Poco::Util::AbstractConfiguration & config) { const auto & node = root_nodes.emplace_back(parseNode(setup_key + "." + key, config)); - std::cout << "---- Will create tree ----" << std::endl; + std::cout << "Tree to create:" << std::endl; node->dumpTree(); + std::cout << std::endl; } } + std::cout << "---- Done parsing data setup ----\n" << std::endl; - std::cout << "\n---- Collecting request generators ----" << std::endl; + std::cout << "---- Collecting request generators ----" << std::endl; static const std::string requests_key = generator_key + ".requests"; request_getter = RequestGetter::fromConfig(requests_key, config); std::cout << request_getter.description() << std::endl; - std::cout << "---- Done collecting request generators ----" << std::endl; + std::cout << "---- Done collecting request generators ----\n" << std::endl; } std::shared_ptr Generator::parseNode(const std::string & key, const Poco::Util::AbstractConfiguration & config) @@ -741,7 +744,7 @@ void Generator::Node::createNode(Coordination::ZooKeeper & zookeeper, const std: void Generator::startup(Coordination::ZooKeeper & zookeeper) { - std::cout << "\n---- Creating test data ----" << std::endl; + std::cout << "---- Creating test data ----" << std::endl; for (const auto & node : root_nodes) { auto node_name = node->name.getString(); @@ -753,7 +756,7 @@ void Generator::startup(Coordination::ZooKeeper & zookeeper) node->createNode(zookeeper, "/", default_acls); } - std::cout << "---- Created test data ----" << std::endl; + std::cout << "---- Created test data ----\n" << std::endl; std::cout << "---- Initializing generators ----" << std::endl; diff --git a/utils/keeper-bench/Runner.cpp b/utils/keeper-bench/Runner.cpp index 3076bf425583..72d80e478db1 100644 --- a/utils/keeper-bench/Runner.cpp +++ b/utils/keeper-bench/Runner.cpp @@ -3,6 +3,7 @@ #include "Common/ZooKeeper/ZooKeeperCommon.h" #include "Common/ZooKeeper/ZooKeeperConstants.h" +#include #include namespace DB::ErrorCodes @@ -11,22 +12,15 @@ namespace DB::ErrorCodes } Runner::Runner( - size_t concurrency_, + std::optional concurrency_, const std::string & generator_name, const std::string & config_path, const Strings & hosts_strings_, - double max_time_, - double delay_, - bool continue_on_error_, - size_t max_iterations_) - : concurrency(concurrency_) - , pool(CurrentMetrics::LocalThread, CurrentMetrics::LocalThreadActive, concurrency) - , max_time(max_time_) - , delay(delay_) - , continue_on_error(continue_on_error_) - , max_iterations(max_iterations_) - , info(std::make_shared()) - , queue(concurrency) + std::optional max_time_, + std::optional delay_, + std::optional continue_on_error_, + std::optional max_iterations_) + : info(std::make_shared()) { DB::ConfigurationPtr config = nullptr; @@ -39,7 +33,7 @@ Runner::Runner( if (!generator_name.empty()) { - generator = getGenerator(generator_name); + //generator = getGenerator(generator_name); if (!generator) throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, "Failed to create generator"); @@ -49,7 +43,7 @@ Runner::Runner( if (!config) throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, "No config file or generator name defined"); - generator = std::make_unique(*config); + generator.emplace(*config); } if (!hosts_strings_.empty()) @@ -64,6 +58,41 @@ Runner::Runner( parseHostsFromConfig(*config); } + + std::cout << "---- Run options ---- " << std::endl; + if (concurrency_) + concurrency = *concurrency_; + else + concurrency = config->getUInt64("concurrency", 1); + std::cout << "Concurrency: " << concurrency << std::endl; + + if (max_iterations_) + max_iterations = *max_iterations_; + else + max_iterations = config->getUInt64("iterations", 0); + std::cout << "Iterations: " << max_iterations << std::endl; + + if (delay_) + delay = *delay_; + else + delay = config->getDouble("report_delay", 1); + std::cout << "Report delay: " << delay << std::endl; + + if (max_time_) + max_time = *max_time_; + else + max_time = config->getDouble("timelimit", 1.0); + std::cout << "Time limit: " << max_time << std::endl; + + if (continue_on_error_) + continue_on_error = *continue_on_error_; + else + continue_on_error = config->getBool("continue_on_error", 1.0); + std::cout << "Continue on error: " << continue_on_error << std::endl; + std::cout << "---- Run options ----\n" << std::endl; + + pool.emplace(CurrentMetrics::LocalThread, CurrentMetrics::LocalThreadActive, concurrency); + queue.emplace(concurrency); } void Runner::parseHostsFromConfig(const Poco::Util::AbstractConfiguration & config) @@ -134,7 +163,7 @@ void Runner::thread(std::vector> zookee while (!extracted) { - extracted = queue.tryPop(request, 100); + extracted = queue->tryPop(request, 100); if (shutdown || (max_iterations && requests_executed >= max_iterations)) @@ -211,12 +240,12 @@ void Runner::thread(std::vector> zookee } } -bool Runner::tryPushRequestInteractively(const Coordination::ZooKeeperRequestPtr & request, DB::InterruptListener & interrupt_listener) +bool Runner::tryPushRequestInteractively(Coordination::ZooKeeperRequestPtr && request, DB::InterruptListener & interrupt_listener) { - static std::unordered_map counts; - static size_t i = 0; - - counts[request->getOpNum()]++; + //static std::unordered_map counts; + //static size_t i = 0; + // + //counts[request->getOpNum()]++; //if (request->getOpNum() == Coordination::OpNum::Multi) //{ @@ -224,18 +253,18 @@ bool Runner::tryPushRequestInteractively(const Coordination::ZooKeeperRequestPtr // counts[dynamic_cast(*multi_request).getOpNum()]++; //} - ++i; - if (i % 10000 == 0) - { - for (const auto & [op_num, count] : counts) - std::cout << fmt::format("{}: {}", op_num, count) << std::endl; - } + //++i; + //if (i % 10000 == 0) + //{ + // for (const auto & [op_num, count] : counts) + // std::cout << fmt::format("{}: {}", op_num, count) << std::endl; + //} bool inserted = false; while (!inserted) { - inserted = queue.tryPush(request, 100); + inserted = queue->tryPush(std::move(request), 100); if (shutdown) { @@ -281,13 +310,13 @@ void Runner::runBenchmark() for (size_t i = 0; i < concurrency; ++i) { auto thread_connections = connections; - pool.scheduleOrThrowOnError([this, connections = std::move(thread_connections)]() mutable { thread(connections); }); + pool->scheduleOrThrowOnError([this, connections = std::move(thread_connections)]() mutable { thread(connections); }); } } catch (...) { shutdown = true; - pool.wait(); + pool->wait(); throw; } @@ -304,7 +333,7 @@ void Runner::runBenchmark() } } - pool.wait(); + pool->wait(); total_watch.stop(); printNumberOfRequestsExecuted(requests_executed); @@ -316,6 +345,8 @@ void Runner::runBenchmark() void Runner::createConnections() { + DB::EventNotifier::init(); + std::cout << "---- Creating connections ---- " << std::endl; for (size_t connection_info_idx = 0; connection_info_idx < connection_infos.size(); ++connection_info_idx) { const auto & connection_info = connection_infos[connection_info_idx]; @@ -338,6 +369,7 @@ void Runner::createConnections() connections_to_info_map[connections.size() - 1] = connection_info_idx; } } + std::cout << "---- Done creating connections ----\n" << std::endl; } std::shared_ptr Runner::getConnection(const ConnectionInfo & connection_info) @@ -366,3 +398,11 @@ std::vector> Runner::refreshConnections } return connections; } + +Runner::~Runner() +{ + queue->clearAndFinish(); + shutdown = true; + pool->wait(); +} + diff --git a/utils/keeper-bench/Runner.h b/utils/keeper-bench/Runner.h index e36089d55196..ebd2d702019c 100644 --- a/utils/keeper-bench/Runner.h +++ b/utils/keeper-bench/Runner.h @@ -34,14 +34,14 @@ class Runner { public: Runner( - size_t concurrency_, + std::optional concurrency_, const std::string & generator_name, const std::string & config_path, const Strings & hosts_strings_, - double max_time_, - double delay_, - bool continue_on_error_, - size_t max_iterations_); + std::optional max_time_, + std::optional delay_, + std::optional continue_on_error_, + std::optional max_iterations_); void thread(std::vector> zookeepers); @@ -50,19 +50,19 @@ class Runner std::cerr << "Requests executed: " << num << ".\n"; } - bool tryPushRequestInteractively(const Coordination::ZooKeeperRequestPtr & request, DB::InterruptListener & interrupt_listener); + bool tryPushRequestInteractively(Coordination::ZooKeeperRequestPtr && request, DB::InterruptListener & interrupt_listener); void runBenchmark(); - + ~Runner(); private: void parseHostsFromConfig(const Poco::Util::AbstractConfiguration & config); size_t concurrency = 1; - ThreadPool pool; + std::optional pool; - std::unique_ptr generator; + std::optional generator; double max_time = 0; double delay = 1; bool continue_on_error = false; @@ -78,7 +78,7 @@ class Runner std::mutex mutex; using Queue = ConcurrentBoundedQueue; - Queue queue; + std::optional queue; struct ConnectionInfo { diff --git a/utils/keeper-bench/main.cpp b/utils/keeper-bench/main.cpp index bec91dc6ad17..cb25aa7d0a52 100644 --- a/utils/keeper-bench/main.cpp +++ b/utils/keeper-bench/main.cpp @@ -3,10 +3,24 @@ #include "Runner.h" #include "Stats.h" #include "Generator.h" +#include "Common/Exception.h" #include #include +#include -using namespace std; +namespace +{ + +template +std::optional valueToOptional(const boost::program_options::variable_value & value) +{ + if (value.empty()) + return std::nullopt; + + return value.as(); +} + +} int main(int argc, char *argv[]) { @@ -19,16 +33,15 @@ int main(int argc, char *argv[]) boost::program_options::options_description desc = createOptionsDescription("Allowed options", getTerminalWidth()); desc.add_options() - ("help", "produce help message") - ("generator", value()->default_value(""), "query to execute") - ("config", value()->default_value(""), "xml file containing generator configuration") - ("concurrency,c", value()->default_value(1), "number of parallel queries") - ("delay,d", value()->default_value(1), "delay between intermediate reports in seconds (set 0 to disable reports)") - ("iterations,i", value()->default_value(0), "amount of queries to be executed") - ("timelimit,t", value()->default_value(0.), "stop launch of queries after specified time limit") - ("hosts,h", value()->multitoken()->default_value(Strings{}, ""), "") + ("help", "produce help message") + ("generator", value()->default_value(""), "query to execute") + ("config", value()->default_value(""), "yaml/xml file containing configuration") + ("concurrency,c", value(), "number of parallel queries") + ("report-delay,d", value(), "delay between intermediate reports in seconds (set 0 to disable reports)") + ("iterations,i", value(), "amount of queries to be executed") + ("time-limit,t", value(), "stop launch of queries after specified time limit") + ("hosts,h", value()->multitoken()->default_value(Strings{}, ""), "") ("continue_on_errors", "continue testing even if a query fails") - ("reconnect", "establish new connection for every query") ; boost::program_options::variables_map options; @@ -42,16 +55,23 @@ int main(int argc, char *argv[]) return 1; } - Runner runner(options["concurrency"].as(), - options["generator"].as(), - options["config"].as(), - options["hosts"].as(), - options["timelimit"].as(), - options["delay"].as(), - options.count("continue_on_errors"), - options["iterations"].as()); + Runner runner(valueToOptional(options["concurrency"]), + options["generator"].as(), + options["config"].as(), + options["hosts"].as(), + valueToOptional(options["time-limit"]), + valueToOptional(options["report-delay"]), + options.count("continue_on_errors") ? std::optional(true) : std::nullopt, + valueToOptional(options["iterations"])); - runner.runBenchmark(); + try + { + runner.runBenchmark(); + } + catch (const DB::Exception & e) + { + std::cout << "Got exception while trying to run benchmark: " << e.message() << std::endl; + } return 0; } From 46533c1ea7928a62dfa7455c194b4ce7d794e21e Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Fri, 7 Apr 2023 14:51:25 +0000 Subject: [PATCH 104/406] CreateRequest more stable --- utils/keeper-bench/Generator.cpp | 23 +++++++++++++++++++---- utils/keeper-bench/Generator.h | 4 +++- utils/keeper-bench/Runner.cpp | 31 +++++++++++++++++++++++++++++-- 3 files changed, 51 insertions(+), 7 deletions(-) diff --git a/utils/keeper-bench/Generator.cpp b/utils/keeper-bench/Generator.cpp index f9f684e49efc..280593ed511e 100644 --- a/utils/keeper-bench/Generator.cpp +++ b/utils/keeper-bench/Generator.cpp @@ -468,23 +468,26 @@ void CreateRequestGenerator::getFromConfigImpl(const std::string & key, const Po if (config.has(key + ".data")) data = StringGetter::fromConfig(key + ".data", config); - remove_factor = config.getDouble(key + ".remove_factor", 0.0); + if (config.has(key + ".remove_factor")) + remove_factor = config.getDouble(key + ".remove_factor"); } std::string CreateRequestGenerator::descriptionImpl() { std::string data_string = data.has_value() ? fmt::format("data for created nodes: {}", data->description()) : "no data for created nodes"; + std::string remove_factor_string + = remove_factor.has_value() ? fmt::format("- remove factor: {}", *remove_factor) : "- without removes"; return fmt::format( "Create Request Generator\n" "- parent path(s) for created nodes: {}\n" "- name for created nodes: {}\n" "- {}\n" - "- remove factor: {}", + "{}", parent_path.description(), name.description(), data_string, - remove_factor); + remove_factor_string); } void CreateRequestGenerator::startupImpl(Coordination::ZooKeeper & zookeeper) @@ -494,7 +497,7 @@ void CreateRequestGenerator::startupImpl(Coordination::ZooKeeper & zookeeper) Coordination::ZooKeeperRequestPtr CreateRequestGenerator::generateImpl(const Coordination::ACLs & acls) { - if (!paths_created.empty() && remove_picker(rng) < remove_factor) + if (remove_factor.has_value() && !paths_created.empty() && remove_picker(rng) < *remove_factor) { auto request = std::make_shared(); auto it = paths_created.begin(); @@ -767,3 +770,15 @@ Coordination::ZooKeeperRequestPtr Generator::generate() { return request_getter.getRequestGenerator()->generate(default_acls); } + +void Generator::cleanup(Coordination::ZooKeeper & zookeeper) +{ + std::cout << "---- Cleaning up test data ----" << std::endl; + for (const auto & node : root_nodes) + { + auto node_name = node->name.getString(); + std::string root_path = std::filesystem::path("/") / node_name; + std::cout << "Cleaning up " << root_path << std::endl; + removeRecursive(zookeeper, root_path); + } +} diff --git a/utils/keeper-bench/Generator.h b/utils/keeper-bench/Generator.h index d2925e8650a2..5c15d2bce9ad 100644 --- a/utils/keeper-bench/Generator.h +++ b/utils/keeper-bench/Generator.h @@ -204,7 +204,7 @@ struct CreateRequestGenerator final : public RequestGenerator StringGetter name; std::optional data; - double remove_factor; + std::optional remove_factor; pcg64 rng; std::uniform_real_distribution remove_picker; @@ -278,6 +278,8 @@ class Generator void startup(Coordination::ZooKeeper & zookeeper); Coordination::ZooKeeperRequestPtr generate(); + void cleanup(Coordination::ZooKeeper & zookeeper); + private: struct Node { diff --git a/utils/keeper-bench/Runner.cpp b/utils/keeper-bench/Runner.cpp index 72d80e478db1..2e3cd9116980 100644 --- a/utils/keeper-bench/Runner.cpp +++ b/utils/keeper-bench/Runner.cpp @@ -177,9 +177,35 @@ void Runner::thread(std::vector> zookee auto promise = std::make_shared>(); auto future = promise->get_future(); - Coordination::ResponseCallback callback = [promise](const Coordination::Response & response) + Coordination::ResponseCallback callback = [&request, promise](const Coordination::Response & response) { - if (response.error != Coordination::Error::ZOK) + bool set_exception = true; + + if (response.error == Coordination::Error::ZOK) + { + set_exception = false; + } + else if (response.error == Coordination::Error::ZNONODE) + { + /// remove can fail with ZNONODE because of different order of execution + /// of generated create and remove requests + /// this is okay for concurrent runs + if (dynamic_cast(&response)) + set_exception = false; + else if (const auto * multi_response = dynamic_cast(&response)) + { + const auto & responses = multi_response->responses; + size_t i = 0; + while (responses[i]->error != Coordination::Error::ZNONODE) + ++i; + + const auto & multi_request = dynamic_cast(*request); + if (dynamic_cast(&*multi_request.requests[i])) + set_exception = false; + } + } + + if (set_exception) promise->set_exception(std::make_exception_ptr(zkutil::KeeperException(response.error))); else promise->set_value(response.bytesSize()); @@ -404,5 +430,6 @@ Runner::~Runner() queue->clearAndFinish(); shutdown = true; pool->wait(); + generator->cleanup(*connections[0]); } From e9d43a8f6a2746ef101287fc92a6c13dcf1b3b65 Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Fri, 7 Apr 2023 15:20:36 +0000 Subject: [PATCH 105/406] Remove generator name config --- utils/keeper-bench/Generator.cpp | 64 ++--------------- utils/keeper-bench/Generator.h | 116 ++----------------------------- utils/keeper-bench/Runner.cpp | 39 ++++------- utils/keeper-bench/Runner.h | 1 - utils/keeper-bench/main.cpp | 2 - 5 files changed, 22 insertions(+), 200 deletions(-) diff --git a/utils/keeper-bench/Generator.cpp b/utils/keeper-bench/Generator.cpp index 280593ed511e..12e628ed1e52 100644 --- a/utils/keeper-bench/Generator.cpp +++ b/utils/keeper-bench/Generator.cpp @@ -87,64 +87,6 @@ void removeRecursive(Coordination::ZooKeeper & zookeeper, const std::string & pa remove_future.get(); } -std::unique_ptr getGenerator(const std::string & name) -{ - //if (name == "create_no_data") - //{ - // return std::make_unique(); - //} - //else if (name == "create_small_data") - //{ - // return std::make_unique("/create_generator", 5, 32); - //} - //else if (name == "create_medium_data") - //{ - // return std::make_unique("/create_generator", 5, 1024); - //} - //else if (name == "create_big_data") - //{ - // return std::make_unique("/create_generator", 5, 512 * 1024); - //} - //else if (name == "get_no_data") - //{ - // return std::make_unique("/get_generator", 10, 0); - //} - //else if (name == "get_small_data") - //{ - // return std::make_unique("/get_generator", 10, 32); - //} - //else if (name == "get_medium_data") - //{ - // return std::make_unique("/get_generator", 10, 1024); - //} - //else if (name == "get_big_data") - //{ - // return std::make_unique("/get_generator", 10, 512 * 1024); - //} - //else if (name == "list_no_nodes") - //{ - // return std::make_unique("/list_generator", 0, 1); - //} - //else if (name == "list_few_nodes") - //{ - // return std::make_unique("/list_generator", 10, 5); - //} - //else if (name == "list_medium_nodes") - //{ - // return std::make_unique("/list_generator", 1000, 5); - //} - //else if (name == "list_a_lot_nodes") - //{ - // return std::make_unique("/list_generator", 100000, 5); - //} - //else if (name == "set_small_data") - //{ - // return std::make_unique("/set_generator", 5); - //} - - throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "Unknown generator {}", name); -} - NumberGetter NumberGetter::fromConfig(const std::string & key, const Poco::Util::AbstractConfiguration & config, std::optional default_value) { @@ -321,6 +263,10 @@ std::string PathGetter::description() const return description; } +RequestGetter::RequestGetter(std::vector request_generators_) + : request_generators(std::move(request_generators_)) +{} + RequestGetter RequestGetter::fromConfig(const std::string & key, const Poco::Util::AbstractConfiguration & config, bool for_multi) { RequestGetter request_getter; @@ -658,7 +604,7 @@ Generator::Generator(const Poco::Util::AbstractConfiguration & config) static const std::string generator_key = "generator"; - std::cout << "---- Parsing setup ---- " << std::endl; + std::cout << "---- Parsing setup ---- " << std::endl; static const std::string setup_key = generator_key + ".setup"; Poco::Util::AbstractConfiguration::Keys keys; config.keys(setup_key, keys); diff --git a/utils/keeper-bench/Generator.h b/utils/keeper-bench/Generator.h index 5c15d2bce9ad..60c4fcb3cc4c 100644 --- a/utils/keeper-bench/Generator.h +++ b/utils/keeper-bench/Generator.h @@ -9,115 +9,6 @@ #include #include - -std::string generateRandomPath(const std::string & prefix, size_t length = 5); - -std::string generateRandomData(size_t size); - -// -//class CreateRequestGenerator final : public IGenerator -//{ -//public: -// explicit CreateRequestGenerator( -// std::string path_prefix_ = "/create_generator", -// std::optional path_length_ = std::nullopt, -// std::optional data_size_ = std::nullopt) -// : path_prefix(path_prefix_) -// , path_length(path_length_) -// , data_size(data_size_) -// {} -// -// void startup(Coordination::ZooKeeper & zookeeper) override; -// Coordination::ZooKeeperRequestPtr generate() override; -// -//private: -// std::string path_prefix; -// std::optional path_length; -// std::optional data_size; -// std::unordered_set paths_created; -//}; -// -// -//class GetRequestGenerator final : public IGenerator -//{ -//public: -// explicit GetRequestGenerator( -// std::string path_prefix_ = "/get_generator", -// std::optional num_nodes_ = std::nullopt, -// std::optional nodes_data_size_ = std::nullopt) -// : path_prefix(path_prefix_) -// , num_nodes(num_nodes_) -// , nodes_data_size(nodes_data_size_) -// , rng(randomSeed()) -// , distribution(0, num_nodes ? *num_nodes - 1 : 0) -// {} -// -// void startup(Coordination::ZooKeeper & zookeeper) override; -// Coordination::ZooKeeperRequestPtr generate() override; -// -//private: -// std::string path_prefix; -// std::optional num_nodes; -// std::optional nodes_data_size; -// std::vector paths_to_get; -// -// pcg64 rng; -// std::uniform_int_distribution distribution; -//}; -// -//class ListRequestGenerator final : public IGenerator -//{ -//public: -// explicit ListRequestGenerator( -// std::string path_prefix_ = "/list_generator", -// std::optional num_nodes_ = std::nullopt, -// std::optional paths_length_ = std::nullopt) -// : path_prefix(path_prefix_) -// , num_nodes(num_nodes_) -// , paths_length(paths_length_) -// {} -// -// void startup(Coordination::ZooKeeper & zookeeper) override; -// Coordination::ZooKeeperRequestPtr generate() override; -// -//private: -// std::string path_prefix; -// std::optional num_nodes; -// std::optional paths_length; -//}; -// -//class SetRequestGenerator final : public IGenerator -//{ -//public: -// explicit SetRequestGenerator( -// std::string path_prefix_ = "/set_generator", -// uint64_t data_size_ = 5) -// : path_prefix(path_prefix_) -// , data_size(data_size_) -// {} -// -// void startup(Coordination::ZooKeeper & zookeeper) override; -// Coordination::ZooKeeperRequestPtr generate() override; -// -//private: -// std::string path_prefix; -// uint64_t data_size; -//}; -// -//class MixedRequestGenerator final : public IGenerator -//{ -//public: -// explicit MixedRequestGenerator(std::vector> generators_) -// : generators(std::move(generators_)) -// {} -// -// void startup(Coordination::ZooKeeper & zookeeper) override; -// Coordination::ZooKeeperRequestPtr generate() override; -// -//private: -// std::vector> generators; -//}; - struct NumberGetter { static NumberGetter fromConfig(const std::string & key, const Poco::Util::AbstractConfiguration & config, std::optional default_value = std::nullopt); @@ -247,6 +138,10 @@ struct ListRequestGenerator final : public RequestGenerator struct RequestGetter { + explicit RequestGetter(std::vector request_generators_); + + RequestGetter() = default; + static RequestGetter fromConfig(const std::string & key, const Poco::Util::AbstractConfiguration & config, bool for_multi = false); RequestGeneratorPtr getRequestGenerator() const; @@ -279,7 +174,6 @@ class Generator void startup(Coordination::ZooKeeper & zookeeper); Coordination::ZooKeeperRequestPtr generate(); void cleanup(Coordination::ZooKeeper & zookeeper); - private: struct Node { @@ -299,4 +193,4 @@ class Generator Coordination::ACLs default_acls; }; -std::unique_ptr getGenerator(const std::string & name); +std::optional getGenerator(const std::string & name); diff --git a/utils/keeper-bench/Runner.cpp b/utils/keeper-bench/Runner.cpp index 2e3cd9116980..2b645ddc6c3c 100644 --- a/utils/keeper-bench/Runner.cpp +++ b/utils/keeper-bench/Runner.cpp @@ -13,7 +13,6 @@ namespace DB::ErrorCodes Runner::Runner( std::optional concurrency_, - const std::string & generator_name, const std::string & config_path, const Strings & hosts_strings_, std::optional max_time_, @@ -23,28 +22,10 @@ Runner::Runner( : info(std::make_shared()) { - DB::ConfigurationPtr config = nullptr; + DB::ConfigProcessor config_processor(config_path, true, false); + auto config = config_processor.loadConfig().configuration; - if (!config_path.empty()) - { - DB::ConfigProcessor config_processor(config_path, true, false); - config = config_processor.loadConfig().configuration; - } - - if (!generator_name.empty()) - { - //generator = getGenerator(generator_name); - - if (!generator) - throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, "Failed to create generator"); - } - else - { - if (!config) - throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, "No config file or generator name defined"); - - generator.emplace(*config); - } + generator.emplace(*config); if (!hosts_strings_.empty()) { @@ -60,34 +41,38 @@ Runner::Runner( } std::cout << "---- Run options ---- " << std::endl; + static constexpr uint64_t DEFAULT_CONCURRENCY = 1; if (concurrency_) concurrency = *concurrency_; else - concurrency = config->getUInt64("concurrency", 1); + concurrency = config->getUInt64("concurrency", DEFAULT_CONCURRENCY); std::cout << "Concurrency: " << concurrency << std::endl; + static constexpr uint64_t DEFAULT_ITERATIONS = 0; if (max_iterations_) max_iterations = *max_iterations_; else - max_iterations = config->getUInt64("iterations", 0); + max_iterations = config->getUInt64("iterations", DEFAULT_ITERATIONS); std::cout << "Iterations: " << max_iterations << std::endl; + static constexpr double DEFAULT_DELAY = 1.0; if (delay_) delay = *delay_; else - delay = config->getDouble("report_delay", 1); + delay = config->getDouble("report_delay", DEFAULT_DELAY); std::cout << "Report delay: " << delay << std::endl; + static constexpr double DEFAULT_TIME_LIMIT = 1.0; if (max_time_) max_time = *max_time_; else - max_time = config->getDouble("timelimit", 1.0); + max_time = config->getDouble("timelimit", DEFAULT_TIME_LIMIT); std::cout << "Time limit: " << max_time << std::endl; if (continue_on_error_) continue_on_error = *continue_on_error_; else - continue_on_error = config->getBool("continue_on_error", 1.0); + continue_on_error = config->getBool("continue_on_error", false); std::cout << "Continue on error: " << continue_on_error << std::endl; std::cout << "---- Run options ----\n" << std::endl; diff --git a/utils/keeper-bench/Runner.h b/utils/keeper-bench/Runner.h index ebd2d702019c..d85dc9e86585 100644 --- a/utils/keeper-bench/Runner.h +++ b/utils/keeper-bench/Runner.h @@ -35,7 +35,6 @@ class Runner public: Runner( std::optional concurrency_, - const std::string & generator_name, const std::string & config_path, const Strings & hosts_strings_, std::optional max_time_, diff --git a/utils/keeper-bench/main.cpp b/utils/keeper-bench/main.cpp index cb25aa7d0a52..0753d66850f5 100644 --- a/utils/keeper-bench/main.cpp +++ b/utils/keeper-bench/main.cpp @@ -34,7 +34,6 @@ int main(int argc, char *argv[]) boost::program_options::options_description desc = createOptionsDescription("Allowed options", getTerminalWidth()); desc.add_options() ("help", "produce help message") - ("generator", value()->default_value(""), "query to execute") ("config", value()->default_value(""), "yaml/xml file containing configuration") ("concurrency,c", value(), "number of parallel queries") ("report-delay,d", value(), "delay between intermediate reports in seconds (set 0 to disable reports)") @@ -56,7 +55,6 @@ int main(int argc, char *argv[]) } Runner runner(valueToOptional(options["concurrency"]), - options["generator"].as(), options["config"].as(), options["hosts"].as(), valueToOptional(options["time-limit"]), From ca1e6ac5cac47a15ca7e0b5c38f543cd358d44d5 Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Fri, 7 Apr 2023 15:22:03 +0000 Subject: [PATCH 106/406] Add example yaml --- utils/keeper-bench/example.yaml | 111 ++++++++++++++++++++++++++++++++ 1 file changed, 111 insertions(+) create mode 100644 utils/keeper-bench/example.yaml diff --git a/utils/keeper-bench/example.yaml b/utils/keeper-bench/example.yaml new file mode 100644 index 000000000000..2febb881634f --- /dev/null +++ b/utils/keeper-bench/example.yaml @@ -0,0 +1,111 @@ +concurrency: 20 +iterations: 10000 +delay: 4 +timelimit: 300 +continue_on_errors: true + +connections: + operation_timeout_ms: 3000 + connection_timeout_ms: 40000 + + connection: + secure: false + operation_timeout_ms: 2000 + session_timeout_ms: 2000 + connection_timeout_ms: 50000 + host: "localhost:9181" + sessions: 1 + + host: "localhost:9181" + +generator: + setup: + node: + name: "test3" + node: + name: "test_create" + node: + name: "test4" + node: + name: "test" + data: "somedata" + node: + repeat: 4 + name: + random_string: + size: 15 + data: + random_string: + size: + min_value: 10 + max_value: 20 + node: + repeat: 2 + node: + repeat: 2 + name: + random_string: + size: 12 + name: + random_string: + size: 15 + data: + random_string: + size: + min_value: 10 + max_value: 20 + node: + name: "test2" + data: "somedata" + requests: + create: + path: "/test_create" + name_length: 10 + remove_factor: 0.5 + multi: + size: 20 + create: + path: "/test" + data: + random_string: + size: + min_value: 10 + max_value: 20 + remove_factor: 0.8 + set: + weight: 2 + path: + - "/test3" + - "/test4" + path: + children_of: "/test" + data: + random_string: + size: 10 + get: + path: + - "/test3" + - "/test4" + path: + children_of: "/test" + + multi: + weight: 10 + get: + path: + - "/test3" + - "/test4" + path: + children_of: "/test" + list: + path: + - "/test3" + path: + children_of: "/test" + + list: + path: + - "/test3" + - "/test4" + path: + children_of: "/test" From 9ec91acef385e322e2f51a844e762a13b58d0de3 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Fri, 7 Apr 2023 18:26:23 +0200 Subject: [PATCH 107/406] add some columns to system.clusters --- src/Databases/DatabaseReplicated.cpp | 61 +++++- src/Databases/DatabaseReplicated.h | 5 +- src/Functions/hasColumnInTable.cpp | 12 +- src/Interpreters/Cluster.cpp | 202 ++++++++---------- src/Interpreters/Cluster.h | 60 ++++-- src/Interpreters/ClusterDiscovery.cpp | 13 +- src/Interpreters/InterpreterSystemQuery.cpp | 9 +- src/Parsers/ASTSystemQuery.cpp | 6 + src/Parsers/ASTSystemQuery.h | 1 + src/Parsers/ParserSystemQuery.cpp | 8 + src/Storages/System/StorageSystemClusters.cpp | 22 +- src/Storages/System/StorageSystemClusters.h | 2 +- src/TableFunctions/TableFunctionRemote.cpp | 13 +- .../02447_drop_database_replica.reference | 14 +- .../02447_drop_database_replica.sh | 26 ++- 15 files changed, 284 insertions(+), 170 deletions(-) diff --git a/src/Databases/DatabaseReplicated.cpp b/src/Databases/DatabaseReplicated.cpp index 5f5cd2667cb3..3f3cb10e9e33 100644 --- a/src/Databases/DatabaseReplicated.cpp +++ b/src/Databases/DatabaseReplicated.cpp @@ -117,9 +117,14 @@ DatabaseReplicated::DatabaseReplicated( fillClusterAuthInfo(db_settings.collection_name.value, context_->getConfigRef()); } +String DatabaseReplicated::getFullReplicaName(const String & shard, const String & replica) +{ + return shard + '|' + replica; +} + String DatabaseReplicated::getFullReplicaName() const { - return shard_name + '|' + replica_name; + return getFullReplicaName(shard_name, replica_name); } std::pair DatabaseReplicated::parseFullReplicaName(const String & name) @@ -216,7 +221,7 @@ ClusterPtr DatabaseReplicated::getClusterImpl() const assert(!hosts.empty()); assert(hosts.size() == host_ids.size()); String current_shard = parseFullReplicaName(hosts.front()).first; - std::vector shards; + std::vector> shards; shards.emplace_back(); for (size_t i = 0; i < hosts.size(); ++i) { @@ -232,25 +237,61 @@ ClusterPtr DatabaseReplicated::getClusterImpl() const if (!shards.back().empty()) shards.emplace_back(); } - shards.back().emplace_back(unescapeForFileName(host_port)); + String hostname = unescapeForFileName(host_port); + shards.back().push_back(DatabaseReplicaInfo{std::move(hostname), std::move(shard), std::move(replica)}); } UInt16 default_port = getContext()->getTCPPort(); bool treat_local_as_remote = false; bool treat_local_port_as_remote = getContext()->getApplicationType() == Context::ApplicationType::LOCAL; - return std::make_shared( - getContext()->getSettingsRef(), - shards, + ClusterConnectionParameters params{ cluster_auth_info.cluster_username, cluster_auth_info.cluster_password, default_port, treat_local_as_remote, treat_local_port_as_remote, cluster_auth_info.cluster_secure_connection, - /*priority=*/1, + /*priority=*/ 1, TSA_SUPPRESS_WARNING_FOR_READ(database_name), /// FIXME - cluster_auth_info.cluster_secret); + cluster_auth_info.cluster_secret}; + + return std::make_shared(getContext()->getSettingsRef(), shards, params); +} + +std::vector DatabaseReplicated::tryGetAreReplicasActive(const ClusterPtr & cluster_) const +{ + Strings paths; + const auto & addresses_with_failover = cluster->getShardsAddresses(); + const auto & shards_info = cluster_->getShardsInfo(); + for (size_t shard_index = 0; shard_index < shards_info.size(); ++shard_index) + { + for (const auto & replica : addresses_with_failover[shard_index]) + { + String full_name = getFullReplicaName(replica.database_shard_name, replica.database_replica_name); + paths.emplace_back(fs::path(zookeeper_path) / "replicas" / full_name / "active"); + } + } + + try + { + auto current_zookeeper = getZooKeeper(); + auto res = current_zookeeper->exists(paths); + + std::vector statuses; + statuses.resize(paths.size()); + + for (size_t i = 0; i < res.size(); ++i) + if (res[i].error == Coordination::Error::ZOK) + statuses[i] = 1; + + return statuses; + } + catch (...) + { + tryLogCurrentException(log); + return {}; + } } @@ -1043,10 +1084,12 @@ ASTPtr DatabaseReplicated::parseQueryFromMetadataInZooKeeper(const String & node } void DatabaseReplicated::dropReplica( - DatabaseReplicated * database, const String & database_zookeeper_path, const String & full_replica_name) + DatabaseReplicated * database, const String & database_zookeeper_path, const String & shard, const String & replica) { assert(!database || database_zookeeper_path == database->zookeeper_path); + String full_replica_name = shard.empty() ? replica : getFullReplicaName(shard, replica); + if (full_replica_name.find('/') != std::string::npos) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Invalid replica name: {}", full_replica_name); diff --git a/src/Databases/DatabaseReplicated.h b/src/Databases/DatabaseReplicated.h index 6a897f7322a0..b3397a832f24 100644 --- a/src/Databases/DatabaseReplicated.h +++ b/src/Databases/DatabaseReplicated.h @@ -55,6 +55,7 @@ class DatabaseReplicated : public DatabaseAtomic String getShardName() const { return shard_name; } String getReplicaName() const { return replica_name; } String getFullReplicaName() const; + static String getFullReplicaName(const String & shard, const String & replica); static std::pair parseFullReplicaName(const String & name); const String & getZooKeeperPath() const { return zookeeper_path; } @@ -77,7 +78,9 @@ class DatabaseReplicated : public DatabaseAtomic bool shouldReplicateQuery(const ContextPtr & query_context, const ASTPtr & query_ptr) const override; - static void dropReplica(DatabaseReplicated * database, const String & database_zookeeper_path, const String & full_replica_name); + static void dropReplica(DatabaseReplicated * database, const String & database_zookeeper_path, const String & shard, const String & replica); + + std::vector tryGetAreReplicasActive(const ClusterPtr & cluster_) const; friend struct DatabaseReplicatedTask; friend class DatabaseReplicatedDDLWorker; diff --git a/src/Functions/hasColumnInTable.cpp b/src/Functions/hasColumnInTable.cpp index 824056a452b2..4676b4083b70 100644 --- a/src/Functions/hasColumnInTable.cpp +++ b/src/Functions/hasColumnInTable.cpp @@ -130,14 +130,18 @@ ColumnPtr FunctionHasColumnInTable::executeImpl(const ColumnsWithTypeAndName & a bool treat_local_as_remote = false; bool treat_local_port_as_remote = getContext()->getApplicationType() == Context::ApplicationType::LOCAL; - auto cluster = std::make_shared( - getContext()->getSettings(), - host_names, + ClusterConnectionParameters params{ !user_name.empty() ? user_name : "default", password, getContext()->getTCPPort(), treat_local_as_remote, - treat_local_port_as_remote); + treat_local_port_as_remote, + /* secure= */ false, + /* priority= */ 1, + /* cluster_name= */ "", + /* password= */ "" + }; + auto cluster = std::make_shared(getContext()->getSettings(), host_names, params); // FIXME this (probably) needs a non-constant access to query context, // because it might initialized a storage. Ideally, the tables required diff --git a/src/Interpreters/Cluster.cpp b/src/Interpreters/Cluster.cpp index 445c227dd294..b696b5390136 100644 --- a/src/Interpreters/Cluster.cpp +++ b/src/Interpreters/Cluster.cpp @@ -127,24 +127,17 @@ Cluster::Address::Address( Cluster::Address::Address( - const String & host_port_, - const String & user_, - const String & password_, - UInt16 clickhouse_port, - bool treat_local_port_as_remote, - bool secure_, - Int64 priority_, + const DatabaseReplicaInfo & info, + const ClusterConnectionParameters & params, UInt32 shard_index_, - UInt32 replica_index_, - String cluster_name_, - String cluster_secret_) - : user(user_), password(password_) + UInt32 replica_index_) + : user(params.username), password(params.password) { bool can_be_local = true; std::pair parsed_host_port; - if (!treat_local_port_as_remote) + if (!params.treat_local_port_as_remote) { - parsed_host_port = parseAddress(host_port_, clickhouse_port); + parsed_host_port = parseAddress(info.hostname, params.clickhouse_port); } else { @@ -154,23 +147,25 @@ Cluster::Address::Address( /// If it doesn't include a port then use the default one and it could be local (if the address is) try { - parsed_host_port = parseAddress(host_port_, 0); + parsed_host_port = parseAddress(info.hostname, 0); can_be_local = false; } catch (...) { - parsed_host_port = parseAddress(host_port_, clickhouse_port); + parsed_host_port = parseAddress(info.hostname, params.clickhouse_port); } } host_name = parsed_host_port.first; + database_shard_name = info.shard_name; + database_replica_name = info.replica_name; port = parsed_host_port.second; - secure = secure_ ? Protocol::Secure::Enable : Protocol::Secure::Disable; - priority = priority_; - is_local = can_be_local && isLocal(clickhouse_port); + secure = params.secure ? Protocol::Secure::Enable : Protocol::Secure::Disable; + priority = params.priority; + is_local = can_be_local && isLocal(params.clickhouse_port); shard_index = shard_index_; replica_index = replica_index_; - cluster = cluster_name_; - cluster_secret = cluster_secret_; + cluster = params.cluster_name; + cluster_secret = params.cluster_secret; } @@ -492,44 +487,8 @@ Cluster::Cluster(const Poco::Util::AbstractConfiguration & config, throw Exception(ErrorCodes::UNKNOWN_ELEMENT_IN_CONFIG, "Unknown element in config: {}", replica_key); } - Addresses shard_local_addresses; - Addresses shard_all_addresses; - - ConnectionPoolPtrs all_replicas_pools; - all_replicas_pools.reserve(replica_addresses.size()); - - for (const auto & replica : replica_addresses) - { - auto replica_pool = ConnectionPoolFactory::instance().get( - static_cast(settings.distributed_connections_pool_size), - replica.host_name, replica.port, - replica.default_database, replica.user, replica.password, replica.quota_key, - replica.cluster, replica.cluster_secret, - "server", replica.compression, - replica.secure, replica.priority); - - all_replicas_pools.emplace_back(replica_pool); - if (replica.is_local) - shard_local_addresses.push_back(replica); - shard_all_addresses.push_back(replica); - } - ConnectionPoolWithFailoverPtr shard_pool = std::make_shared( - all_replicas_pools, settings.load_balancing, - settings.distributed_replica_error_half_life.totalSeconds(), settings.distributed_replica_error_cap); - - if (weight) - slot_to_shard.insert(std::end(slot_to_shard), weight, shards_info.size()); - - shards_info.push_back({ - std::move(insert_paths), - current_shard_num, - weight, - std::move(shard_local_addresses), - std::move(shard_all_addresses), - std::move(shard_pool), - std::move(all_replicas_pools), - internal_replication - }); + addShard(settings, std::move(replica_addresses), false, current_shard_num, + std::move(insert_paths), /* treat_local_as_remote */ weight, internal_replication); } else throw Exception(ErrorCodes::UNKNOWN_ELEMENT_IN_CONFIG, "Unknown element in config: {}", key); @@ -547,79 +506,102 @@ Cluster::Cluster(const Poco::Util::AbstractConfiguration & config, Cluster::Cluster( const Settings & settings, const std::vector> & names, - const String & username, - const String & password, - UInt16 clickhouse_port, - bool treat_local_as_remote, - bool treat_local_port_as_remote, - bool secure, - Int64 priority, - String cluster_name, - String cluster_secret) + const ClusterConnectionParameters & params) { UInt32 current_shard_num = 1; - secret = cluster_secret; + secret = params.cluster_secret; for (const auto & shard : names) { Addresses current; for (const auto & replica : shard) current.emplace_back( - replica, - username, - password, - clickhouse_port, - treat_local_port_as_remote, - secure, - priority, + DatabaseReplicaInfo{replica, "", ""}, + params, current_shard_num, - current.size() + 1, - cluster_name, - cluster_secret); + current.size() + 1); addresses_with_failover.emplace_back(current); - Addresses shard_local_addresses; - Addresses all_addresses; - ConnectionPoolPtrs all_replicas; - all_replicas.reserve(current.size()); + addShard(settings, std::move(current), params.treat_local_as_remote, current_shard_num); + ++current_shard_num; + } - for (const auto & replica : current) - { - auto replica_pool = ConnectionPoolFactory::instance().get( - static_cast(settings.distributed_connections_pool_size), - replica.host_name, replica.port, - replica.default_database, replica.user, replica.password, replica.quota_key, - replica.cluster, replica.cluster_secret, - "server", replica.compression, replica.secure, replica.priority); - all_replicas.emplace_back(replica_pool); - if (replica.is_local && !treat_local_as_remote) - shard_local_addresses.push_back(replica); - all_addresses.push_back(replica); - } + initMisc(); +} + +Cluster::Cluster( + const Settings & settings, + const std::vector> & infos, + const ClusterConnectionParameters & params) +{ + UInt32 current_shard_num = 1; + + secret = params.cluster_secret; + + for (const auto & shard : infos) + { + Addresses current; + for (const auto & replica : shard) + current.emplace_back( + replica, + params, + current_shard_num, + current.size() + 1); + + addresses_with_failover.emplace_back(current); - ConnectionPoolWithFailoverPtr shard_pool = std::make_shared( - all_replicas, settings.load_balancing, - settings.distributed_replica_error_half_life.totalSeconds(), settings.distributed_replica_error_cap); - - slot_to_shard.insert(std::end(slot_to_shard), default_weight, shards_info.size()); - shards_info.push_back({ - {}, // insert_path_for_internal_replication - current_shard_num, - default_weight, - std::move(shard_local_addresses), - std::move(all_addresses), - std::move(shard_pool), - std::move(all_replicas), - false // has_internal_replication - }); + addShard(settings, std::move(current), params.treat_local_as_remote, current_shard_num); ++current_shard_num; } initMisc(); } +void Cluster::addShard(const Settings & settings, Addresses && addresses, bool treat_local_as_remote, UInt32 current_shard_num, + ShardInfoInsertPathForInternalReplication && insert_paths, UInt32 weight, bool internal_replication) +{ + Addresses shard_local_addresses; + Addresses shard_all_addresses; + + ConnectionPoolPtrs all_replicas_pools; + all_replicas_pools.reserve(addresses.size()); + + for (const auto & replica : addresses) + { + auto replica_pool = ConnectionPoolFactory::instance().get( + static_cast(settings.distributed_connections_pool_size), + replica.host_name, replica.port, + replica.default_database, replica.user, replica.password, replica.quota_key, + replica.cluster, replica.cluster_secret, + "server", replica.compression, + replica.secure, replica.priority); + + all_replicas_pools.emplace_back(replica_pool); + if (replica.is_local && !treat_local_as_remote) + shard_local_addresses.push_back(replica); + shard_all_addresses.push_back(replica); + } + ConnectionPoolWithFailoverPtr shard_pool = std::make_shared( + all_replicas_pools, settings.load_balancing, + settings.distributed_replica_error_half_life.totalSeconds(), settings.distributed_replica_error_cap); + + if (weight) + slot_to_shard.insert(std::end(slot_to_shard), weight, shards_info.size()); + + shards_info.push_back({ + std::move(insert_paths), + current_shard_num, + weight, + std::move(shard_local_addresses), + std::move(shard_all_addresses), + std::move(shard_pool), + std::move(all_replicas_pools), + internal_replication + }); +} + Poco::Timespan Cluster::saturate(Poco::Timespan v, Poco::Timespan limit) { diff --git a/src/Interpreters/Cluster.h b/src/Interpreters/Cluster.h index d74f75c941e6..f8437b7e027b 100644 --- a/src/Interpreters/Cluster.h +++ b/src/Interpreters/Cluster.h @@ -29,6 +29,26 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; } +struct DatabaseReplicaInfo +{ + String hostname; + String shard_name; + String replica_name; +}; + +struct ClusterConnectionParameters +{ + const String & username; + const String & password; + UInt16 clickhouse_port; + bool treat_local_as_remote; + bool treat_local_port_as_remote; + bool secure = false; + Int64 priority = 1; + String cluster_name; + String cluster_secret; +}; + /// Cluster contains connection pools to each node /// With the local nodes, the connection is not established, but the request is executed directly. /// Therefore we store only the number of local nodes @@ -51,15 +71,13 @@ class Cluster Cluster( const Settings & settings, const std::vector> & names, - const String & username, - const String & password, - UInt16 clickhouse_port, - bool treat_local_as_remote, - bool treat_local_port_as_remote, - bool secure = false, - Int64 priority = 1, - String cluster_name = "", - String cluster_secret = ""); + const ClusterConnectionParameters & params); + + + Cluster( + const Settings & settings, + const std::vector> & infos, + const ClusterConnectionParameters & params); Cluster(const Cluster &)= delete; Cluster & operator=(const Cluster &) = delete; @@ -90,6 +108,8 @@ class Cluster */ String host_name; + String database_shard_name; + String database_replica_name; UInt16 port{0}; String user; String password; @@ -125,16 +145,15 @@ class Cluster Address( const String & host_port_, - const String & user_, - const String & password_, - UInt16 clickhouse_port, - bool treat_local_port_as_remote, - bool secure_ = false, - Int64 priority_ = 1, - UInt32 shard_index_ = 0, - UInt32 replica_index_ = 0, - String cluster_name = "", - String cluster_secret_ = ""); + const ClusterConnectionParameters & params, + UInt32 shard_index_, + UInt32 replica_index_); + + Address( + const DatabaseReplicaInfo & host_port_, + const ClusterConnectionParameters & params, + UInt32 shard_index_, + UInt32 replica_index_); /// Returns 'escaped_host_name:port' String toString() const; @@ -276,6 +295,9 @@ class Cluster struct ReplicasAsShardsTag {}; Cluster(ReplicasAsShardsTag, const Cluster & from, const Settings & settings, size_t max_replicas_from_shard); + void addShard(const Settings & settings, Addresses && addresses, bool treat_local_as_remote, UInt32 current_shard_num, + ShardInfoInsertPathForInternalReplication && insert_paths = {}, UInt32 weight = 1, bool internal_replication = false); + /// Inter-server secret String secret; diff --git a/src/Interpreters/ClusterDiscovery.cpp b/src/Interpreters/ClusterDiscovery.cpp index 36b2f17e8a11..d006b8f29b16 100644 --- a/src/Interpreters/ClusterDiscovery.cpp +++ b/src/Interpreters/ClusterDiscovery.cpp @@ -237,15 +237,20 @@ ClusterPtr ClusterDiscovery::makeCluster(const ClusterInfo & cluster_info) } bool secure = cluster_info.current_node.secure; - auto cluster = std::make_shared( - context->getSettingsRef(), - shards, + ClusterConnectionParameters params{ /* username= */ context->getUserName(), /* password= */ "", /* clickhouse_port= */ secure ? context->getTCPPortSecure().value_or(DBMS_DEFAULT_SECURE_PORT) : context->getTCPPort(), /* treat_local_as_remote= */ false, /* treat_local_port_as_remote= */ false, /// should be set only for clickhouse-local, but cluster discovery is not used there - /* secure= */ secure); + /* secure= */ secure, + /* priority= */ 1, + /* cluster_name= */ "", + /* password= */ ""}; + auto cluster = std::make_shared( + context->getSettingsRef(), + shards, + params); return cluster; } diff --git a/src/Interpreters/InterpreterSystemQuery.cpp b/src/Interpreters/InterpreterSystemQuery.cpp index e9905821fd15..5ef171d194c3 100644 --- a/src/Interpreters/InterpreterSystemQuery.cpp +++ b/src/Interpreters/InterpreterSystemQuery.cpp @@ -829,7 +829,8 @@ void InterpreterSystemQuery::dropDatabaseReplica(ASTSystemQuery & query) { if (!query.replica_zk_path.empty() && fs::path(replicated->getZooKeeperPath()) != fs::path(query.replica_zk_path)) return; - if (replicated->getFullReplicaName() != query.replica) + String full_replica_name = query.shard.empty() ? query.replica : DatabaseReplicated::getFullReplicaName(query.shard, query.replica); + if (replicated->getFullReplicaName() != full_replica_name) return; throw Exception(ErrorCodes::TABLE_WAS_NOT_DROPPED, "There is a local database {}, which has the same path in ZooKeeper " @@ -844,7 +845,7 @@ void InterpreterSystemQuery::dropDatabaseReplica(ASTSystemQuery & query) if (auto * replicated = dynamic_cast(database.get())) { check_not_local_replica(replicated, query); - DatabaseReplicated::dropReplica(replicated, replicated->getZooKeeperPath(), query.replica); + DatabaseReplicated::dropReplica(replicated, replicated->getZooKeeperPath(), query.shard, query.replica); } else throw Exception(ErrorCodes::BAD_ARGUMENTS, "Database {} is not Replicated, cannot drop replica", query.getDatabase()); @@ -869,7 +870,7 @@ void InterpreterSystemQuery::dropDatabaseReplica(ASTSystemQuery & query) } check_not_local_replica(replicated, query); - DatabaseReplicated::dropReplica(replicated, replicated->getZooKeeperPath(), query.replica); + DatabaseReplicated::dropReplica(replicated, replicated->getZooKeeperPath(), query.shard, query.replica); LOG_TRACE(log, "Dropped replica {} of Replicated database {}", query.replica, backQuoteIfNeed(database->getDatabaseName())); } } @@ -882,7 +883,7 @@ void InterpreterSystemQuery::dropDatabaseReplica(ASTSystemQuery & query) if (auto * replicated = dynamic_cast(elem.second.get())) check_not_local_replica(replicated, query); - DatabaseReplicated::dropReplica(nullptr, query.replica_zk_path, query.replica); + DatabaseReplicated::dropReplica(nullptr, query.replica_zk_path, query.shard, query.replica); LOG_INFO(log, "Dropped replica {} of Replicated database with path {}", query.replica, query.replica_zk_path); } else diff --git a/src/Parsers/ASTSystemQuery.cpp b/src/Parsers/ASTSystemQuery.cpp index 4bd5cdb5ebe7..ee46ab11f846 100644 --- a/src/Parsers/ASTSystemQuery.cpp +++ b/src/Parsers/ASTSystemQuery.cpp @@ -104,6 +104,12 @@ void ASTSystemQuery::formatImpl(const FormatSettings & settings, FormatState &, auto print_drop_replica = [&] { settings.ostr << " " << quoteString(replica); + if (!shard.empty()) + { + settings.ostr << (settings.hilite ? hilite_keyword : "") << " FROM SHARD " + << (settings.hilite ? hilite_none : "") << quoteString(shard); + } + if (table) { settings.ostr << (settings.hilite ? hilite_keyword : "") << " FROM TABLE" diff --git a/src/Parsers/ASTSystemQuery.h b/src/Parsers/ASTSystemQuery.h index e5824911645c..3e5d5465a9cf 100644 --- a/src/Parsers/ASTSystemQuery.h +++ b/src/Parsers/ASTSystemQuery.h @@ -96,6 +96,7 @@ class ASTSystemQuery : public IAST, public ASTQueryWithOnCluster String target_model; String target_function; String replica; + String shard; String replica_zk_path; bool is_drop_whole_replica{}; String storage_policy; diff --git a/src/Parsers/ParserSystemQuery.cpp b/src/Parsers/ParserSystemQuery.cpp index 26819f0ee6c5..792de02a6ea4 100644 --- a/src/Parsers/ParserSystemQuery.cpp +++ b/src/Parsers/ParserSystemQuery.cpp @@ -159,6 +159,14 @@ enum class SystemQueryTargetType if (!ParserStringLiteral{}.parse(pos, ast, expected)) return false; res->replica = ast->as().value.safeGet(); + + if (ParserKeyword{"FROM SHARD"}.ignore(pos, expected)) + { + if (!ParserStringLiteral{}.parse(pos, ast, expected)) + return false; + res->shard = ast->as().value.safeGet(); + } + if (ParserKeyword{"FROM"}.ignore(pos, expected)) { // way 1. parse replica database diff --git a/src/Storages/System/StorageSystemClusters.cpp b/src/Storages/System/StorageSystemClusters.cpp index 096e2aa206de..879fc92a010e 100644 --- a/src/Storages/System/StorageSystemClusters.cpp +++ b/src/Storages/System/StorageSystemClusters.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include #include @@ -24,7 +25,10 @@ NamesAndTypesList StorageSystemClusters::getNamesAndTypes() {"default_database", std::make_shared()}, {"errors_count", std::make_shared()}, {"slowdowns_count", std::make_shared()}, - {"estimated_recovery_time", std::make_shared()} + {"estimated_recovery_time", std::make_shared()}, + {"database_shard_name", std::make_shared()}, + {"database_replica_name", std::make_shared()}, + {"is_active", std::make_shared(std::make_shared())}, }; } @@ -32,26 +36,30 @@ NamesAndTypesList StorageSystemClusters::getNamesAndTypes() void StorageSystemClusters::fillData(MutableColumns & res_columns, ContextPtr context, const SelectQueryInfo &) const { for (const auto & name_and_cluster : context->getClusters()->getContainer()) - writeCluster(res_columns, name_and_cluster); + writeCluster(res_columns, name_and_cluster, {}); const auto databases = DatabaseCatalog::instance().getDatabases(); for (const auto & name_and_database : databases) { if (const auto * replicated = typeid_cast(name_and_database.second.get())) { + if (auto database_cluster = replicated->tryGetCluster()) - writeCluster(res_columns, {name_and_database.first, database_cluster}); + writeCluster(res_columns, {name_and_database.first, database_cluster}, + replicated->tryGetAreReplicasActive(database_cluster)); } } } -void StorageSystemClusters::writeCluster(MutableColumns & res_columns, const NameAndCluster & name_and_cluster) +void StorageSystemClusters::writeCluster(MutableColumns & res_columns, const NameAndCluster & name_and_cluster, + const std::vector & is_active) { const String & cluster_name = name_and_cluster.first; const ClusterPtr & cluster = name_and_cluster.second; const auto & shards_info = cluster->getShardsInfo(); const auto & addresses_with_failover = cluster->getShardsAddresses(); + size_t replica_idx = 0; for (size_t shard_index = 0; shard_index < shards_info.size(); ++shard_index) { const auto & shard_info = shards_info[shard_index]; @@ -77,6 +85,12 @@ void StorageSystemClusters::writeCluster(MutableColumns & res_columns, const Nam res_columns[i++]->insert(pool_status[replica_index].error_count); res_columns[i++]->insert(pool_status[replica_index].slowdown_count); res_columns[i++]->insert(pool_status[replica_index].estimated_recovery_time.count()); + res_columns[i++]->insert(address.database_shard_name); + res_columns[i++]->insert(address.database_replica_name); + if (is_active.empty()) + res_columns[i++]->insertDefault(); + else + res_columns[i++]->insert(is_active[replica_idx++]); } } } diff --git a/src/Storages/System/StorageSystemClusters.h b/src/Storages/System/StorageSystemClusters.h index f14446bf4d30..9aa1a6a51836 100644 --- a/src/Storages/System/StorageSystemClusters.h +++ b/src/Storages/System/StorageSystemClusters.h @@ -27,7 +27,7 @@ class StorageSystemClusters final : public IStorageSystemOneBlock>; void fillData(MutableColumns & res_columns, ContextPtr context, const SelectQueryInfo & query_info) const override; - static void writeCluster(MutableColumns & res_columns, const NameAndCluster & name_and_cluster); + static void writeCluster(MutableColumns & res_columns, const NameAndCluster & name_and_cluster, const std::vector & is_active); }; } diff --git a/src/TableFunctions/TableFunctionRemote.cpp b/src/TableFunctions/TableFunctionRemote.cpp index 1ee51bcb0400..b2f09adf7737 100644 --- a/src/TableFunctions/TableFunctionRemote.cpp +++ b/src/TableFunctions/TableFunctionRemote.cpp @@ -255,15 +255,18 @@ void TableFunctionRemote::parseArguments(const ASTPtr & ast_function, ContextPtr bool treat_local_as_remote = false; bool treat_local_port_as_remote = context->getApplicationType() == Context::ApplicationType::LOCAL; - cluster = std::make_shared( - context->getSettingsRef(), - names, + ClusterConnectionParameters params{ username, password, - (secure ? (maybe_secure_port ? *maybe_secure_port : DBMS_DEFAULT_SECURE_PORT) : context->getTCPPort()), + static_cast(secure ? (maybe_secure_port ? *maybe_secure_port : DBMS_DEFAULT_SECURE_PORT) : context->getTCPPort()), treat_local_as_remote, treat_local_port_as_remote, - secure); + secure, + /* priority= */ 1, + /* cluster_name= */ "", + /* password= */ "" + }; + cluster = std::make_shared(context->getSettingsRef(), names, params); } if (!remote_table_function_ptr && table.empty()) diff --git a/tests/queries/0_stateless/02447_drop_database_replica.reference b/tests/queries/0_stateless/02447_drop_database_replica.reference index 1d65fe66c6e8..2ae878f11d05 100644 --- a/tests/queries/0_stateless/02447_drop_database_replica.reference +++ b/tests/queries/0_stateless/02447_drop_database_replica.reference @@ -6,10 +6,18 @@ t 2 2 2 -rdb_default 1 1 -rdb_default 1 2 2 2 2 +2 +rdb_default 1 1 s1 r1 1 +rdb_default 1 2 s1 r2 1 +rdb_default 2 1 s2 r1 1 +2 +2 +rdb_default 1 1 s1 r1 1 +rdb_default 1 2 s1 r2 0 +2 +2 t -rdb_default_3 1 1 +rdb_default_4 1 1 s1 r1 1 diff --git a/tests/queries/0_stateless/02447_drop_database_replica.sh b/tests/queries/0_stateless/02447_drop_database_replica.sh index 4bfd6243c2ed..00e56be69613 100755 --- a/tests/queries/0_stateless/02447_drop_database_replica.sh +++ b/tests/queries/0_stateless/02447_drop_database_replica.sh @@ -13,35 +13,49 @@ $CLICKHOUSE_CLIENT -q "show tables from $db" $CLICKHOUSE_CLIENT -q "system drop database replica 's1|r1' from table t" 2>&1| grep -Fac "SYNTAX_ERROR" $CLICKHOUSE_CLIENT -q "system drop database replica 's1|r1' from database $db" 2>&1| grep -Fac "There is a local database" +$CLICKHOUSE_CLIENT -q "system drop database replica 'r1' from shard 's1' from database $db" 2>&1| grep -Fac "There is a local database" $CLICKHOUSE_CLIENT -q "system drop database replica 's1|r1' from zkpath '/test/$CLICKHOUSE_DATABASE/rdb'" 2>&1| grep -Fac "There is a local database" $CLICKHOUSE_CLIENT -q "system drop database replica 's1|r1' from zkpath '/test/$CLICKHOUSE_DATABASE/rdb/'" 2>&1| grep -Fac "There is a local database" +$CLICKHOUSE_CLIENT -q "system drop database replica 'r1' from shard 's1' from zkpath '/test/$CLICKHOUSE_DATABASE/rdb/'" 2>&1| grep -Fac "There is a local database" $CLICKHOUSE_CLIENT -q "system drop database replica 's1|r1' from zkpath '/test/$CLICKHOUSE_DATABASE/'" 2>&1| grep -Fac "does not look like a path of Replicated database" $CLICKHOUSE_CLIENT -q "system drop database replica 's2|r1' from zkpath '/test/$CLICKHOUSE_DATABASE/rdb'" 2>&1| grep -Fac "does not exist" +$CLICKHOUSE_CLIENT -q "system drop database replica 's1' from shard 'r1' from zkpath '/test/$CLICKHOUSE_DATABASE/rdb'" 2>&1| grep -Fac "does not exist" +$CLICKHOUSE_CLIENT -q "system drop database replica 's1|r1' from shard 's1' from zkpath '/test/$CLICKHOUSE_DATABASE/rdb'" 2>&1| grep -Fac "does not exist" $CLICKHOUSE_CLIENT -q "system drop database replica 's2/r1' from zkpath '/test/$CLICKHOUSE_DATABASE/rdb'" 2>&1| grep -Fac "Invalid replica name" db2="${db}_2" +db3="${db}_3" $CLICKHOUSE_CLIENT --allow_experimental_database_replicated=1 -q "create database $db2 engine=Replicated('/test/$CLICKHOUSE_DATABASE/rdb', 's1', 'r2')" +$CLICKHOUSE_CLIENT --allow_experimental_database_replicated=1 -q "create database $db3 engine=Replicated('/test/$CLICKHOUSE_DATABASE/rdb', 's2', 'r1')" $CLICKHOUSE_CLIENT -q "system sync database replica $db" -$CLICKHOUSE_CLIENT -q "select cluster, shard_num, replica_num from system.clusters where cluster='$db' order by shard_num, replica_num" +$CLICKHOUSE_CLIENT -q "select cluster, shard_num, replica_num, database_shard_name, database_replica_name, is_active from system.clusters where cluster='$db' order by shard_num, replica_num" $CLICKHOUSE_CLIENT -q "system drop database replica 's1|r1' from database $db2" 2>&1| grep -Fac "is active, cannot drop it" +$CLICKHOUSE_CLIENT -q "detach database $db3" +$CLICKHOUSE_CLIENT -q "system drop database replica 'r1' from shard 's2' from database $db" +$CLICKHOUSE_CLIENT -q "attach database $db3" 2>/dev/null +$CLICKHOUSE_CLIENT --distributed_ddl_output_mode=none -q "create table $db3.t2 as system.query_log" 2>&1| grep -Fac "Database is in readonly mode" # Suppress style check: current_database=$CLICKHOUSE_DATABASE + $CLICKHOUSE_CLIENT -q "detach database $db2" +$CLICKHOUSE_CLIENT -q "system sync database replica $db" +$CLICKHOUSE_CLIENT -q "select cluster, shard_num, replica_num, database_shard_name, database_replica_name, is_active from system.clusters where cluster='$db' order by shard_num, replica_num" $CLICKHOUSE_CLIENT -q "system drop database replica 's1|r2' from database $db" $CLICKHOUSE_CLIENT -q "attach database $db2" 2>/dev/null $CLICKHOUSE_CLIENT --distributed_ddl_output_mode=none -q "create table $db2.t2 as system.query_log" 2>&1| grep -Fac "Database is in readonly mode" # Suppress style check: current_database=$CLICKHOUSE_DATABASE $CLICKHOUSE_CLIENT -q "detach database $db" -$CLICKHOUSE_CLIENT -q "system drop database replica 's1|r1' from zkpath '/test/$CLICKHOUSE_DATABASE/rdb/'" +$CLICKHOUSE_CLIENT -q "system drop database replica 'r1' from shard 's1' from zkpath '/test/$CLICKHOUSE_DATABASE/rdb/'" $CLICKHOUSE_CLIENT -q "attach database $db" 2>/dev/null $CLICKHOUSE_CLIENT --distributed_ddl_output_mode=none -q "create table $db.t2 as system.query_log" 2>&1| grep -Fac "Database is in readonly mode" # Suppress style check: current_database=$CLICKHOUSE_DATABASE $CLICKHOUSE_CLIENT -q "show tables from $db" -db3="${db}_3" -$CLICKHOUSE_CLIENT --allow_experimental_database_replicated=1 -q "create database $db3 engine=Replicated('/test/$CLICKHOUSE_DATABASE/rdb', 's1', 'r1')" -$CLICKHOUSE_CLIENT -q "system sync database replica $db3" -$CLICKHOUSE_CLIENT -q "select cluster, shard_num, replica_num from system.clusters where cluster='$db3'" +db4="${db}_4" +$CLICKHOUSE_CLIENT --allow_experimental_database_replicated=1 -q "create database $db4 engine=Replicated('/test/$CLICKHOUSE_DATABASE/rdb', 's1', 'r1')" +$CLICKHOUSE_CLIENT -q "system sync database replica $db4" +$CLICKHOUSE_CLIENT -q "select cluster, shard_num, replica_num, database_shard_name, database_replica_name, is_active from system.clusters where cluster='$db4'" $CLICKHOUSE_CLIENT -q "drop database $db" $CLICKHOUSE_CLIENT -q "drop database $db2" $CLICKHOUSE_CLIENT -q "drop database $db3" +$CLICKHOUSE_CLIENT -q "drop database $db4" From b141d020f62abf6f92eb9693fe3b1c846aa8c260 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Fri, 7 Apr 2023 21:00:05 +0300 Subject: [PATCH 108/406] Update 02117_show_create_table_system.reference --- .../0_stateless/02117_show_create_table_system.reference | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/02117_show_create_table_system.reference b/tests/queries/0_stateless/02117_show_create_table_system.reference index b07d6e011613..b68ca364c395 100644 --- a/tests/queries/0_stateless/02117_show_create_table_system.reference +++ b/tests/queries/0_stateless/02117_show_create_table_system.reference @@ -47,7 +47,10 @@ CREATE TABLE system.clusters `default_database` String, `errors_count` UInt32, `slowdowns_count` UInt32, - `estimated_recovery_time` UInt32 + `estimated_recovery_time` UInt32, + `database_shard_name` String, + `database_replica_name` String, + `is_active` Nullable(UInt8) ) ENGINE = SystemClusters COMMENT 'SYSTEM TABLE is built on the fly.' From 64dc60a8d13157940ec6bf419db62c284ee48bd3 Mon Sep 17 00:00:00 2001 From: Nikolay Degterinsky Date: Mon, 10 Apr 2023 02:20:57 +0000 Subject: [PATCH 109/406] Better version, introduce ASTAuthenticationData --- programs/client/Client.cpp | 1 - src/Access/AccessControl.cpp | 10 +- src/Access/AccessControl.h | 1 - src/Access/Common/AuthenticationData.cpp | 38 -- src/Access/Common/AuthenticationData.h | 2 - src/Client/ClientBase.cpp | 29 +- src/Client/Connection.cpp | 7 - src/Client/Connection.h | 3 - src/Client/IServerConnection.h | 1 - src/Client/LocalConnection.h | 1 - src/Core/ProtocolDefines.h | 3 +- .../Access/InterpreterCreateUserQuery.cpp | 42 +-- ...InterpreterShowCreateAccessEntityQuery.cpp | 3 +- src/Parsers/ASTSubquery.h | 12 +- src/Parsers/Access/ASTAuthenticationData.cpp | 329 ++++++++++++++++++ src/Parsers/Access/ASTAuthenticationData.h | 43 +++ src/Parsers/Access/ASTCreateUserQuery.cpp | 131 +------ src/Parsers/Access/ASTCreateUserQuery.h | 5 +- src/Parsers/Access/ParserCreateUserQuery.cpp | 101 +++--- src/Server/TCPHandler.cpp | 6 - 20 files changed, 469 insertions(+), 299 deletions(-) create mode 100644 src/Parsers/Access/ASTAuthenticationData.cpp create mode 100644 src/Parsers/Access/ASTAuthenticationData.h diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp index b04748dd1614..4d460c7ac48c 100644 --- a/programs/client/Client.cpp +++ b/programs/client/Client.cpp @@ -332,7 +332,6 @@ try /// Set user password complexity rules auto & access_control = global_context->getAccessControl(); access_control.setPasswordComplexityRules(connection->getPasswordComplexityRules()); - access_control.setDefaultPasswordType(connection->getDefaultPasswordType()); if (is_interactive && !delayed_interactive) { diff --git a/src/Access/AccessControl.cpp b/src/Access/AccessControl.cpp index 234885fefff3..835673bc8a13 100644 --- a/src/Access/AccessControl.cpp +++ b/src/Access/AccessControl.cpp @@ -661,15 +661,7 @@ void AccessControl::setDefaultPasswordTypeFromConfig(const String & type_) else if (type_ == "sha256") default_password_type = AuthenticationType::SHA256_PASSWORD; else - throw Exception("Unknown password type in 'default_password_type' in config", ErrorCodes::UNKNOWN_ELEMENT_IN_CONFIG); -} - -void AccessControl::setDefaultPasswordType(const AuthenticationType & type_) -{ - if (type_ < AuthenticationType::MAX) - default_password_type = type_; - else - default_password_type = AuthenticationType::SHA256_PASSWORD; + throw Exception(ErrorCodes::UNKNOWN_ELEMENT_IN_CONFIG, "Unknown password type in 'default_password_type' in config"); } AuthenticationType AccessControl::getDefaultPasswordType() const diff --git a/src/Access/AccessControl.h b/src/Access/AccessControl.h index e37f22498dec..b1ac9671d3cf 100644 --- a/src/Access/AccessControl.h +++ b/src/Access/AccessControl.h @@ -150,7 +150,6 @@ class AccessControl : public MultipleAccessStorage /// Default password type when the user does not specify it. void setDefaultPasswordTypeFromConfig(const String & type_); - void setDefaultPasswordType(const AuthenticationType & type_); AuthenticationType getDefaultPasswordType() const; /// Check complexity requirements for passwords diff --git a/src/Access/Common/AuthenticationData.cpp b/src/Access/Common/AuthenticationData.cpp index 44a58512b622..ac604a789e24 100644 --- a/src/Access/Common/AuthenticationData.cpp +++ b/src/Access/Common/AuthenticationData.cpp @@ -98,44 +98,6 @@ AuthenticationData::Digest AuthenticationData::Util::encodeSHA1(std::string_view } -AuthenticationData AuthenticationData::makePasswordAuthenticationData(AuthenticationType type, String password) -{ - AuthenticationData auth_data(type); - - if (type == AuthenticationType::SHA256_PASSWORD) - { -#if USE_SSL - ///random generator FIPS complaint - uint8_t key[32]; - if (RAND_bytes(key, sizeof(key)) != 1) - { - char buf[512] = {0}; - ERR_error_string_n(ERR_get_error(), buf, sizeof(buf)); - throw Exception(ErrorCodes::OPENSSL_ERROR, "Cannot generate salt for password. OpenSSL {}", buf); - } - - String salt; - salt.resize(sizeof(key) * 2); - char * buf_pos = salt.data(); - for (uint8_t k : key) - { - writeHexByteUppercase(k, buf_pos); - buf_pos += 2; - } - password.append(salt); - auth_data.setSalt(salt); -#else - throw DB::Exception( - "SHA256 passwords support is disabled, because ClickHouse was built without SSL library", - DB::ErrorCodes::SUPPORT_IS_DISABLED); -#endif - } - - auth_data.setPassword(password); - return auth_data; -} - - bool operator ==(const AuthenticationData & lhs, const AuthenticationData & rhs) { return (lhs.type == rhs.type) && (lhs.password_hash == rhs.password_hash) diff --git a/src/Access/Common/AuthenticationData.h b/src/Access/Common/AuthenticationData.h index 85486dea17f8..ced9fcd4b6da 100644 --- a/src/Access/Common/AuthenticationData.h +++ b/src/Access/Common/AuthenticationData.h @@ -54,8 +54,6 @@ class AuthenticationData public: using Digest = std::vector; - static AuthenticationData makePasswordAuthenticationData(AuthenticationType type, String password); - explicit AuthenticationData(AuthenticationType type_ = AuthenticationType::NO_PASSWORD) : type(type_) {} AuthenticationData(const AuthenticationData & src) = default; AuthenticationData & operator =(const AuthenticationData & src) = default; diff --git a/src/Client/ClientBase.cpp b/src/Client/ClientBase.cpp index 9c989e27c7fd..6219e6f2d742 100644 --- a/src/Client/ClientBase.cpp +++ b/src/Client/ClientBase.cpp @@ -33,6 +33,7 @@ #include #include #include +#include #include #include #include @@ -351,16 +352,6 @@ ASTPtr ClientBase::parseQuery(const char *& pos, const char * end, bool allow_mu res = parseQueryAndMovePosition(*parser, pos, end, "", allow_multi_statements, max_length, settings.max_parser_depth); } - /// We need to change the auth_data before the query will be formatted - if (const auto * create_user_query = res->as()) - { - if (create_user_query->auth_data->getType() == AuthenticationType::NO_PASSWORD && create_user_query->temporary_password) - { - AuthenticationType default_password_type = global_context->getAccessControl().getDefaultPasswordType(); - create_user_query->auth_data = AuthenticationData::makePasswordAuthenticationData(default_password_type, create_user_query->temporary_password.value()); - } - } - if (is_interactive) { std::cout << std::endl; @@ -804,13 +795,8 @@ void ClientBase::processTextAsSingleQuery(const String & full_query) /// But for asynchronous inserts we don't extract data, because it's needed /// to be done on server side in that case (for coalescing the data from multiple inserts on server side). const auto * insert = parsed_query->as(); - /// In the case of a CREATE USER query, we should serialize the parsed AST to send - /// the hash of the password, if applicable, to the server instead of the actual password. - const auto * create_user_query = parsed_query->as(); if (insert && isSyncInsertWithData(*insert, global_context)) query_to_execute = full_query.substr(0, insert->data - full_query.data()); - else if (create_user_query) - query_to_execute = serializeAST(*parsed_query); else query_to_execute = full_query; @@ -1602,10 +1588,10 @@ void ClientBase::processParsedSingleQuery(const String & full_query, const Strin if (const auto * create_user_query = parsed_query->as()) { - if (!create_user_query->attach && create_user_query->temporary_password) + if (!create_user_query->attach && create_user_query->auth_data) { - global_context->getAccessControl().checkPasswordComplexityRules(create_user_query->temporary_password.value()); - create_user_query->temporary_password.reset(); + if (const auto * auth_data = create_user_query->auth_data->as()) + auth_data->checkPasswordComplexityRules(global_context); } } @@ -1826,12 +1812,7 @@ MultiQueryProcessingStage ClientBase::analyzeMultiQueryText( query_to_execute_end = isSyncInsertWithData(*insert_ast, global_context) ? insert_ast->data : this_query_end; } - /// In the case of a CREATE USER query, we should serialize the parsed AST to send - /// the hash of the password, if applicable, to the server instead of the actual password. - if (const auto * create_user_query = parsed_query->as()) - query_to_execute = serializeAST(*parsed_query); - else - query_to_execute = all_queries_text.substr(this_query_begin - all_queries_text.data(), query_to_execute_end - this_query_begin); + query_to_execute = all_queries_text.substr(this_query_begin - all_queries_text.data(), query_to_execute_end - this_query_begin); // Try to include the trailing comment with test hints. It is just // a guess for now, because we don't yet know where the query ends diff --git a/src/Client/Connection.cpp b/src/Client/Connection.cpp index 92b8e2abd982..eea007a8608f 100644 --- a/src/Client/Connection.cpp +++ b/src/Client/Connection.cpp @@ -324,13 +324,6 @@ void Connection::receiveHello() password_complexity_rules.push_back({std::move(original_pattern), std::move(exception_message)}); } } - - if (server_revision >= DBMS_MIN_PROTOCOL_VERSION_WITH_DEFAULT_PASSWORD_TYPE) - { - UInt64 default_password_type_uint; - readVarUInt(default_password_type_uint, *in); - default_password_type = static_cast(default_password_type_uint); - } } else if (packet_type == Protocol::Server::Exception) receiveException()->rethrow(); diff --git a/src/Client/Connection.h b/src/Client/Connection.h index c3721b3f9d9b..d806c5e8b1fc 100644 --- a/src/Client/Connection.h +++ b/src/Client/Connection.h @@ -94,7 +94,6 @@ class Connection : public IServerConnection Protocol::Compression getCompression() const { return compression; } std::vector> getPasswordComplexityRules() const override { return password_complexity_rules; } - AuthenticationType getDefaultPasswordType() const override { return default_password_type; } void sendQuery( const ConnectionTimeouts & timeouts, @@ -212,8 +211,6 @@ class Connection : public IServerConnection std::vector> password_complexity_rules; - AuthenticationType default_password_type = AuthenticationType::MAX; - /// From where to read query execution result. std::shared_ptr maybe_compressed_in; std::unique_ptr block_in; diff --git a/src/Client/IServerConnection.h b/src/Client/IServerConnection.h index 33681b5dcbe5..98a9cf64aa7a 100644 --- a/src/Client/IServerConnection.h +++ b/src/Client/IServerConnection.h @@ -87,7 +87,6 @@ class IServerConnection : boost::noncopyable virtual const String & getDescription() const = 0; virtual std::vector> getPasswordComplexityRules() const = 0; - virtual AuthenticationType getDefaultPasswordType() const = 0; /// If last flag is true, you need to call sendExternalTablesData after. virtual void sendQuery( diff --git a/src/Client/LocalConnection.h b/src/Client/LocalConnection.h index 6b7ae207c02f..3e6fc007fb94 100644 --- a/src/Client/LocalConnection.h +++ b/src/Client/LocalConnection.h @@ -92,7 +92,6 @@ class LocalConnection : public IServerConnection, WithContext const String & getDescription() const override { return description; } std::vector> getPasswordComplexityRules() const override { return {}; } - AuthenticationType getDefaultPasswordType() const override { return AuthenticationType::MAX; } void sendQuery( const ConnectionTimeouts & timeouts, diff --git a/src/Core/ProtocolDefines.h b/src/Core/ProtocolDefines.h index b5bb2d39cc19..005d8d9edf1a 100644 --- a/src/Core/ProtocolDefines.h +++ b/src/Core/ProtocolDefines.h @@ -54,7 +54,7 @@ /// NOTE: DBMS_TCP_PROTOCOL_VERSION has nothing common with VERSION_REVISION, /// later is just a number for server version (one number instead of commit SHA) /// for simplicity (sometimes it may be more convenient in some use cases). -#define DBMS_TCP_PROTOCOL_VERSION 54462 +#define DBMS_TCP_PROTOCOL_VERSION 54461 #define DBMS_MIN_PROTOCOL_VERSION_WITH_INITIAL_QUERY_START_TIME 54449 @@ -73,4 +73,3 @@ #define DBMS_MIN_PROTOCOL_VERSION_WITH_PASSWORD_COMPLEXITY_RULES 54461 -#define DBMS_MIN_PROTOCOL_VERSION_WITH_DEFAULT_PASSWORD_TYPE 54462 diff --git a/src/Interpreters/Access/InterpreterCreateUserQuery.cpp b/src/Interpreters/Access/InterpreterCreateUserQuery.cpp index 52af0a92e77a..dc4764e28f20 100644 --- a/src/Interpreters/Access/InterpreterCreateUserQuery.cpp +++ b/src/Interpreters/Access/InterpreterCreateUserQuery.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include #include #include @@ -23,10 +24,12 @@ namespace void updateUserFromQueryImpl( User & user, const ASTCreateUserQuery & query, + const std::optional auth_data, const std::shared_ptr & override_name, const std::optional & override_default_roles, const std::optional & override_settings, const std::optional & override_grantees, + bool allow_implicit_no_password, bool allow_no_password, bool allow_plaintext_password) { @@ -37,10 +40,16 @@ namespace else if (query.names->size() == 1) user.setName(query.names->front()->toString()); - if (query.auth_data) - user.auth_data = *query.auth_data; + if (!query.attach && !query.alter && !auth_data && !allow_implicit_no_password) + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "Authentication type NO_PASSWORD must " + "be explicitly specified, check the setting allow_implicit_no_password " + "in the server configuration"); - if (query.auth_data || !query.alter) + if (auth_data) + user.auth_data = *auth_data; + + if (auth_data || !query.alter) { auto auth_type = user.auth_data.getType(); if (((auth_type == AuthenticationType::NO_PASSWORD) && !allow_no_password) || @@ -103,21 +112,10 @@ BlockIO InterpreterCreateUserQuery::execute() bool implicit_no_password_allowed = access_control.isImplicitNoPasswordAllowed(); bool no_password_allowed = access_control.isNoPasswordAllowed(); bool plaintext_password_allowed = access_control.isPlaintextPasswordAllowed(); - AuthenticationType default_password_type = access_control.getDefaultPasswordType(); - - if (!query.attach && !query.alter && !query.auth_data && !implicit_no_password_allowed) - throw Exception(ErrorCodes::BAD_ARGUMENTS, - "Authentication type NO_PASSWORD must " - "be explicitly specified, check the setting allow_implicit_no_password " - "in the server configuration"); - - if (query.auth_data->getType() == AuthenticationType::NO_PASSWORD && query.temporary_password) - query.auth_data = AuthenticationData::makePasswordAuthenticationData(default_password_type, query.temporary_password.value()); - if (!query.attach && query.temporary_password) - access_control.checkPasswordComplexityRules(query.temporary_password.value()); - - query.temporary_password.reset(); + std::optional auth_data; + if (query.auth_data) + auth_data = query.auth_data->makeAuthenticationData(getContext(), !query.attach); std::optional default_roles_from_query; if (query.default_roles) @@ -151,7 +149,7 @@ BlockIO InterpreterCreateUserQuery::execute() auto update_func = [&](const AccessEntityPtr & entity) -> AccessEntityPtr { auto updated_user = typeid_cast>(entity->clone()); - updateUserFromQueryImpl(*updated_user, query, {}, default_roles_from_query, settings_from_query, grantees_from_query, no_password_allowed, plaintext_password_allowed); + updateUserFromQueryImpl(*updated_user, query, auth_data, {}, default_roles_from_query, settings_from_query, grantees_from_query, implicit_no_password_allowed, no_password_allowed, plaintext_password_allowed); return updated_user; }; @@ -170,7 +168,7 @@ BlockIO InterpreterCreateUserQuery::execute() for (const auto & name : *query.names) { auto new_user = std::make_shared(); - updateUserFromQueryImpl(*new_user, query, name, default_roles_from_query, settings_from_query, RolesOrUsersSet::AllTag{}, no_password_allowed, plaintext_password_allowed); + updateUserFromQueryImpl(*new_user, query, auth_data, name, default_roles_from_query, settings_from_query, RolesOrUsersSet::AllTag{}, implicit_no_password_allowed, no_password_allowed, plaintext_password_allowed); new_users.emplace_back(std::move(new_user)); } @@ -200,7 +198,11 @@ BlockIO InterpreterCreateUserQuery::execute() void InterpreterCreateUserQuery::updateUserFromQuery(User & user, const ASTCreateUserQuery & query, bool allow_no_password, bool allow_plaintext_password) { - updateUserFromQueryImpl(user, query, {}, {}, {}, {}, allow_no_password, allow_plaintext_password); + std::optional auth_data; + if (query.auth_data) + auth_data = query.auth_data->makeAuthenticationData({}, !query.attach); + + updateUserFromQueryImpl(user, query, auth_data, {}, {}, {}, {}, allow_no_password, allow_plaintext_password, true); } } diff --git a/src/Interpreters/Access/InterpreterShowCreateAccessEntityQuery.cpp b/src/Interpreters/Access/InterpreterShowCreateAccessEntityQuery.cpp index 7b9a8f98c8f5..5790e1107bbc 100644 --- a/src/Interpreters/Access/InterpreterShowCreateAccessEntityQuery.cpp +++ b/src/Interpreters/Access/InterpreterShowCreateAccessEntityQuery.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -62,7 +63,7 @@ namespace } if (user.auth_data.getType() != AuthenticationType::NO_PASSWORD) - query->auth_data = user.auth_data; + query->auth_data = ASTAuthenticationData::makeASTAuthenticationData(user.auth_data); if (!user.settings.empty()) { diff --git a/src/Parsers/ASTSubquery.h b/src/Parsers/ASTSubquery.h index 7d0fabf3ed4a..e4de766621a0 100644 --- a/src/Parsers/ASTSubquery.h +++ b/src/Parsers/ASTSubquery.h @@ -21,15 +21,9 @@ class ASTSubquery : public ASTWithAlias ASTPtr clone() const override { - const auto res = std::make_shared(*this); - ASTPtr ptr{res}; - - res->children.clear(); - - for (const auto & child : children) - res->children.emplace_back(child->clone()); - - return ptr; + auto clone = std::make_shared(*this); + clone->cloneChildren(); + return clone; } void updateTreeHashImpl(SipHash & hash_state) const override; diff --git a/src/Parsers/Access/ASTAuthenticationData.cpp b/src/Parsers/Access/ASTAuthenticationData.cpp new file mode 100644 index 000000000000..0403c81fb18f --- /dev/null +++ b/src/Parsers/Access/ASTAuthenticationData.cpp @@ -0,0 +1,329 @@ +#include "config.h" + +#include + +#include +#include +#include +#include +#include +#include + +#if USE_SSL +# include +# include +# include +#endif + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int SUPPORT_IS_DISABLED; + extern const int LOGICAL_ERROR; + extern const int OPENSSL_ERROR; +} + +void ASTAuthenticationData::checkPasswordComplexityRules(ContextPtr context) const +{ + if (expect_password) + { + if (const auto * password = children[0]->as()) + { + context->getAccessControl().checkPasswordComplexityRules(password->value.safeGet()); + } + } +} + +AuthenticationData ASTAuthenticationData::makeAuthenticationData(ContextPtr context, bool check_password_rules) const +{ + if (type && *type == AuthenticationType::NO_PASSWORD) + return AuthenticationData(); + + boost::container::flat_set common_names; + String value; + + if (expect_common_names) + { + for (const auto & ast_child : children[0]->children) + common_names.insert(ast_child->as().value.safeGet()); + } + else if (!children.empty()) + { + value = children[0]->as().value.safeGet(); + } + + if (expect_password) + { + if (!context) + throw Exception(DB::ErrorCodes::LOGICAL_ERROR, "Cannot get necessary parameters without context"); + + auto & access_control = context->getAccessControl(); + AuthenticationType default_password_type = access_control.getDefaultPasswordType(); + + /// NOTE: We will also extract bcrypt workfactor from access_control + + AuthenticationType current_type; + + if (type) + current_type = *type; + else + current_type = default_password_type; + + AuthenticationData auth_data(current_type); + + if (check_password_rules) + access_control.checkPasswordComplexityRules(value); + + if (type == AuthenticationType::SHA256_PASSWORD) + { +#if USE_SSL + ///random generator FIPS complaint + uint8_t key[32]; + if (RAND_bytes(key, sizeof(key)) != 1) + { + char buf[512] = {0}; + ERR_error_string_n(ERR_get_error(), buf, sizeof(buf)); + throw Exception(ErrorCodes::OPENSSL_ERROR, "Cannot generate salt for password. OpenSSL {}", buf); + } + + String salt; + salt.resize(sizeof(key) * 2); + char * buf_pos = salt.data(); + for (uint8_t k : key) + { + writeHexByteUppercase(k, buf_pos); + buf_pos += 2; + } + value.append(salt); + auth_data.setSalt(salt); +#else + throw DB::Exception( + "SHA256 passwords support is disabled, because ClickHouse was built without SSL library", + DB::ErrorCodes::SUPPORT_IS_DISABLED); +#endif + } + + auth_data.setPassword(value); + return auth_data; + } + + AuthenticationData auth_data(*type); + + if (expect_hash) + { + auth_data.setPasswordHashHex(value); + + if (type == AuthenticationType::SHA256_PASSWORD && children.size() == 2) + { + String parsed_salt = children[1]->as().value.safeGet(); + auth_data.setSalt(parsed_salt); + } + } + else if (expect_ldap_server_name) + { + auth_data.setLDAPServerName(value); + } + else if (expect_kerberos_realm) + { + auth_data.setKerberosRealm(value); + } + else if (expect_common_names) + { + auth_data.setSSLCertificateCommonNames(std::move(common_names)); + } + + return auth_data; +} + +std::shared_ptr ASTAuthenticationData::makeASTAuthenticationData(AuthenticationData auth_data) +{ + auto node = std::make_shared(); + auto auth_type = auth_data.getType(); + node->type = auth_type; + + switch (auth_type) + { + case AuthenticationType::PLAINTEXT_PASSWORD: + { + node->children.push_back(std::make_shared(auth_data.getPassword())); + break; + } + case AuthenticationType::SHA256_PASSWORD: + { + node->expect_hash = true; + node->children.push_back(std::make_shared(auth_data.getPasswordHashHex())); + + if (!auth_data.getSalt().empty()) + node->children.push_back(std::make_shared(auth_data.getSalt())); + break; + } + case AuthenticationType::DOUBLE_SHA1_PASSWORD: + { + node->expect_hash = true; + node->children.push_back(std::make_shared(auth_data.getPasswordHashHex())); + break; + } + case AuthenticationType::LDAP: + { + node->expect_ldap_server_name = true; + node->children.push_back(std::make_shared(auth_data.getLDAPServerName())); + break; + } + case AuthenticationType::KERBEROS: + { + node->expect_kerberos_realm = true; + const auto & realm = auth_data.getKerberosRealm(); + + if (!realm.empty()) + node->children.push_back(std::make_shared(realm)); + + break; + } + case AuthenticationType::SSL_CERTIFICATE: + { + node->expect_common_names = true; + + for (const auto & name : auth_data.getSSLCertificateCommonNames()) + node->children.push_back(std::make_shared(name)); + + break; + } + + case AuthenticationType::NO_PASSWORD: [[fallthrough]]; + case AuthenticationType::MAX: + throw Exception(ErrorCodes::LOGICAL_ERROR, "AST: Unexpected authentication type {}", toString(auth_type)); + } + + return node; +} + +void ASTAuthenticationData::formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const +{ + if (type && *type == AuthenticationType::NO_PASSWORD) + { + settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " NOT IDENTIFIED" + << (settings.hilite ? IAST::hilite_none : ""); + return; + } + + String auth_type_name; + String prefix; /// "BY" or "SERVER" or "REALM" + ASTPtr password; /// either a password or hash + ASTPtr salt; + ASTPtr parameter; + ASTPtr parameters; + + if (type) + { + auth_type_name = AuthenticationTypeInfo::get(*type).name; + + switch (*type) + { + case AuthenticationType::PLAINTEXT_PASSWORD: + { + prefix = "BY"; + password = children[0]; + break; + } + case AuthenticationType::SHA256_PASSWORD: + { + if (expect_hash) + auth_type_name = "sha256_hash"; + + prefix = "BY"; + password = children[0]; + if (children.size() == 2) + salt = children[1]; + break; + } + case AuthenticationType::DOUBLE_SHA1_PASSWORD: + { + if (expect_hash) + auth_type_name = "double_sha1_hash"; + + prefix = "BY"; + password = children[0]; + break; + } + case AuthenticationType::LDAP: + { + prefix = "SERVER"; + parameter = children[0]; + break; + } + case AuthenticationType::KERBEROS: + { + if (!children.empty()) + { + prefix = "REALM"; + parameter = children[0]; + } + break; + } + case AuthenticationType::SSL_CERTIFICATE: + { + prefix = "CN"; + parameters = children[0]; + break; + } + case AuthenticationType::NO_PASSWORD: [[fallthrough]]; + case AuthenticationType::MAX: + throw Exception(ErrorCodes::LOGICAL_ERROR, "AST: Unexpected authentication type {}", toString(*type)); + } + } + else + { + /// Default password type + prefix = "BY"; + password = children[0]; + } + + if (password && !settings.show_secrets) + { + prefix = ""; + password.reset(); + salt.reset(); + if (type) + auth_type_name = AuthenticationTypeInfo::get(*type).name; + } + + settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " IDENTIFIED" << (settings.hilite ? IAST::hilite_none : ""); + + if (!auth_type_name.empty()) + { + settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " WITH " << auth_type_name + << (settings.hilite ? IAST::hilite_none : ""); + } + + if (!prefix.empty()) + { + settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " " << prefix << (settings.hilite ? IAST::hilite_none : ""); + } + + if (password) + { + settings.ostr << " "; + password->format(settings); + } + + if (salt) + { + settings.ostr << " SALT "; + salt->format(settings); + } + + if (parameter) + { + settings.ostr << " "; + parameter->format(settings); + } + else if (parameters) + { + settings.ostr << " "; + parameters->format(settings); + } +} + +} diff --git a/src/Parsers/Access/ASTAuthenticationData.h b/src/Parsers/Access/ASTAuthenticationData.h new file mode 100644 index 000000000000..4168075f28c9 --- /dev/null +++ b/src/Parsers/Access/ASTAuthenticationData.h @@ -0,0 +1,43 @@ +#pragma once + +#include +#include +#include +#include + + +namespace DB +{ + +class ASTAuthenticationData : public IAST +{ +public: + String getID(char) const override { return "AuthenticationData"; } + + ASTPtr clone() const override + { + auto clone = std::make_shared(*this); + clone->cloneChildren(); + return clone; + } + + static std::shared_ptr makeASTAuthenticationData(AuthenticationData auth_data); + AuthenticationData makeAuthenticationData(ContextPtr context, bool check_password_rules) const; + void checkPasswordComplexityRules(ContextPtr context) const; + + /// If type is empty we use the default password type. + /// AuthenticationType::NO_PASSWORD is specified explicitly. + std::optional type; + + /// TODO: Only expect_password and expect_hash are actually needed + bool expect_password = false; + bool expect_hash = false; + bool expect_ldap_server_name = false; + bool expect_kerberos_realm = false; + bool expect_common_names = false; + +protected: + void formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override; +}; + +} diff --git a/src/Parsers/Access/ASTCreateUserQuery.cpp b/src/Parsers/Access/ASTCreateUserQuery.cpp index b0d4aef38b83..335a12aef833 100644 --- a/src/Parsers/Access/ASTCreateUserQuery.cpp +++ b/src/Parsers/Access/ASTCreateUserQuery.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include #include @@ -23,122 +24,9 @@ namespace } - void formatAuthenticationData(const AuthenticationData & auth_data, const IAST::FormatSettings & settings) + void formatAuthenticationData(const ASTAuthenticationData & auth_data, const IAST::FormatSettings & settings) { - auto auth_type = auth_data.getType(); - if (auth_type == AuthenticationType::NO_PASSWORD) - { - settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " NOT IDENTIFIED" - << (settings.hilite ? IAST::hilite_none : ""); - return; - } - - String auth_type_name = AuthenticationTypeInfo::get(auth_type).name; - String prefix; /// "BY" or "SERVER" or "REALM" - std::optional password; /// either a password or hash - std::optional salt; - std::optional parameter; - const boost::container::flat_set * parameters = nullptr; - - switch (auth_type) - { - case AuthenticationType::PLAINTEXT_PASSWORD: - { - prefix = "BY"; - password = auth_data.getPassword(); - break; - } - case AuthenticationType::SHA256_PASSWORD: - { - auth_type_name = "sha256_hash"; - prefix = "BY"; - password = auth_data.getPasswordHashHex(); - if (!auth_data.getSalt().empty()) - salt = auth_data.getSalt(); - break; - } - case AuthenticationType::DOUBLE_SHA1_PASSWORD: - { - auth_type_name = "double_sha1_hash"; - prefix = "BY"; - password = auth_data.getPasswordHashHex(); - break; - } - case AuthenticationType::LDAP: - { - prefix = "SERVER"; - parameter = auth_data.getLDAPServerName(); - break; - } - case AuthenticationType::KERBEROS: - { - const auto & realm = auth_data.getKerberosRealm(); - if (!realm.empty()) - { - prefix = "REALM"; - parameter = realm; - } - break; - } - - case AuthenticationType::SSL_CERTIFICATE: - { - prefix = "CN"; - parameters = &auth_data.getSSLCertificateCommonNames(); - break; - } - - case AuthenticationType::NO_PASSWORD: [[fallthrough]]; - case AuthenticationType::MAX: - throw Exception(ErrorCodes::LOGICAL_ERROR, "AST: Unexpected authentication type {}", toString(auth_type)); - } - - if (password && !settings.show_secrets) - { - prefix = ""; - password.reset(); - salt.reset(); - auth_type_name = AuthenticationTypeInfo::get(auth_type).name; - } - - settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " IDENTIFIED" << (settings.hilite ? IAST::hilite_none : ""); - - if (!auth_type_name.empty()) - { - settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " WITH " << auth_type_name - << (settings.hilite ? IAST::hilite_none : ""); - } - - if (!prefix.empty()) - { - settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " " << prefix << (settings.hilite ? IAST::hilite_none : ""); - } - - if (password) - { - settings.ostr << " " << quoteString(*password); - } - - if (salt) - { - settings.ostr << " SALT " << quoteString(*salt); - } - - if (parameter) - { - settings.ostr << " " << quoteString(*parameter); - } - else if (parameters) - { - settings.ostr << " "; - bool need_comma = false; - for (const auto & param : *parameters) - { - if (std::exchange(need_comma, true)) - settings.ostr << ", "; - settings.ostr << quoteString(param); - } - } + auth_data.format(settings); } @@ -276,6 +164,7 @@ String ASTCreateUserQuery::getID(char) const ASTPtr ASTCreateUserQuery::clone() const { auto res = std::make_shared(*this); + res->children.clear(); if (names) res->names = std::static_pointer_cast(names->clone()); @@ -292,6 +181,12 @@ ASTPtr ASTCreateUserQuery::clone() const if (settings) res->settings = std::static_pointer_cast(settings->clone()); + if (auth_data) + { + res->auth_data = std::static_pointer_cast(auth_data->clone()); + res->children.push_back(res->auth_data); + } + return res; } @@ -350,7 +245,11 @@ bool ASTCreateUserQuery::hasSecretParts() const { if (auth_data) { - auto auth_type = auth_data->getType(); + /// Default password type is used hence has secret part + if (!auth_data->type) + return true; + + auto auth_type = *(auth_data->type); if ((auth_type == AuthenticationType::PLAINTEXT_PASSWORD) || (auth_type == AuthenticationType::SHA256_PASSWORD) || (auth_type == AuthenticationType::DOUBLE_SHA1_PASSWORD)) diff --git a/src/Parsers/Access/ASTCreateUserQuery.h b/src/Parsers/Access/ASTCreateUserQuery.h index f6064c7203d1..646bc30b679c 100644 --- a/src/Parsers/Access/ASTCreateUserQuery.h +++ b/src/Parsers/Access/ASTCreateUserQuery.h @@ -13,6 +13,8 @@ class ASTUserNamesWithHost; class ASTRolesOrUsersSet; class ASTDatabaseOrNone; class ASTSettingsProfileElements; +class ASTAuthenticationData; + /** CREATE USER [IF NOT EXISTS | OR REPLACE] name * [NOT IDENTIFIED | IDENTIFIED {[WITH {no_password|plaintext_password|sha256_password|sha256_hash|double_sha1_password|double_sha1_hash}] BY {'password'|'hash'}}|{WITH ldap SERVER 'server_name'}|{WITH kerberos [REALM 'realm']}] @@ -44,8 +46,7 @@ class ASTCreateUserQuery : public IAST, public ASTQueryWithOnCluster std::shared_ptr names; std::optional new_name; - mutable std::optional auth_data; - mutable std::optional temporary_password; + std::shared_ptr auth_data; std::optional hosts; std::optional add_hosts; diff --git a/src/Parsers/Access/ParserCreateUserQuery.cpp b/src/Parsers/Access/ParserCreateUserQuery.cpp index d1a91669dcc8..3e794d167079 100644 --- a/src/Parsers/Access/ParserCreateUserQuery.cpp +++ b/src/Parsers/Access/ParserCreateUserQuery.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -39,14 +40,29 @@ namespace }); } + class ParserStringAndSubstitution : public IParserBase + { + private: + const char * getName() const override { return "ParserStringAndSubstitution"; } + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override + { + return ParserStringLiteral{}.parse(pos, node, expected) || ParserSubstitution{}.parse(pos, node, expected); + } + + public: + explicit ParserStringAndSubstitution() = default; + }; - bool parseAuthenticationData(IParserBase::Pos & pos, Expected & expected, AuthenticationData & auth_data, std::optional & temporary_password) + + bool parseAuthenticationData(IParserBase::Pos & pos, Expected & expected, std::shared_ptr & auth_data) { return IParserBase::wrapParseImpl(pos, [&] { if (ParserKeyword{"NOT IDENTIFIED"}.ignore(pos, expected)) { - auth_data = AuthenticationData{AuthenticationType::NO_PASSWORD}; + auth_data = std::make_shared(); + auth_data->type = AuthenticationType::NO_PASSWORD; + return true; } @@ -54,6 +70,7 @@ namespace return false; std::optional type; + bool expect_password = false; bool expect_hash = false; bool expect_ldap_server_name = false; @@ -98,44 +115,37 @@ namespace } } + /// If authentication type is not specied, then the default password type is used if (!type) expect_password = true; - String value; - String parsed_salt; - boost::container::flat_set common_names; + ASTPtr value; + ASTPtr parsed_salt; if (expect_password || expect_hash) { - ASTPtr ast; - if (!ParserKeyword{"BY"}.ignore(pos, expected) || !ParserStringLiteral{}.parse(pos, ast, expected)) + if (!ParserKeyword{"BY"}.ignore(pos, expected) || !ParserStringAndSubstitution{}.parse(pos, value, expected)) return false; - value = ast->as().value.safeGet(); if (expect_hash && type == AuthenticationType::SHA256_PASSWORD) { - if (ParserKeyword{"SALT"}.ignore(pos, expected) && ParserStringLiteral{}.parse(pos, ast, expected)) + if (ParserKeyword{"SALT"}.ignore(pos, expected)) { - parsed_salt = ast->as().value.safeGet(); + if (!ParserStringAndSubstitution{}.parse(pos, parsed_salt, expected)) + return false; } } } else if (expect_ldap_server_name) { - ASTPtr ast; - if (!ParserKeyword{"SERVER"}.ignore(pos, expected) || !ParserStringLiteral{}.parse(pos, ast, expected)) + if (!ParserKeyword{"SERVER"}.ignore(pos, expected) || !ParserStringAndSubstitution{}.parse(pos, value, expected)) return false; - - value = ast->as().value.safeGet(); } else if (expect_kerberos_realm) { if (ParserKeyword{"REALM"}.ignore(pos, expected)) { - ASTPtr ast; - if (!ParserStringLiteral{}.parse(pos, ast, expected)) + if (!ParserStringAndSubstitution{}.parse(pos, value, expected)) return false; - - value = ast->as().value.safeGet(); } } else if (expect_common_names) @@ -143,44 +153,23 @@ namespace if (!ParserKeyword{"CN"}.ignore(pos, expected)) return false; - ASTPtr ast; - if (!ParserList{std::make_unique(), std::make_unique(TokenType::Comma), false}.parse(pos, ast, expected)) + if (!ParserList{std::make_unique(), std::make_unique(TokenType::Comma), false}.parse(pos, value, expected)) return false; - - for (const auto & ast_child : ast->children) - common_names.insert(ast_child->as().value.safeGet()); } - /// Save password separately for future complexity rules check - /// In the case when user did not specified password type we will fill it later - if (expect_password) - temporary_password = value; + auth_data = std::make_shared(); - if (!type) - { - auth_data = AuthenticationData{AuthenticationType::NO_PASSWORD}; - return true; - } + auth_data->type = type; + auth_data->expect_password = expect_password; + auth_data->expect_hash = expect_hash; + auth_data->expect_ldap_server_name = expect_ldap_server_name; + auth_data->expect_kerberos_realm = expect_kerberos_realm; + auth_data->expect_common_names = expect_common_names; - if (expect_password) - { - auth_data = AuthenticationData::makePasswordAuthenticationData(*type, value); - return true; - } + auth_data->children.push_back(value); - auth_data = AuthenticationData{*type}; - - if (*type == AuthenticationType::SHA256_PASSWORD) - auth_data.setSalt(parsed_salt); - - if (expect_hash) - auth_data.setPasswordHashHex(value); - else if (expect_ldap_server_name) - auth_data.setLDAPServerName(value); - else if (expect_kerberos_realm) - auth_data.setKerberosRealm(value); - else if (expect_common_names) - auth_data.setSSLCertificateCommonNames(std::move(common_names)); + if (parsed_salt) + auth_data->children.push_back(parsed_salt); return true; }); @@ -408,11 +397,11 @@ bool ParserCreateUserQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expec auto names_ref = names->names; std::optional new_name; - std::optional auth_data; std::optional temporary_password; std::optional hosts; std::optional add_hosts; std::optional remove_hosts; + std::shared_ptr auth_data; std::shared_ptr default_roles; std::shared_ptr settings; std::shared_ptr grantees; @@ -423,12 +412,10 @@ bool ParserCreateUserQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expec { if (!auth_data) { - AuthenticationData new_auth_data; - std::optional new_temporary_password; - if (parseAuthenticationData(pos, expected, new_auth_data, new_temporary_password)) + std::shared_ptr new_auth_data; + if (parseAuthenticationData(pos, expected, new_auth_data)) { auth_data = std::move(new_auth_data); - temporary_password = std::move(new_temporary_password); continue; } } @@ -513,7 +500,6 @@ bool ParserCreateUserQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expec query->names = std::move(names); query->new_name = std::move(new_name); query->auth_data = std::move(auth_data); - query->temporary_password = std::move(temporary_password); query->hosts = std::move(hosts); query->add_hosts = std::move(add_hosts); query->remove_hosts = std::move(remove_hosts); @@ -522,6 +508,9 @@ bool ParserCreateUserQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expec query->grantees = std::move(grantees); query->default_database = std::move(default_database); + if (query->auth_data) + query->children.push_back(query->auth_data); + return true; } } diff --git a/src/Server/TCPHandler.cpp b/src/Server/TCPHandler.cpp index 78ccfe79d31a..ed28db860ca9 100644 --- a/src/Server/TCPHandler.cpp +++ b/src/Server/TCPHandler.cpp @@ -1283,12 +1283,6 @@ void TCPHandler::sendHello() writeStringBinary(exception_message, *out); } } - if (client_tcp_protocol_version >= DBMS_MIN_PROTOCOL_VERSION_WITH_DEFAULT_PASSWORD_TYPE) - { - auto default_password_type = server.context()->getAccessControl().getDefaultPasswordType(); - auto default_password_type_uint = static_cast(default_password_type); - writeVarUInt(default_password_type_uint, *out); - } out->next(); } From dbc620f7e28b0f03a7d3c15a086a8f9bf564e0ce Mon Sep 17 00:00:00 2001 From: Nikolay Degterinsky Date: Mon, 10 Apr 2023 02:42:05 +0000 Subject: [PATCH 110/406] Fix style --- src/Access/Common/AuthenticationData.cpp | 8 +------- src/Parsers/Access/ASTAuthenticationData.cpp | 5 ++--- src/Parsers/Access/ASTCreateUserQuery.cpp | 5 ----- src/Parsers/Access/ParserCreateUserQuery.cpp | 2 +- 4 files changed, 4 insertions(+), 16 deletions(-) diff --git a/src/Access/Common/AuthenticationData.cpp b/src/Access/Common/AuthenticationData.cpp index ac604a789e24..0a22eeb92b39 100644 --- a/src/Access/Common/AuthenticationData.cpp +++ b/src/Access/Common/AuthenticationData.cpp @@ -5,12 +5,7 @@ #include #include #include -#include -#if USE_SSL -# include -# include -# include -#endif + namespace DB { @@ -20,7 +15,6 @@ namespace ErrorCodes extern const int BAD_ARGUMENTS; extern const int LOGICAL_ERROR; extern const int NOT_IMPLEMENTED; - extern const int OPENSSL_ERROR; } diff --git a/src/Parsers/Access/ASTAuthenticationData.cpp b/src/Parsers/Access/ASTAuthenticationData.cpp index 0403c81fb18f..f20d4e27fa66 100644 --- a/src/Parsers/Access/ASTAuthenticationData.cpp +++ b/src/Parsers/Access/ASTAuthenticationData.cpp @@ -99,9 +99,8 @@ AuthenticationData ASTAuthenticationData::makeAuthenticationData(ContextPtr cont value.append(salt); auth_data.setSalt(salt); #else - throw DB::Exception( - "SHA256 passwords support is disabled, because ClickHouse was built without SSL library", - DB::ErrorCodes::SUPPORT_IS_DISABLED); + throw DB::Exception(DB::ErrorCodes::SUPPORT_IS_DISABLED, + "SHA256 passwords support is disabled, because ClickHouse was built without SSL library"); #endif } diff --git a/src/Parsers/Access/ASTCreateUserQuery.cpp b/src/Parsers/Access/ASTCreateUserQuery.cpp index 335a12aef833..b0f6df0bceb8 100644 --- a/src/Parsers/Access/ASTCreateUserQuery.cpp +++ b/src/Parsers/Access/ASTCreateUserQuery.cpp @@ -9,11 +9,6 @@ namespace DB { -namespace ErrorCodes -{ - extern const int LOGICAL_ERROR; -} - namespace { diff --git a/src/Parsers/Access/ParserCreateUserQuery.cpp b/src/Parsers/Access/ParserCreateUserQuery.cpp index 3f017ca59226..de4245cd34ba 100644 --- a/src/Parsers/Access/ParserCreateUserQuery.cpp +++ b/src/Parsers/Access/ParserCreateUserQuery.cpp @@ -116,7 +116,7 @@ namespace } } - /// If authentication type is not specied, then the default password type is used + /// If authentication type is not specified, then the default password type is used if (!type) expect_password = true; From 02d2f5c37c5097acc7526d67385ce953361727ac Mon Sep 17 00:00:00 2001 From: Ilya Golshtein Date: Sun, 19 Mar 2023 22:09:01 +0100 Subject: [PATCH 111/406] sequence state fix - works, tests added --- .../AggregateFunctionSequenceMatch.h | 7 +++- ...sequence_match_serialization_fix.reference | 3 ++ ...02713_sequence_match_serialization_fix.sql | 36 +++++++++++++++++++ 3 files changed, 45 insertions(+), 1 deletion(-) create mode 100644 tests/queries/0_stateless/02713_sequence_match_serialization_fix.reference create mode 100644 tests/queries/0_stateless/02713_sequence_match_serialization_fix.sql diff --git a/src/AggregateFunctions/AggregateFunctionSequenceMatch.h b/src/AggregateFunctions/AggregateFunctionSequenceMatch.h index 563d3d6aa8ab..f2e17940d358 100644 --- a/src/AggregateFunctions/AggregateFunctionSequenceMatch.h +++ b/src/AggregateFunctions/AggregateFunctionSequenceMatch.h @@ -50,7 +50,7 @@ struct AggregateFunctionSequenceMatchData final bool sorted = true; PODArrayWithStackMemory events_list; /// sequenceMatch conditions met at least once in events_list - std::bitset conditions_met; + Events conditions_met; void add(const Timestamp timestamp, const Events & events) { @@ -101,6 +101,11 @@ struct AggregateFunctionSequenceMatchData final size_t size; readBinary(size, buf); + /// If we lose these flags, functionality is broken + /// If we serialize/deserialize these flags, we have compatibility issues + /// If we set these flags to 1, we have a minor performance penalty, which seems acceptable + conditions_met.set(); + events_list.clear(); events_list.reserve(size); diff --git a/tests/queries/0_stateless/02713_sequence_match_serialization_fix.reference b/tests/queries/0_stateless/02713_sequence_match_serialization_fix.reference new file mode 100644 index 000000000000..2a1c127e635a --- /dev/null +++ b/tests/queries/0_stateless/02713_sequence_match_serialization_fix.reference @@ -0,0 +1,3 @@ +serialized state is not used 1 +serialized state is used 1 +via Distributed 1 diff --git a/tests/queries/0_stateless/02713_sequence_match_serialization_fix.sql b/tests/queries/0_stateless/02713_sequence_match_serialization_fix.sql new file mode 100644 index 000000000000..3521cb8470fc --- /dev/null +++ b/tests/queries/0_stateless/02713_sequence_match_serialization_fix.sql @@ -0,0 +1,36 @@ +DROP TABLE IF EXISTS 02713_seqt; +DROP TABLE IF EXISTS 02713_seqt_distr; + +SELECT + 'serialized state is not used', sequenceMatch('(?1)(?2)')(time, number_ = 1, number_ = 0) AS seq +FROM +( + SELECT + number AS time, + number % 2 AS number_ + FROM numbers_mt(100) +); + + +CREATE TABLE 02713_seqt +ENGINE = MergeTree +ORDER BY n AS +SELECT + sequenceMatchState('(?1)(?2)')(time, number_ = 1, number_ = 0) AS seq, + 1 AS n +FROM +( + SELECT + number AS time, + number % 2 AS number_ + FROM numbers_mt(100) +); + + +SELECT 'serialized state is used', sequenceMatchMerge('(?1)(?2)')(seq) AS seq +FROM 02713_seqt; + + +CREATE TABLE 02713_seqt_distr ( seq AggregateFunction(sequenceMatch('(?1)(?2)'), UInt64, UInt8, UInt8) , n UInt8) ENGINE = Distributed(test_shard_localhost, currentDatabase(), '02713_seqt'); + +SELECT 'via Distributed', sequenceMatchMerge('(?1)(?2)')(seq) AS seq FROM 02713_seqt_distr; From 6f57a952d9b6a9a7d2a69838ef9b30b9c3742d76 Mon Sep 17 00:00:00 2001 From: Nikolay Degterinsky Date: Mon, 10 Apr 2023 16:40:49 +0000 Subject: [PATCH 112/406] Improvements --- src/Client/ClientBase.cpp | 7 +- .../Access/InterpreterCreateUserQuery.cpp | 113 ++++++++++- ...InterpreterShowCreateAccessEntityQuery.cpp | 65 +++++- src/Parsers/Access/ASTAuthenticationData.cpp | 189 +++--------------- src/Parsers/Access/ASTAuthenticationData.h | 8 +- src/Parsers/Access/ASTCreateUserQuery.cpp | 17 -- src/Parsers/Access/ASTCreateUserQuery.h | 1 - src/Parsers/Access/ParserCreateUserQuery.cpp | 5 +- src/Parsers/tests/gtest_Parser.cpp | 3 +- 9 files changed, 213 insertions(+), 195 deletions(-) diff --git a/src/Client/ClientBase.cpp b/src/Client/ClientBase.cpp index b261c46a1085..2852ab8ed0b6 100644 --- a/src/Client/ClientBase.cpp +++ b/src/Client/ClientBase.cpp @@ -1614,7 +1614,12 @@ void ClientBase::processParsedSingleQuery(const String & full_query, const Strin if (!create_user_query->attach && create_user_query->auth_data) { if (const auto * auth_data = create_user_query->auth_data->as()) - auth_data->checkPasswordComplexityRules(global_context); + { + auto password = auth_data->getPassword(); + + if (password) + global_context->getAccessControl().checkPasswordComplexityRules(*password); + } } } diff --git a/src/Interpreters/Access/InterpreterCreateUserQuery.cpp b/src/Interpreters/Access/InterpreterCreateUserQuery.cpp index dc4764e28f20..44736c598b2b 100644 --- a/src/Interpreters/Access/InterpreterCreateUserQuery.cpp +++ b/src/Interpreters/Access/InterpreterCreateUserQuery.cpp @@ -10,17 +10,126 @@ #include #include #include +#include #include +#include "config.h" + +#if USE_SSL +# include +# include +# include +#endif namespace DB { namespace ErrorCodes { extern const int BAD_ARGUMENTS; + extern const int SUPPORT_IS_DISABLED; + extern const int OPENSSL_ERROR; } namespace { + AuthenticationData makeAuthenticationData(const ASTAuthenticationData & query, ContextPtr context, bool check_password_rules) + { + if (query.type && *query.type == AuthenticationType::NO_PASSWORD) + return AuthenticationData(); + + if (query.expect_password) + { + if (!context) + throw Exception(DB::ErrorCodes::LOGICAL_ERROR, "Cannot get necessary parameters without context"); + + String value = checkAndGetLiteralArgument(query.children[0], "password"); + + const auto & access_control = context->getAccessControl(); + AuthenticationType default_password_type = access_control.getDefaultPasswordType(); + + /// NOTE: We will also extract bcrypt workfactor from access_control + + AuthenticationType current_type; + + if (query.type) + current_type = *query.type; + else + current_type = default_password_type; + + AuthenticationData auth_data(current_type); + + if (check_password_rules) + access_control.checkPasswordComplexityRules(value); + + if (query.type == AuthenticationType::SHA256_PASSWORD) + { + #if USE_SSL + ///random generator FIPS complaint + uint8_t key[32]; + if (RAND_bytes(key, sizeof(key)) != 1) + { + char buf[512] = {0}; + ERR_error_string_n(ERR_get_error(), buf, sizeof(buf)); + throw Exception(ErrorCodes::OPENSSL_ERROR, "Cannot generate salt for password. OpenSSL {}", buf); + } + + String salt; + salt.resize(sizeof(key) * 2); + char * buf_pos = salt.data(); + for (uint8_t k : key) + { + writeHexByteUppercase(k, buf_pos); + buf_pos += 2; + } + value.append(salt); + auth_data.setSalt(salt); + #else + throw DB::Exception(DB::ErrorCodes::SUPPORT_IS_DISABLED, + "SHA256 passwords support is disabled, because ClickHouse was built without SSL library"); + #endif + } + + auth_data.setPassword(value); + return auth_data; + } + + AuthenticationData auth_data(*query.type); + + if (query.expect_hash) + { + String value = checkAndGetLiteralArgument(query.children[0], "hash"); + auth_data.setPasswordHashHex(value); + + if (*query.type == AuthenticationType::SHA256_PASSWORD && query.children.size() == 2) + { + String parsed_salt = checkAndGetLiteralArgument(query.children[1], "salt"); + auth_data.setSalt(parsed_salt); + } + } + else if (query.expect_ldap_server_name) + { + String value = checkAndGetLiteralArgument(query.children[0], "ldap_server_name"); + auth_data.setLDAPServerName(value); + } + else if (query.expect_kerberos_realm) + { + if (!query.children.empty()) + { + String value = checkAndGetLiteralArgument(query.children[0], "kerberos_realm"); + auth_data.setKerberosRealm(value); + } + } + else if (query.expect_common_names) + { + boost::container::flat_set common_names; + for (const auto & ast_child : query.children[0]->children) + common_names.insert(checkAndGetLiteralArgument(ast_child, "common_name")); + + auth_data.setSSLCertificateCommonNames(std::move(common_names)); + } + + return auth_data; + } + void updateUserFromQueryImpl( User & user, const ASTCreateUserQuery & query, @@ -115,7 +224,7 @@ BlockIO InterpreterCreateUserQuery::execute() std::optional auth_data; if (query.auth_data) - auth_data = query.auth_data->makeAuthenticationData(getContext(), !query.attach); + auth_data = makeAuthenticationData(*query.auth_data, getContext(), !query.attach); std::optional default_roles_from_query; if (query.default_roles) @@ -200,7 +309,7 @@ void InterpreterCreateUserQuery::updateUserFromQuery(User & user, const ASTCreat { std::optional auth_data; if (query.auth_data) - auth_data = query.auth_data->makeAuthenticationData({}, !query.attach); + auth_data = makeAuthenticationData(*query.auth_data, {}, !query.attach); updateUserFromQueryImpl(user, query, auth_data, {}, {}, {}, {}, allow_no_password, allow_plaintext_password, true); } diff --git a/src/Interpreters/Access/InterpreterShowCreateAccessEntityQuery.cpp b/src/Interpreters/Access/InterpreterShowCreateAccessEntityQuery.cpp index 5790e1107bbc..9262ae66db4f 100644 --- a/src/Interpreters/Access/InterpreterShowCreateAccessEntityQuery.cpp +++ b/src/Interpreters/Access/InterpreterShowCreateAccessEntityQuery.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -41,6 +42,68 @@ namespace ErrorCodes namespace { + std::shared_ptr makeASTAuthenticationData(AuthenticationData auth_data) + { + auto node = std::make_shared(); + auto auth_type = auth_data.getType(); + node->type = auth_type; + + switch (auth_type) + { + case AuthenticationType::PLAINTEXT_PASSWORD: + { + node->children.push_back(std::make_shared(auth_data.getPassword())); + break; + } + case AuthenticationType::SHA256_PASSWORD: + { + node->expect_hash = true; + node->children.push_back(std::make_shared(auth_data.getPasswordHashHex())); + + if (!auth_data.getSalt().empty()) + node->children.push_back(std::make_shared(auth_data.getSalt())); + break; + } + case AuthenticationType::DOUBLE_SHA1_PASSWORD: + { + node->expect_hash = true; + node->children.push_back(std::make_shared(auth_data.getPasswordHashHex())); + break; + } + case AuthenticationType::LDAP: + { + node->expect_ldap_server_name = true; + node->children.push_back(std::make_shared(auth_data.getLDAPServerName())); + break; + } + case AuthenticationType::KERBEROS: + { + node->expect_kerberos_realm = true; + const auto & realm = auth_data.getKerberosRealm(); + + if (!realm.empty()) + node->children.push_back(std::make_shared(realm)); + + break; + } + case AuthenticationType::SSL_CERTIFICATE: + { + node->expect_common_names = true; + + for (const auto & name : auth_data.getSSLCertificateCommonNames()) + node->children.push_back(std::make_shared(name)); + + break; + } + + case AuthenticationType::NO_PASSWORD: [[fallthrough]]; + case AuthenticationType::MAX: + throw Exception(ErrorCodes::LOGICAL_ERROR, "AST: Unexpected authentication type {}", toString(auth_type)); + } + + return node; + } + ASTPtr getCreateQueryImpl( const User & user, const AccessControl * access_control /* not used if attach_mode == true */, @@ -63,7 +126,7 @@ namespace } if (user.auth_data.getType() != AuthenticationType::NO_PASSWORD) - query->auth_data = ASTAuthenticationData::makeASTAuthenticationData(user.auth_data); + query->auth_data = makeASTAuthenticationData(user.auth_data); if (!user.settings.empty()) { diff --git a/src/Parsers/Access/ASTAuthenticationData.cpp b/src/Parsers/Access/ASTAuthenticationData.cpp index f20d4e27fa66..aa4642c10cbd 100644 --- a/src/Parsers/Access/ASTAuthenticationData.cpp +++ b/src/Parsers/Access/ASTAuthenticationData.cpp @@ -1,201 +1,43 @@ -#include "config.h" - #include #include #include #include -#include #include #include -#if USE_SSL -# include -# include -# include -#endif namespace DB { namespace ErrorCodes { - extern const int SUPPORT_IS_DISABLED; extern const int LOGICAL_ERROR; - extern const int OPENSSL_ERROR; } -void ASTAuthenticationData::checkPasswordComplexityRules(ContextPtr context) const +std::optional ASTAuthenticationData::getPassword() const { if (expect_password) { if (const auto * password = children[0]->as()) { - context->getAccessControl().checkPasswordComplexityRules(password->value.safeGet()); - } - } -} - -AuthenticationData ASTAuthenticationData::makeAuthenticationData(ContextPtr context, bool check_password_rules) const -{ - if (type && *type == AuthenticationType::NO_PASSWORD) - return AuthenticationData(); - - boost::container::flat_set common_names; - String value; - - if (expect_common_names) - { - for (const auto & ast_child : children[0]->children) - common_names.insert(ast_child->as().value.safeGet()); - } - else if (!children.empty()) - { - value = children[0]->as().value.safeGet(); - } - - if (expect_password) - { - if (!context) - throw Exception(DB::ErrorCodes::LOGICAL_ERROR, "Cannot get necessary parameters without context"); - - auto & access_control = context->getAccessControl(); - AuthenticationType default_password_type = access_control.getDefaultPasswordType(); - - /// NOTE: We will also extract bcrypt workfactor from access_control - - AuthenticationType current_type; - - if (type) - current_type = *type; - else - current_type = default_password_type; - - AuthenticationData auth_data(current_type); - - if (check_password_rules) - access_control.checkPasswordComplexityRules(value); - - if (type == AuthenticationType::SHA256_PASSWORD) - { -#if USE_SSL - ///random generator FIPS complaint - uint8_t key[32]; - if (RAND_bytes(key, sizeof(key)) != 1) - { - char buf[512] = {0}; - ERR_error_string_n(ERR_get_error(), buf, sizeof(buf)); - throw Exception(ErrorCodes::OPENSSL_ERROR, "Cannot generate salt for password. OpenSSL {}", buf); - } - - String salt; - salt.resize(sizeof(key) * 2); - char * buf_pos = salt.data(); - for (uint8_t k : key) - { - writeHexByteUppercase(k, buf_pos); - buf_pos += 2; - } - value.append(salt); - auth_data.setSalt(salt); -#else - throw DB::Exception(DB::ErrorCodes::SUPPORT_IS_DISABLED, - "SHA256 passwords support is disabled, because ClickHouse was built without SSL library"); -#endif + return password->value.safeGet(); } - - auth_data.setPassword(value); - return auth_data; } - AuthenticationData auth_data(*type); - - if (expect_hash) - { - auth_data.setPasswordHashHex(value); - - if (type == AuthenticationType::SHA256_PASSWORD && children.size() == 2) - { - String parsed_salt = children[1]->as().value.safeGet(); - auth_data.setSalt(parsed_salt); - } - } - else if (expect_ldap_server_name) - { - auth_data.setLDAPServerName(value); - } - else if (expect_kerberos_realm) - { - auth_data.setKerberosRealm(value); - } - else if (expect_common_names) - { - auth_data.setSSLCertificateCommonNames(std::move(common_names)); - } - - return auth_data; + return {}; } - -std::shared_ptr ASTAuthenticationData::makeASTAuthenticationData(AuthenticationData auth_data) +std::optional ASTAuthenticationData::getSalt() const { - auto node = std::make_shared(); - auto auth_type = auth_data.getType(); - node->type = auth_type; - - switch (auth_type) + if (type && *type == AuthenticationType::SHA256_PASSWORD && children.size() == 2) { - case AuthenticationType::PLAINTEXT_PASSWORD: - { - node->children.push_back(std::make_shared(auth_data.getPassword())); - break; - } - case AuthenticationType::SHA256_PASSWORD: - { - node->expect_hash = true; - node->children.push_back(std::make_shared(auth_data.getPasswordHashHex())); - - if (!auth_data.getSalt().empty()) - node->children.push_back(std::make_shared(auth_data.getSalt())); - break; - } - case AuthenticationType::DOUBLE_SHA1_PASSWORD: + if (const auto * salt = children[0]->as()) { - node->expect_hash = true; - node->children.push_back(std::make_shared(auth_data.getPasswordHashHex())); - break; + return salt->value.safeGet(); } - case AuthenticationType::LDAP: - { - node->expect_ldap_server_name = true; - node->children.push_back(std::make_shared(auth_data.getLDAPServerName())); - break; - } - case AuthenticationType::KERBEROS: - { - node->expect_kerberos_realm = true; - const auto & realm = auth_data.getKerberosRealm(); - - if (!realm.empty()) - node->children.push_back(std::make_shared(realm)); - - break; - } - case AuthenticationType::SSL_CERTIFICATE: - { - node->expect_common_names = true; - - for (const auto & name : auth_data.getSSLCertificateCommonNames()) - node->children.push_back(std::make_shared(name)); - - break; - } - - case AuthenticationType::NO_PASSWORD: [[fallthrough]]; - case AuthenticationType::MAX: - throw Exception(ErrorCodes::LOGICAL_ERROR, "AST: Unexpected authentication type {}", toString(auth_type)); } - return node; + return {}; } void ASTAuthenticationData::formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const @@ -325,4 +167,19 @@ void ASTAuthenticationData::formatImpl(const FormatSettings & settings, FormatSt } } +bool ASTAuthenticationData::hasSecretParts() const +{ + /// Default password type is used hence secret part + if (!type) + return true; + + auto auth_type = *type; + if ((auth_type == AuthenticationType::PLAINTEXT_PASSWORD) + || (auth_type == AuthenticationType::SHA256_PASSWORD) + || (auth_type == AuthenticationType::DOUBLE_SHA1_PASSWORD)) + return true; + + return childrenHaveSecretParts(); +} + } diff --git a/src/Parsers/Access/ASTAuthenticationData.h b/src/Parsers/Access/ASTAuthenticationData.h index 4168075f28c9..e2bbb99122c0 100644 --- a/src/Parsers/Access/ASTAuthenticationData.h +++ b/src/Parsers/Access/ASTAuthenticationData.h @@ -2,7 +2,6 @@ #include #include -#include #include @@ -21,9 +20,10 @@ class ASTAuthenticationData : public IAST return clone; } - static std::shared_ptr makeASTAuthenticationData(AuthenticationData auth_data); - AuthenticationData makeAuthenticationData(ContextPtr context, bool check_password_rules) const; - void checkPasswordComplexityRules(ContextPtr context) const; + bool hasSecretParts() const override; + + std::optional getPassword() const; + std::optional getSalt() const; /// If type is empty we use the default password type. /// AuthenticationType::NO_PASSWORD is specified explicitly. diff --git a/src/Parsers/Access/ASTCreateUserQuery.cpp b/src/Parsers/Access/ASTCreateUserQuery.cpp index b0f6df0bceb8..0611545adf02 100644 --- a/src/Parsers/Access/ASTCreateUserQuery.cpp +++ b/src/Parsers/Access/ASTCreateUserQuery.cpp @@ -236,21 +236,4 @@ void ASTCreateUserQuery::formatImpl(const FormatSettings & format, FormatState & formatGrantees(*grantees, format); } -bool ASTCreateUserQuery::hasSecretParts() const -{ - if (auth_data) - { - /// Default password type is used hence has secret part - if (!auth_data->type) - return true; - - auto auth_type = *(auth_data->type); - if ((auth_type == AuthenticationType::PLAINTEXT_PASSWORD) - || (auth_type == AuthenticationType::SHA256_PASSWORD) - || (auth_type == AuthenticationType::DOUBLE_SHA1_PASSWORD)) - return true; - } - return childrenHaveSecretParts(); -} - } diff --git a/src/Parsers/Access/ASTCreateUserQuery.h b/src/Parsers/Access/ASTCreateUserQuery.h index 646bc30b679c..3c0390bf81ee 100644 --- a/src/Parsers/Access/ASTCreateUserQuery.h +++ b/src/Parsers/Access/ASTCreateUserQuery.h @@ -61,7 +61,6 @@ class ASTCreateUserQuery : public IAST, public ASTQueryWithOnCluster String getID(char) const override; ASTPtr clone() const override; void formatImpl(const FormatSettings & format, FormatState &, FormatStateStacked) const override; - bool hasSecretParts() const override; ASTPtr getRewrittenASTWithoutOnCluster(const WithoutOnClusterASTRewriteParams &) const override { return removeOnCluster(clone()); } QueryKind getQueryKind() const override { return QueryKind::Create; } diff --git a/src/Parsers/Access/ParserCreateUserQuery.cpp b/src/Parsers/Access/ParserCreateUserQuery.cpp index de4245cd34ba..367e137b6c8a 100644 --- a/src/Parsers/Access/ParserCreateUserQuery.cpp +++ b/src/Parsers/Access/ParserCreateUserQuery.cpp @@ -167,10 +167,11 @@ namespace auth_data->expect_kerberos_realm = expect_kerberos_realm; auth_data->expect_common_names = expect_common_names; - auth_data->children.push_back(value); + if (value) + auth_data->children.push_back(std::move(value)); if (parsed_salt) - auth_data->children.push_back(parsed_salt); + auth_data->children.push_back(std::move(parsed_salt)); return true; }); diff --git a/src/Parsers/tests/gtest_Parser.cpp b/src/Parsers/tests/gtest_Parser.cpp index b452bd276429..e71547df82e0 100644 --- a/src/Parsers/tests/gtest_Parser.cpp +++ b/src/Parsers/tests/gtest_Parser.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -69,7 +70,7 @@ TEST_P(ParserTest, parseQuery) { if (input_text.starts_with("ATTACH")) { - auto salt = (dynamic_cast(ast.get())->auth_data)->getSalt(); + auto salt = (dynamic_cast(ast.get())->auth_data)->getSalt().value_or(""); EXPECT_TRUE(std::regex_match(salt, std::regex(expected_ast))); } else From def8611d2c31fa8ae2244904e4ff0c6d3776fe6a Mon Sep 17 00:00:00 2001 From: Nikolay Degterinsky Date: Mon, 10 Apr 2023 17:05:31 +0000 Subject: [PATCH 113/406] Fix style --- src/Interpreters/Access/InterpreterCreateUserQuery.cpp | 7 ++++--- .../Access/InterpreterShowCreateAccessEntityQuery.cpp | 1 + 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/Interpreters/Access/InterpreterCreateUserQuery.cpp b/src/Interpreters/Access/InterpreterCreateUserQuery.cpp index 44736c598b2b..6546f014732f 100644 --- a/src/Interpreters/Access/InterpreterCreateUserQuery.cpp +++ b/src/Interpreters/Access/InterpreterCreateUserQuery.cpp @@ -28,6 +28,7 @@ namespace ErrorCodes extern const int BAD_ARGUMENTS; extern const int SUPPORT_IS_DISABLED; extern const int OPENSSL_ERROR; + extern const int LOGICAL_ERROR; } namespace { @@ -39,7 +40,7 @@ namespace if (query.expect_password) { if (!context) - throw Exception(DB::ErrorCodes::LOGICAL_ERROR, "Cannot get necessary parameters without context"); + throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot get necessary parameters without context"); String value = checkAndGetLiteralArgument(query.children[0], "password"); @@ -83,8 +84,8 @@ namespace value.append(salt); auth_data.setSalt(salt); #else - throw DB::Exception(DB::ErrorCodes::SUPPORT_IS_DISABLED, - "SHA256 passwords support is disabled, because ClickHouse was built without SSL library"); + throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, + "SHA256 passwords support is disabled, because ClickHouse was built without SSL library"); #endif } diff --git a/src/Interpreters/Access/InterpreterShowCreateAccessEntityQuery.cpp b/src/Interpreters/Access/InterpreterShowCreateAccessEntityQuery.cpp index 9262ae66db4f..887d008a8a90 100644 --- a/src/Interpreters/Access/InterpreterShowCreateAccessEntityQuery.cpp +++ b/src/Interpreters/Access/InterpreterShowCreateAccessEntityQuery.cpp @@ -37,6 +37,7 @@ namespace DB namespace ErrorCodes { extern const int NOT_IMPLEMENTED; + extern const int LOGICAL_ERROR; } From 908be1e324b72c7b7ff78486ae569dcc73f94df4 Mon Sep 17 00:00:00 2001 From: Nikolay Degterinsky Date: Mon, 10 Apr 2023 22:04:10 +0000 Subject: [PATCH 114/406] Fix tests --- .../Access/InterpreterShowCreateAccessEntityQuery.cpp | 5 ++++- src/Parsers/tests/gtest_Parser.cpp | 4 ++-- tests/integration/test_mask_sensitive_info/test.py | 8 ++++---- tests/integration/test_password_constraints/test.py | 2 +- 4 files changed, 11 insertions(+), 8 deletions(-) diff --git a/src/Interpreters/Access/InterpreterShowCreateAccessEntityQuery.cpp b/src/Interpreters/Access/InterpreterShowCreateAccessEntityQuery.cpp index 887d008a8a90..c86b5597f5b1 100644 --- a/src/Interpreters/Access/InterpreterShowCreateAccessEntityQuery.cpp +++ b/src/Interpreters/Access/InterpreterShowCreateAccessEntityQuery.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -91,9 +92,11 @@ namespace { node->expect_common_names = true; + auto list = std::make_shared(); for (const auto & name : auth_data.getSSLCertificateCommonNames()) - node->children.push_back(std::make_shared(name)); + list->children.push_back(std::make_shared(name)); + node->children.push_back(std::move(list)); break; } diff --git a/src/Parsers/tests/gtest_Parser.cpp b/src/Parsers/tests/gtest_Parser.cpp index e71547df82e0..19872c4189ad 100644 --- a/src/Parsers/tests/gtest_Parser.cpp +++ b/src/Parsers/tests/gtest_Parser.cpp @@ -261,7 +261,7 @@ INSTANTIATE_TEST_SUITE_P(ParserCreateUserQuery, ParserTest, ::testing::ValuesIn(std::initializer_list{ { "CREATE USER user1 IDENTIFIED WITH sha256_password BY 'qwe123'", - "CREATE USER user1 IDENTIFIED WITH sha256_hash BY '[A-Za-z0-9]{64}' SALT '[A-Za-z0-9]{64}'" + "CREATE USER user1 IDENTIFIED WITH sha256_password BY 'qwe123'" }, { "CREATE USER user1 IDENTIFIED WITH sha256_hash BY '7A37B85C8918EAC19A9089C0FA5A2AB4DCE3F90528DCDEEC108B23DDF3607B99' SALT 'salt'", @@ -269,7 +269,7 @@ INSTANTIATE_TEST_SUITE_P(ParserCreateUserQuery, ParserTest, }, { "ALTER USER user1 IDENTIFIED WITH sha256_password BY 'qwe123'", - "ALTER USER user1 IDENTIFIED WITH sha256_hash BY '[A-Za-z0-9]{64}' SALT '[A-Za-z0-9]{64}'" + "ALTER USER user1 IDENTIFIED WITH sha256_password BY 'qwe123'" }, { "ALTER USER user1 IDENTIFIED WITH sha256_hash BY '7A37B85C8918EAC19A9089C0FA5A2AB4DCE3F90528DCDEEC108B23DDF3607B99' SALT 'salt'", diff --git a/tests/integration/test_mask_sensitive_info/test.py b/tests/integration/test_mask_sensitive_info/test.py index 92232f7e6a8b..69144c0eb074 100644 --- a/tests/integration/test_mask_sensitive_info/test.py +++ b/tests/integration/test_mask_sensitive_info/test.py @@ -95,14 +95,14 @@ def test_create_alter_user(): check_logs( must_contain=[ - "CREATE USER u1 IDENTIFIED WITH sha256_password", - "ALTER USER u1 IDENTIFIED WITH sha256_password", + "CREATE USER u1 IDENTIFIED", + "ALTER USER u1 IDENTIFIED", "CREATE USER u2 IDENTIFIED WITH plaintext_password", ], must_not_contain=[ password, - "IDENTIFIED WITH sha256_password BY", - "IDENTIFIED WITH sha256_hash BY", + "IDENTIFIED BY", + "IDENTIFIED BY", "IDENTIFIED WITH plaintext_password BY", ], ) diff --git a/tests/integration/test_password_constraints/test.py b/tests/integration/test_password_constraints/test.py index 51b5cfcbda08..94e10ed5f9e3 100644 --- a/tests/integration/test_password_constraints/test.py +++ b/tests/integration/test_password_constraints/test.py @@ -47,5 +47,5 @@ def test_complexity_rules(start_cluster): def test_default_password_type(start_cluster): node2.query("CREATE USER u1 IDENTIFIED BY 'pwd'") - required_type = "DOUBLE_SHA1_PASSWORD" + required_type = "double_sha1_password" assert required_type in node2.query("SHOW CREATE USER u1") From f34b304707892dee0668c5690b1bc06bfc901429 Mon Sep 17 00:00:00 2001 From: Nikolay Degterinsky Date: Tue, 11 Apr 2023 12:54:54 +0000 Subject: [PATCH 115/406] Fixes, add test --- .../Access/InterpreterCreateUserQuery.cpp | 16 +++++------ ...InterpreterShowCreateAccessEntityQuery.cpp | 1 + .../02713_create_user_substitutions.reference | 7 +++++ .../02713_create_user_substitutions.sh | 27 +++++++++++++++++++ 4 files changed, 43 insertions(+), 8 deletions(-) create mode 100644 tests/queries/0_stateless/02713_create_user_substitutions.reference create mode 100755 tests/queries/0_stateless/02713_create_user_substitutions.sh diff --git a/src/Interpreters/Access/InterpreterCreateUserQuery.cpp b/src/Interpreters/Access/InterpreterCreateUserQuery.cpp index 6546f014732f..222ab2081ca3 100644 --- a/src/Interpreters/Access/InterpreterCreateUserQuery.cpp +++ b/src/Interpreters/Access/InterpreterCreateUserQuery.cpp @@ -39,27 +39,27 @@ namespace if (query.expect_password) { - if (!context) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot get necessary parameters without context"); + if (!query.type && !context) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot get default password type without context"); - String value = checkAndGetLiteralArgument(query.children[0], "password"); + if (check_password_rules && !context) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot check password complexity rules without context"); - const auto & access_control = context->getAccessControl(); - AuthenticationType default_password_type = access_control.getDefaultPasswordType(); + /// NOTE: We will also extract bcrypt workfactor from context - /// NOTE: We will also extract bcrypt workfactor from access_control + String value = checkAndGetLiteralArgument(query.children[0], "password"); AuthenticationType current_type; if (query.type) current_type = *query.type; else - current_type = default_password_type; + current_type = context->getAccessControl().getDefaultPasswordType(); AuthenticationData auth_data(current_type); if (check_password_rules) - access_control.checkPasswordComplexityRules(value); + context->getAccessControl().checkPasswordComplexityRules(value); if (query.type == AuthenticationType::SHA256_PASSWORD) { diff --git a/src/Interpreters/Access/InterpreterShowCreateAccessEntityQuery.cpp b/src/Interpreters/Access/InterpreterShowCreateAccessEntityQuery.cpp index c86b5597f5b1..de7df523a3ec 100644 --- a/src/Interpreters/Access/InterpreterShowCreateAccessEntityQuery.cpp +++ b/src/Interpreters/Access/InterpreterShowCreateAccessEntityQuery.cpp @@ -54,6 +54,7 @@ namespace { case AuthenticationType::PLAINTEXT_PASSWORD: { + node->expect_password = true; node->children.push_back(std::make_shared(auth_data.getPassword())); break; } diff --git a/tests/queries/0_stateless/02713_create_user_substitutions.reference b/tests/queries/0_stateless/02713_create_user_substitutions.reference new file mode 100644 index 000000000000..02f7d1d87934 --- /dev/null +++ b/tests/queries/0_stateless/02713_create_user_substitutions.reference @@ -0,0 +1,7 @@ +1 +2 +3 +4 +CREATE USER user5_02713 IDENTIFIED WITH ldap SERVER \'qwerty5\' +CREATE USER user6_02713 IDENTIFIED WITH kerberos REALM \'qwerty6\' +CREATE USER user7_02713 IDENTIFIED WITH ssl_certificate CN \'qwerty7\', \'qwerty8\' diff --git a/tests/queries/0_stateless/02713_create_user_substitutions.sh b/tests/queries/0_stateless/02713_create_user_substitutions.sh new file mode 100755 index 000000000000..2d7fef56a21e --- /dev/null +++ b/tests/queries/0_stateless/02713_create_user_substitutions.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, no-parallel + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +$CLICKHOUSE_CLIENT -q "DROP USER IF EXISTS user1_02713, user2_02713, user3_02713, user4_02713, user5_02713, user6_02713, user7_02713"; + +$CLICKHOUSE_CLIENT --param_password=qwerty1 -q "CREATE USER user1_02713 IDENTIFIED BY {password:String}"; +$CLICKHOUSE_CLIENT --param_password=qwerty2 -q "CREATE USER user2_02713 IDENTIFIED WITH PLAINTEXT_PASSWORD BY {password:String}"; +$CLICKHOUSE_CLIENT --param_password=qwerty3 -q "CREATE USER user3_02713 IDENTIFIED WITH SHA256_PASSWORD BY {password:String}"; +$CLICKHOUSE_CLIENT --param_password=qwerty4 -q "CREATE USER user4_02713 IDENTIFIED WITH DOUBLE_SHA1_PASSWORD BY {password:String}"; +$CLICKHOUSE_CLIENT --param_server=qwerty5 -q "CREATE USER user5_02713 IDENTIFIED WITH LDAP SERVER {server:String}"; +$CLICKHOUSE_CLIENT --param_realm=qwerty6 -q "CREATE USER user6_02713 IDENTIFIED WITH KERBEROS REALM {realm:String}"; +$CLICKHOUSE_CLIENT --param_cert1=qwerty7 --param_cert2=qwerty8 -q "CREATE USER user7_02713 IDENTIFIED WITH SSL_CERTIFICATE CN {cert1:String}, {cert2:String}"; + +$CLICKHOUSE_CLIENT --user=user1_02713 --password=qwerty1 -q "SELECT 1"; +$CLICKHOUSE_CLIENT --user=user2_02713 --password=qwerty2 -q "SELECT 2"; +$CLICKHOUSE_CLIENT --user=user3_02713 --password=qwerty3 -q "SELECT 3"; +$CLICKHOUSE_CLIENT --user=user4_02713 --password=qwerty4 -q "SELECT 4"; + +$CLICKHOUSE_CLIENT -q "SHOW CREATE USER user5_02713"; +$CLICKHOUSE_CLIENT -q "SHOW CREATE USER user6_02713"; +$CLICKHOUSE_CLIENT -q "SHOW CREATE USER user7_02713"; + +$CLICKHOUSE_CLIENT -q "DROP USER user1_02713, user2_02713, user3_02713, user4_02713, user5_02713, user6_02713, user7_02713"; From 0621222737ef596e17d2e297ef43da49560bf84d Mon Sep 17 00:00:00 2001 From: Nikolay Degterinsky Date: Tue, 11 Apr 2023 14:19:45 +0000 Subject: [PATCH 116/406] Fix crashes with incorrect query parameters --- .../Access/InterpreterCreateUserQuery.cpp | 24 ++++++---- ...InterpreterShowCreateAccessEntityQuery.cpp | 5 +-- src/Parsers/Access/ASTAuthenticationData.cpp | 44 +++++++++++-------- src/Parsers/Access/ParserCreateUserQuery.cpp | 6 ++- 4 files changed, 46 insertions(+), 33 deletions(-) diff --git a/src/Interpreters/Access/InterpreterCreateUserQuery.cpp b/src/Interpreters/Access/InterpreterCreateUserQuery.cpp index 222ab2081ca3..8924d7d887c8 100644 --- a/src/Interpreters/Access/InterpreterCreateUserQuery.cpp +++ b/src/Interpreters/Access/InterpreterCreateUserQuery.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include "config.h" @@ -37,6 +38,11 @@ namespace if (query.type && *query.type == AuthenticationType::NO_PASSWORD) return AuthenticationData(); + size_t args_size = query.children.size(); + ASTs args(args_size); + for (size_t i = 0; i < args_size; ++i) + args[i] = evaluateConstantExpressionAsLiteral(query.children[i], context); + if (query.expect_password) { if (!query.type && !context) @@ -47,7 +53,7 @@ namespace /// NOTE: We will also extract bcrypt workfactor from context - String value = checkAndGetLiteralArgument(query.children[0], "password"); + String value = checkAndGetLiteralArgument(args[0], "password"); AuthenticationType current_type; @@ -97,33 +103,33 @@ namespace if (query.expect_hash) { - String value = checkAndGetLiteralArgument(query.children[0], "hash"); + String value = checkAndGetLiteralArgument(args[0], "hash"); auth_data.setPasswordHashHex(value); - if (*query.type == AuthenticationType::SHA256_PASSWORD && query.children.size() == 2) + if (*query.type == AuthenticationType::SHA256_PASSWORD && args_size == 2) { - String parsed_salt = checkAndGetLiteralArgument(query.children[1], "salt"); + String parsed_salt = checkAndGetLiteralArgument(args[1], "salt"); auth_data.setSalt(parsed_salt); } } else if (query.expect_ldap_server_name) { - String value = checkAndGetLiteralArgument(query.children[0], "ldap_server_name"); + String value = checkAndGetLiteralArgument(args[0], "ldap_server_name"); auth_data.setLDAPServerName(value); } else if (query.expect_kerberos_realm) { - if (!query.children.empty()) + if (!args.empty()) { - String value = checkAndGetLiteralArgument(query.children[0], "kerberos_realm"); + String value = checkAndGetLiteralArgument(args[0], "kerberos_realm"); auth_data.setKerberosRealm(value); } } else if (query.expect_common_names) { boost::container::flat_set common_names; - for (const auto & ast_child : query.children[0]->children) - common_names.insert(checkAndGetLiteralArgument(ast_child, "common_name")); + for (const auto & arg : args) + common_names.insert(checkAndGetLiteralArgument(arg, "common_name")); auth_data.setSSLCertificateCommonNames(std::move(common_names)); } diff --git a/src/Interpreters/Access/InterpreterShowCreateAccessEntityQuery.cpp b/src/Interpreters/Access/InterpreterShowCreateAccessEntityQuery.cpp index de7df523a3ec..0878c9d32074 100644 --- a/src/Interpreters/Access/InterpreterShowCreateAccessEntityQuery.cpp +++ b/src/Interpreters/Access/InterpreterShowCreateAccessEntityQuery.cpp @@ -92,12 +92,9 @@ namespace case AuthenticationType::SSL_CERTIFICATE: { node->expect_common_names = true; - - auto list = std::make_shared(); for (const auto & name : auth_data.getSSLCertificateCommonNames()) - list->children.push_back(std::make_shared(name)); + node->children.push_back(std::make_shared(name)); - node->children.push_back(std::move(list)); break; } diff --git a/src/Parsers/Access/ASTAuthenticationData.cpp b/src/Parsers/Access/ASTAuthenticationData.cpp index aa4642c10cbd..b214be07f422 100644 --- a/src/Parsers/Access/ASTAuthenticationData.cpp +++ b/src/Parsers/Access/ASTAuthenticationData.cpp @@ -31,7 +31,7 @@ std::optional ASTAuthenticationData::getSalt() const { if (type && *type == AuthenticationType::SHA256_PASSWORD && children.size() == 2) { - if (const auto * salt = children[0]->as()) + if (const auto * salt = children[1]->as()) { return salt->value.safeGet(); } @@ -51,10 +51,10 @@ void ASTAuthenticationData::formatImpl(const FormatSettings & settings, FormatSt String auth_type_name; String prefix; /// "BY" or "SERVER" or "REALM" - ASTPtr password; /// either a password or hash - ASTPtr salt; - ASTPtr parameter; - ASTPtr parameters; + bool password = false; /// either a password or hash + bool salt = false; + bool parameter = false; + bool parameters = false; if (type) { @@ -65,7 +65,7 @@ void ASTAuthenticationData::formatImpl(const FormatSettings & settings, FormatSt case AuthenticationType::PLAINTEXT_PASSWORD: { prefix = "BY"; - password = children[0]; + password = true; break; } case AuthenticationType::SHA256_PASSWORD: @@ -74,9 +74,9 @@ void ASTAuthenticationData::formatImpl(const FormatSettings & settings, FormatSt auth_type_name = "sha256_hash"; prefix = "BY"; - password = children[0]; + password = true; if (children.size() == 2) - salt = children[1]; + salt = true; break; } case AuthenticationType::DOUBLE_SHA1_PASSWORD: @@ -85,13 +85,13 @@ void ASTAuthenticationData::formatImpl(const FormatSettings & settings, FormatSt auth_type_name = "double_sha1_hash"; prefix = "BY"; - password = children[0]; + password = true; break; } case AuthenticationType::LDAP: { prefix = "SERVER"; - parameter = children[0]; + parameter = true; break; } case AuthenticationType::KERBEROS: @@ -99,14 +99,14 @@ void ASTAuthenticationData::formatImpl(const FormatSettings & settings, FormatSt if (!children.empty()) { prefix = "REALM"; - parameter = children[0]; + parameter = true; } break; } case AuthenticationType::SSL_CERTIFICATE: { prefix = "CN"; - parameters = children[0]; + parameters = true; break; } case AuthenticationType::NO_PASSWORD: [[fallthrough]]; @@ -118,14 +118,14 @@ void ASTAuthenticationData::formatImpl(const FormatSettings & settings, FormatSt { /// Default password type prefix = "BY"; - password = children[0]; + password = true; } if (password && !settings.show_secrets) { prefix = ""; - password.reset(); - salt.reset(); + password = false; + salt = false; if (type) auth_type_name = AuthenticationTypeInfo::get(*type).name; } @@ -146,24 +146,30 @@ void ASTAuthenticationData::formatImpl(const FormatSettings & settings, FormatSt if (password) { settings.ostr << " "; - password->format(settings); + children[0]->format(settings); } if (salt) { settings.ostr << " SALT "; - salt->format(settings); + children[1]->format(settings); } if (parameter) { settings.ostr << " "; - parameter->format(settings); + children[0]->format(settings); } else if (parameters) { settings.ostr << " "; - parameters->format(settings); + bool need_comma = false; + for (const auto & child : children) + { + if (std::exchange(need_comma, true)) + settings.ostr << ", "; + child->format(settings); + } } } diff --git a/src/Parsers/Access/ParserCreateUserQuery.cpp b/src/Parsers/Access/ParserCreateUserQuery.cpp index 367e137b6c8a..4f26c624e9b0 100644 --- a/src/Parsers/Access/ParserCreateUserQuery.cpp +++ b/src/Parsers/Access/ParserCreateUserQuery.cpp @@ -122,6 +122,7 @@ namespace ASTPtr value; ASTPtr parsed_salt; + ASTPtr common_names; if (expect_password || expect_hash) { if (!ParserKeyword{"BY"}.ignore(pos, expected) || !ParserStringAndSubstitution{}.parse(pos, value, expected)) @@ -154,7 +155,7 @@ namespace if (!ParserKeyword{"CN"}.ignore(pos, expected)) return false; - if (!ParserList{std::make_unique(), std::make_unique(TokenType::Comma), false}.parse(pos, value, expected)) + if (!ParserList{std::make_unique(), std::make_unique(TokenType::Comma), false}.parse(pos, common_names, expected)) return false; } @@ -173,6 +174,9 @@ namespace if (parsed_salt) auth_data->children.push_back(std::move(parsed_salt)); + if (common_names) + auth_data->children = std::move(common_names->children); + return true; }); } From a1fda047f6aa561b0a6ed4b6e15fc57888002570 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Tue, 11 Apr 2023 16:32:40 +0200 Subject: [PATCH 117/406] add docs --- docs/en/operations/system-tables/clusters.md | 9 +++++++++ docs/en/sql-reference/statements/system.md | 16 ++++++++++++++-- src/Interpreters/Cluster.h | 2 +- 3 files changed, 24 insertions(+), 3 deletions(-) diff --git a/docs/en/operations/system-tables/clusters.md b/docs/en/operations/system-tables/clusters.md index 2c5e2699b4fc..b25427419c87 100644 --- a/docs/en/operations/system-tables/clusters.md +++ b/docs/en/operations/system-tables/clusters.md @@ -20,6 +20,9 @@ Columns: - `errors_count` ([UInt32](../../sql-reference/data-types/int-uint.md)) — The number of times this host failed to reach replica. - `slowdowns_count` ([UInt32](../../sql-reference/data-types/int-uint.md)) — The number of slowdowns that led to changing replica when establishing a connection with hedged requests. - `estimated_recovery_time` ([UInt32](../../sql-reference/data-types/int-uint.md)) — Seconds remaining until the replica error count is zeroed and it is considered to be back to normal. +- `database_shard_name` ([String](../../sql-reference/data-types/string.md)) — The name of the `Replicated` database shard (for clusters that belong to a `Replicated` database). +- `database_replica_name` ([String](../../sql-reference/data-types/string.md)) — The name of the `Replicated` database replica (for clusters that belong to a `Replicated` database). +- `is_active` ([Nullable(UInt8)](../../sql-reference/data-types/int-uint.md)) — The status of the `Replicated` database replica (for clusters that belong to a `Replicated` database): 1 means "replica is online", 0 means "replica is offline", `NULL` means "unknown". **Example** @@ -47,6 +50,9 @@ default_database: errors_count: 0 slowdowns_count: 0 estimated_recovery_time: 0 +database_shard_name: +database_replica_name: +is_active: NULL Row 2: ────── @@ -63,6 +69,9 @@ default_database: errors_count: 0 slowdowns_count: 0 estimated_recovery_time: 0 +database_shard_name: +database_replica_name: +is_active: NULL ``` **See Also** diff --git a/docs/en/sql-reference/statements/system.md b/docs/en/sql-reference/statements/system.md index 5a5a771f2393..c5596b7ba5f6 100644 --- a/docs/en/sql-reference/statements/system.md +++ b/docs/en/sql-reference/statements/system.md @@ -76,7 +76,7 @@ Resets the mark cache. ## DROP REPLICA -Dead replicas can be dropped using following syntax: +Dead replicas of `ReplicatedMergeTree` tables can be dropped using following syntax: ``` sql SYSTEM DROP REPLICA 'replica_name' FROM TABLE database.table; @@ -85,13 +85,25 @@ SYSTEM DROP REPLICA 'replica_name'; SYSTEM DROP REPLICA 'replica_name' FROM ZKPATH '/path/to/table/in/zk'; ``` -Queries will remove the replica path in ZooKeeper. It is useful when the replica is dead and its metadata cannot be removed from ZooKeeper by `DROP TABLE` because there is no such table anymore. It will only drop the inactive/stale replica, and it cannot drop local replica, please use `DROP TABLE` for that. `DROP REPLICA` does not drop any tables and does not remove any data or metadata from disk. +Queries will remove the `ReplicatedMergeTree` replica path in ZooKeeper. It is useful when the replica is dead and its metadata cannot be removed from ZooKeeper by `DROP TABLE` because there is no such table anymore. It will only drop the inactive/stale replica, and it cannot drop local replica, please use `DROP TABLE` for that. `DROP REPLICA` does not drop any tables and does not remove any data or metadata from disk. The first one removes metadata of `'replica_name'` replica of `database.table` table. The second one does the same for all replicated tables in the database. The third one does the same for all replicated tables on the local server. The fourth one is useful to remove metadata of dead replica when all other replicas of a table were dropped. It requires the table path to be specified explicitly. It must be the same path as was passed to the first argument of `ReplicatedMergeTree` engine on table creation. +## DROP DATABASE REPLICA + +Dead replicas of `Replicated` databases can be dropped using following syntax: + +``` sql +SYSTEM DROP DATABASE REPLICA 'replica_name' [FROM SHARD 'shard_name'] FROM DATABASE database; +SYSTEM DROP DATABASE REPLICA 'replica_name' [FROM SHARD 'shard_name']; +SYSTEM DROP DATABASE REPLICA 'replica_name' [FROM SHARD 'shard_name'] FROM ZKPATH '/path/to/table/in/zk'; +``` + +Similar to `SYSTEM DROP REPLICA`, but removes the `Replicated` database replica path from ZooKeeper when there's no database to run `DROP DATABASE`. Please note that it does not remove `ReplicatedMergeTree` replicas (so you may need `SYSTEM DROP REPLICA` as well). Shard and replica names are the names that were specified in `Replicated` engine arguments when creating the database. Also, these names can be obtained from `database_shard_name` and `database_replica_name` columns in `system.clusters`. If the `FROM SHARD` clause is missing, then `replica_name` must be a full replica name in `shard_name|replica_name` format. + ## DROP UNCOMPRESSED CACHE Reset the uncompressed data cache. diff --git a/src/Interpreters/Cluster.h b/src/Interpreters/Cluster.h index f8437b7e027b..4798384f29c7 100644 --- a/src/Interpreters/Cluster.h +++ b/src/Interpreters/Cluster.h @@ -150,7 +150,7 @@ class Cluster UInt32 replica_index_); Address( - const DatabaseReplicaInfo & host_port_, + const DatabaseReplicaInfo & info, const ClusterConnectionParameters & params, UInt32 shard_index_, UInt32 replica_index_); From b773b506b274a9eb75c3427bd7aafe30f46b3cc6 Mon Sep 17 00:00:00 2001 From: kssenii Date: Wed, 12 Apr 2023 23:27:06 +0200 Subject: [PATCH 118/406] Better --- .../IO/CachedOnDiskReadBufferFromFile.cpp | 1 + src/Interpreters/Cache/FileCache.cpp | 513 +++++++++--------- src/Interpreters/Cache/FileCache.h | 99 ++-- src/Interpreters/Cache/FileCacheKey.h | 11 + src/Interpreters/Cache/FileCache_fwd.h | 2 +- src/Interpreters/Cache/FileSegment.cpp | 130 +++-- src/Interpreters/Cache/FileSegment.h | 62 +-- src/Interpreters/Cache/IFileCachePriority.h | 54 +- .../Cache/LRUFileCachePriority.cpp | 42 +- src/Interpreters/Cache/LRUFileCachePriority.h | 19 +- .../Cache/LockedFileCachePriority.h | 60 -- src/Interpreters/Cache/Metadata.cpp | 243 +++++---- src/Interpreters/Cache/Metadata.h | 132 ++--- src/Interpreters/Cache/QueryLimit.cpp | 53 +- src/Interpreters/Cache/QueryLimit.h | 72 ++- .../tests/gtest_lru_file_cache.cpp | 65 ++- 16 files changed, 823 insertions(+), 735 deletions(-) delete mode 100644 src/Interpreters/Cache/LockedFileCachePriority.h diff --git a/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp b/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp index 066d33364865..a2ec899938db 100644 --- a/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp +++ b/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp @@ -489,6 +489,7 @@ bool CachedOnDiskReadBufferFromFile::completeFileSegmentAndGetNext() return false; current_file_segment = &file_segments->front(); + current_file_segment->use(); implementation_buffer = getImplementationBuffer(*current_file_segment); if (read_type == ReadType::CACHED) diff --git a/src/Interpreters/Cache/FileCache.cpp b/src/Interpreters/Cache/FileCache.cpp index 9182e3887bfa..76145afe35bf 100644 --- a/src/Interpreters/Cache/FileCache.cpp +++ b/src/Interpreters/Cache/FileCache.cpp @@ -22,8 +22,7 @@ namespace ErrorCodes FileCache::FileCache( const String & cache_base_path_, const FileCacheSettings & cache_settings_) - : cache_base_path(cache_base_path_) - , max_file_segment_size(cache_settings_.max_file_segment_size) + : max_file_segment_size(cache_settings_.max_file_segment_size) , allow_persistent_files(cache_settings_.do_not_evict_index_and_mark_files) , bypass_cache_threshold(cache_settings_.enable_bypass_cache_with_threashold ? cache_settings_.bypass_cache_threashold : 0) , delayed_cleanup_interval_ms(cache_settings_.delayed_cleanup_interval_ms) @@ -46,6 +45,11 @@ FileCache::Key FileCache::createKeyForPath(const String & path) return Key(path); } +const String & FileCache::getBasePath() const +{ + return metadata.getBaseDirectory(); +} + String FileCache::getPathInLocalCache(const Key & key, size_t offset, FileSegmentKind segment_kind) const { return metadata.getPathInLocalCache(key, offset, segment_kind); @@ -94,7 +98,7 @@ void FileCache::initialize() cleanup_task->scheduleAfter(delayed_cleanup_interval_ms); } -FileSegments FileCache::getImpl(const LockedKeyMetadata & locked_key, const FileSegment::Range & range) +FileSegments FileCache::getImpl(const LockedKey & locked_key, const FileSegment::Range & range) const { /// Given range = [left, right] and non-overlapping ordered set of file segments, /// find list [segment1, ..., segmentN] of segments which intersect with given range. @@ -102,8 +106,7 @@ FileSegments FileCache::getImpl(const LockedKeyMetadata & locked_key, const File if (bypass_cache_threshold && range.size() > bypass_cache_threshold) { auto file_segment = std::make_shared( - locked_key.getKey(), range.left, range.size(), std::weak_ptr(), nullptr, this, - FileSegment::State::DETACHED, CreateFileSegmentSettings{}); + locked_key.getKey(), range.left, range.size(), FileSegment::State::DETACHED); return { file_segment }; } @@ -112,42 +115,55 @@ FileSegments FileCache::getImpl(const LockedKeyMetadata & locked_key, const File return {}; FileSegments result; - auto add_to_result = [&](const FileSegmentPtr & file_segment) + auto add_to_result = [&](const FileSegmentMetadata & file_segment_metadata) { - if (file_segment->isDownloaded()) + FileSegmentPtr file_segment; + if (file_segment_metadata.valid()) { - if (file_segment->getDownloadedSize(true) == 0) + file_segment = file_segment_metadata.file_segment; + if (file_segment->isDownloaded()) { - throw Exception( - ErrorCodes::LOGICAL_ERROR, - "Cannot have zero size downloaded file segments. {}", - file_segment->getInfoForLog()); - } - - #ifndef NDEBUG - /** - * Check that in-memory state of the cache is consistent with the state on disk. - * Check only in debug build, because such checks can be done often and can be quite - * expensive compared to overall query execution time. - */ + if (file_segment->getDownloadedSize(true) == 0) + { + throw Exception( + ErrorCodes::LOGICAL_ERROR, + "Cannot have zero size downloaded file segments. {}", + file_segment->getInfoForLog()); + } - fs::path path = file_segment->getPathInLocalCache(); - if (!fs::exists(path)) - { - throw Exception( - ErrorCodes::LOGICAL_ERROR, - "File path does not exist, but file has DOWNLOADED state. {}", - file_segment->getInfoForLog()); - } +#ifndef NDEBUG + /** + * Check that in-memory state of the cache is consistent with the state on disk. + * Check only in debug build, because such checks can be done often and can be quite + * expensive compared to overall query execution time. + */ + + fs::path path = file_segment->getPathInLocalCache(); + if (!fs::exists(path)) + { + throw Exception( + ErrorCodes::LOGICAL_ERROR, + "File path does not exist, but file has DOWNLOADED state. {}", + file_segment->getInfoForLog()); + } - if (fs::file_size(path) == 0) - { - throw Exception( - ErrorCodes::LOGICAL_ERROR, - "Cannot have zero size downloaded file segments. {}", - file_segment->getInfoForLog()); + if (fs::file_size(path) == 0) + { + throw Exception( + ErrorCodes::LOGICAL_ERROR, + "Cannot have zero size downloaded file segments. {}", + file_segment->getInfoForLog()); + } +#endif } - #endif + } + else + { + file_segment = std::make_shared( + locked_key.getKey(), + file_segment_metadata.file_segment->offset(), + file_segment_metadata.file_segment->range().size(), + FileSegment::State::DETACHED); } result.push_back(file_segment); @@ -163,17 +179,17 @@ FileSegments FileCache::getImpl(const LockedKeyMetadata & locked_key, const File /// ^ ^ /// range.left range.left - const auto & file_segment_metadata = file_segments.rbegin()->second; + const auto & file_segment_metadata = *file_segments.rbegin()->second; if (file_segment_metadata.file_segment->range().right < range.left) return {}; - add_to_result(file_segment_metadata.file_segment); + add_to_result(file_segment_metadata); } else /// segment_it <-- segmment{k} { if (segment_it != file_segments.begin()) { - const auto & prev_file_segment_metadata = std::prev(segment_it)->second; + const auto & prev_file_segment_metadata = *std::prev(segment_it)->second; const auto & prev_range = prev_file_segment_metadata.file_segment->range(); if (range.left <= prev_range.right) @@ -183,7 +199,7 @@ FileSegments FileCache::getImpl(const LockedKeyMetadata & locked_key, const File /// [___________ /// ^ /// range.left - add_to_result(prev_file_segment_metadata.file_segment); + add_to_result(prev_file_segment_metadata); } } @@ -195,11 +211,11 @@ FileSegments FileCache::getImpl(const LockedKeyMetadata & locked_key, const File while (segment_it != file_segments.end()) { - const auto & file_segment_metadata = segment_it->second; + const auto & file_segment_metadata = *segment_it->second; if (range.right < file_segment_metadata.file_segment->range().left) break; - add_to_result(file_segment_metadata.file_segment); + add_to_result(file_segment_metadata); ++segment_it; } } @@ -207,8 +223,8 @@ FileSegments FileCache::getImpl(const LockedKeyMetadata & locked_key, const File return result; } -FileSegments FileCache::splitRangeInfoFileSegments( - LockedKeyMetadata & locked_key, +FileSegments FileCache::splitRangeIntoFileSegments( + LockedKey & locked_key, size_t offset, size_t size, FileSegment::State state, @@ -228,8 +244,9 @@ FileSegments FileCache::splitRangeInfoFileSegments( current_file_segment_size = std::min(remaining_size, max_file_segment_size); remaining_size -= current_file_segment_size; - auto file_segment_metadata_it = addFileSegment(locked_key, current_pos, current_file_segment_size, state, settings, nullptr); - file_segments.push_back(file_segment_metadata_it->second.file_segment); + auto file_segment_metadata_it = addFileSegment( + locked_key, current_pos, current_file_segment_size, state, settings, nullptr); + file_segments.push_back(file_segment_metadata_it->second->file_segment); current_pos += current_file_segment_size; } @@ -239,7 +256,7 @@ FileSegments FileCache::splitRangeInfoFileSegments( } void FileCache::fillHolesWithEmptyFileSegments( - LockedKeyMetadata & locked_key, + LockedKey & locked_key, FileSegments & file_segments, const FileSegment::Range & range, bool fill_with_detached_file_segments, @@ -292,15 +309,15 @@ void FileCache::fillHolesWithEmptyFileSegments( if (fill_with_detached_file_segments) { auto file_segment = std::make_shared( - locked_key.getKey(), current_pos, hole_size, std::weak_ptr(), - nullptr, this, FileSegment::State::DETACHED, settings); + locked_key.getKey(), current_pos, hole_size, FileSegment::State::DETACHED, settings); file_segments.insert(it, file_segment); } else { - file_segments.splice( - it, splitRangeInfoFileSegments(locked_key, current_pos, hole_size, FileSegment::State::EMPTY, settings)); + auto splitted = splitRangeIntoFileSegments( + locked_key, current_pos, hole_size, FileSegment::State::EMPTY, settings); + file_segments.splice(it, std::move(splitted)); } current_pos = segment_range.right + 1; @@ -319,21 +336,24 @@ void FileCache::fillHolesWithEmptyFileSegments( if (fill_with_detached_file_segments) { auto file_segment = std::make_shared( - locked_key.getKey(), current_pos, hole_size, std::weak_ptr(), - nullptr, this, FileSegment::State::DETACHED, settings); + locked_key.getKey(), current_pos, hole_size, FileSegment::State::DETACHED, settings); file_segments.insert(file_segments.end(), file_segment); } else { - file_segments.splice( - file_segments.end(), - splitRangeInfoFileSegments(locked_key, current_pos, hole_size, FileSegment::State::EMPTY, settings)); + auto splitted = splitRangeIntoFileSegments( + locked_key, current_pos, hole_size, FileSegment::State::EMPTY, settings); + file_segments.splice(file_segments.end(), std::move(splitted)); } } } -FileSegmentsHolderPtr FileCache::set(const Key & key, size_t offset, size_t size, const CreateFileSegmentSettings & settings) +FileSegmentsHolderPtr FileCache::set( + const Key & key, + size_t offset, + size_t size, + const CreateFileSegmentSettings & settings) { assertInitialized(); @@ -349,10 +369,13 @@ FileSegmentsHolderPtr FileCache::set(const Key & key, size_t offset, size_t size /// If the file is unbounded, we can create a single file_segment_metadata for it. auto file_segment_metadata_it = addFileSegment( *locked_key, offset, size, FileSegment::State::EMPTY, settings, nullptr); - file_segments = {file_segment_metadata_it->second.file_segment}; + file_segments = {file_segment_metadata_it->second->file_segment}; } else - file_segments = splitRangeInfoFileSegments(*locked_key, offset, size, FileSegment::State::EMPTY, settings); + { + file_segments = splitRangeIntoFileSegments( + *locked_key, offset, size, FileSegment::State::EMPTY, settings); + } return std::make_unique(std::move(file_segments)); } @@ -373,11 +396,13 @@ FileSegmentsHolderPtr FileCache::getOrSet( auto file_segments = getImpl(*locked_key, range); if (file_segments.empty()) { - file_segments = splitRangeInfoFileSegments(*locked_key, offset, size, FileSegment::State::EMPTY, settings); + file_segments = splitRangeIntoFileSegments( + *locked_key, offset, size, FileSegment::State::EMPTY, settings); } else { - fillHolesWithEmptyFileSegments(*locked_key, file_segments, range, /* fill_with_detached */false, settings); + fillHolesWithEmptyFileSegments( + *locked_key, file_segments, range, /* fill_with_detached */false, settings); } chassert(!file_segments.empty()); @@ -404,15 +429,12 @@ FileSegmentsHolderPtr FileCache::get(const Key & key, size_t offset, size_t size } } - auto file_segment = std::make_shared( - key, offset, size, std::weak_ptr(), nullptr, - this, FileSegment::State::DETACHED, CreateFileSegmentSettings{}); - - return std::make_unique(FileSegments{file_segment}); + return std::make_unique(FileSegments{ + std::make_shared(key, offset, size, FileSegment::State::DETACHED)}); } KeyMetadata::iterator FileCache::addFileSegment( - LockedKeyMetadata & locked_key, + LockedKey & locked_key, size_t offset, size_t size, FileSegment::State state, @@ -429,7 +451,7 @@ KeyMetadata::iterator FileCache::addFileSegment( throw Exception( ErrorCodes::LOGICAL_ERROR, "Cache entry already exists for key: `{}`, offset: {}, size: {}.", - key.toString(), offset, size); + key, offset, size); } FileSegment::State result_state; @@ -446,19 +468,19 @@ KeyMetadata::iterator FileCache::addFileSegment( auto record_it = stash->records.find(stash_key); if (record_it == stash->records.end()) { - auto stash_queue = LockedCachePriority(*lock, *stash->queue); auto & stash_records = stash->records; - stash_records.emplace(stash_key, stash_queue.add(key, offset, 0, locked_key.getKeyMetadata())); + stash_records.emplace( + stash_key, stash->queue->add(key, offset, 0, locked_key.getKeyMetadata(), *lock)); - if (stash_queue.getElementsCount() > stash->queue->getElementsLimit()) - stash_queue.pop(); + if (stash->queue->getElementsCount(*lock) > stash->queue->getElementsLimit()) + stash->queue->pop(*lock); result_state = FileSegment::State::DETACHED; } else { - result_state = LockedCachePriorityIterator(*lock, record_it->second).use() >= stash->hits_threshold + result_state = record_it->second->use(*lock) >= stash->hits_threshold ? FileSegment::State::EMPTY : FileSegment::State::DETACHED; } @@ -468,164 +490,162 @@ KeyMetadata::iterator FileCache::addFileSegment( result_state = state; } - auto file_segment = std::make_shared(key, offset, size, locked_key.getKeyMetadata(), nullptr, this, result_state, settings); - - std::optional locked_queue( - lock ? LockedCachePriority(*lock, *main_priority) : std::optional{}); - - auto [file_segment_metadata_it, inserted] = locked_key.getKeyMetadata()->emplace( - std::piecewise_construct, - std::forward_as_tuple(offset), - std::forward_as_tuple(std::move(file_segment), locked_key, locked_queue ? &*locked_queue : nullptr)); + PriorityIterator cache_it; + if (state == FileSegment::State::DOWNLOADED) + { + cache_it = main_priority->add(key, offset, size, locked_key.getKeyMetadata(), *lock); + } - assert(inserted); + try + { + auto file_segment = std::make_shared( + key, offset, size, result_state, settings, this, locked_key.getKeyMetadata(), cache_it); + auto file_segment_metadata = std::make_shared(std::move(file_segment)); - return file_segment_metadata_it; -} + auto [file_segment_metadata_it, inserted] = locked_key.getKeyMetadata()->emplace(offset, file_segment_metadata); + if (!inserted) + { + if (cache_it) + cache_it->remove(*lock); -void FileCache::removeFileSegment(LockedKeyMetadata & locked_key, FileSegmentPtr file_segment, const CacheGuard::Lock & cache_lock) -{ - /// We must hold pointer to file segment while removing it - /// (because we remove file segment under file segment lock). + throw Exception( + ErrorCodes::LOGICAL_ERROR, + "Failed to insert {}:{}: entry already exists", key, offset); + } - chassert(file_segment->key() == locked_key.getKey()); - locked_key.removeFileSegment(file_segment->offset(), file_segment->lock(), cache_lock); + if (state == FileSegment::State::DOWNLOADED) + chassert(file_segment_metadata_it->second->file_segment->getQueueIterator()); + return file_segment_metadata_it; + } + catch (...) + { + if (cache_it) + cache_it->remove(*lock); + throw; + } } -bool FileCache::tryReserve(const FileSegment & file_segment, size_t size) +bool FileCache::tryReserve(FileSegment & file_segment, size_t size) { assertInitialized(); - auto lock = cache_guard.lock(); + auto cache_lock = cache_guard.lock(); - LOG_TEST(log, "Reserving {} byts for key {} at offset {}", size, file_segment.key().toString(), file_segment.offset()); - - auto query_context = query_limit ? query_limit->tryGetQueryContext(lock) : nullptr; - bool reserved; + /// In case of per query cache limit (by default disabled), we add/remove entries from both + /// (main_priority and query_priority) priority queues, but iterate entries in order of query_priority, + /// while checking the limits in both. + Priority * query_priority = nullptr; + auto query_context = query_limit ? query_limit->tryGetQueryContext(cache_lock) : nullptr; if (query_context) { - const bool query_limit_exceeded = query_context->getSize() + size > query_context->getSizeLimit(); - reserved = (!query_limit_exceeded || query_context->recacheOnFileCacheQueryLimitExceeded()) - && tryReserveImpl(file_segment, size, query_context->getPriority(), query_context.get(), lock); - } - else - { - reserved = tryReserveImpl(file_segment, size, *main_priority, nullptr, lock); - } + query_priority = &query_context->getPriority(); - const auto & key_metadata = file_segment.getKeyMetadata(); - if (reserved && !key_metadata->created_base_directory) - { - fs::create_directories(metadata.getPathInLocalCache(file_segment.key())); - key_metadata->created_base_directory = true; + const bool query_limit_exceeded = query_priority->getSize(cache_lock) + size > query_priority->getSizeLimit(); + if (query_limit_exceeded && !query_context->recacheOnFileCacheQueryLimitExceeded()) + return false; } - return reserved; -} - -bool FileCache::tryReserveImpl( - const FileSegment & file_segment, - size_t size, - IFileCachePriority & priority_queue, - FileCacheQueryLimit::LockedQueryContext * query_context, - const CacheGuard::Lock & cache_lock) -{ - /// In case of per query cache limit (by default disabled). - /// We add/remove entries from both (global and local) priority queues, - /// but iterate only local, though check the limits in both. - - LOG_TEST(log, "Reserving space {} for {}:{}", size, file_segment.key().toString(), file_segment.offset()); - - LockedCachePriority locked_priority_queue(cache_lock, priority_queue); - LockedCachePriority locked_main_priority(cache_lock, *main_priority); - - size_t queue_size = locked_priority_queue.getElementsCount(); - chassert(queue_size <= locked_priority_queue.getElementsLimit()); + size_t queue_size = main_priority->getElementsCount(cache_lock); + chassert(queue_size <= main_priority->getElementsLimit()); /// A file_segment_metadata acquires a LRUQueue iterator on first successful space reservation attempt. - auto & queue_iterator = file_segment.getQueueIterator(); + auto queue_iterator = file_segment.getQueueIterator(); if (queue_iterator) chassert(file_segment.getReservedSize() > 0); else queue_size += 1; size_t removed_size = 0; - auto is_overflow = [&] - { - /// max_size == 0 means unlimited cache size, - /// max_element_size means unlimited number of cache elements. - return (main_priority->getSizeLimit() != 0 && locked_main_priority.getSize() + size - removed_size > main_priority->getSizeLimit()) - || (main_priority->getElementsLimit() != 0 && queue_size > main_priority->getElementsLimit()) - || (query_context && query_context->getSize() + size - removed_size > query_context->getSizeLimit()); - }; - - struct DeletionInfo - { - KeyMetadataPtr key_metadata; - std::vector> offsets; - }; - std::unordered_map to_delete; + std::unordered_map to_delete; - using IterationResult = IFileCachePriority::IterationResult; - auto iterate = [this, &removed_size, &queue_size, &cache_lock, &to_delete](const IFileCachePriority::Entry & entry) + auto iterate_func = [&](const PriorityEntry & entry, LockedKey & locked_key) { - auto current_locked_key = metadata.lockKeyMetadata(entry.key, entry.getKeyMetadata()); - auto & file_segment_metadata = *current_locked_key->getByOffset(entry.offset); + auto file_segment_metadata = locked_key.tryGetByOffset(entry.offset); + if (!file_segment_metadata) + return PriorityIterationResult::REMOVE_AND_CONTINUE; - chassert(file_segment_metadata.getQueueIterator()); - chassert((entry.size == file_segment_metadata.size()) || (file_segment_metadata.file_segment->state() == FileSegment::State::DOWNLOADING)); - chassert(entry.offset == file_segment_metadata.file_segment->offset()); + chassert(file_segment_metadata->file_segment->getQueueIterator()); + chassert(entry.offset == file_segment_metadata->file_segment->offset()); - auto iteration_result = IterationResult::CONTINUE; - - const bool is_persistent = allow_persistent_files && file_segment_metadata.file_segment->isPersistent(); - const bool releasable = file_segment_metadata.releasable() && !is_persistent; + auto iteration_result = PriorityIterationResult::CONTINUE; + const bool is_persistent = allow_persistent_files && file_segment_metadata->file_segment->isPersistent(); + const bool releasable = file_segment_metadata->releasable() && !is_persistent; if (releasable) { + auto current_file_segment = file_segment_metadata->file_segment; const size_t file_segment_size = entry.size; - const auto & current_file_segment = file_segment_metadata.file_segment; if (current_file_segment->state() == FileSegment::State::DOWNLOADED) { - /// file_segment_metadata will actually be removed only if we managed to reserve enough space. - auto & deletion_info = to_delete[current_file_segment->key()]; - deletion_info.offsets.emplace_back(file_segment_metadata.getEvictHolder(), current_file_segment->offset()); - deletion_info.key_metadata = current_locked_key->getKeyMetadata(); + const auto & key = current_file_segment->key(); + auto it = to_delete.find(key); + if (it == to_delete.end()) + it = to_delete.emplace(key, locked_key.getKeyMetadata()).first; + it->second.add(file_segment_metadata); } else { - iteration_result = IterationResult::REMOVE_AND_CONTINUE; - file_segment_metadata.getQueueIterator() = {}; - removeFileSegment(*current_locked_key, current_file_segment, cache_lock); + /// TODO: we can resize if partially downloaded instead. + iteration_result = PriorityIterationResult::REMOVE_AND_CONTINUE; + locked_key.removeFileSegment(current_file_segment->offset(), current_file_segment->lock()); } removed_size += file_segment_size; --queue_size; } + return iteration_result; }; - locked_priority_queue.iterate([&](const IFileCachePriority::Entry & entry) + if (query_priority) { - if (!is_overflow()) - return IterationResult::BREAK; + auto is_query_priority_overflow = [&] + { + const size_t new_size = query_priority->getSize(cache_lock) + size - removed_size; + return new_size > query_priority->getSizeLimit(); + }; - return iterate(entry); - }); + query_priority->iterate( + [&](const auto & entry, LockedKey & locked_key) + { return is_query_priority_overflow() ? iterate_func(entry, locked_key) : PriorityIterationResult::BREAK; }, + cache_lock); + + if (is_query_priority_overflow()) + return false; + } - if (is_overflow()) + auto is_main_priority_overflow = [&] + { + /// max_size == 0 means unlimited cache size, + /// max_element_size means unlimited number of cache elements. + return (main_priority->getSizeLimit() != 0 && main_priority->getSize(cache_lock) + size - removed_size > main_priority->getSizeLimit()) + || (main_priority->getElementsLimit() != 0 && queue_size > main_priority->getElementsLimit()); + }; + + main_priority->iterate( + [&](const auto & entry, LockedKey & locked_key) + { return is_main_priority_overflow() ? iterate_func(entry, locked_key) : PriorityIterationResult::BREAK; }, + cache_lock); + + if (is_main_priority_overflow()) return false; - for (const auto & [current_key, deletion_info] : to_delete) + for (auto & [current_key, deletion_info] : to_delete) { - auto current_locked_metadata = metadata.lockKeyMetadata(current_key, deletion_info.key_metadata); - for (const auto & offset_to_delete : deletion_info.offsets) + auto locked_key = deletion_info.getMetadata().lock(); + for (auto it = deletion_info.begin(); it != deletion_info.end();) { - auto * file_segment_metadata = current_locked_metadata->getByOffset(offset_to_delete.second); - removeFileSegment(*current_locked_metadata, file_segment_metadata->file_segment, cache_lock); + chassert((*it)->releasable()); + + auto segment = (*it)->file_segment; + locked_key->removeFileSegment(segment->offset(), segment->lock()); + segment->getQueueIterator()->remove(cache_lock); + if (query_context) - query_context->remove(current_key, offset_to_delete.second); + query_context->remove(current_key, segment->offset(), cache_lock); + + it = deletion_info.erase(it); } } @@ -633,33 +653,35 @@ bool FileCache::tryReserveImpl( /// acquires queue iterator on first successful space reservation attempt. /// If queue iterator already exists, we need to update the size after each space reservation. if (queue_iterator) - LockedCachePriorityIterator(cache_lock, queue_iterator).incrementSize(size); + { + queue_iterator->updateSize(size); + } else { /// Space reservation is incremental, so file_segment_metadata is created first (with state empty), /// and getQueueIterator() is assigned on first space reservation attempt. - queue_iterator = locked_main_priority.add(file_segment.key(), file_segment.offset(), size, file_segment.getKeyMetadata()); + file_segment.setQueueIterator(main_priority->add( + file_segment.key(), file_segment.offset(), size, file_segment.getKeyMetadata(), cache_lock)); } if (query_context) { - auto query_queue_it = query_context->tryGet(file_segment.key(), file_segment.offset()); + auto query_queue_it = query_context->tryGet(file_segment.key(), file_segment.offset(), cache_lock); if (query_queue_it) - { - LockedCachePriorityIterator(cache_lock, query_queue_it).incrementSize(size); - } + query_queue_it->updateSize(size); else - { - auto it = LockedCachePriority( - cache_lock, query_context->getPriority()).add(file_segment.key(), file_segment.offset(), size, file_segment.getKeyMetadata()); - - query_context->add(file_segment.key(), file_segment.offset(), it); - } + query_context->add(file_segment, cache_lock); } - if (locked_main_priority.getSize() > (1ull << 63)) + if (main_priority->getSize(cache_lock) > (1ull << 63)) throw Exception(ErrorCodes::LOGICAL_ERROR, "Cache became inconsistent. There must be a bug"); + const auto & key_metadata = file_segment.getKeyMetadata(); + if (!key_metadata->created_base_directory.exchange(true)) + { + fs::create_directories(metadata.getPathInLocalCache(file_segment.key())); + } + return true; } @@ -667,77 +689,57 @@ void FileCache::removeKeyIfExists(const Key & key) { assertInitialized(); - auto lock = cache_guard.lock(); auto locked_key = metadata.lockKeyMetadata(key, CacheMetadata::KeyNotFoundPolicy::RETURN_NULL); if (!locked_key) return; - auto & key_metadata = *locked_key->getKeyMetadata(); - if (!key_metadata.empty()) - { - std::vector file_segments_metadata_to_remove; - file_segments_metadata_to_remove.reserve(key_metadata.size()); - for (auto & [offset, file_segment_metadata] : key_metadata) - file_segments_metadata_to_remove.push_back(&file_segment_metadata); - - for (auto & file_segment_metadata : file_segments_metadata_to_remove) - { - /// In ordinary case we remove data from cache when it's not used by anyone. - /// But if we have multiple replicated zero-copy tables on the same server - /// it became possible to start removing something from cache when it is used - /// by other "zero-copy" tables. That is why it's not an error. - if (!file_segment_metadata->releasable()) - continue; - - removeFileSegment(*locked_key, file_segment_metadata->file_segment, lock); - } - } + /// In ordinary case we remove data from cache when it's not used by anyone. + /// But if we have multiple replicated zero-copy tables on the same server + /// it became possible to start removing something from cache when it is used + /// by other "zero-copy" tables. That is why it's not an error. + locked_key->removeAllReleasable(); } void FileCache::removeAllReleasable() { assertInitialized(); - using QueueEntry = IFileCachePriority::Entry; - using IterationResult = IFileCachePriority::IterationResult; - - /// Try remove all cached files by cache_base_path. /// Only releasable file segments are evicted. /// `remove_persistent_files` defines whether non-evictable by some criteria files /// (they do not comply with the cache eviction policy) should also be removed. auto lock = cache_guard.lock(); - LockedCachePriority(lock, *main_priority).iterate([&](const QueueEntry & entry) -> IterationResult + main_priority->iterate([&](const PriorityEntry & entry, LockedKey & locked_key) { - auto locked_key = metadata.lockKeyMetadata(entry.key, entry.getKeyMetadata()); - auto * file_segment_metadata = locked_key->getByOffset(entry.offset); + auto file_segment_metadata = locked_key.tryGetByOffset(entry.offset); + if (!file_segment_metadata) + return PriorityIterationResult::REMOVE_AND_CONTINUE; if (file_segment_metadata->releasable()) { - file_segment_metadata->getQueueIterator() = {}; - removeFileSegment(*locked_key, file_segment_metadata->file_segment, lock); - return IterationResult::REMOVE_AND_CONTINUE; + auto file_segment = file_segment_metadata->file_segment; + locked_key.removeFileSegment(file_segment->offset(), file_segment->lock()); + return PriorityIterationResult::REMOVE_AND_CONTINUE; } - return IterationResult::CONTINUE; - }); + return PriorityIterationResult::CONTINUE; + }, lock); if (stash) { /// Remove all access information. stash->records.clear(); - LockedCachePriority(lock, *stash->queue).removeAll(); + stash->queue->removeAll(lock); } } void FileCache::loadMetadata() { auto lock = cache_guard.lock(); - LockedCachePriority queue(lock, *main_priority); UInt64 offset = 0; size_t size = 0; - std::vector>> queue_entries; + std::vector>> queue_entries; /// cache_base_path / key_prefix / key / offset if (!metadata.empty()) @@ -750,7 +752,8 @@ void FileCache::loadMetadata() } size_t total_size = 0; - for (auto key_prefix_it = fs::directory_iterator{cache_base_path}; key_prefix_it != fs::directory_iterator();) + for (auto key_prefix_it = fs::directory_iterator{metadata.getBaseDirectory()}; + key_prefix_it != fs::directory_iterator();) { const fs::path key_prefix_directory = key_prefix_it->path(); key_prefix_it++; @@ -833,18 +836,19 @@ void FileCache::loadMetadata() continue; } - if ((queue.getSizeLimit() == 0 || queue.getSize() + size <= queue.getSizeLimit()) - && (queue.getElementsLimit() == 0 || queue.getElementsCount() + 1 <= queue.getElementsLimit())) + if ((main_priority->getSizeLimit() == 0 || main_priority->getSize(lock) + size <= main_priority->getSizeLimit()) + && (main_priority->getElementsLimit() == 0 || main_priority->getElementsCount(lock) + 1 <= main_priority->getElementsLimit())) { auto file_segment_metadata_it = addFileSegment( *locked_key, offset, size, FileSegment::State::DOWNLOADED, CreateFileSegmentSettings(segment_kind), &lock); - chassert(file_segment_metadata_it->second.getQueueIterator()); - chassert(file_segment_metadata_it->second.size() == size); + chassert(file_segment_metadata_it->second->file_segment->getQueueIterator()); + chassert(file_segment_metadata_it->second->size() == size); total_size += size; queue_entries.emplace_back( - file_segment_metadata_it->second.getQueueIterator(), file_segment_metadata_it->second.file_segment); + file_segment_metadata_it->second->file_segment->getQueueIterator(), + file_segment_metadata_it->second->file_segment); } else { @@ -852,7 +856,7 @@ void FileCache::loadMetadata() log, "Cache capacity changed (max size: {}, used: {}), " "cached file `{}` does not fit in cache anymore (size: {})", - queue.getSizeLimit(), queue.getSize(), key_directory.string(), size); + main_priority->getSizeLimit(), main_priority->getSize(lock), key_directory.string(), size); fs::remove(offset_it->path()); } @@ -860,8 +864,8 @@ void FileCache::loadMetadata() } } - chassert(total_size == queue.getSize()); - chassert(total_size <= queue.getSizeLimit()); + chassert(total_size == main_priority->getSize(lock)); + chassert(total_size <= main_priority->getSizeLimit()); /// Shuffle file_segment_metadatas to have random order in LRUQueue /// as at startup all file_segment_metadatas have the same priority. @@ -875,15 +879,10 @@ void FileCache::loadMetadata() if (file_segment.expired()) continue; - LockedCachePriorityIterator(lock, it).use(); + it->use(lock); } } -LockedKeyMetadataPtr FileCache::lockKeyMetadata(const Key & key, KeyMetadataPtr key_metadata, bool return_null_if_in_cleanup_queue) const -{ - return metadata.lockKeyMetadata(key, key_metadata, return_null_if_in_cleanup_queue); -} - FileCache::~FileCache() { deactivateBackgroundOperations(); @@ -926,10 +925,10 @@ FileSegmentsHolderPtr FileCache::getSnapshot() #endif FileSegments file_segments; - metadata.iterate([&](const LockedKeyMetadata & locked_key) + metadata.iterate([&](const LockedKey & locked_key) { for (const auto & [_, file_segment_metadata] : locked_key) - file_segments.push_back(FileSegment::getSnapshot(file_segment_metadata.file_segment)); + file_segments.push_back(FileSegment::getSnapshot(file_segment_metadata->file_segment)); }); return std::make_unique(std::move(file_segments)); } @@ -939,24 +938,24 @@ FileSegmentsHolderPtr FileCache::getSnapshot(const Key & key) FileSegments file_segments; auto locked_key = metadata.lockKeyMetadata(key, CacheMetadata::KeyNotFoundPolicy::THROW); for (const auto & [_, file_segment_metadata] : *locked_key->getKeyMetadata()) - file_segments.push_back(FileSegment::getSnapshot(file_segment_metadata.file_segment)); + file_segments.push_back(FileSegment::getSnapshot(file_segment_metadata->file_segment)); return std::make_unique(std::move(file_segments)); } FileSegmentsHolderPtr FileCache::dumpQueue() { assertInitialized(); - using QueueEntry = IFileCachePriority::Entry; - using IterationResult = IFileCachePriority::IterationResult; FileSegments file_segments; - LockedCachePriority(cache_guard.lock(), *main_priority).iterate([&](const QueueEntry & entry) + main_priority->iterate([&](const PriorityEntry & entry, LockedKey & locked_key) { - auto locked_metadata = metadata.lockKeyMetadata(entry.key, entry.getKeyMetadata()); - auto * file_segment_metadata = locked_metadata->getByOffset(entry.offset); + auto file_segment_metadata = locked_key.tryGetByOffset(entry.offset); + if (!file_segment_metadata) + return PriorityIterationResult::REMOVE_AND_CONTINUE; + file_segments.push_back(FileSegment::getSnapshot(file_segment_metadata->file_segment)); - return IterationResult::CONTINUE; - }); + return PriorityIterationResult::CONTINUE; + }, cache_guard.lock()); return std::make_unique(std::move(file_segments)); } @@ -973,29 +972,29 @@ std::vector FileCache::tryGetCachePaths(const Key & key) for (const auto & [offset, file_segment_metadata] : *locked_key->getKeyMetadata()) { - if (file_segment_metadata.file_segment->state() == FileSegment::State::DOWNLOADED) - cache_paths.push_back(metadata.getPathInLocalCache(key, offset, file_segment_metadata.file_segment->getKind())); + if (file_segment_metadata->file_segment->state() == FileSegment::State::DOWNLOADED) + cache_paths.push_back(metadata.getPathInLocalCache(key, offset, file_segment_metadata->file_segment->getKind())); } return cache_paths; } size_t FileCache::getUsedCacheSize() const { - return LockedCachePriority(cache_guard.lock(), *main_priority).getSize(); + return main_priority->getSize(cache_guard.lock()); } size_t FileCache::getFileSegmentsNum() const { - return LockedCachePriority(cache_guard.lock(), *main_priority).getElementsCount(); + return main_priority->getElementsCount(cache_guard.lock()); } void FileCache::assertCacheCorrectness() { - metadata.iterate([&](const LockedKeyMetadata & locked_key) + metadata.iterate([&](const LockedKey & locked_key) { for (const auto & [offset, file_segment_metadata] : locked_key) { - locked_key.assertFileSegmentCorrectness(*file_segment_metadata.file_segment); + locked_key.assertFileSegmentCorrectness(*file_segment_metadata->file_segment); } }); } diff --git a/src/Interpreters/Cache/FileCache.h b/src/Interpreters/Cache/FileCache.h index 6490ed8f99ae..4e01751daaf7 100644 --- a/src/Interpreters/Cache/FileCache.h +++ b/src/Interpreters/Cache/FileCache.h @@ -13,7 +13,6 @@ #include #include -#include #include #include #include @@ -21,14 +20,12 @@ #include #include -namespace fs = std::filesystem; - namespace DB { -struct LockedKeyMetadata; -using LockedKeyMetadataPtr = std::shared_ptr; +struct LockedKey; +using LockedKeyPtr = std::shared_ptr; namespace ErrorCodes { @@ -42,6 +39,10 @@ class FileCache : private boost::noncopyable public: using Key = DB::FileCacheKey; using QueryLimit = DB::FileCacheQueryLimit; + using Priority = IFileCachePriority; + using PriorityEntry = IFileCachePriority::Entry; + using PriorityIterator = IFileCachePriority::Iterator; + using PriorityIterationResult = IFileCachePriority::IterationResult; FileCache(const String & cache_base_path_, const FileCacheSettings & cache_settings_); @@ -49,10 +50,14 @@ class FileCache : private boost::noncopyable void initialize(); - const String & getBasePath() const { return cache_base_path; } + const String & getBasePath() const; static Key createKeyForPath(const String & path); + String getPathInLocalCache(const Key & key, size_t offset, FileSegmentKind segment_kind) const; + + String getPathInLocalCache(const Key & key) const; + /** * Given an `offset` and `size` representing [offset, offset + size) bytes interval, * return list of cached non-overlapping non-empty @@ -85,10 +90,6 @@ class FileCache : private boost::noncopyable /// Remove files by `key`. Will not remove files which are used at the moment. void removeAllReleasable(); - String getPathInLocalCache(const Key & key, size_t offset, FileSegmentKind segment_kind) const; - - String getPathInLocalCache(const Key & key) const; - std::vector tryGetCachePaths(const Key & key); size_t getUsedCacheSize() const; @@ -97,7 +98,7 @@ class FileCache : private boost::noncopyable size_t getMaxFileSegmentSize() const { return max_file_segment_size; } - bool tryReserve(const FileSegment & file_segment, size_t size); + bool tryReserve(FileSegment & file_segment, size_t size); FileSegmentsHolderPtr getSnapshot(); @@ -105,16 +106,10 @@ class FileCache : private boost::noncopyable FileSegmentsHolderPtr dumpQueue(); - bool isInitialized() const { return is_initialized; } - - CacheGuard::Lock cacheLock() { return cache_guard.lock(); } - void cleanup(); void deactivateBackgroundOperations(); - LockedKeyMetadataPtr lockKeyMetadata(const Key & key, KeyMetadataPtr key_metadata, bool return_null_if_in_cleanup_queue = true) const; - /// For per query cache limit. struct QueryContextHolder : private boost::noncopyable { @@ -131,10 +126,11 @@ class FileCache : private boost::noncopyable using QueryContextHolderPtr = std::unique_ptr; QueryContextHolderPtr getQueryContextHolder(const String & query_id, const ReadSettings & settings); + CacheGuard::Lock lockCache() { return cache_guard.lock(); } + private: using KeyAndOffset = FileCacheKeyAndOffset; - const String cache_base_path; const size_t max_file_segment_size; const bool allow_persistent_files; const size_t bypass_cache_threshold = 0; @@ -162,65 +158,82 @@ class FileCache : private boost::noncopyable const size_t hits_threshold; FileCachePriorityPtr queue; - using Records = std::unordered_map; + using Records = std::unordered_map; Records records; }; + /** + * A HitsCountStash allows to cache certain data only after it reached + * a certain hit rate, e.g. if hit rate it 5, then data is cached on 6th cache hit. + */ mutable std::unique_ptr stash; - + /** + * A QueryLimit allows to control cache write limit per query. + * E.g. if a query needs n bytes from cache, but it has only k bytes, where 0 <= k <= n + * then allowed loaded cache size is std::min(n - k, max_query_cache_size). + */ FileCacheQueryLimitPtr query_limit; + /** + * A background cleanup task. + * Clears removed cache entries from metadata. + */ + BackgroundSchedulePool::TaskHolder cleanup_task; void assertInitialized() const; + void assertCacheCorrectness(); + void loadMetadata(); - FileSegments getImpl(const LockedKeyMetadata & locked_key, const FileSegment::Range & range); + FileSegments getImpl(const LockedKey & locked_key, const FileSegment::Range & range) const; - FileSegments splitRangeInfoFileSegments( - LockedKeyMetadata & locked_key, + FileSegments splitRangeIntoFileSegments( + LockedKey & locked_key, size_t offset, size_t size, FileSegment::State state, const CreateFileSegmentSettings & create_settings); void fillHolesWithEmptyFileSegments( - LockedKeyMetadata & locked_key, + LockedKey & locked_key, FileSegments & file_segments, const FileSegment::Range & range, bool fill_with_detached_file_segments, const CreateFileSegmentSettings & settings); KeyMetadata::iterator addFileSegment( - LockedKeyMetadata & locked_key, + LockedKey & locked_key, size_t offset, size_t size, FileSegment::State state, const CreateFileSegmentSettings & create_settings, const CacheGuard::Lock *); - static void removeFileSegment( - LockedKeyMetadata & locked_key, - FileSegmentPtr file_segment, - const CacheGuard::Lock &); - - bool tryReserveImpl( - const FileSegment & file_segment, - size_t size, - IFileCachePriority & priority_queue, - QueryLimit::LockedQueryContext * query_context, - const CacheGuard::Lock &); + void cleanupThreadFunc(); - struct IterateAndLockResult + class EvictionCandidates : public std::vector { - IFileCachePriority::IterationResult iteration_result; - bool lock_key = false; - }; + public: + explicit EvictionCandidates(KeyMetadataPtr key_metadata_) : key_metadata(key_metadata_) {} - void assertCacheCorrectness(); + KeyMetadata & getMetadata() { return *key_metadata; } - BackgroundSchedulePool::TaskHolder cleanup_task; + void add(FileSegmentMetadataPtr candidate) + { + candidate->removal_candidate = true; + push_back(candidate); + } + + ~EvictionCandidates() + { + for (const auto & candidate : *this) + candidate->removal_candidate = false; + } + + private: + KeyMetadataPtr key_metadata; +}; - void cleanupThreadFunc(); }; } diff --git a/src/Interpreters/Cache/FileCacheKey.h b/src/Interpreters/Cache/FileCacheKey.h index e4037b6f9aff..bab8359732cd 100644 --- a/src/Interpreters/Cache/FileCacheKey.h +++ b/src/Interpreters/Cache/FileCacheKey.h @@ -1,5 +1,6 @@ #pragma once #include +#include namespace DB { @@ -42,3 +43,13 @@ struct hash }; } + +template <> +struct fmt::formatter : fmt::formatter +{ + template + auto format(const DB::FileCacheKey & key, FormatCtx & ctx) const + { + return fmt::formatter::format(key.toString(), ctx); + } +}; diff --git a/src/Interpreters/Cache/FileCache_fwd.h b/src/Interpreters/Cache/FileCache_fwd.h index 47166ffd602c..afd8d86074ef 100644 --- a/src/Interpreters/Cache/FileCache_fwd.h +++ b/src/Interpreters/Cache/FileCache_fwd.h @@ -8,7 +8,7 @@ static constexpr int FILECACHE_DEFAULT_MAX_FILE_SEGMENT_SIZE = 100 * 1024 * 1024 static constexpr int FILECACHE_DEFAULT_MAX_ELEMENTS = 1024 * 1024; static constexpr int FILECACHE_DEFAULT_HITS_THRESHOLD = 0; static constexpr size_t FILECACHE_BYPASS_THRESHOLD = 256 * 1024 * 1024; -static constexpr size_t FILECACHE_DELAYED_CLEANUP_INTERVAL_MS = 1000 * 60 * 4; /// 4 min +static constexpr size_t FILECACHE_DELAYED_CLEANUP_INTERVAL_MS = 1000 * 60; /// 1 min class FileCache; using FileCachePtr = std::shared_ptr; diff --git a/src/Interpreters/Cache/FileSegment.cpp b/src/Interpreters/Cache/FileSegment.cpp index 93bb76281b77..16220f692dc6 100644 --- a/src/Interpreters/Cache/FileSegment.cpp +++ b/src/Interpreters/Cache/FileSegment.cpp @@ -25,26 +25,27 @@ String toString(FileSegmentKind kind) } FileSegment::FileSegment( - const Key & key_, size_t offset_, size_t size_, - std::weak_ptr key_metadata_, - IFileCachePriority::Iterator queue_iterator_, - FileCache * cache_, + const Key & key_, + size_t offset_, + size_t size_, State download_state_, - const CreateFileSegmentSettings & settings) - : segment_range(offset_, offset_ + size_ - 1) - , queue_iterator(queue_iterator_) + const CreateFileSegmentSettings & settings, + FileCache * cache_, + std::weak_ptr key_metadata_, + CachePriorityIterator queue_iterator_) + : file_key(key_) + , segment_range(offset_, offset_ + size_ - 1) + , segment_kind(settings.kind) + , is_unbound(settings.unbounded) , download_state(download_state_) , key_metadata(key_metadata_) - , file_key(key_) - , file_path(cache_->getPathInLocalCache(key(), offset(), settings.kind)) + , queue_iterator(queue_iterator_) , cache(cache_) #ifndef NDEBUG , log(&Poco::Logger::get(fmt::format("FileSegment({}) : {}", key_.toString(), range().toString()))) #else , log(&Poco::Logger::get("FileSegment")) #endif - , segment_kind(settings.kind) - , is_unbound(settings.unbounded) { /// On creation, file segment state can be EMPTY, DOWNLOADED, DOWNLOADING. switch (download_state) @@ -60,7 +61,8 @@ FileSegment::FileSegment( case (State::DOWNLOADED): { reserved_size = downloaded_size = size_; - chassert(std::filesystem::file_size(file_path) == size_); + chassert(std::filesystem::file_size(getPathInLocalCache()) == size_); + chassert(queue_iterator); break; } case (State::DETACHED): @@ -82,6 +84,11 @@ FileSegment::State FileSegment::state() const return download_state; } +String FileSegment::getPathInLocalCache() const +{ + return getKeyMetadata()->getFileSegmentPath(*this); +} + void FileSegment::setDownloadState(State state, const FileSegmentGuard::Lock &) { LOG_TEST(log, "Updated state from {} to {}", stateToString(download_state), stateToString(state)); @@ -94,6 +101,20 @@ size_t FileSegment::getReservedSize() const return reserved_size; } +FileSegment::CachePriorityIterator FileSegment::getQueueIterator() const +{ + auto lock = segment_guard.lock(); + return queue_iterator; +} + +void FileSegment::setQueueIterator(CachePriorityIterator iterator) +{ + auto lock = segment_guard.lock(); + if (queue_iterator) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Queue iterator cannot be set twice"); + queue_iterator = iterator; +} + size_t FileSegment::getFirstNonDownloadedOffset(bool sync) const { return range().left + getDownloadedSize(sync); @@ -325,7 +346,7 @@ void FileSegment::write(const char * from, size_t size, size_t offset) "Cache writer was finalized (downloaded size: {}, state: {})", current_downloaded_size, stateToString(download_state)); - cache_writer = std::make_unique(file_path); + cache_writer = std::make_unique(getPathInLocalCache()); } } @@ -339,7 +360,7 @@ void FileSegment::write(const char * from, size_t size, size_t offset) downloaded_size += size; - chassert(std::filesystem::file_size(file_path) == downloaded_size); + chassert(std::filesystem::file_size(getPathInLocalCache()) == downloaded_size); } catch (Exception & e) { @@ -378,9 +399,6 @@ FileSegment::State FileSegment::wait(size_t offset) { return download_state != State::DOWNLOADING || offset < getCurrentWriteOffset(true); }); - /// If we exited by timeout, it means we are missing notify somewhere. - /// Make sure this case is caught by stress tests. - chassert(ok); } return download_state; @@ -391,7 +409,7 @@ KeyMetadataPtr FileSegment::getKeyMetadata() const auto metadata = key_metadata.lock(); if (metadata) return metadata; - throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot lock key, key metadata is not set"); + throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot lock key, key metadata is not set ({})", stateToString(download_state)); } KeyMetadataPtr FileSegment::tryGetKeyMetadata() const @@ -402,7 +420,7 @@ KeyMetadataPtr FileSegment::tryGetKeyMetadata() const return nullptr; } -LockedKeyMetadataPtr FileSegment::lockKeyMetadata(bool assert_exists) const +LockedKeyPtr FileSegment::lockKeyMetadata(bool assert_exists) const { KeyMetadataPtr metadata; if (assert_exists) @@ -413,7 +431,7 @@ LockedKeyMetadataPtr FileSegment::lockKeyMetadata(bool assert_exists) const if (!metadata) return nullptr; } - return cache->lockKeyMetadata(key(), metadata); + return metadata->lock(); } bool FileSegment::reserve(size_t size_to_reserve) @@ -494,7 +512,7 @@ void FileSegment::setDownloadedUnlocked([[maybe_unused]] const FileSegmentGuard: } chassert(getDownloadedSize(false) > 0); - chassert(std::filesystem::file_size(file_path) > 0); + chassert(std::filesystem::file_size(getPathInLocalCache()) > 0); } void FileSegment::setDownloadFailedUnlocked(const FileSegmentGuard::Lock & lock) @@ -544,12 +562,24 @@ void FileSegment::setBroken() void FileSegment::complete() { - auto cache_lock = cache->cacheLock(); - auto locked_key = lockKeyMetadata(); - return completeUnlocked(*locked_key, cache_lock); + if (isCompleted()) + return; + + auto locked_key = lockKeyMetadata(false); + if (locked_key) + { + completeUnlocked(*locked_key); + return; + } + + /// If we failed to lock a key, it must be in detached state. + if (isDetached()) + return; + + throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot complete file segment: {}", getInfoForLog()); } -void FileSegment::completeUnlocked(LockedKeyMetadata & locked_key, const CacheGuard::Lock & cache_lock) +void FileSegment::completeUnlocked(LockedKey & locked_key) { auto segment_lock = segment_guard.lock(); @@ -590,7 +620,7 @@ void FileSegment::completeUnlocked(LockedKeyMetadata & locked_key, const CacheGu LOG_TEST(log, "Removing temporary file segment: {}", getInfoForLogUnlocked(segment_lock)); detach(segment_lock, locked_key); setDownloadState(State::DETACHED, segment_lock); - locked_key.removeFileSegment(offset(), segment_lock, cache_lock); + locked_key.removeFileSegment(offset(), segment_lock); return; } @@ -599,7 +629,7 @@ void FileSegment::completeUnlocked(LockedKeyMetadata & locked_key, const CacheGu case State::DOWNLOADED: { chassert(current_downloaded_size == range().size()); - chassert(current_downloaded_size == std::filesystem::file_size(file_path)); + chassert(current_downloaded_size == std::filesystem::file_size(getPathInLocalCache())); chassert(!cache_writer); break; } @@ -619,7 +649,7 @@ void FileSegment::completeUnlocked(LockedKeyMetadata & locked_key, const CacheGu if (current_downloaded_size == 0) { LOG_TEST(log, "Remove file segment {} (nothing downloaded)", range().toString()); - locked_key.removeFileSegment(offset(), segment_lock, cache_lock); + locked_key.removeFileSegment(offset(), segment_lock); } else { @@ -636,7 +666,7 @@ void FileSegment::completeUnlocked(LockedKeyMetadata & locked_key, const CacheGu /// but current file segment should remain PARRTIALLY_DOWNLOADED_NO_CONTINUATION and with detached state, /// because otherwise an invariant that getOrSet() returns a contiguous range of file segments will be broken /// (this will be crucial for other file segment holder, not for current one). - locked_key.shrinkFileSegmentToDownloadedSize(offset(), segment_lock, cache_lock); + locked_key.shrinkFileSegmentToDownloadedSize(offset(), segment_lock); /// We mark current file segment with state DETACHED, even though the data is still in cache /// (but a separate file segment) because is_last_holder is satisfied, so it does not matter. @@ -708,18 +738,18 @@ bool FileSegment::assertCorrectness() const auto current_downloader = getDownloaderUnlocked(lock); chassert(current_downloader.empty() == (download_state != FileSegment::State::DOWNLOADING)); chassert(!current_downloader.empty() == (download_state == FileSegment::State::DOWNLOADING)); - chassert(download_state != FileSegment::State::DOWNLOADED || std::filesystem::file_size(file_path) > 0); + chassert(download_state != FileSegment::State::DOWNLOADED || std::filesystem::file_size(getPathInLocalCache()) > 0); chassert(reserved_size == 0 || queue_iterator); if (queue_iterator) { - const auto & entry = *queue_iterator; + const auto & entry = queue_iterator->getEntry(); if (isCompleted(false)) - chassert(reserved_size == entry.getEntry().size); + chassert(reserved_size == entry.size); else /// We cannot check == here because reserved_size is not /// guarded by any mutex, it is just an atomic. - chassert(reserved_size <= entry.getEntry().size); + chassert(reserved_size <= entry.size); } return true; } @@ -753,9 +783,6 @@ FileSegmentPtr FileSegment::getSnapshot(const FileSegmentPtr & file_segment) file_segment->key(), file_segment->offset(), file_segment->range().size(), - std::weak_ptr(), - nullptr, - file_segment->cache, State::DETACHED, CreateFileSegmentSettings(file_segment->getKind())); @@ -783,6 +810,9 @@ bool FileSegment::isCompleted(bool sync) const if (sync) { + if (is_completed_state()) + return true; + auto lock = segment_guard.lock(); return is_completed_state(); } @@ -790,7 +820,7 @@ bool FileSegment::isCompleted(bool sync) const return is_completed_state(); } -void FileSegment::detach(const FileSegmentGuard::Lock & lock, const LockedKeyMetadata &) +void FileSegment::detach(const FileSegmentGuard::Lock & lock, const LockedKey &) { if (download_state == State::DETACHED) return; @@ -807,27 +837,19 @@ void FileSegment::detachAssumeStateFinalized(const FileSegmentGuard::Lock & lock LOG_TEST(log, "Detached file segment: {}", getInfoForLogUnlocked(lock)); } -FileSegments::iterator FileSegmentsHolder::completeAndPopFrontImpl() +void FileSegment::use() { - auto & file_segment = front(); - if (file_segment.isDetached()) - return file_segments.erase(file_segments.begin()); - - auto cache_lock = file_segment.cache->cacheLock(); - - auto queue_iter = file_segment.getQueueIterator(); - if (queue_iter) - LockedCachePriorityIterator(cache_lock, queue_iter).use(); - - if (!file_segment.isCompleted()) + auto it = getQueueIterator(); + if (it && cache) { - /// File segment pointer must be reset right after calling complete() and - /// under the same mutex, because complete() checks for segment pointers. - auto locked_key = file_segment.lockKeyMetadata(/* assert_exists */false); - if (locked_key) - file_segment.completeUnlocked(*locked_key, cache_lock); + auto cache_lock = cache->lockCache(); + it->use(cache_lock); } +} +FileSegments::iterator FileSegmentsHolder::completeAndPopFrontImpl() +{ + front().complete(); return file_segments.erase(file_segments.begin()); } diff --git a/src/Interpreters/Cache/FileSegment.h b/src/Interpreters/Cache/FileSegment.h index c5dd9e1c4446..3e0406e6c0a0 100644 --- a/src/Interpreters/Cache/FileSegment.h +++ b/src/Interpreters/Cache/FileSegment.h @@ -32,8 +32,8 @@ class FileSegment; using FileSegmentPtr = std::shared_ptr; using FileSegments = std::list; struct FileSegmentMetadata; -struct LockedKeyMetadata; -using LockedKeyMetadataPtr = std::shared_ptr; +struct LockedKey; +using LockedKeyPtr = std::shared_ptr; struct KeyMetadata; using KeyMetadataPtr = std::shared_ptr; @@ -73,13 +73,7 @@ struct CreateFileSegmentSettings class FileSegment : private boost::noncopyable, public std::enable_shared_from_this { - -friend class FileCache; -friend struct FileSegmentsHolder; -friend class FileSegmentRangeWriter; -friend class StorageSystemFilesystemCache; -friend struct LockedKeyMetadata; -friend struct FileSegmentMetadata; +friend struct LockedKey; public: using Key = FileCacheKey; @@ -87,6 +81,7 @@ friend struct FileSegmentMetadata; using LocalCacheWriterPtr = std::unique_ptr; using Downloader = std::string; using DownloaderId = std::string; + using CachePriorityIterator = IFileCachePriority::Iterator; enum class State { @@ -126,11 +121,11 @@ friend struct FileSegmentMetadata; const Key & key_, size_t offset_, size_t size_, - std::weak_ptr key_metadata, - IFileCachePriority::Iterator queue_iterator_, - FileCache * cache_, State download_state_, - const CreateFileSegmentSettings & create_settings); + const CreateFileSegmentSettings & create_settings = {}, + FileCache * cache_ = nullptr, + std::weak_ptr key_metadata_ = std::weak_ptr(), + CachePriorityIterator queue_iterator_ = CachePriorityIterator{}); ~FileSegment() = default; @@ -173,7 +168,7 @@ friend struct FileSegmentMetadata; using UniqueId = std::pair; UniqueId getUniqueId() const { return std::pair(key(), offset()); } - String getPathInLocalCache() const { return file_path; } + String getPathInLocalCache() const; /** * ========== Methods for _any_ file segment's owner ======================== @@ -216,7 +211,7 @@ friend struct FileSegmentMetadata; /// 2. Detached file segment can still be hold by some cache users, but it's state became /// immutable at the point it was detached, any non-const / stateful method will throw an /// exception. - void detach(const FileSegmentGuard::Lock &, const LockedKeyMetadata &); + void detach(const FileSegmentGuard::Lock &, const LockedKey &); static FileSegmentPtr getSnapshot(const FileSegmentPtr & file_segment); @@ -267,7 +262,13 @@ friend struct FileSegmentMetadata; size_t getReservedSize() const; - IFileCachePriority::Iterator & getQueueIterator() const { return queue_iterator; } + CachePriorityIterator getQueueIterator() const; + + void setQueueIterator(CachePriorityIterator iterator); + + KeyMetadataPtr getKeyMetadata() const; + + void use(); private: String getInfoForLogUnlocked(const FileSegmentGuard::Lock &) const; @@ -289,30 +290,29 @@ friend struct FileSegmentMetadata; void assertNotDetachedUnlocked(const FileSegmentGuard::Lock &) const; void assertIsDownloaderUnlocked(const std::string & operation, const FileSegmentGuard::Lock &) const; - LockedKeyMetadataPtr lockKeyMetadata(bool assert_exists = true) const; - KeyMetadataPtr getKeyMetadata() const; + LockedKeyPtr lockKeyMetadata(bool assert_exists = true) const; KeyMetadataPtr tryGetKeyMetadata() const; /// completeWithoutStateUnlocked() is called from destructor of FileSegmentsHolder. /// Function might check if the caller of the method /// is the last alive holder of the segment. Therefore, completion and destruction /// of the file segment pointer must be done under the same cache mutex. - void completeUnlocked(LockedKeyMetadata & locked_key, const CacheGuard::Lock &); + void completeUnlocked(LockedKey & locked_key); void completePartAndResetDownloaderUnlocked(const FileSegmentGuard::Lock & segment_lock); bool isDownloaderUnlocked(const FileSegmentGuard::Lock & segment_lock) const; void wrapWithCacheInfo(Exception & e, const String & message, const FileSegmentGuard::Lock & segment_lock) const; + Key file_key; Range segment_range; - - /// Iterator is put here on first reservation attempt, if successful. - mutable IFileCachePriority::Iterator queue_iterator; + const FileSegmentKind segment_kind; + /// Size of the segment is not known until it is downloaded and + /// can be bigger than max_file_segment_size. + const bool is_unbound = false; std::atomic download_state; - - /// The one who prepares the download - DownloaderId downloader_id; + DownloaderId downloader_id; /// The one who prepares the download RemoteFileReaderPtr remote_file_reader; LocalCacheWriterPtr cache_writer; @@ -324,25 +324,19 @@ friend struct FileSegmentMetadata; mutable FileSegmentGuard segment_guard; std::weak_ptr key_metadata; - std::condition_variable cv; - - Key file_key; - const std::string file_path; + mutable CachePriorityIterator queue_iterator; /// Iterator is put here on first reservation attempt, if successful. FileCache * cache; + std::condition_variable cv; Poco::Logger * log; std::atomic hits_count = 0; /// cache hits. std::atomic ref_count = 0; /// Used for getting snapshot state - const FileSegmentKind segment_kind; - - /// Size of the segment is not known until it is downloaded and can be bigger than max_file_segment_size. - const bool is_unbound = false; - CurrentMetrics::Increment metric_increment{CurrentMetrics::CacheFileSegments}; }; + struct FileSegmentsHolder : private boost::noncopyable { FileSegmentsHolder() = default; diff --git a/src/Interpreters/Cache/IFileCachePriority.h b/src/Interpreters/Cache/IFileCachePriority.h index d0c369f88530..fb0ac1521487 100644 --- a/src/Interpreters/Cache/IFileCachePriority.h +++ b/src/Interpreters/Cache/IFileCachePriority.h @@ -14,6 +14,7 @@ class IFileCachePriority; using FileCachePriorityPtr = std::unique_ptr; struct KeyMetadata; using KeyMetadataPtr = std::shared_ptr; +struct LockedKey; namespace ErrorCodes { @@ -21,34 +22,25 @@ namespace ErrorCodes } /// IFileCachePriority is used to maintain the priority of cached data. -class IFileCachePriority +class IFileCachePriority : private boost::noncopyable { -friend class LockedCachePriority; public: using Key = FileCacheKey; using KeyAndOffset = FileCacheKeyAndOffset; struct Entry { - Entry(const Key & key_, size_t offset_, size_t size_, std::weak_ptr key_metadata_) - : key(key_) , offset(offset_) , size(size_) , key_metadata(key_metadata_) {} - - KeyMetadataPtr getKeyMetadata() const - { - auto result = key_metadata.lock(); - if (!result) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Metadata expired"); - return result; - } - - Key key; - size_t offset; + Entry(const Key & key_, size_t offset_, size_t size_, KeyMetadataPtr key_metadata_) + : key(key_), offset(offset_), size(size_), key_metadata(key_metadata_) {} + + const Key key; + const size_t offset; size_t size; size_t hits = 0; /// In fact, it is guaranteed that the lifetime of key metadata is longer /// than Entry, but it is made as weak_ptr to avoid cycle in shared pointer /// references (because entry actually lies in key metadata). - const std::weak_ptr key_metadata; + const KeyMetadataPtr key_metadata; }; /// Provides an iterator to traverse the cache priority. Under normal circumstances, @@ -57,20 +49,18 @@ friend class LockedCachePriority; /// can only traverse the records in the low priority queue. class IIterator { - friend class LockedCachePriorityIterator; public: virtual ~IIterator() = default; virtual const Entry & getEntry() const = 0; - protected: virtual Entry & getEntry() = 0; - virtual size_t use() = 0; + virtual size_t use(const CacheGuard::Lock &) = 0; - virtual void incrementSize(ssize_t) = 0; + virtual void updateSize(ssize_t size) = 0; - virtual std::shared_ptr remove() = 0; + virtual std::shared_ptr remove(const CacheGuard::Lock &) = 0; }; using Iterator = std::shared_ptr; @@ -82,7 +72,7 @@ friend class LockedCachePriority; CONTINUE, REMOVE_AND_CONTINUE, }; - using IterateFunc = std::function; + using IterateFunc = std::function; IFileCachePriority(size_t max_size_, size_t max_elements_) : max_size(max_size_), max_elements(max_elements_) {} @@ -92,21 +82,23 @@ friend class LockedCachePriority; size_t getSizeLimit() const { return max_size; } -protected: - const size_t max_size = 0; - const size_t max_elements = 0; + virtual size_t getSize(const CacheGuard::Lock &) const = 0; - virtual size_t getSize() const = 0; + virtual size_t getElementsCount(const CacheGuard::Lock &) const = 0; - virtual size_t getElementsCount() const = 0; + virtual Iterator add( + const Key & key, size_t offset, size_t size, + KeyMetadataPtr key_metadata, const CacheGuard::Lock &) = 0; - virtual Iterator add(const Key & key, size_t offset, size_t size, std::weak_ptr key_metadata) = 0; + virtual void pop(const CacheGuard::Lock &) = 0; - virtual void pop() = 0; + virtual void removeAll(const CacheGuard::Lock &) = 0; - virtual void removeAll() = 0; + virtual void iterate(IterateFunc && func, const CacheGuard::Lock &) = 0; - virtual void iterate(IterateFunc && func) = 0; +private: + const size_t max_size = 0; + const size_t max_elements = 0; }; }; diff --git a/src/Interpreters/Cache/LRUFileCachePriority.cpp b/src/Interpreters/Cache/LRUFileCachePriority.cpp index 050f5050c292..0e6ccee8efd3 100644 --- a/src/Interpreters/Cache/LRUFileCachePriority.cpp +++ b/src/Interpreters/Cache/LRUFileCachePriority.cpp @@ -1,6 +1,8 @@ #include #include #include +#include + namespace CurrentMetrics { @@ -17,7 +19,11 @@ namespace ErrorCodes } IFileCachePriority::Iterator LRUFileCachePriority::add( - const Key & key, size_t offset, size_t size, std::weak_ptr key_metadata) + const Key & key, + size_t offset, + size_t size, + KeyMetadataPtr key_metadata, + const CacheGuard::Lock &) { #ifndef NDEBUG for (const auto & entry : queue) @@ -26,7 +32,7 @@ IFileCachePriority::Iterator LRUFileCachePriority::add( throw Exception( ErrorCodes::LOGICAL_ERROR, "Attempt to add duplicate queue entry to queue. (Key: {}, offset: {}, size: {})", - entry.key.toString(), entry.offset, entry.size); + entry.key, entry.offset, entry.size); } #endif @@ -36,7 +42,7 @@ IFileCachePriority::Iterator LRUFileCachePriority::add( throw Exception( ErrorCodes::LOGICAL_ERROR, "Not enough space to add {}:{} with size {}: current size: {}/{}", - key.toString(), offset, size, current_size, getSizeLimit()); + key, offset, size, current_size, getSizeLimit()); } current_size += size; @@ -46,12 +52,12 @@ IFileCachePriority::Iterator LRUFileCachePriority::add( CurrentMetrics::add(CurrentMetrics::FilesystemCacheSize, size); CurrentMetrics::add(CurrentMetrics::FilesystemCacheElements); - LOG_TEST(log, "Added entry into LRU queue, key: {}, offset: {}", key.toString(), offset); + LOG_TEST(log, "Added entry into LRU queue, key: {}, offset: {}", key, offset); return std::make_shared(this, iter); } -void LRUFileCachePriority::removeAll() +void LRUFileCachePriority::removeAll(const CacheGuard::Lock &) { CurrentMetrics::sub(CurrentMetrics::FilesystemCacheSize, current_size); CurrentMetrics::sub(CurrentMetrics::FilesystemCacheElements, queue.size()); @@ -62,7 +68,7 @@ void LRUFileCachePriority::removeAll() current_size = 0; } -void LRUFileCachePriority::pop() +void LRUFileCachePriority::pop(const CacheGuard::Lock &) { remove(queue.begin()); } @@ -74,7 +80,7 @@ LRUFileCachePriority::LRUQueueIterator LRUFileCachePriority::remove(LRUQueueIter CurrentMetrics::sub(CurrentMetrics::FilesystemCacheSize, it->size); CurrentMetrics::sub(CurrentMetrics::FilesystemCacheElements); - LOG_TEST(log, "Removed entry from LRU queue, key: {}, offset: {}", it->key.toString(), it->offset); + LOG_TEST(log, "Removed entry from LRU queue, key: {}, offset: {}", it->key, it->offset); return queue.erase(it); } @@ -84,11 +90,18 @@ LRUFileCachePriority::LRUFileCacheIterator::LRUFileCacheIterator( { } -void LRUFileCachePriority::iterate(IterateFunc && func) +void LRUFileCachePriority::iterate(IterateFunc && func, const CacheGuard::Lock &) { for (auto it = queue.begin(); it != queue.end();) { - auto result = func(*it); + auto locked_key = it->key_metadata->lock(); + if (locked_key->getKeyState() != KeyMetadata::KeyState::ACTIVE) + { + it = remove(it); + continue; + } + + auto result = func(*it, *locked_key); switch (result) { case IterationResult::BREAK: @@ -109,19 +122,22 @@ void LRUFileCachePriority::iterate(IterateFunc && func) } } -LRUFileCachePriority::Iterator LRUFileCachePriority::LRUFileCacheIterator::remove() +LRUFileCachePriority::Iterator LRUFileCachePriority::LRUFileCacheIterator::remove(const CacheGuard::Lock &) { return std::make_shared(cache_priority, cache_priority->remove(queue_iter)); } -void LRUFileCachePriority::LRUFileCacheIterator::incrementSize(ssize_t size) +void LRUFileCachePriority::LRUFileCacheIterator::updateSize(ssize_t size) { cache_priority->current_size += size; - CurrentMetrics::add(CurrentMetrics::FilesystemCacheSize, size); + if (size > 0) + CurrentMetrics::add(CurrentMetrics::FilesystemCacheSize, size); + else + CurrentMetrics::sub(CurrentMetrics::FilesystemCacheSize, size); queue_iter->size += size; } -size_t LRUFileCachePriority::LRUFileCacheIterator::use() +size_t LRUFileCachePriority::LRUFileCacheIterator::use(const CacheGuard::Lock &) { cache_priority->queue.splice(cache_priority->queue.end(), cache_priority->queue, queue_iter); return ++queue_iter->hits; diff --git a/src/Interpreters/Cache/LRUFileCachePriority.h b/src/Interpreters/Cache/LRUFileCachePriority.h index 469238d10ab8..af50d00fcb5e 100644 --- a/src/Interpreters/Cache/LRUFileCachePriority.h +++ b/src/Interpreters/Cache/LRUFileCachePriority.h @@ -20,17 +20,17 @@ class LRUFileCachePriority : public IFileCachePriority public: LRUFileCachePriority(size_t max_size_, size_t max_elements_) : IFileCachePriority(max_size_, max_elements_) {} - size_t getSize() const override { return current_size; } + size_t getSize(const CacheGuard::Lock &) const override { return current_size; } - size_t getElementsCount() const override { return queue.size(); } + size_t getElementsCount(const CacheGuard::Lock &) const override { return queue.size(); } - Iterator add(const Key & key, size_t offset, size_t size, std::weak_ptr key_metadata) override; + Iterator add(const Key & key, size_t offset, size_t size, KeyMetadataPtr key_metadata, const CacheGuard::Lock &) override; - void pop() override; + void pop(const CacheGuard::Lock &) override; - void removeAll() override; + void removeAll(const CacheGuard::Lock &) override; - void iterate(IterateFunc && func) override; + void iterate(IterateFunc && func, const CacheGuard::Lock &) override; private: LRUQueue queue; @@ -50,14 +50,13 @@ class LRUFileCachePriority::LRUFileCacheIterator : public IFileCachePriority::II const Entry & getEntry() const override { return *queue_iter; } -protected: Entry & getEntry() override { return *queue_iter; } - size_t use() override; + size_t use(const CacheGuard::Lock &) override; - Iterator remove() override; + Iterator remove(const CacheGuard::Lock &) override; - void incrementSize(ssize_t size) override; + void updateSize(ssize_t size) override; private: LRUFileCachePriority * cache_priority; diff --git a/src/Interpreters/Cache/LockedFileCachePriority.h b/src/Interpreters/Cache/LockedFileCachePriority.h deleted file mode 100644 index 3cc6e2165cbf..000000000000 --- a/src/Interpreters/Cache/LockedFileCachePriority.h +++ /dev/null @@ -1,60 +0,0 @@ -#pragma once -#include - - -namespace DB -{ - -class LockedCachePriority -{ -public: - LockedCachePriority(const CacheGuard::Lock & lock_, IFileCachePriority & priority_queue_) - : lock(lock_), queue(priority_queue_) {} - - size_t getElementsLimit() const { return queue.max_elements; } - - size_t getSizeLimit() const { return queue.max_size; } - - size_t getSize() const { return queue.getSize(); } - - size_t getElementsCount() const { return queue.getElementsCount(); } - - IFileCachePriority::Iterator add(const FileCacheKey & key, size_t offset, size_t size, std::weak_ptr key_metadata) - { - return queue.add(key, offset, size, key_metadata); - } - - void pop() { queue.pop(); } - - void removeAll() { queue.removeAll(); } - - void iterate(IFileCachePriority::IterateFunc && func) { queue.iterate(std::move(func)); } - -private: - [[maybe_unused]] const CacheGuard::Lock & lock; - IFileCachePriority & queue; -}; - -class LockedCachePriorityIterator -{ -public: - LockedCachePriorityIterator(const CacheGuard::Lock & lock_, IFileCachePriority::Iterator & iterator_) - : lock(lock_), iterator(iterator_) {} - - IFileCachePriority::Entry & getEntry() { return iterator->getEntry(); } - const IFileCachePriority::Entry & getEntry() const { return iterator->getEntry(); } - - size_t use() { return iterator->use(); } - - void incrementSize(ssize_t size) { return iterator->incrementSize(size); } - - IFileCachePriority::Iterator remove() { return iterator->remove(); } - -private: - [[maybe_unused]] const CacheGuard::Lock & lock; - IFileCachePriority::Iterator & iterator; -}; - -using FileCachePriorityPtr = std::unique_ptr; - -} diff --git a/src/Interpreters/Cache/Metadata.cpp b/src/Interpreters/Cache/Metadata.cpp index c092d996f59b..926c6de5d951 100644 --- a/src/Interpreters/Cache/Metadata.cpp +++ b/src/Interpreters/Cache/Metadata.cpp @@ -1,7 +1,6 @@ #include #include #include -#include #include #include @@ -15,23 +14,14 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; } -FileSegmentMetadata::FileSegmentMetadata( - FileSegmentPtr file_segment_, LockedKeyMetadata & locked_key, LockedCachePriority * locked_queue) - : file_segment(file_segment_) +FileSegmentMetadata::FileSegmentMetadata(FileSegmentPtr && file_segment_) + : file_segment(std::move(file_segment_)) { switch (file_segment->state()) { case FileSegment::State::DOWNLOADED: { - if (!locked_queue) - { - throw Exception( - ErrorCodes::LOGICAL_ERROR, - "Adding file segment with state DOWNLOADED requires locked queue lock"); - } - file_segment->getQueueIterator() = locked_queue->add( - file_segment->key(), file_segment->offset(), file_segment->range().size(), locked_key.getKeyMetadata()); - + chassert(file_segment->getQueueIterator()); break; } case FileSegment::State::EMPTY: @@ -52,12 +42,42 @@ size_t FileSegmentMetadata::size() const return file_segment->getReservedSize(); } -IFileCachePriority::Iterator & FileSegmentMetadata::getQueueIterator() const +KeyMetadata::KeyMetadata( + const Key & key_, + const std::string & key_path_, + CleanupQueue & cleanup_queue_, + bool created_base_directory_) + : key(key_) + , key_path(key_path_) + , created_base_directory(created_base_directory_) + , cleanup_queue(cleanup_queue_) { - return file_segment->getQueueIterator(); } -String CacheMetadata::getPathInLocalCache(const Key & key, size_t offset, FileSegmentKind segment_kind) const +LockedKeyPtr KeyMetadata::lock() +{ + auto locked = std::make_unique(shared_from_this()); + if (key_state == KeyMetadata::KeyState::ACTIVE) + return locked; + + throw Exception( + ErrorCodes::LOGICAL_ERROR, + "Cannot lock key {} (state: {})", key, magic_enum::enum_name(key_state)); +} + +std::string KeyMetadata::getFileSegmentPath(const FileSegment & file_segment) +{ + return fs::path(key_path) + / CacheMetadata::getFileNameForFileSegment(file_segment.offset(), file_segment.getKind()); +} + +CacheMetadata::CacheMetadata(const std::string & path_) + : path(path_) + , log(&Poco::Logger::get("CacheMetadata")) +{ +} + +String CacheMetadata::getFileNameForFileSegment(size_t offset, FileSegmentKind segment_kind) { String file_suffix; switch (segment_kind) @@ -72,18 +92,27 @@ String CacheMetadata::getPathInLocalCache(const Key & key, size_t offset, FileSe file_suffix = ""; break; } + return std::to_string(offset) + file_suffix; +} + +String CacheMetadata::getPathInLocalCache(const Key & key, size_t offset, FileSegmentKind segment_kind) const +{ + String file_suffix; const auto key_str = key.toString(); - return fs::path(base_directory) / key_str.substr(0, 3) / key_str / (std::to_string(offset) + file_suffix); + return fs::path(path) / key_str.substr(0, 3) / key_str / getFileNameForFileSegment(offset, segment_kind); } String CacheMetadata::getPathInLocalCache(const Key & key) const { const auto key_str = key.toString(); - return fs::path(base_directory) / key_str.substr(0, 3) / key_str; + return fs::path(path) / key_str.substr(0, 3) / key_str; } -LockedKeyMetadataPtr CacheMetadata::lockKeyMetadata(const FileCacheKey & key, KeyNotFoundPolicy key_not_found_policy, bool is_initial_load) +LockedKeyPtr CacheMetadata::lockKeyMetadata( + const FileCacheKey & key, + KeyNotFoundPolicy key_not_found_policy, + bool is_initial_load) { KeyMetadataPtr key_metadata; { @@ -93,27 +122,27 @@ LockedKeyMetadataPtr CacheMetadata::lockKeyMetadata(const FileCacheKey & key, Ke if (it == end()) { if (key_not_found_policy == KeyNotFoundPolicy::THROW) - throw Exception(ErrorCodes::LOGICAL_ERROR, "No such key `{}` in cache", key.toString()); + throw Exception(ErrorCodes::LOGICAL_ERROR, "No such key `{}` in cache", key); else if (key_not_found_policy == KeyNotFoundPolicy::RETURN_NULL) return nullptr; it = emplace( - key, - std::make_shared(/* base_directory_already_exists */is_initial_load, cleanup_queue)).first; + key, std::make_shared( + key, getPathInLocalCache(key), cleanup_queue, is_initial_load)).first; } key_metadata = it->second; } { - auto locked_metadata = std::make_unique(key, key_metadata, getPathInLocalCache(key)); + auto locked_metadata = std::make_unique(key_metadata); const auto key_state = locked_metadata->getKeyState(); if (key_state == KeyMetadata::KeyState::ACTIVE) return locked_metadata; if (key_not_found_policy == KeyNotFoundPolicy::THROW) - throw Exception(ErrorCodes::LOGICAL_ERROR, "No such key `{}` in cache", key.toString()); + throw Exception(ErrorCodes::LOGICAL_ERROR, "No such key `{}` in cache", key); if (key_not_found_policy == KeyNotFoundPolicy::RETURN_NULL) return nullptr; @@ -135,32 +164,25 @@ LockedKeyMetadataPtr CacheMetadata::lockKeyMetadata(const FileCacheKey & key, Ke return lockKeyMetadata(key, key_not_found_policy); } -LockedKeyMetadataPtr CacheMetadata::lockKeyMetadata( - const FileCacheKey & key, KeyMetadataPtr key_metadata, bool skip_if_in_cleanup_queue) const -{ - auto locked_metadata = std::make_unique(key, key_metadata, getPathInLocalCache(key)); - const auto key_state = locked_metadata->getKeyState(); - - if (key_state == KeyMetadata::KeyState::ACTIVE) - return locked_metadata; - - if (skip_if_in_cleanup_queue - && key_state == KeyMetadata::KeyState::REMOVING) - return nullptr; - - throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot lock key {}: it was removed from cache", key.toString()); -} - void CacheMetadata::iterate(IterateCacheMetadataFunc && func) { auto lock = guard.lock(); for (const auto & [key, key_metadata] : *this) { - auto locked_key = lockKeyMetadata(key, key_metadata, /* skip_if_in_cleanup_queue */true); - if (!locked_key) + auto locked_key = std::make_unique(key_metadata); + const auto key_state = locked_key->getKeyState(); + + if (key_state == KeyMetadata::KeyState::ACTIVE) + { + func(*locked_key); + continue; + } + + if (key_state == KeyMetadata::KeyState::REMOVING) continue; - func(*locked_key); + throw Exception( + ErrorCodes::LOGICAL_ERROR, "Cannot lock key {}: key does not exist", key_metadata->key); } } @@ -182,7 +204,7 @@ void CacheMetadata::doCleanup() if (it == end()) continue; - auto locked_metadata = std::make_unique(it->first, it->second, getPathInLocalCache(it->first)); + auto locked_metadata = std::make_unique(it->second); const auto key_state = locked_metadata->getKeyState(); if (key_state == KeyMetadata::KeyState::ACTIVE) @@ -212,28 +234,23 @@ void CacheMetadata::doCleanup() } } -LockedKeyMetadata::LockedKeyMetadata( - const FileCacheKey & key_, - std::shared_ptr key_metadata_, - const std::string & key_path_) - : key(key_) - , key_path(key_path_) - , key_metadata(key_metadata_) - , lock(key_metadata->lock()) - , log(&Poco::Logger::get("LockedKeyMetadata")) +LockedKey::LockedKey(std::shared_ptr key_metadata_) + : key_metadata(key_metadata_) + , lock(key_metadata->guard.lock()) + , log(&Poco::Logger::get("LockedKey")) { } -LockedKeyMetadata::~LockedKeyMetadata() +LockedKey::~LockedKey() { if (!key_metadata->empty()) return; key_metadata->key_state = KeyMetadata::KeyState::REMOVING; - key_metadata->cleanup_queue.add(key); + key_metadata->cleanup_queue.add(getKey()); } -void LockedKeyMetadata::removeFromCleanupQueue() +void LockedKey::removeFromCleanupQueue() { if (key_metadata->key_state != KeyMetadata::KeyState::REMOVING) throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot remove non-removing"); @@ -242,7 +259,7 @@ void LockedKeyMetadata::removeFromCleanupQueue() key_metadata->key_state = KeyMetadata::KeyState::ACTIVE; } -bool LockedKeyMetadata::markAsRemoved() +bool LockedKey::markAsRemoved() { chassert(key_metadata->key_state != KeyMetadata::KeyState::REMOVED); @@ -253,50 +270,61 @@ bool LockedKeyMetadata::markAsRemoved() return true; } -bool LockedKeyMetadata::isLastOwnerOfFileSegment(size_t offset) const +bool LockedKey::isLastOwnerOfFileSegment(size_t offset) const { - const auto * file_segment_metadata = getByOffset(offset); + const auto file_segment_metadata = getByOffset(offset); return file_segment_metadata->file_segment.use_count() == 2; } -void LockedKeyMetadata::removeFileSegment( - size_t offset, - const FileSegmentGuard::Lock & segment_lock, - const CacheGuard::Lock & cache_lock) +void LockedKey::removeAllReleasable() { - LOG_DEBUG( - log, "Remove from cache. Key: {}, offset: {}", - key.toString(), offset); + for (auto it = key_metadata->begin(); it != key_metadata->end();) + { + if (!it->second->releasable()) + { + ++it; + continue; + } - auto * file_segment_metadata = getByOffset(offset); + auto file_segment = it->second->file_segment; + it = removeFileSegment(file_segment->offset(), file_segment->lock()); + } +} - if (file_segment_metadata->getQueueIterator()) - LockedCachePriorityIterator(cache_lock, file_segment_metadata->getQueueIterator()).remove(); +KeyMetadata::iterator LockedKey::removeFileSegment(size_t offset, const FileSegmentGuard::Lock & segment_lock) +{ + LOG_DEBUG(log, "Remove from cache. Key: {}, offset: {}", getKey(), offset); - const auto cache_file_path = file_segment_metadata->file_segment->getPathInLocalCache(); - file_segment_metadata->file_segment->detach(segment_lock, *this); + auto it = key_metadata->find(offset); + if (it == key_metadata->end()) + throw Exception(ErrorCodes::LOGICAL_ERROR, "There is no offset {}", offset); - key_metadata->erase(offset); + auto file_segment = it->second->file_segment; + if (file_segment->queue_iterator) + file_segment->queue_iterator->updateSize(-file_segment->queue_iterator->getEntry().size); - if (fs::exists(cache_file_path)) - fs::remove(cache_file_path); + const auto path = key_metadata->getFileSegmentPath(*it->second->file_segment); + if (fs::exists(path)) + fs::remove(path); + + it->second->file_segment->detach(segment_lock, *this); + return key_metadata->erase(it); } -void LockedKeyMetadata::shrinkFileSegmentToDownloadedSize( +void LockedKey::shrinkFileSegmentToDownloadedSize( size_t offset, - const FileSegmentGuard::Lock & segment_lock, - const CacheGuard::Lock & cache_lock) + const FileSegmentGuard::Lock & segment_lock) { /** * In case file was partially downloaded and it's download cannot be continued * because of no space left in cache, we need to be able to cut file segment's size to downloaded_size. */ - auto * file_segment_metadata = getByOffset(offset); - const auto & file_segment = file_segment_metadata->file_segment; + auto metadata = getByOffset(offset); + const auto & file_segment = metadata->file_segment; - size_t downloaded_size = file_segment->downloaded_size; - size_t full_size = file_segment->range().size(); + const size_t downloaded_size = file_segment->getDownloadedSize(false); + const size_t full_size = file_segment->range().size(); if (downloaded_size == full_size) { @@ -306,64 +334,69 @@ void LockedKeyMetadata::shrinkFileSegmentToDownloadedSize( file_segment->getInfoForLogUnlocked(segment_lock)); } - auto & entry = LockedCachePriorityIterator(cache_lock, file_segment_metadata->getQueueIterator()).getEntry(); - assert(file_segment->downloaded_size <= file_segment->reserved_size); - assert(entry.size == file_segment->reserved_size); - assert(entry.size >= file_segment->downloaded_size); + auto queue_iterator = metadata->file_segment->queue_iterator; + + chassert(downloaded_size <= file_segment->reserved_size); + chassert(queue_iterator->getEntry().size == file_segment->reserved_size); CreateFileSegmentSettings create_settings(file_segment->getKind()); - file_segment_metadata->file_segment = std::make_shared( - key, offset, downloaded_size, key_metadata, file_segment->getQueueIterator(), - file_segment->cache, FileSegment::State::DOWNLOADED, create_settings); + metadata->file_segment = std::make_shared( + getKey(), offset, downloaded_size, FileSegment::State::DOWNLOADED, create_settings, + file_segment->cache, key_metadata, file_segment->queue_iterator); - if (file_segment->reserved_size > file_segment->downloaded_size) - entry.size = downloaded_size; + ssize_t diff = file_segment->reserved_size - file_segment->downloaded_size; + if (diff) + queue_iterator->updateSize(-diff); - assert(file_segment->reserved_size == downloaded_size); - assert(file_segment_metadata->size() == entry.size); + chassert(file_segment->reserved_size == downloaded_size); + chassert(metadata->size() == queue_iterator->getEntry().size); } -void LockedKeyMetadata::assertFileSegmentCorrectness(const FileSegment & file_segment) const +void LockedKey::assertFileSegmentCorrectness(const FileSegment & file_segment) const { - if (file_segment.key() != key) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Expected {} = {}", file_segment.key().toString(), key.toString()); + if (file_segment.key() != getKey()) + { + throw Exception( + ErrorCodes::LOGICAL_ERROR, + "Expected {} = {}", file_segment.key(), getKey()); + } file_segment.assertCorrectness(); } -const FileSegmentMetadata * LockedKeyMetadata::getByOffset(size_t offset) const +std::shared_ptr LockedKey::getByOffset(size_t offset) const { auto it = key_metadata->find(offset); if (it == key_metadata->end()) throw Exception(ErrorCodes::LOGICAL_ERROR, "There is not offset {}", offset); - return &(it->second); + return it->second; } -FileSegmentMetadata * LockedKeyMetadata::getByOffset(size_t offset) +std::shared_ptr LockedKey::getByOffset(size_t offset) { auto it = key_metadata->find(offset); if (it == key_metadata->end()) throw Exception(ErrorCodes::LOGICAL_ERROR, "There is not offset {}", offset); - return &(it->second); + return it->second; } -const FileSegmentMetadata * LockedKeyMetadata::tryGetByOffset(size_t offset) const +std::shared_ptr LockedKey::tryGetByOffset(size_t offset) const { auto it = key_metadata->find(offset); if (it == key_metadata->end()) return nullptr; - return &(it->second); + return it->second; } -FileSegmentMetadata * LockedKeyMetadata::tryGetByOffset(size_t offset) +std::shared_ptr LockedKey::tryGetByOffset(size_t offset) { auto it = key_metadata->find(offset); if (it == key_metadata->end()) return nullptr; - return &(it->second); + return it->second; } -std::string LockedKeyMetadata::toString() const +std::string LockedKey::toString() const { std::string result; for (auto it = key_metadata->begin(); it != key_metadata->end(); ++it) @@ -386,7 +419,7 @@ void CleanupQueue::remove(const FileCacheKey & key) std::lock_guard lock(mutex); bool erased = keys.erase(key); if (!erased) - throw Exception(ErrorCodes::LOGICAL_ERROR, "No such key {} in removal queue", key.toString()); + throw Exception(ErrorCodes::LOGICAL_ERROR, "No such key {} in removal queue", key); } bool CleanupQueue::tryPop(FileCacheKey & key) diff --git a/src/Interpreters/Cache/Metadata.h b/src/Interpreters/Cache/Metadata.h index 4fc0fed99d83..71a905a259da 100644 --- a/src/Interpreters/Cache/Metadata.h +++ b/src/Interpreters/Cache/Metadata.h @@ -9,52 +9,43 @@ namespace DB { class FileSegment; using FileSegmentPtr = std::shared_ptr; -struct LockedKeyMetadata; -using LockedKeyMetadataPtr = std::shared_ptr; -class LockedCachePriority; -struct KeysQueue; +struct LockedKey; +using LockedKeyPtr = std::shared_ptr; struct CleanupQueue; struct FileSegmentMetadata : private boost::noncopyable { - FileSegmentMetadata( - FileSegmentPtr file_segment_, - LockedKeyMetadata & locked_key, - LockedCachePriority * locked_queue); - - /// Pointer to file segment is always hold by the cache itself. - /// Apart from pointer in cache, it can be hold by cache users, when they call - /// getorSet(), but cache users always hold it via FileSegmentsHolder. - bool releasable() const { return file_segment.unique(); } + using Priority = IFileCachePriority; + using PriorityIterator = IFileCachePriority::Iterator; - IFileCachePriority::Iterator & getQueueIterator() const; + explicit FileSegmentMetadata(FileSegmentPtr && file_segment_); - bool valid() const { return *evict_flag == false; } + bool releasable() const { return file_segment.unique(); } size_t size() const; + bool valid() const { return !removal_candidate.load(); } + FileSegmentPtr file_segment; + std::atomic removal_candidate{false}; +}; - using EvictFlag = std::shared_ptr>; - EvictFlag evict_flag = std::make_shared>(false); - struct EvictHolder : boost::noncopyable - { - explicit EvictHolder(EvictFlag evict_flag_) : evict_flag(evict_flag_) { *evict_flag = true; } - ~EvictHolder() { *evict_flag = false; } - EvictFlag evict_flag; - }; - using EvictHolderPtr = std::unique_ptr; +using FileSegmentMetadataPtr = std::shared_ptr; - EvictHolderPtr getEvictHolder() { return std::make_unique(evict_flag); } -}; -struct KeyMetadata : public std::map, private boost::noncopyable +struct KeyMetadata : public std::map, + private boost::noncopyable, + public std::enable_shared_from_this { - friend struct LockedKeyMetadata; -public: - explicit KeyMetadata(bool created_base_directory_, CleanupQueue & cleanup_queue_) - : created_base_directory(created_base_directory_), cleanup_queue(cleanup_queue_) {} + friend struct LockedKey; + using Key = FileCacheKey; + + KeyMetadata( + const Key & key_, + const std::string & key_path_, + CleanupQueue & cleanup_queue_, + bool created_base_directory_ = false); enum class KeyState { @@ -63,19 +54,23 @@ struct KeyMetadata : public std::map, private boost REMOVED, }; - bool created_base_directory = false; + const Key key; + const std::string key_path; + std::atomic created_base_directory = false; -private: - KeyGuard::Lock lock() const { return guard.lock(); } + LockedKeyPtr lock(); - KeyState key_state = KeyState::ACTIVE; + std::string getFileSegmentPath(const FileSegment & file_segment); - mutable KeyGuard guard; +private: + KeyState key_state = KeyState::ACTIVE; + KeyGuard guard; CleanupQueue & cleanup_queue; }; using KeyMetadataPtr = std::shared_ptr; + struct CleanupQueue { friend struct CacheMetadata; @@ -91,16 +86,26 @@ struct CleanupQueue mutable std::mutex mutex; }; + struct CacheMetadata : public std::unordered_map, private boost::noncopyable { public: using Key = FileCacheKey; + using IterateCacheMetadataFunc = std::function; + + explicit CacheMetadata(const std::string & path_); - explicit CacheMetadata(const std::string & base_directory_) : base_directory(base_directory_) {} + const String & getBaseDirectory() const { return path; } - String getPathInLocalCache(const Key & key, size_t offset, FileSegmentKind segment_kind) const; + String getPathInLocalCache( + const Key & key, + size_t offset, + FileSegmentKind segment_kind) const; String getPathInLocalCache(const Key & key) const; + static String getFileNameForFileSegment(size_t offset, FileSegmentKind segment_kind); + + void iterate(IterateCacheMetadataFunc && func); enum class KeyNotFoundPolicy { @@ -108,62 +113,63 @@ struct CacheMetadata : public std::unordered_map, CREATE_EMPTY, RETURN_NULL, }; - LockedKeyMetadataPtr lockKeyMetadata(const Key & key, KeyNotFoundPolicy key_not_found_policy, bool is_initial_load = false); - LockedKeyMetadataPtr lockKeyMetadata(const Key & key, KeyMetadataPtr key_metadata, bool skip_if_in_cleanup_queue = false) const; - - using IterateCacheMetadataFunc = std::function; - void iterate(IterateCacheMetadataFunc && func); + LockedKeyPtr lockKeyMetadata( + const Key & key, + KeyNotFoundPolicy key_not_found_policy, + bool is_initial_load = false); void doCleanup(); private: - const std::string base_directory; + const std::string path; /// Cache base path CacheMetadataGuard guard; CleanupQueue cleanup_queue; + Poco::Logger * log; }; /** - * `LockedKeyMetadata` is an object which makes sure that as long as it exists the following is true: + * `LockedKey` is an object which makes sure that as long as it exists the following is true: * 1. the key cannot be removed from cache - * (Why: this LockedKeyMetadata locks key metadata mutex in ctor, unlocks it in dtor, and so + * (Why: this LockedKey locks key metadata mutex in ctor, unlocks it in dtor, and so * when key is going to be deleted, key mutex is also locked. - * Why it cannot be the other way round? E.g. that ctor of LockedKeyMetadata locks the key - * right after it was deleted? This case it taken into consideration in createLockedKeyMetadata()) + * Why it cannot be the other way round? E.g. that ctor of LockedKey locks the key + * right after it was deleted? This case it taken into consideration in createLockedKey()) * 2. the key cannot be modified, e.g. new offsets cannot be added to key; already existing * offsets cannot be deleted from the key - * And also provides some methods which allow the owner of this LockedKeyMetadata object to do such + * And also provides some methods which allow the owner of this LockedKey object to do such * modification of the key (adding/deleting offsets) and deleting the key from cache. */ -struct LockedKeyMetadata : private boost::noncopyable +struct LockedKey : private boost::noncopyable { - LockedKeyMetadata( - const FileCacheKey & key_, - std::shared_ptr key_metadata_, - const std::string & key_path_); + using Key = FileCacheKey; - ~LockedKeyMetadata(); + explicit LockedKey(std::shared_ptr key_metadata_); - const FileCacheKey & getKey() const { return key; } + ~LockedKey(); + + const Key & getKey() const { return key_metadata->key; } auto begin() const { return key_metadata->begin(); } auto end() const { return key_metadata->end(); } - const FileSegmentMetadata * getByOffset(size_t offset) const; - FileSegmentMetadata * getByOffset(size_t offset); + std::shared_ptr getByOffset(size_t offset) const; + std::shared_ptr getByOffset(size_t offset); - const FileSegmentMetadata * tryGetByOffset(size_t offset) const; - FileSegmentMetadata * tryGetByOffset(size_t offset); + std::shared_ptr tryGetByOffset(size_t offset) const; + std::shared_ptr tryGetByOffset(size_t offset); KeyMetadata::KeyState getKeyState() const { return key_metadata->key_state; } KeyMetadataPtr getKeyMetadata() const { return key_metadata; } KeyMetadataPtr getKeyMetadata() { return key_metadata; } - void removeFileSegment(size_t offset, const FileSegmentGuard::Lock &, const CacheGuard::Lock &); + KeyMetadata::iterator removeFileSegment(size_t offset, const FileSegmentGuard::Lock &); + + void removeAllReleasable(); - void shrinkFileSegmentToDownloadedSize(size_t offset, const FileSegmentGuard::Lock &, const CacheGuard::Lock &); + void shrinkFileSegmentToDownloadedSize(size_t offset, const FileSegmentGuard::Lock &); bool isLastOwnerOfFileSegment(size_t offset) const; @@ -180,8 +186,6 @@ struct LockedKeyMetadata : private boost::noncopyable std::string toString() const; private: - const FileCacheKey key; - const std::string key_path; const std::shared_ptr key_metadata; KeyGuard::Lock lock; /// `lock` must be destructed before `key_metadata`. Poco::Logger * log; diff --git a/src/Interpreters/Cache/QueryLimit.cpp b/src/Interpreters/Cache/QueryLimit.cpp index 3c941cd3dd29..3e36129de43f 100644 --- a/src/Interpreters/Cache/QueryLimit.cpp +++ b/src/Interpreters/Cache/QueryLimit.cpp @@ -15,13 +15,13 @@ static bool isQueryInitialized() && !CurrentThread::getQueryId().empty(); } -FileCacheQueryLimit::LockedQueryContextPtr FileCacheQueryLimit::tryGetQueryContext(const CacheGuard::Lock & lock) +FileCacheQueryLimit::QueryContextPtr FileCacheQueryLimit::tryGetQueryContext(const CacheGuard::Lock &) { if (!isQueryInitialized()) return nullptr; auto query_iter = query_map.find(std::string(CurrentThread::getQueryId())); - return (query_iter == query_map.end()) ? nullptr : std::make_unique(query_iter->second, lock); + return (query_iter == query_map.end()) ? nullptr : query_iter->second; } void FileCacheQueryLimit::removeQueryContext(const std::string & query_id, const CacheGuard::Lock &) @@ -49,38 +49,61 @@ FileCacheQueryLimit::QueryContextPtr FileCacheQueryLimit::getOrSetQueryContext( if (inserted) { it->second = std::make_shared( - settings.filesystem_cache_max_download_size, !settings.skip_download_if_exceeds_query_cache); + settings.filesystem_cache_max_download_size, + !settings.skip_download_if_exceeds_query_cache); } return it->second; } -void FileCacheQueryLimit::LockedQueryContext::add(const FileCacheKey & key, size_t offset, IFileCachePriority::Iterator iterator) +FileCacheQueryLimit::QueryContext::QueryContext( + size_t query_cache_size, + bool recache_on_query_limit_exceeded_) + : priority(LRUFileCachePriority(query_cache_size, 0)) + , recache_on_query_limit_exceeded(recache_on_query_limit_exceeded_) { - auto [_, inserted] = context->records.emplace(FileCacheKeyAndOffset{key, offset}, iterator); +} + +void FileCacheQueryLimit::QueryContext::add( + const FileSegment & file_segment, + const CacheGuard::Lock & lock) +{ + const auto key = file_segment.key(); + const auto offset = file_segment.offset(); + + auto it = getPriority().add( + key, offset, file_segment.range().size(), file_segment.getKeyMetadata(), lock); + + auto [_, inserted] = records.emplace(FileCacheKeyAndOffset{key, offset}, it); if (!inserted) { throw Exception( ErrorCodes::LOGICAL_ERROR, "Cannot add offset {} to query context under key {}, it already exists", - offset, key.toString()); + offset, key); } } -void FileCacheQueryLimit::LockedQueryContext::remove(const FileCacheKey & key, size_t offset) +void FileCacheQueryLimit::QueryContext::remove( + const Key & key, + size_t offset, + const CacheGuard::Lock & lock) { - auto record = context->records.find({key, offset}); - if (record == context->records.end()) - throw Exception(ErrorCodes::LOGICAL_ERROR, "There is no {}:{} in query context", key.toString(), offset); + auto record = records.find({key, offset}); + if (record == records.end()) + throw Exception(ErrorCodes::LOGICAL_ERROR, "There is no {}:{} in query context", key, offset); - LockedCachePriorityIterator(lock, record->second).remove(); - context->records.erase({key, offset}); + record->second->remove(lock); + records.erase({key, offset}); } -IFileCachePriority::Iterator FileCacheQueryLimit::LockedQueryContext::tryGet(const FileCacheKey & key, size_t offset) +IFileCachePriority::Iterator FileCacheQueryLimit::QueryContext::tryGet( + const Key & key, + size_t offset, + const CacheGuard::Lock &) { - auto it = context->records.find({key, offset}); - if (it == context->records.end()) + auto it = records.find({key, offset}); + if (it == records.end()) return nullptr; return it->second; diff --git a/src/Interpreters/Cache/QueryLimit.h b/src/Interpreters/Cache/QueryLimit.h index e66c7d2af769..5c08584bacff 100644 --- a/src/Interpreters/Cache/QueryLimit.h +++ b/src/Interpreters/Cache/QueryLimit.h @@ -1,75 +1,65 @@ #pragma once #include -#include #include namespace DB { struct ReadSettings; +class FileSegment; class FileCacheQueryLimit { public: class QueryContext; using QueryContextPtr = std::shared_ptr; - class LockedQueryContext; - using LockedQueryContextPtr = std::unique_ptr; - LockedQueryContextPtr tryGetQueryContext(const CacheGuard::Lock & lock); + QueryContextPtr tryGetQueryContext(const CacheGuard::Lock & lock); QueryContextPtr getOrSetQueryContext( - const std::string & query_id, const ReadSettings & settings, const CacheGuard::Lock &); + const std::string & query_id, + const ReadSettings & settings, + const CacheGuard::Lock &); void removeQueryContext(const std::string & query_id, const CacheGuard::Lock &); -private: - using QueryContextMap = std::unordered_map; - QueryContextMap query_map; - -public: class QueryContext { public: - QueryContext(size_t query_cache_size, bool recache_on_query_limit_exceeded_) - : priority(std::make_unique(query_cache_size, 0)) - , recache_on_query_limit_exceeded(recache_on_query_limit_exceeded_) {} + using Key = FileCacheKey; + using Priority = IFileCachePriority; + using PriorityIterator = IFileCachePriority::Iterator; - private: - friend class FileCacheQueryLimit::LockedQueryContext; + QueryContext(size_t query_cache_size, bool recache_on_query_limit_exceeded_); - using Records = std::unordered_map; - Records records; - FileCachePriorityPtr priority; - const bool recache_on_query_limit_exceeded; - }; + Priority & getPriority() { return priority; } + const Priority & getPriority() const { return priority; } - /// CacheGuard::Lock protects all priority queues. - class LockedQueryContext - { - public: - LockedQueryContext(QueryContextPtr context_, const CacheGuard::Lock & lock_) - : context(context_), lock(lock_), priority(lock_, *context->priority) {} + bool recacheOnFileCacheQueryLimitExceeded() const { return recache_on_query_limit_exceeded; } - IFileCachePriority & getPriority() { return *context->priority; } - const IFileCachePriority & getPriority() const { return *context->priority; } + IFileCachePriority::Iterator tryGet( + const Key & key, + size_t offset, + const CacheGuard::Lock &); - size_t getSize() const { return priority.getSize(); } + void add( + const FileSegment & file_segment, + const CacheGuard::Lock &); - size_t getSizeLimit() const { return priority.getSizeLimit(); } - - bool recacheOnFileCacheQueryLimitExceeded() const { return context->recache_on_query_limit_exceeded; } - - IFileCachePriority::Iterator tryGet(const FileCacheKey & key, size_t offset); - - void add(const FileCacheKey & key, size_t offset, IFileCachePriority::Iterator iterator); - - void remove(const FileCacheKey & key, size_t offset); + void remove( + const Key & key, + size_t offset, + const CacheGuard::Lock &); private: - QueryContextPtr context; - const CacheGuard::Lock & lock; - LockedCachePriority priority; + using Records = std::unordered_map; + Records records; + LRUFileCachePriority priority; + const bool recache_on_query_limit_exceeded; }; + +private: + using QueryContextMap = std::unordered_map; + QueryContextMap query_map; }; using FileCacheQueryLimitPtr = std::unique_ptr; diff --git a/src/Interpreters/tests/gtest_lru_file_cache.cpp b/src/Interpreters/tests/gtest_lru_file_cache.cpp index 03a5854f904f..8e2a5225a461 100644 --- a/src/Interpreters/tests/gtest_lru_file_cache.cpp +++ b/src/Interpreters/tests/gtest_lru_file_cache.cpp @@ -132,6 +132,12 @@ void download(const HolderPtr & holder) } } +void increasePriority(const HolderPtr & holder) +{ + for (auto & it : *holder) + it->use(); +} + class FileCacheTest : public ::testing::Test { public: @@ -198,6 +204,7 @@ TEST_F(FileCacheTest, get) assertEqual(holder, { Range(0, 9) }, { State::EMPTY }); download(holder->front()); assertEqual(holder, { Range(0, 9) }, { State::DOWNLOADED }); + increasePriority(holder); } /// Current cache: [__________] @@ -216,6 +223,7 @@ TEST_F(FileCacheTest, get) assertEqual(holder, { Range(0, 9), Range(10, 14) }, { State::DOWNLOADED, State::EMPTY }); download(get(holder, 1)); assertEqual(holder, { Range(0, 9), Range(10, 14) }, { State::DOWNLOADED, State::DOWNLOADED }); + increasePriority(holder); } /// Current cache: [__________][_____] @@ -229,14 +237,24 @@ TEST_F(FileCacheTest, get) std::cerr << "Step 3\n"; /// Get [9, 9] - assertEqual(cache.getOrSet(key, 9, 1, {}), { Range(0, 9) }, { State::DOWNLOADED }); + { + auto holder = cache.getOrSet(key, 9, 1, {}); + assertEqual(holder, { Range(0, 9) }, { State::DOWNLOADED }); + increasePriority(holder); + } + assertEqual(cache.dumpQueue(), { Range(10, 14), Range(0, 9) }); /// Get [9, 10] assertEqual(cache.getOrSet(key, 9, 2, {}), { Range(0, 9), Range(10, 14) }, { State::DOWNLOADED, State::DOWNLOADED }); + /// Get [10, 10] - assertEqual(cache.getOrSet(key, 10, 1, {}), { Range(10, 14) }, { State::DOWNLOADED }); + { + auto holder = cache.getOrSet(key, 10, 1, {}); + assertEqual(holder, { Range(10, 14) }, { State::DOWNLOADED }); + increasePriority(holder); + } assertEqual(cache.getSnapshot(key), { Range(0, 9), Range(10, 14) }); assertEqual(cache.dumpQueue(), { Range(0, 9), Range(10, 14) }); @@ -245,9 +263,23 @@ TEST_F(FileCacheTest, get) std::cerr << "Step 4\n"; - download(cache.getOrSet(key, 17, 4, {})); /// Get [17, 20] - download(cache.getOrSet(key, 24, 3, {})); /// Get [24, 26] - download(cache.getOrSet(key, 27, 1, {})); /// Get [27, 27] + { + auto holder = cache.getOrSet(key, 17, 4, {}); + download(holder); /// Get [17, 20] + increasePriority(holder); + } + + { + auto holder = cache.getOrSet(key, 24, 3, {}); + download(holder); /// Get [24, 26] + increasePriority(holder); + } + + { + auto holder = cache.getOrSet(key, 27, 1, {}); + download(holder); /// Get [27, 27] + increasePriority(holder); + } /// Current cache: [__________][_____] [____] [___][] /// ^ ^^ ^ ^ ^ ^ ^^^ @@ -286,6 +318,10 @@ TEST_F(FileCacheTest, get) assertEqual(holder3, { Range(28, 30) }, { State::EMPTY }); assertDownloadFails(holder3->front()); assertEqual(holder3, { Range(28, 30) }, { State::DETACHED }); + + increasePriority(holder); + increasePriority(holder2); + increasePriority(holder3); } /// Current cache: [__________][_____][ ][____] [___] @@ -308,6 +344,7 @@ TEST_F(FileCacheTest, get) assertEqual(holder, { Range(10, 14), Range(15, 16), Range(17, 20), Range(21, 21) }, { State::DOWNLOADED, State::DOWNLOADED, State::DOWNLOADED, State::DOWNLOADED }); + increasePriority(holder); } /// Current cache: [_____][__][____][_] [___] @@ -326,6 +363,7 @@ TEST_F(FileCacheTest, get) { State::EMPTY, State::DOWNLOADED, State::EMPTY }); download(get(holder, 0)); download(get(holder, 2)); + increasePriority(holder); } /// Current cache: [____][_] [][___][__] @@ -365,18 +403,29 @@ TEST_F(FileCacheTest, get) assertDownloadFails(get(holder6, 2)); assertDownloadFails(get(holder6, 6)); assertDownloadFails(get(holder6, 8)); + + increasePriority(holder); + increasePriority(holder2); + increasePriority(holder3); + increasePriority(holder4); + increasePriority(holder5); + increasePriority(holder6); } /// Current cache: [___] [_][___][_] [__] /// ^ ^ ^ ^ ^ ^ ^ ^ /// 2 4 23 24 26 27 30 31 assertEqual(cache.getSnapshot(key), { Range(2, 4), Range(23, 23), Range(24, 26), Range(27, 27), Range(30, 31) }); - assertEqual(cache.dumpQueue(), { Range(27, 27), Range(24, 26), Range(23, 23), Range(30, 31), Range(2, 4) }); + assertEqual(cache.dumpQueue(), { Range(2, 4), Range(23, 23), Range(24, 26), Range(27, 27), Range(30, 31) }); std::cerr << "Step 9\n"; /// Get [2, 4] - assertEqual(cache.getOrSet(key, 2, 3, {}), { Range(2, 4) }, { State::DOWNLOADED }); + { + auto holder = cache.getOrSet(key, 2, 3, {}); + assertEqual(holder, { Range(2, 4) }, { State::DOWNLOADED }); + increasePriority(holder); + } { @@ -429,6 +478,8 @@ TEST_F(FileCacheTest, get) ASSERT_TRUE(file_segment.state() == State::DOWNLOADED); other_1.join(); + + increasePriority(holder); } /// Current cache: [___] [___][_][__][__] From ce45105448cb2f3cafa27d9269c8e04a9330c427 Mon Sep 17 00:00:00 2001 From: kssenii Date: Thu, 13 Apr 2023 13:27:01 +0200 Subject: [PATCH 119/406] Cleanup a bit --- src/Interpreters/Cache/FileCache.cpp | 49 +++---- src/Interpreters/Cache/FileSegment.cpp | 130 +++++++----------- src/Interpreters/Cache/FileSegment.h | 86 ++++++------ src/Interpreters/Cache/IFileCachePriority.h | 11 +- .../Cache/LRUFileCachePriority.cpp | 12 +- src/Interpreters/Cache/LRUFileCachePriority.h | 2 + src/Interpreters/Cache/Metadata.cpp | 72 +++++----- src/Interpreters/Cache/Metadata.h | 39 ++---- 8 files changed, 180 insertions(+), 221 deletions(-) diff --git a/src/Interpreters/Cache/FileCache.cpp b/src/Interpreters/Cache/FileCache.cpp index 76145afe35bf..82128e681e46 100644 --- a/src/Interpreters/Cache/FileCache.cpp +++ b/src/Interpreters/Cache/FileCache.cpp @@ -513,8 +513,6 @@ KeyMetadata::iterator FileCache::addFileSegment( "Failed to insert {}:{}: entry already exists", key, offset); } - if (state == FileSegment::State::DOWNLOADED) - chassert(file_segment_metadata_it->second->file_segment->getQueueIterator()); return file_segment_metadata_it; } catch (...) @@ -565,37 +563,30 @@ bool FileCache::tryReserve(FileSegment & file_segment, size_t size) return PriorityIterationResult::REMOVE_AND_CONTINUE; chassert(file_segment_metadata->file_segment->getQueueIterator()); - chassert(entry.offset == file_segment_metadata->file_segment->offset()); - - auto iteration_result = PriorityIterationResult::CONTINUE; - const bool is_persistent = allow_persistent_files && file_segment_metadata->file_segment->isPersistent(); const bool releasable = file_segment_metadata->releasable() && !is_persistent; + if (releasable) { - auto current_file_segment = file_segment_metadata->file_segment; - const size_t file_segment_size = entry.size; + removed_size += entry.size; + --queue_size; - if (current_file_segment->state() == FileSegment::State::DOWNLOADED) + auto segment = file_segment_metadata->file_segment; + if (segment->state() == FileSegment::State::DOWNLOADED) { - const auto & key = current_file_segment->key(); + const auto & key = segment->key(); auto it = to_delete.find(key); if (it == to_delete.end()) it = to_delete.emplace(key, locked_key.getKeyMetadata()).first; it->second.add(file_segment_metadata); - } - else - { - /// TODO: we can resize if partially downloaded instead. - iteration_result = PriorityIterationResult::REMOVE_AND_CONTINUE; - locked_key.removeFileSegment(current_file_segment->offset(), current_file_segment->lock()); + return PriorityIterationResult::CONTINUE; } - removed_size += file_segment_size; - --queue_size; + /// TODO: we can resize if partially downloaded instead. + locked_key.removeFileSegment(segment->offset(), segment->lock()); + return PriorityIterationResult::REMOVE_AND_CONTINUE; } - - return iteration_result; + return PriorityIterationResult::CONTINUE; }; if (query_priority) @@ -676,12 +667,7 @@ bool FileCache::tryReserve(FileSegment & file_segment, size_t size) if (main_priority->getSize(cache_lock) > (1ull << 63)) throw Exception(ErrorCodes::LOGICAL_ERROR, "Cache became inconsistent. There must be a bug"); - const auto & key_metadata = file_segment.getKeyMetadata(); - if (!key_metadata->created_base_directory.exchange(true)) - { - fs::create_directories(metadata.getPathInLocalCache(file_segment.key())); - } - + file_segment.getKeyMetadata()->createBaseDirectory(); return true; } @@ -994,7 +980,16 @@ void FileCache::assertCacheCorrectness() { for (const auto & [offset, file_segment_metadata] : locked_key) { - locked_key.assertFileSegmentCorrectness(*file_segment_metadata->file_segment); + const auto & file_segment = *file_segment_metadata->file_segment; + + if (file_segment.key() != locked_key.getKey()) + { + throw Exception( + ErrorCodes::LOGICAL_ERROR, + "Expected {} = {}", file_segment.key(), locked_key.getKey()); + } + + file_segment.assertCorrectness(); } }); } diff --git a/src/Interpreters/Cache/FileSegment.cpp b/src/Interpreters/Cache/FileSegment.cpp index cd6fed39c32f..34de2c71bde5 100644 --- a/src/Interpreters/Cache/FileSegment.cpp +++ b/src/Interpreters/Cache/FileSegment.cpp @@ -138,11 +138,6 @@ size_t FileSegment::getDownloadedSize(bool sync) const void FileSegment::setDownloadedSize(size_t delta) { auto lock = segment_guard.lock(); - setDownloadedSizeUnlocked(delta, lock); -} - -void FileSegment::setDownloadedSizeUnlocked(size_t delta, const FileSegmentGuard::Lock &) -{ downloaded_size += delta; assert(downloaded_size == std::filesystem::file_size(getPathInLocalCache())); } @@ -196,7 +191,7 @@ String FileSegment::getOrSetDownloader() return current_downloader; } -void FileSegment::resetDownloadingStateUnlocked([[maybe_unused]] const FileSegmentGuard::Lock & lock) +void FileSegment::resetDownloadingStateUnlocked(const FileSegmentGuard::Lock & lock) { assert(isDownloaderUnlocked(lock)); assert(download_state == State::DOWNLOADING); @@ -213,6 +208,8 @@ void FileSegment::resetDownloader() { auto lock = segment_guard.lock(); + SCOPE_EXIT({ cv.notify_all(); }); + assertNotDetachedUnlocked(lock); assertIsDownloaderUnlocked("resetDownloader", lock); @@ -224,7 +221,6 @@ void FileSegment::resetDownloaderUnlocked(const FileSegmentGuard::Lock &) { LOG_TEST(log, "Resetting downloader from {}", downloader_id); downloader_id.clear(); - cv.notify_all(); } void FileSegment::assertIsDownloaderUnlocked(const std::string & operation, const FileSegmentGuard::Lock & lock) const @@ -292,17 +288,6 @@ void FileSegment::setRemoteFileReader(RemoteFileReaderPtr remote_file_reader_) remote_file_reader = remote_file_reader_; } -void FileSegment::resetRemoteFileReader() -{ - auto lock = segment_guard.lock(); - assertIsDownloaderUnlocked("resetRemoteFileReader", lock); - - if (!remote_file_reader) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Remote file reader does not exist"); - - remote_file_reader.reset(); -} - void FileSegment::write(const char * from, size_t size, size_t offset) { if (!size) @@ -366,7 +351,7 @@ void FileSegment::write(const char * from, size_t size, size_t offset) { auto lock = segment_guard.lock(); - wrapWithCacheInfo(e, "while writing into cache", lock); + e.addMessage(fmt::format("{}, current cache state: {}", e.what(), getInfoForLogUnlocked(lock))); setDownloadFailedUnlocked(lock); @@ -382,7 +367,7 @@ FileSegment::State FileSegment::wait(size_t offset) { auto lock = segment_guard.lock(); - if (downloader_id.empty()) + if (downloader_id.empty() || offset < getCurrentWriteOffset(true)) return download_state; if (download_state == State::EMPTY) @@ -395,7 +380,7 @@ FileSegment::State FileSegment::wait(size_t offset) chassert(!getDownloaderUnlocked(lock).empty()); chassert(!isDownloaderUnlocked(lock)); - [[maybe_unused]] const bool ok = cv.wait_for(lock, std::chrono::seconds(60), [&, this]() + cv.wait_for(lock, std::chrono::seconds(60), [&, this]() { return download_state != State::DOWNLOADING || offset < getCurrentWriteOffset(true); }); @@ -445,8 +430,6 @@ bool FileSegment::reserve(size_t size_to_reserve) { auto lock = segment_guard.lock(); - LOG_TRACE(log, "Try reserve for {}", getInfoForLogUnlocked(lock)); - assertNotDetachedUnlocked(lock); assertIsDownloaderUnlocked("reserve", lock); @@ -497,7 +480,7 @@ bool FileSegment::reserve(size_t size_to_reserve) return reserved; } -void FileSegment::setDownloadedUnlocked([[maybe_unused]] const FileSegmentGuard::Lock & lock) +void FileSegment::setDownloadedUnlocked(const FileSegmentGuard::Lock &) { if (download_state == State::DOWNLOADED) return; @@ -517,7 +500,9 @@ void FileSegment::setDownloadedUnlocked([[maybe_unused]] const FileSegmentGuard: void FileSegment::setDownloadFailedUnlocked(const FileSegmentGuard::Lock & lock) { - LOG_INFO(log, "Settings download as failed: {}", getInfoForLogUnlocked(lock)); + LOG_INFO(log, "Setting download as failed: {}", getInfoForLogUnlocked(lock)); + + SCOPE_EXIT({ cv.notify_all(); }); setDownloadState(State::PARTIALLY_DOWNLOADED_NO_CONTINUATION, lock); @@ -532,11 +517,9 @@ void FileSegment::setDownloadFailedUnlocked(const FileSegmentGuard::Lock & lock) void FileSegment::completePartAndResetDownloader() { auto lock = segment_guard.lock(); - completePartAndResetDownloaderUnlocked(lock); -} -void FileSegment::completePartAndResetDownloaderUnlocked(const FileSegmentGuard::Lock & lock) -{ + SCOPE_EXIT({ cv.notify_all(); }); + assertNotDetachedUnlocked(lock); assertIsDownloaderUnlocked("completePartAndResetDownloader", lock); @@ -550,6 +533,8 @@ void FileSegment::setBroken() { auto lock = segment_guard.lock(); + SCOPE_EXIT({ cv.notify_all(); }); + assertNotDetachedUnlocked(lock); assertIsDownloaderUnlocked("setBroken", lock); @@ -566,35 +551,27 @@ void FileSegment::complete() return; auto locked_key = lockKeyMetadata(false); - if (locked_key) + if (!locked_key) { - completeUnlocked(*locked_key); - return; - } - - /// If we failed to lock a key, it must be in detached state. - if (isDetached()) - return; + /// If we failed to lock a key, it must be in detached state. + if (isDetached()) + return; - throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot complete file segment: {}", getInfoForLog()); -} + throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot complete file segment: {}", getInfoForLog()); + } -void FileSegment::completeUnlocked(LockedKey & locked_key) -{ auto segment_lock = segment_guard.lock(); if (isCompleted(false)) return; const bool is_downloader = isDownloaderUnlocked(segment_lock); - const bool is_last_holder = locked_key.isLastOwnerOfFileSegment(offset()); + const bool is_last_holder = locked_key->isLastOwnerOfFileSegment(offset()); const size_t current_downloaded_size = getDownloadedSize(true); SCOPE_EXIT({ if (is_downloader) - { cv.notify_all(); - } }); LOG_TEST( @@ -618,9 +595,9 @@ void FileSegment::completeUnlocked(LockedKey & locked_key) if (segment_kind == FileSegmentKind::Temporary && is_last_holder) { LOG_TEST(log, "Removing temporary file segment: {}", getInfoForLogUnlocked(segment_lock)); - detach(segment_lock, locked_key); + detach(segment_lock, *locked_key); setDownloadState(State::DETACHED, segment_lock); - locked_key.removeFileSegment(offset(), segment_lock); + locked_key->removeFileSegment(offset(), segment_lock); return; } @@ -644,12 +621,10 @@ void FileSegment::completeUnlocked(LockedKey & locked_key) { if (is_last_holder) { - setDownloadState(State::DETACHED, segment_lock); - if (current_downloaded_size == 0) { LOG_TEST(log, "Remove file segment {} (nothing downloaded)", range().toString()); - locked_key.removeFileSegment(offset(), segment_lock); + locked_key->removeFileSegment(offset(), segment_lock); } else { @@ -666,13 +641,13 @@ void FileSegment::completeUnlocked(LockedKey & locked_key) /// but current file segment should remain PARRTIALLY_DOWNLOADED_NO_CONTINUATION and with detached state, /// because otherwise an invariant that getOrSet() returns a contiguous range of file segments will be broken /// (this will be crucial for other file segment holder, not for current one). - locked_key.shrinkFileSegmentToDownloadedSize(offset(), segment_lock); + locked_key->shrinkFileSegmentToDownloadedSize(offset(), segment_lock); /// We mark current file segment with state DETACHED, even though the data is still in cache /// (but a separate file segment) because is_last_holder is satisfied, so it does not matter. } - detachAssumeStateFinalized(segment_lock); + setDetachedState(segment_lock); } break; } @@ -707,11 +682,6 @@ String FileSegment::getInfoForLogUnlocked(const FileSegmentGuard::Lock &) const return info.str(); } -void FileSegment::wrapWithCacheInfo(Exception & e, const String & message, const FileSegmentGuard::Lock & lock) const -{ - e.addMessage(fmt::format("{}, current cache state: {}", message, getInfoForLogUnlocked(lock))); -} - String FileSegment::stateToString(FileSegment::State state) { switch (state) @@ -755,15 +725,6 @@ bool FileSegment::assertCorrectness() const return true; } -void FileSegment::throwIfDetachedUnlocked(const FileSegmentGuard::Lock & lock) const -{ - throw Exception( - ErrorCodes::LOGICAL_ERROR, - "Cache file segment is in detached state, operation not allowed. " - "It can happen when cache was concurrently dropped with SYSTEM DROP FILESYSTEM CACHE FORCE. " - "Please, retry. File segment info: {}", getInfoForLogUnlocked(lock)); -} - void FileSegment::assertNotDetached() const { auto lock = segment_guard.lock(); @@ -773,7 +734,13 @@ void FileSegment::assertNotDetached() const void FileSegment::assertNotDetachedUnlocked(const FileSegmentGuard::Lock & lock) const { if (download_state == State::DETACHED) - throwIfDetachedUnlocked(lock); + { + throw Exception( + ErrorCodes::LOGICAL_ERROR, + "Cache file segment is in detached state, operation not allowed. " + "It can happen when cache was concurrently dropped with SYSTEM DROP FILESYSTEM CACHE FORCE. " + "Please, retry. File segment info: {}", getInfoForLogUnlocked(lock)); + } } FileSegmentPtr FileSegment::getSnapshot(const FileSegmentPtr & file_segment) @@ -785,14 +752,12 @@ FileSegmentPtr FileSegment::getSnapshot(const FileSegmentPtr & file_segment) file_segment->offset(), file_segment->range().size(), State::DETACHED, - CreateFileSegmentSettings(file_segment->getKind())); + CreateFileSegmentSettings(file_segment->getKind(), file_segment->is_unbound)); snapshot->hits_count = file_segment->getHitsCount(); snapshot->downloaded_size = file_segment->getDownloadedSize(false); snapshot->download_state = file_segment->download_state.load(); - snapshot->ref_count = file_segment.use_count(); - snapshot->is_unbound = file_segment->is_unbound; return snapshot; } @@ -822,27 +787,32 @@ bool FileSegment::isCompleted(bool sync) const return is_completed_state(); } +void FileSegment::setDetachedState(const FileSegmentGuard::Lock & lock) +{ + setDownloadState(State::DETACHED, lock); + key_metadata.reset(); + cache = nullptr; +} + void FileSegment::detach(const FileSegmentGuard::Lock & lock, const LockedKey &) { if (download_state == State::DETACHED) return; - setDownloadState(State::DETACHED, lock); resetDownloaderUnlocked(lock); - - detachAssumeStateFinalized(lock); -} - -void FileSegment::detachAssumeStateFinalized(const FileSegmentGuard::Lock & lock) -{ - key_metadata.reset(); - LOG_TEST(log, "Detached file segment: {}", getInfoForLogUnlocked(lock)); + setDetachedState(lock); } void FileSegment::use() { + if (!cache) + { + chassert(isCompleted(true)); + return; + } + auto it = getQueueIterator(); - if (it && cache) + if (it) { auto cache_lock = cache->lockCache(); it->use(cache_lock); @@ -869,7 +839,7 @@ String FileSegmentsHolder::toString() if (!ranges.empty()) ranges += ", "; ranges += file_segment->range().toString(); - if (file_segment->is_unbound) + if (file_segment->isUnbound()) ranges += "(unbound)"; } return ranges; diff --git a/src/Interpreters/Cache/FileSegment.h b/src/Interpreters/Cache/FileSegment.h index 5fc3b45cc5ec..67432a1de019 100644 --- a/src/Interpreters/Cache/FileSegment.h +++ b/src/Interpreters/Cache/FileSegment.h @@ -163,11 +163,10 @@ friend struct LockedKey; size_t offset() const { return range().left; } FileSegmentKind getKind() const { return segment_kind; } + bool isPersistent() const { return segment_kind == FileSegmentKind::Persistent; } - bool isUnbound() const { return is_unbound; } - using UniqueId = std::pair; - UniqueId getUniqueId() const { return std::pair(key(), offset()); } + bool isUnbound() const { return is_unbound; } String getPathInLocalCache() const; @@ -198,6 +197,8 @@ friend struct LockedKey; size_t getDownloadedSize(bool sync) const; + size_t getReservedSize() const; + /// Now detached status can be used in the following cases: /// 1. there is only 1 remaining file segment holder /// && it does not need this segment anymore @@ -218,12 +219,40 @@ friend struct LockedKey; bool isDetached() const; - /// File segment has a completed state, if this state is final and is not going to be changed. - /// Completed states: DOWNALODED, DETACHED. + /// File segment has a completed state, if this state is final and + /// is not going to be changed. Completed states: DOWNALODED, DETACHED. bool isCompleted(bool sync = false) const; + void use(); + + /** + * ========== Methods used by `cache` ======================== + */ + + FileSegmentGuard::Lock lock() const { return segment_guard.lock(); } + + CachePriorityIterator getQueueIterator() const; + + void setQueueIterator(CachePriorityIterator iterator); + + KeyMetadataPtr tryGetKeyMetadata() const; + + KeyMetadataPtr getKeyMetadata() const; + bool assertCorrectness() const; + /** + * ========== Methods that must do cv.notify() ================== + */ + + void setBroken(); + + void complete(); + + void completePartAndResetDownloader(); + + void resetDownloader(); + /** * ========== Methods for _only_ file segment's `downloader` ================== */ @@ -240,70 +269,33 @@ friend struct LockedKey; /// Write data into reserved space. void write(const char * from, size_t size, size_t offset); - void setBroken(); - - void complete(); - - /// Complete file segment's part which was last written. - void completePartAndResetDownloader(); - - void resetDownloader(); - RemoteFileReaderPtr getRemoteFileReader(); RemoteFileReaderPtr extractRemoteFileReader(); void setRemoteFileReader(RemoteFileReaderPtr remote_file_reader_); - void resetRemoteFileReader(); - - FileSegmentGuard::Lock lock() const { return segment_guard.lock(); } - void setDownloadedSize(size_t delta); - size_t getReservedSize() const; - - CachePriorityIterator getQueueIterator() const; - - void setQueueIterator(CachePriorityIterator iterator); - - KeyMetadataPtr getKeyMetadata() const; - - void use(); - private: - String getInfoForLogUnlocked(const FileSegmentGuard::Lock &) const; String getDownloaderUnlocked(const FileSegmentGuard::Lock &) const; + bool isDownloaderUnlocked(const FileSegmentGuard::Lock & segment_lock) const; void resetDownloaderUnlocked(const FileSegmentGuard::Lock &); - void resetDownloadingStateUnlocked(const FileSegmentGuard::Lock &); void setDownloadState(State state, const FileSegmentGuard::Lock &); - void setDownloadedSizeUnlocked(size_t delta, const FileSegmentGuard::Lock &); + void resetDownloadingStateUnlocked(const FileSegmentGuard::Lock &); + void setDetachedState(const FileSegmentGuard::Lock &); + + String getInfoForLogUnlocked(const FileSegmentGuard::Lock &) const; void setDownloadedUnlocked(const FileSegmentGuard::Lock &); void setDownloadFailedUnlocked(const FileSegmentGuard::Lock &); - bool isDetached(const FileSegmentGuard::Lock &) const { return download_state == State::DETACHED; } - void detachAssumeStateFinalized(const FileSegmentGuard::Lock &); - [[noreturn]] void throwIfDetachedUnlocked(const FileSegmentGuard::Lock &) const; - void assertNotDetached() const; void assertNotDetachedUnlocked(const FileSegmentGuard::Lock &) const; void assertIsDownloaderUnlocked(const std::string & operation, const FileSegmentGuard::Lock &) const; LockedKeyPtr lockKeyMetadata(bool assert_exists = true) const; - KeyMetadataPtr tryGetKeyMetadata() const; - - /// completeWithoutStateUnlocked() is called from destructor of FileSegmentsHolder. - /// Function might check if the caller of the method - /// is the last alive holder of the segment. Therefore, completion and destruction - /// of the file segment pointer must be done under the same cache mutex. - void completeUnlocked(LockedKey & locked_key); - - void completePartAndResetDownloaderUnlocked(const FileSegmentGuard::Lock & segment_lock); - bool isDownloaderUnlocked(const FileSegmentGuard::Lock & segment_lock) const; - - void wrapWithCacheInfo(Exception & e, const String & message, const FileSegmentGuard::Lock & segment_lock) const; Key file_key; Range segment_range; diff --git a/src/Interpreters/Cache/IFileCachePriority.h b/src/Interpreters/Cache/IFileCachePriority.h index fb0ac1521487..31a23e137ca5 100644 --- a/src/Interpreters/Cache/IFileCachePriority.h +++ b/src/Interpreters/Cache/IFileCachePriority.h @@ -37,9 +37,6 @@ class IFileCachePriority : private boost::noncopyable const size_t offset; size_t size; size_t hits = 0; - /// In fact, it is guaranteed that the lifetime of key metadata is longer - /// than Entry, but it is made as weak_ptr to avoid cycle in shared pointer - /// references (because entry actually lies in key metadata). const KeyMetadataPtr key_metadata; }; @@ -52,15 +49,17 @@ class IFileCachePriority : private boost::noncopyable public: virtual ~IIterator() = default; + virtual size_t use(const CacheGuard::Lock &) = 0; + + virtual std::shared_ptr remove(const CacheGuard::Lock &) = 0; + virtual const Entry & getEntry() const = 0; virtual Entry & getEntry() = 0; - virtual size_t use(const CacheGuard::Lock &) = 0; + virtual void annul() = 0; virtual void updateSize(ssize_t size) = 0; - - virtual std::shared_ptr remove(const CacheGuard::Lock &) = 0; }; using Iterator = std::shared_ptr; diff --git a/src/Interpreters/Cache/LRUFileCachePriority.cpp b/src/Interpreters/Cache/LRUFileCachePriority.cpp index 5b37191fba77..131db6714c3f 100644 --- a/src/Interpreters/Cache/LRUFileCachePriority.cpp +++ b/src/Interpreters/Cache/LRUFileCachePriority.cpp @@ -95,7 +95,8 @@ void LRUFileCachePriority::iterate(IterateFunc && func, const CacheGuard::Lock & for (auto it = queue.begin(); it != queue.end();) { auto locked_key = it->key_metadata->lock(); - if (locked_key->getKeyState() != KeyMetadata::KeyState::ACTIVE) + if (it->size == 0 + || locked_key->getKeyState() != KeyMetadata::KeyState::ACTIVE) { it = remove(it); continue; @@ -127,6 +128,12 @@ LRUFileCachePriority::Iterator LRUFileCachePriority::LRUFileCacheIterator::remov return std::make_shared(cache_priority, cache_priority->remove(queue_iter)); } +void LRUFileCachePriority::LRUFileCacheIterator::annul() +{ + cache_priority->current_size -= queue_iter->size; + queue_iter->size = 0; +} + void LRUFileCachePriority::LRUFileCacheIterator::updateSize(ssize_t size) { cache_priority->current_size += size; @@ -135,6 +142,9 @@ void LRUFileCachePriority::LRUFileCacheIterator::updateSize(ssize_t size) else CurrentMetrics::sub(CurrentMetrics::FilesystemCacheSize, size); queue_iter->size += size; + + chassert(cache_priority->current_size >= 0); + chassert(queue_iter->size >= 0); } size_t LRUFileCachePriority::LRUFileCacheIterator::use(const CacheGuard::Lock &) diff --git a/src/Interpreters/Cache/LRUFileCachePriority.h b/src/Interpreters/Cache/LRUFileCachePriority.h index af50d00fcb5e..20e3a8ffe4bd 100644 --- a/src/Interpreters/Cache/LRUFileCachePriority.h +++ b/src/Interpreters/Cache/LRUFileCachePriority.h @@ -56,6 +56,8 @@ class LRUFileCachePriority::LRUFileCacheIterator : public IFileCachePriority::II Iterator remove(const CacheGuard::Lock &) override; + void annul() override; + void updateSize(ssize_t size) override; private: diff --git a/src/Interpreters/Cache/Metadata.cpp b/src/Interpreters/Cache/Metadata.cpp index 926c6de5d951..67a5f61ad6f5 100644 --- a/src/Interpreters/Cache/Metadata.cpp +++ b/src/Interpreters/Cache/Metadata.cpp @@ -49,9 +49,11 @@ KeyMetadata::KeyMetadata( bool created_base_directory_) : key(key_) , key_path(key_path_) - , created_base_directory(created_base_directory_) , cleanup_queue(cleanup_queue_) + , created_base_directory(created_base_directory_) { + if (created_base_directory) + chassert(fs::exists(key_path)); } LockedKeyPtr KeyMetadata::lock() @@ -65,14 +67,40 @@ LockedKeyPtr KeyMetadata::lock() "Cannot lock key {} (state: {})", key, magic_enum::enum_name(key_state)); } +void KeyMetadata::createBaseDirectory() +{ + if (!created_base_directory.exchange(true)) + { + fs::create_directories(key_path); + } +} + std::string KeyMetadata::getFileSegmentPath(const FileSegment & file_segment) { return fs::path(key_path) / CacheMetadata::getFileNameForFileSegment(file_segment.offset(), file_segment.getKind()); } + +struct CleanupQueue +{ + friend struct CacheMetadata; +public: + void add(const FileCacheKey & key); + void remove(const FileCacheKey & key); + size_t getSize() const; + +private: + bool tryPop(FileCacheKey & key); + + std::unordered_set keys; + mutable std::mutex mutex; +}; + + CacheMetadata::CacheMetadata(const std::string & path_) : path(path_) + , cleanup_queue(std::make_unique()) , log(&Poco::Logger::get("CacheMetadata")) { } @@ -128,7 +156,7 @@ LockedKeyPtr CacheMetadata::lockKeyMetadata( it = emplace( key, std::make_shared( - key, getPathInLocalCache(key), cleanup_queue, is_initial_load)).first; + key, getPathInLocalCache(key), *cleanup_queue, is_initial_load)).first; } key_metadata = it->second; @@ -198,7 +226,7 @@ void CacheMetadata::doCleanup() /// we perform this delayed removal. FileCacheKey cleanup_key; - while (cleanup_queue.tryPop(cleanup_key)) + while (cleanup_queue->tryPop(cleanup_key)) { auto it = find(cleanup_key); if (it == end()) @@ -259,15 +287,9 @@ void LockedKey::removeFromCleanupQueue() key_metadata->key_state = KeyMetadata::KeyState::ACTIVE; } -bool LockedKey::markAsRemoved() +void LockedKey::markAsRemoved() { - chassert(key_metadata->key_state != KeyMetadata::KeyState::REMOVED); - - if (key_metadata->key_state == KeyMetadata::KeyState::ACTIVE) - return false; - key_metadata->key_state = KeyMetadata::KeyState::REMOVED; - return true; } bool LockedKey::isLastOwnerOfFileSegment(size_t offset) const @@ -301,13 +323,13 @@ KeyMetadata::iterator LockedKey::removeFileSegment(size_t offset, const FileSegm auto file_segment = it->second->file_segment; if (file_segment->queue_iterator) - file_segment->queue_iterator->updateSize(-file_segment->queue_iterator->getEntry().size); + file_segment->queue_iterator->annul(); - const auto path = key_metadata->getFileSegmentPath(*it->second->file_segment); + const auto path = key_metadata->getFileSegmentPath(*file_segment); if (fs::exists(path)) fs::remove(path); - it->second->file_segment->detach(segment_lock, *this); + file_segment->detach(segment_lock, *this); return key_metadata->erase(it); } @@ -326,6 +348,8 @@ void LockedKey::shrinkFileSegmentToDownloadedSize( const size_t downloaded_size = file_segment->getDownloadedSize(false); const size_t full_size = file_segment->range().size(); + chassert(downloaded_size <= file_segment->reserved_size); + if (downloaded_size == full_size) { throw Exception( @@ -334,16 +358,14 @@ void LockedKey::shrinkFileSegmentToDownloadedSize( file_segment->getInfoForLogUnlocked(segment_lock)); } - auto queue_iterator = metadata->file_segment->queue_iterator; - - chassert(downloaded_size <= file_segment->reserved_size); - chassert(queue_iterator->getEntry().size == file_segment->reserved_size); - CreateFileSegmentSettings create_settings(file_segment->getKind()); + auto queue_iterator = file_segment->queue_iterator; + metadata->file_segment = std::make_shared( getKey(), offset, downloaded_size, FileSegment::State::DOWNLOADED, create_settings, - file_segment->cache, key_metadata, file_segment->queue_iterator); + file_segment->cache, key_metadata, queue_iterator); + chassert(queue_iterator->getEntry().size == file_segment->reserved_size); ssize_t diff = file_segment->reserved_size - file_segment->downloaded_size; if (diff) queue_iterator->updateSize(-diff); @@ -352,18 +374,6 @@ void LockedKey::shrinkFileSegmentToDownloadedSize( chassert(metadata->size() == queue_iterator->getEntry().size); } -void LockedKey::assertFileSegmentCorrectness(const FileSegment & file_segment) const -{ - if (file_segment.key() != getKey()) - { - throw Exception( - ErrorCodes::LOGICAL_ERROR, - "Expected {} = {}", file_segment.key(), getKey()); - } - - file_segment.assertCorrectness(); -} - std::shared_ptr LockedKey::getByOffset(size_t offset) const { auto it = key_metadata->find(offset); diff --git a/src/Interpreters/Cache/Metadata.h b/src/Interpreters/Cache/Metadata.h index 71a905a259da..54dde8c9aee8 100644 --- a/src/Interpreters/Cache/Metadata.h +++ b/src/Interpreters/Cache/Metadata.h @@ -12,6 +12,7 @@ using FileSegmentPtr = std::shared_ptr; struct LockedKey; using LockedKeyPtr = std::shared_ptr; struct CleanupQueue; +using CleanupQueuePtr = std::shared_ptr; struct FileSegmentMetadata : private boost::noncopyable @@ -56,37 +57,23 @@ struct KeyMetadata : public std::map, const Key key; const std::string key_path; - std::atomic created_base_directory = false; LockedKeyPtr lock(); + void createBaseDirectory(); + std::string getFileSegmentPath(const FileSegment & file_segment); private: KeyState key_state = KeyState::ACTIVE; KeyGuard guard; CleanupQueue & cleanup_queue; + std::atomic created_base_directory = false; }; using KeyMetadataPtr = std::shared_ptr; -struct CleanupQueue -{ - friend struct CacheMetadata; -public: - void add(const FileCacheKey & key); - void remove(const FileCacheKey & key); - size_t getSize() const; - -private: - bool tryPop(FileCacheKey & key); - - std::unordered_set keys; - mutable std::mutex mutex; -}; - - struct CacheMetadata : public std::unordered_map, private boost::noncopyable { public: @@ -124,7 +111,7 @@ struct CacheMetadata : public std::unordered_map, private: const std::string path; /// Cache base path CacheMetadataGuard guard; - CleanupQueue cleanup_queue; + const CleanupQueuePtr cleanup_queue; Poco::Logger * log; }; @@ -162,26 +149,20 @@ struct LockedKey : private boost::noncopyable KeyMetadata::KeyState getKeyState() const { return key_metadata->key_state; } - KeyMetadataPtr getKeyMetadata() const { return key_metadata; } - KeyMetadataPtr getKeyMetadata() { return key_metadata; } - - KeyMetadata::iterator removeFileSegment(size_t offset, const FileSegmentGuard::Lock &); + std::shared_ptr getKeyMetadata() const { return key_metadata; } + std::shared_ptr getKeyMetadata() { return key_metadata; } void removeAllReleasable(); + KeyMetadata::iterator removeFileSegment(size_t offset, const FileSegmentGuard::Lock &); + void shrinkFileSegmentToDownloadedSize(size_t offset, const FileSegmentGuard::Lock &); bool isLastOwnerOfFileSegment(size_t offset) const; - void assertFileSegmentCorrectness(const FileSegment & file_segment) const; - - bool isRemovalCandidate() const; - - bool markAsRemovalCandidate(size_t offset); - void removeFromCleanupQueue(); - bool markAsRemoved(); + void markAsRemoved(); std::string toString() const; From 58a30213c9f5c7f515a3ab1390dd6b9bcbb04d11 Mon Sep 17 00:00:00 2001 From: kssenii Date: Thu, 13 Apr 2023 13:34:19 +0200 Subject: [PATCH 120/406] Fix after merge --- src/Interpreters/Cache/FileSegment.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Interpreters/Cache/FileSegment.cpp b/src/Interpreters/Cache/FileSegment.cpp index 34de2c71bde5..77749ed3233d 100644 --- a/src/Interpreters/Cache/FileSegment.cpp +++ b/src/Interpreters/Cache/FileSegment.cpp @@ -538,7 +538,8 @@ void FileSegment::setBroken() assertNotDetachedUnlocked(lock); assertIsDownloaderUnlocked("setBroken", lock); - resetDownloadingStateUnlocked(lock); + if (download_state == State::DOWNLOADING) + resetDownloadingStateUnlocked(lock); if (download_state != State::DOWNLOADED) download_state = State::PARTIALLY_DOWNLOADED_NO_CONTINUATION; From ce723ec32dcc128b1a39057d9f475f71b82e2446 Mon Sep 17 00:00:00 2001 From: kssenii Date: Thu, 13 Apr 2023 14:44:06 +0200 Subject: [PATCH 121/406] Fix style check, better priority->iterate --- .../IO/CachedOnDiskReadBufferFromFile.cpp | 2 +- src/Interpreters/Cache/FileCache.cpp | 52 +++++++------------ src/Interpreters/Cache/FileCache.h | 4 +- .../Cache/FileCache_fwd_internal.h | 25 +++++++++ src/Interpreters/Cache/FileSegment.cpp | 3 +- src/Interpreters/Cache/FileSegment.h | 12 +---- src/Interpreters/Cache/IFileCachePriority.h | 14 +---- .../Cache/LRUFileCachePriority.cpp | 17 +++++- src/Interpreters/Cache/Metadata.h | 5 +- src/Interpreters/Context.cpp | 1 - 10 files changed, 69 insertions(+), 66 deletions(-) create mode 100644 src/Interpreters/Cache/FileCache_fwd_internal.h diff --git a/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp b/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp index 528e26743833..7827e537708d 100644 --- a/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp +++ b/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp @@ -632,7 +632,7 @@ void CachedOnDiskReadBufferFromFile::predownload(FileSegment & file_segment) bytes_to_predownload = 0; file_segment.setBroken(); - chassert(file_segment->state() == FileSegment::State::PARTIALLY_DOWNLOADED_NO_CONTINUATION); + chassert(file_segment.state() == FileSegment::State::PARTIALLY_DOWNLOADED_NO_CONTINUATION); LOG_TEST(log, "Bypassing cache because for {}", file_segment.getInfoForLog()); diff --git a/src/Interpreters/Cache/FileCache.cpp b/src/Interpreters/Cache/FileCache.cpp index 82128e681e46..7bdb97e527d5 100644 --- a/src/Interpreters/Cache/FileCache.cpp +++ b/src/Interpreters/Cache/FileCache.cpp @@ -315,9 +315,9 @@ void FileCache::fillHolesWithEmptyFileSegments( } else { - auto splitted = splitRangeIntoFileSegments( + auto split = splitRangeIntoFileSegments( locked_key, current_pos, hole_size, FileSegment::State::EMPTY, settings); - file_segments.splice(it, std::move(splitted)); + file_segments.splice(it, std::move(split)); } current_pos = segment_range.right + 1; @@ -342,9 +342,9 @@ void FileCache::fillHolesWithEmptyFileSegments( } else { - auto splitted = splitRangeIntoFileSegments( + auto split = splitRangeIntoFileSegments( locked_key, current_pos, hole_size, FileSegment::State::EMPTY, settings); - file_segments.splice(file_segments.end(), std::move(splitted)); + file_segments.splice(file_segments.end(), std::move(split)); } } } @@ -556,29 +556,25 @@ bool FileCache::tryReserve(FileSegment & file_segment, size_t size) size_t removed_size = 0; std::unordered_map to_delete; - auto iterate_func = [&](const PriorityEntry & entry, LockedKey & locked_key) + auto iterate_func = [&](LockedKey & locked_key, FileSegmentMetadataPtr segment_metadata) { - auto file_segment_metadata = locked_key.tryGetByOffset(entry.offset); - if (!file_segment_metadata) - return PriorityIterationResult::REMOVE_AND_CONTINUE; - - chassert(file_segment_metadata->file_segment->getQueueIterator()); - const bool is_persistent = allow_persistent_files && file_segment_metadata->file_segment->isPersistent(); - const bool releasable = file_segment_metadata->releasable() && !is_persistent; + chassert(segment_metadata->file_segment->getQueueIterator()); + const bool is_persistent = allow_persistent_files && segment_metadata->file_segment->isPersistent(); + const bool releasable = segment_metadata->releasable() && !is_persistent; if (releasable) { - removed_size += entry.size; + removed_size += segment_metadata->size(); --queue_size; - auto segment = file_segment_metadata->file_segment; + auto segment = segment_metadata->file_segment; if (segment->state() == FileSegment::State::DOWNLOADED) { const auto & key = segment->key(); auto it = to_delete.find(key); if (it == to_delete.end()) it = to_delete.emplace(key, locked_key.getKeyMetadata()).first; - it->second.add(file_segment_metadata); + it->second.add(segment_metadata); return PriorityIterationResult::CONTINUE; } @@ -598,8 +594,8 @@ bool FileCache::tryReserve(FileSegment & file_segment, size_t size) }; query_priority->iterate( - [&](const auto & entry, LockedKey & locked_key) - { return is_query_priority_overflow() ? iterate_func(entry, locked_key) : PriorityIterationResult::BREAK; }, + [&](LockedKey & locked_key, FileSegmentMetadataPtr segment_metadata) + { return is_query_priority_overflow() ? iterate_func(locked_key, segment_metadata) : PriorityIterationResult::BREAK; }, cache_lock); if (is_query_priority_overflow()) @@ -615,8 +611,8 @@ bool FileCache::tryReserve(FileSegment & file_segment, size_t size) }; main_priority->iterate( - [&](const auto & entry, LockedKey & locked_key) - { return is_main_priority_overflow() ? iterate_func(entry, locked_key) : PriorityIterationResult::BREAK; }, + [&](LockedKey & locked_key, FileSegmentMetadataPtr segment_metadata) + { return is_main_priority_overflow() ? iterate_func(locked_key, segment_metadata) : PriorityIterationResult::BREAK; }, cache_lock); if (is_main_priority_overflow()) @@ -696,15 +692,11 @@ void FileCache::removeAllReleasable() auto lock = cache_guard.lock(); - main_priority->iterate([&](const PriorityEntry & entry, LockedKey & locked_key) + main_priority->iterate([&](LockedKey & locked_key, FileSegmentMetadataPtr segment_metadata) { - auto file_segment_metadata = locked_key.tryGetByOffset(entry.offset); - if (!file_segment_metadata) - return PriorityIterationResult::REMOVE_AND_CONTINUE; - - if (file_segment_metadata->releasable()) + if (segment_metadata->releasable()) { - auto file_segment = file_segment_metadata->file_segment; + auto file_segment = segment_metadata->file_segment; locked_key.removeFileSegment(file_segment->offset(), file_segment->lock()); return PriorityIterationResult::REMOVE_AND_CONTINUE; } @@ -933,13 +925,9 @@ FileSegmentsHolderPtr FileCache::dumpQueue() assertInitialized(); FileSegments file_segments; - main_priority->iterate([&](const PriorityEntry & entry, LockedKey & locked_key) + main_priority->iterate([&](LockedKey &, FileSegmentMetadataPtr segment_metadata) { - auto file_segment_metadata = locked_key.tryGetByOffset(entry.offset); - if (!file_segment_metadata) - return PriorityIterationResult::REMOVE_AND_CONTINUE; - - file_segments.push_back(FileSegment::getSnapshot(file_segment_metadata->file_segment)); + file_segments.push_back(FileSegment::getSnapshot(segment_metadata->file_segment)); return PriorityIterationResult::CONTINUE; }, cache_guard.lock()); diff --git a/src/Interpreters/Cache/FileCache.h b/src/Interpreters/Cache/FileCache.h index 4e01751daaf7..49c5dade3bfc 100644 --- a/src/Interpreters/Cache/FileCache.h +++ b/src/Interpreters/Cache/FileCache.h @@ -18,15 +18,13 @@ #include #include #include +#include #include namespace DB { -struct LockedKey; -using LockedKeyPtr = std::shared_ptr; - namespace ErrorCodes { extern const int BAD_ARGUMENTS; diff --git a/src/Interpreters/Cache/FileCache_fwd_internal.h b/src/Interpreters/Cache/FileCache_fwd_internal.h new file mode 100644 index 000000000000..8c59f1f78b72 --- /dev/null +++ b/src/Interpreters/Cache/FileCache_fwd_internal.h @@ -0,0 +1,25 @@ +#include + +namespace DB +{ + +class FileCache; +using FileCachePtr = std::shared_ptr; + +class IFileCachePriority; +using FileCachePriorityPtr = std::unique_ptr; + +class FileSegment; +using FileSegmentPtr = std::shared_ptr; +using FileSegments = std::list; + +struct FileSegmentMetadata; +using FileSegmentMetadataPtr = std::shared_ptr; + +struct LockedKey; +using LockedKeyPtr = std::shared_ptr; + +struct KeyMetadata; +using KeyMetadataPtr = std::shared_ptr; + +} diff --git a/src/Interpreters/Cache/FileSegment.cpp b/src/Interpreters/Cache/FileSegment.cpp index 77749ed3233d..63c666d04c58 100644 --- a/src/Interpreters/Cache/FileSegment.cpp +++ b/src/Interpreters/Cache/FileSegment.cpp @@ -380,10 +380,11 @@ FileSegment::State FileSegment::wait(size_t offset) chassert(!getDownloaderUnlocked(lock).empty()); chassert(!isDownloaderUnlocked(lock)); - cv.wait_for(lock, std::chrono::seconds(60), [&, this]() + [[maybe_unused]] auto ok = cv.wait_for(lock, std::chrono::seconds(60), [&, this]() { return download_state != State::DOWNLOADING || offset < getCurrentWriteOffset(true); }); + chassert(ok); } return download_state; diff --git a/src/Interpreters/Cache/FileSegment.h b/src/Interpreters/Cache/FileSegment.h index 67432a1de019..6defe969ba45 100644 --- a/src/Interpreters/Cache/FileSegment.h +++ b/src/Interpreters/Cache/FileSegment.h @@ -11,7 +11,7 @@ #include #include #include -#include +#include #include @@ -25,18 +25,8 @@ extern const Metric CacheFileSegments; namespace DB { -class FileCache; class ReadBufferFromFileBase; -class FileSegment; -using FileSegmentPtr = std::shared_ptr; -using FileSegments = std::list; -struct FileSegmentMetadata; -struct LockedKey; -using LockedKeyPtr = std::shared_ptr; -struct KeyMetadata; -using KeyMetadataPtr = std::shared_ptr; - /* * FileSegmentKind is used to specify the eviction policy for file segments. */ diff --git a/src/Interpreters/Cache/IFileCachePriority.h b/src/Interpreters/Cache/IFileCachePriority.h index 31a23e137ca5..748089fbd1ba 100644 --- a/src/Interpreters/Cache/IFileCachePriority.h +++ b/src/Interpreters/Cache/IFileCachePriority.h @@ -6,21 +6,11 @@ #include #include #include +#include namespace DB { -class IFileCachePriority; -using FileCachePriorityPtr = std::unique_ptr; -struct KeyMetadata; -using KeyMetadataPtr = std::shared_ptr; -struct LockedKey; - -namespace ErrorCodes -{ - extern const int LOGICAL_ERROR; -} - /// IFileCachePriority is used to maintain the priority of cached data. class IFileCachePriority : private boost::noncopyable { @@ -71,7 +61,7 @@ class IFileCachePriority : private boost::noncopyable CONTINUE, REMOVE_AND_CONTINUE, }; - using IterateFunc = std::function; + using IterateFunc = std::function; IFileCachePriority(size_t max_size_, size_t max_elements_) : max_size(max_size_), max_elements(max_elements_) {} diff --git a/src/Interpreters/Cache/LRUFileCachePriority.cpp b/src/Interpreters/Cache/LRUFileCachePriority.cpp index 131db6714c3f..9a0e2d8fb057 100644 --- a/src/Interpreters/Cache/LRUFileCachePriority.cpp +++ b/src/Interpreters/Cache/LRUFileCachePriority.cpp @@ -102,7 +102,22 @@ void LRUFileCachePriority::iterate(IterateFunc && func, const CacheGuard::Lock & continue; } - auto result = func(*it, *locked_key); + auto metadata = locked_key->tryGetByOffset(it->offset); + if (!metadata) + { + it = remove(it); + continue; + } + + if (metadata->size() != it->size) + { + throw Exception( + ErrorCodes::LOGICAL_ERROR, + "Mismatch of file segment size in file segment metadata and priority queue: {} != {}", + it->size, metadata->size()); + } + + auto result = func(*locked_key, metadata); switch (result) { case IterationResult::BREAK: diff --git a/src/Interpreters/Cache/Metadata.h b/src/Interpreters/Cache/Metadata.h index 54dde8c9aee8..cb2c68d929ce 100644 --- a/src/Interpreters/Cache/Metadata.h +++ b/src/Interpreters/Cache/Metadata.h @@ -4,13 +4,10 @@ #include #include #include +#include namespace DB { -class FileSegment; -using FileSegmentPtr = std::shared_ptr; -struct LockedKey; -using LockedKeyPtr = std::shared_ptr; struct CleanupQueue; using CleanupQueuePtr = std::shared_ptr; diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index 49b50ce94b10..ba250ed3e5bb 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -19,7 +19,6 @@ #include #include #include -#include #include #include #include From 461477dabfb31c1aff9634fe2cee4360c2c2ef76 Mon Sep 17 00:00:00 2001 From: kssenii Date: Thu, 13 Apr 2023 16:38:38 +0200 Subject: [PATCH 122/406] Fix style check --- src/Interpreters/Cache/FileCache_fwd_internal.h | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Interpreters/Cache/FileCache_fwd_internal.h b/src/Interpreters/Cache/FileCache_fwd_internal.h index 8c59f1f78b72..5ded018a674e 100644 --- a/src/Interpreters/Cache/FileCache_fwd_internal.h +++ b/src/Interpreters/Cache/FileCache_fwd_internal.h @@ -1,3 +1,4 @@ +#pragma once #include namespace DB From 599a02ca0da1a97c996638454e58a62456089384 Mon Sep 17 00:00:00 2001 From: kssenii Date: Thu, 13 Apr 2023 18:53:16 +0200 Subject: [PATCH 123/406] Fix --- programs/local/LocalServer.cpp | 1 + src/Disks/ObjectStorages/DiskObjectStorage.cpp | 1 + src/Interpreters/Cache/FileCache.cpp | 1 + src/Interpreters/Cache/FileSegment.cpp | 10 +--------- src/Interpreters/Cache/FileSegment.h | 1 + src/Interpreters/Cache/LRUFileCachePriority.cpp | 4 ++-- 6 files changed, 7 insertions(+), 11 deletions(-) diff --git a/programs/local/LocalServer.cpp b/programs/local/LocalServer.cpp index 5768e744f94e..bbfd73af476f 100644 --- a/programs/local/LocalServer.cpp +++ b/programs/local/LocalServer.cpp @@ -26,6 +26,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Disks/ObjectStorages/DiskObjectStorage.cpp b/src/Disks/ObjectStorages/DiskObjectStorage.cpp index 6143f0620b81..0dc070ee9172 100644 --- a/src/Disks/ObjectStorages/DiskObjectStorage.cpp +++ b/src/Disks/ObjectStorages/DiskObjectStorage.cpp @@ -15,6 +15,7 @@ #include #include #include +#include #include #include diff --git a/src/Interpreters/Cache/FileCache.cpp b/src/Interpreters/Cache/FileCache.cpp index 7bdb97e527d5..d7497f068b6a 100644 --- a/src/Interpreters/Cache/FileCache.cpp +++ b/src/Interpreters/Cache/FileCache.cpp @@ -664,6 +664,7 @@ bool FileCache::tryReserve(FileSegment & file_segment, size_t size) throw Exception(ErrorCodes::LOGICAL_ERROR, "Cache became inconsistent. There must be a bug"); file_segment.getKeyMetadata()->createBaseDirectory(); + file_segment.reserved_size += size; return true; } diff --git a/src/Interpreters/Cache/FileSegment.cpp b/src/Interpreters/Cache/FileSegment.cpp index 63c666d04c58..35d9c0e16cd8 100644 --- a/src/Interpreters/Cache/FileSegment.cpp +++ b/src/Interpreters/Cache/FileSegment.cpp @@ -380,7 +380,7 @@ FileSegment::State FileSegment::wait(size_t offset) chassert(!getDownloaderUnlocked(lock).empty()); chassert(!isDownloaderUnlocked(lock)); - [[maybe_unused]] auto ok = cv.wait_for(lock, std::chrono::seconds(60), [&, this]() + [[maybe_unused]] const auto ok = cv.wait_for(lock, std::chrono::seconds(60), [&, this]() { return download_state != State::DOWNLOADING || offset < getCurrentWriteOffset(true); }); @@ -467,14 +467,6 @@ bool FileSegment::reserve(size_t size_to_reserve) segment_range.right = range().left + expected_downloaded_size + size_to_reserve; reserved = cache->tryReserve(*this, size_to_reserve); - if (reserved) - { - /// No lock is required because reserved size is always - /// mananaged (read/modified) by the downloader only - /// or in isLastOwnerOfFileSegment() case. - /// It is made atomic because of getInfoForLog. - reserved_size += size_to_reserve; - } chassert(assertCorrectness()); } diff --git a/src/Interpreters/Cache/FileSegment.h b/src/Interpreters/Cache/FileSegment.h index 6defe969ba45..6a40be3280c7 100644 --- a/src/Interpreters/Cache/FileSegment.h +++ b/src/Interpreters/Cache/FileSegment.h @@ -64,6 +64,7 @@ struct CreateFileSegmentSettings class FileSegment : private boost::noncopyable, public std::enable_shared_from_this { friend struct LockedKey; +friend class FileCache; /// Because of reserved_size in tryReserve(). public: using Key = FileCacheKey; diff --git a/src/Interpreters/Cache/LRUFileCachePriority.cpp b/src/Interpreters/Cache/LRUFileCachePriority.cpp index 9a0e2d8fb057..9853a09a3d3e 100644 --- a/src/Interpreters/Cache/LRUFileCachePriority.cpp +++ b/src/Interpreters/Cache/LRUFileCachePriority.cpp @@ -113,8 +113,8 @@ void LRUFileCachePriority::iterate(IterateFunc && func, const CacheGuard::Lock & { throw Exception( ErrorCodes::LOGICAL_ERROR, - "Mismatch of file segment size in file segment metadata and priority queue: {} != {}", - it->size, metadata->size()); + "Mismatch of file segment size in file segment metadata and priority queue: {} != {} ({})", + it->size, metadata->size(), metadata->file_segment->getInfoForLog()); } auto result = func(*locked_key, metadata); From f7232ef5372175d0a652a5c61c6eeb8631ad8e17 Mon Sep 17 00:00:00 2001 From: kssenii Date: Fri, 14 Apr 2023 12:46:36 +0200 Subject: [PATCH 124/406] Fix --- src/Interpreters/Cache/LRUFileCachePriority.cpp | 5 ++--- src/Interpreters/Cache/Metadata.cpp | 13 +++++++++++-- src/Interpreters/Cache/Metadata.h | 2 ++ 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/src/Interpreters/Cache/LRUFileCachePriority.cpp b/src/Interpreters/Cache/LRUFileCachePriority.cpp index 9853a09a3d3e..17828013c6c1 100644 --- a/src/Interpreters/Cache/LRUFileCachePriority.cpp +++ b/src/Interpreters/Cache/LRUFileCachePriority.cpp @@ -94,9 +94,8 @@ void LRUFileCachePriority::iterate(IterateFunc && func, const CacheGuard::Lock & { for (auto it = queue.begin(); it != queue.end();) { - auto locked_key = it->key_metadata->lock(); - if (it->size == 0 - || locked_key->getKeyState() != KeyMetadata::KeyState::ACTIVE) + auto locked_key = it->key_metadata->tryLock(); + if (!locked_key || it->size == 0) { it = remove(it); continue; diff --git a/src/Interpreters/Cache/Metadata.cpp b/src/Interpreters/Cache/Metadata.cpp index 67a5f61ad6f5..9396b8c6f449 100644 --- a/src/Interpreters/Cache/Metadata.cpp +++ b/src/Interpreters/Cache/Metadata.cpp @@ -58,8 +58,8 @@ KeyMetadata::KeyMetadata( LockedKeyPtr KeyMetadata::lock() { - auto locked = std::make_unique(shared_from_this()); - if (key_state == KeyMetadata::KeyState::ACTIVE) + auto locked = tryLock(); + if (locked) return locked; throw Exception( @@ -67,6 +67,15 @@ LockedKeyPtr KeyMetadata::lock() "Cannot lock key {} (state: {})", key, magic_enum::enum_name(key_state)); } +LockedKeyPtr KeyMetadata::tryLock() +{ + auto locked = std::make_unique(shared_from_this()); + if (key_state == KeyMetadata::KeyState::ACTIVE) + return locked; + + return nullptr; +} + void KeyMetadata::createBaseDirectory() { if (!created_base_directory.exchange(true)) diff --git a/src/Interpreters/Cache/Metadata.h b/src/Interpreters/Cache/Metadata.h index cb2c68d929ce..ee6506504886 100644 --- a/src/Interpreters/Cache/Metadata.h +++ b/src/Interpreters/Cache/Metadata.h @@ -57,6 +57,8 @@ struct KeyMetadata : public std::map, LockedKeyPtr lock(); + LockedKeyPtr tryLock(); + void createBaseDirectory(); std::string getFileSegmentPath(const FileSegment & file_segment); From ed2a5d25e5ac719d63f0857907a126c97a6ba2ac Mon Sep 17 00:00:00 2001 From: kssenii Date: Fri, 14 Apr 2023 15:26:25 +0200 Subject: [PATCH 125/406] Fix --- src/Disks/IO/CachedOnDiskWriteBufferFromFile.cpp | 2 +- src/Interpreters/Cache/FileCache.cpp | 2 +- src/Interpreters/Cache/FileSegment.cpp | 7 ++++++- src/Interpreters/Cache/FileSegment.h | 4 +++- src/Interpreters/Cache/LRUFileCachePriority.h | 2 +- 5 files changed, 12 insertions(+), 5 deletions(-) diff --git a/src/Disks/IO/CachedOnDiskWriteBufferFromFile.cpp b/src/Disks/IO/CachedOnDiskWriteBufferFromFile.cpp index 0a993cc503d9..9b351df041bb 100644 --- a/src/Disks/IO/CachedOnDiskWriteBufferFromFile.cpp +++ b/src/Disks/IO/CachedOnDiskWriteBufferFromFile.cpp @@ -142,7 +142,7 @@ FileSegment & FileSegmentRangeWriter::allocateFileSegment(size_t offset, FileSeg * File segment capacity will equal `max_file_segment_size`, but actual size is 0. */ - CreateFileSegmentSettings create_settings(segment_kind, true); + CreateFileSegmentSettings create_settings(segment_kind, false); /// We set max_file_segment_size to be downloaded, /// if we have less size to write, file segment will be resized in complete() method. diff --git a/src/Interpreters/Cache/FileCache.cpp b/src/Interpreters/Cache/FileCache.cpp index d7497f068b6a..9db634a81887 100644 --- a/src/Interpreters/Cache/FileCache.cpp +++ b/src/Interpreters/Cache/FileCache.cpp @@ -909,7 +909,7 @@ FileSegmentsHolderPtr FileCache::getSnapshot() for (const auto & [_, file_segment_metadata] : locked_key) file_segments.push_back(FileSegment::getSnapshot(file_segment_metadata->file_segment)); }); - return std::make_unique(std::move(file_segments)); + return std::make_unique(std::move(file_segments), /* complete_on_dtor */false); } FileSegmentsHolderPtr FileCache::getSnapshot(const Key & key) diff --git a/src/Interpreters/Cache/FileSegment.cpp b/src/Interpreters/Cache/FileSegment.cpp index 35d9c0e16cd8..b06bc521ac09 100644 --- a/src/Interpreters/Cache/FileSegment.cpp +++ b/src/Interpreters/Cache/FileSegment.cpp @@ -392,7 +392,7 @@ FileSegment::State FileSegment::wait(size_t offset) KeyMetadataPtr FileSegment::getKeyMetadata() const { - auto metadata = key_metadata.lock(); + auto metadata = tryGetKeyMetadata(); if (metadata) return metadata; throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot lock key, key metadata is not set ({})", stateToString(download_state)); @@ -613,6 +613,8 @@ void FileSegment::complete() case State::PARTIALLY_DOWNLOADED: case State::PARTIALLY_DOWNLOADED_NO_CONTINUATION: { + chassert(current_downloaded_size != range().size()); + if (is_last_holder) { if (current_downloaded_size == 0) @@ -821,6 +823,9 @@ FileSegments::iterator FileSegmentsHolder::completeAndPopFrontImpl() FileSegmentsHolder::~FileSegmentsHolder() { + if (!complete_on_dtor) + return; + for (auto file_segment_it = file_segments.begin(); file_segment_it != file_segments.end();) file_segment_it = completeAndPopFrontImpl(); } diff --git a/src/Interpreters/Cache/FileSegment.h b/src/Interpreters/Cache/FileSegment.h index 6a40be3280c7..e6aee5827e71 100644 --- a/src/Interpreters/Cache/FileSegment.h +++ b/src/Interpreters/Cache/FileSegment.h @@ -325,7 +325,8 @@ struct FileSegmentsHolder : private boost::noncopyable { FileSegmentsHolder() = default; - explicit FileSegmentsHolder(FileSegments && file_segments_) : file_segments(std::move(file_segments_)) {} + explicit FileSegmentsHolder(FileSegments && file_segments_, bool complete_on_dtor_ = true) + : file_segments(std::move(file_segments_)), complete_on_dtor(complete_on_dtor_) {} ~FileSegmentsHolder(); @@ -361,6 +362,7 @@ struct FileSegmentsHolder : private boost::noncopyable private: FileSegments file_segments{}; + const bool complete_on_dtor = true; FileSegments::iterator completeAndPopFrontImpl(); }; diff --git a/src/Interpreters/Cache/LRUFileCachePriority.h b/src/Interpreters/Cache/LRUFileCachePriority.h index 20e3a8ffe4bd..7d98da4f5991 100644 --- a/src/Interpreters/Cache/LRUFileCachePriority.h +++ b/src/Interpreters/Cache/LRUFileCachePriority.h @@ -36,7 +36,7 @@ class LRUFileCachePriority : public IFileCachePriority LRUQueue queue; Poco::Logger * log = &Poco::Logger::get("LRUFileCachePriority"); - size_t current_size = 0; + std::atomic current_size = 0; LRUQueueIterator remove(LRUQueueIterator it); }; From f12e11cd5c97f976ba9bc1cbe980709f01f85381 Mon Sep 17 00:00:00 2001 From: kssenii Date: Fri, 14 Apr 2023 16:08:44 +0200 Subject: [PATCH 126/406] Ping CI From 70c1000c227072722dd14f731d9f1549edb7a1c9 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Fri, 14 Apr 2023 19:42:17 +0200 Subject: [PATCH 127/406] fix some tests --- src/Interpreters/MergeTreeTransaction.cpp | 2 +- src/Storages/IStorage.h | 2 +- src/Storages/MergeTree/MergeTreeData.cpp | 9 ++++++++ src/Storages/StorageMergeTree.cpp | 23 +++++++++++-------- src/Storages/StorageMergeTree.h | 9 ++++---- .../config.d/merge_tree_old_dirs_cleanup.xml | 2 +- 6 files changed, 30 insertions(+), 17 deletions(-) diff --git a/src/Interpreters/MergeTreeTransaction.cpp b/src/Interpreters/MergeTreeTransaction.cpp index bfdda354c9bd..1358e3ed3c24 100644 --- a/src/Interpreters/MergeTreeTransaction.cpp +++ b/src/Interpreters/MergeTreeTransaction.cpp @@ -184,7 +184,7 @@ scope_guard MergeTreeTransaction::beforeCommit() /// We should wait for mutations to finish before committing transaction, because some mutation may fail and cause rollback. for (const auto & table_and_mutation : mutations_to_wait) - table_and_mutation.first->waitForMutation(table_and_mutation.second); + table_and_mutation.first->waitForMutation(table_and_mutation.second, /* wait_for_another_mutation */ false); assert([&]() { diff --git a/src/Storages/IStorage.h b/src/Storages/IStorage.h index 351e147e6cdc..a65b0f1442bc 100644 --- a/src/Storages/IStorage.h +++ b/src/Storages/IStorage.h @@ -503,7 +503,7 @@ class IStorage : public std::enable_shared_from_this, public TypePromo throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Mutations are not supported by storage {}", getName()); } - virtual void waitForMutation(const String & /*mutation_id*/) + virtual void waitForMutation(const String & /*mutation_id*/, bool /*wait_for_another_mutation*/) { throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Mutations are not supported by storage {}", getName()); } diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 45759c449f6c..ce42faccc791 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -2001,6 +2002,7 @@ size_t MergeTreeData::clearOldTemporaryDirectories(size_t custom_directories_lif { if (isOldPartDirectory(disk, it->path(), deadline)) { + ThreadFuzzer::maybeInjectSleep(); if (temporary_parts.contains(basename)) { /// Actually we don't rely on temporary_directories_lifetime when removing old temporaries directories, @@ -2008,6 +2010,13 @@ size_t MergeTreeData::clearOldTemporaryDirectories(size_t custom_directories_lif LOG_INFO(LogFrequencyLimiter(log, 10), "{} is in use (by merge/mutation/INSERT) (consider increasing temporary_directories_lifetime setting)", full_path); continue; } + else if (!disk->exists(it->path())) + { + /// We should recheck that the dir exists, otherwise we can get "No such file or directory" + /// due to a race condition with "Renaming temporary part" (temporary part holder could be already released, so the check above is not enough) + LOG_WARNING(log, "Temporary directory {} suddenly disappeared while iterating, assuming it was concurrently renamed to persistent", it->path()); + continue; + } else { LOG_WARNING(log, "Removing temporary directory {}", full_path); diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp index 6cb3ce35e5b7..9d43bb77a925 100644 --- a/src/Storages/StorageMergeTree.cpp +++ b/src/Storages/StorageMergeTree.cpp @@ -341,7 +341,7 @@ void StorageMergeTree::alter( if (prev_mutation != 0) { LOG_DEBUG(log, "Cannot change metadata with barrier alter query, will wait for mutation {}", prev_mutation); - waitForMutation(prev_mutation); + waitForMutation(prev_mutation, /* from_another_mutation */ true); LOG_DEBUG(log, "Mutation {} finished", prev_mutation); } } @@ -537,19 +537,19 @@ void StorageMergeTree::updateMutationEntriesErrors(FutureMergedMutatedPartPtr re mutation_wait_event.notify_all(); } -void StorageMergeTree::waitForMutation(Int64 version) +void StorageMergeTree::waitForMutation(Int64 version, bool wait_for_another_mutation) { String mutation_id = MergeTreeMutationEntry::versionToFileName(version); - waitForMutation(version, mutation_id); + waitForMutation(version, mutation_id, wait_for_another_mutation); } -void StorageMergeTree::waitForMutation(const String & mutation_id) +void StorageMergeTree::waitForMutation(const String & mutation_id, bool wait_for_another_mutation) { Int64 version = MergeTreeMutationEntry::parseFileName(mutation_id); - waitForMutation(version, mutation_id); + waitForMutation(version, mutation_id, wait_for_another_mutation); } -void StorageMergeTree::waitForMutation(Int64 version, const String & mutation_id) +void StorageMergeTree::waitForMutation(Int64 version, const String & mutation_id, bool wait_for_another_mutation) { LOG_INFO(log, "Waiting mutation: {}", mutation_id); { @@ -569,7 +569,7 @@ void StorageMergeTree::waitForMutation(Int64 version, const String & mutation_id std::set mutation_ids; mutation_ids.insert(mutation_id); - auto mutation_status = getIncompleteMutationsStatus(version, &mutation_ids); + auto mutation_status = getIncompleteMutationsStatus(version, &mutation_ids, wait_for_another_mutation); checkMutationStatus(mutation_status, mutation_ids); LOG_INFO(log, "Mutation {} done", mutation_id); @@ -619,7 +619,8 @@ bool comparator(const PartVersionWithName & f, const PartVersionWithName & s) } -std::optional StorageMergeTree::getIncompleteMutationsStatus(Int64 mutation_version, std::set * mutation_ids) const +std::optional StorageMergeTree::getIncompleteMutationsStatus( + Int64 mutation_version, std::set * mutation_ids, bool from_another_mutation) const { std::lock_guard lock(currently_processing_in_background_mutex); @@ -633,7 +634,9 @@ std::optional StorageMergeTree::getIncompleteMutationsS const auto & mutation_entry = current_mutation_it->second; auto txn = tryGetTransactionForMutation(mutation_entry, log); - assert(txn || mutation_entry.tid.isPrehistoric()); + /// There's no way a transaction may finish before a mutation that was started by the transaction. + /// But sometimes we need to check status of an unrelated mutation, in this case we don't care about transactions. + assert(txn || mutation_entry.tid.isPrehistoric() || from_another_mutation); auto data_parts = getVisibleDataPartsVector(txn); for (const auto & data_part : data_parts) { @@ -658,7 +661,7 @@ std::optional StorageMergeTree::getIncompleteMutationsS mutation_ids->insert(it->second.file_name); } } - else if (txn) + else if (txn && !from_another_mutation) { /// Part is locked by concurrent transaction, most likely it will never be mutated TIDHash part_locked = data_part->version.removal_tid_lock.load(); diff --git a/src/Storages/StorageMergeTree.h b/src/Storages/StorageMergeTree.h index a0629bb8d3ef..0ea727437538 100644 --- a/src/Storages/StorageMergeTree.h +++ b/src/Storages/StorageMergeTree.h @@ -183,9 +183,9 @@ class StorageMergeTree final : public MergeTreeData /// and into in-memory structures. Wake up merge-mutation task. Int64 startMutation(const MutationCommands & commands, ContextPtr query_context); /// Wait until mutation with version will finish mutation for all parts - void waitForMutation(Int64 version); - void waitForMutation(const String & mutation_id) override; - void waitForMutation(Int64 version, const String & mutation_id); + void waitForMutation(Int64 version, bool wait_for_another_mutation = false); + void waitForMutation(const String & mutation_id, bool wait_for_another_mutation) override; + void waitForMutation(Int64 version, const String & mutation_id, bool wait_for_another_mutation = false); void setMutationCSN(const String & mutation_id, CSN csn) override; @@ -246,7 +246,8 @@ class StorageMergeTree final : public MergeTreeData /// because we can execute several mutations at once. Order is important for /// better readability of exception message. If mutation was killed doesn't /// return any ids. - std::optional getIncompleteMutationsStatus(Int64 mutation_version, std::set * mutation_ids = nullptr) const; + std::optional getIncompleteMutationsStatus(Int64 mutation_version, std::set * mutation_ids = nullptr, + bool from_another_mutation = true) const; void fillNewPartName(MutableDataPartPtr & part, DataPartsLock & lock); diff --git a/tests/config/config.d/merge_tree_old_dirs_cleanup.xml b/tests/config/config.d/merge_tree_old_dirs_cleanup.xml index 41932cb6d61d..2b8ea63b63de 100644 --- a/tests/config/config.d/merge_tree_old_dirs_cleanup.xml +++ b/tests/config/config.d/merge_tree_old_dirs_cleanup.xml @@ -3,6 +3,6 @@ 1 - 10 + 5 From 6bc1ab7ab186a1ffdc35786b14f298c2aa97cb05 Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Fri, 14 Apr 2023 13:32:08 +0000 Subject: [PATCH 128/406] Add JSON output --- utils/check-style/check-style | 2 +- utils/keeper-bench/CMakeLists.txt | 2 +- utils/keeper-bench/Generator.cpp | 72 +++++++++---- utils/keeper-bench/Generator.h | 3 + utils/keeper-bench/Runner.cpp | 51 +++++---- utils/keeper-bench/Runner.h | 11 -- utils/keeper-bench/Stats.cpp | 171 ++++++++++++++++++++++++------ utils/keeper-bench/Stats.h | 58 +++++----- 8 files changed, 253 insertions(+), 117 deletions(-) diff --git a/utils/check-style/check-style b/utils/check-style/check-style index a6cc20bb7c85..988c6acd8a76 100755 --- a/utils/check-style/check-style +++ b/utils/check-style/check-style @@ -13,7 +13,7 @@ # and then to run formatter only for the specified files. ROOT_PATH=$(git rev-parse --show-toplevel) -EXCLUDE_DIRS='build/|integration/|widechar_width/|glibc-compatibility/|poco/|memcpy/|consistent-hashing|benchmark|tests/' +EXCLUDE_DIRS='build/|integration/|widechar_width/|glibc-compatibility/|poco/|memcpy/|consistent-hashing|benchmark|tests/|utils/keeper-bench/example.yaml' # From [1]: # But since array_to_string_internal() in array.c still loops over array diff --git a/utils/keeper-bench/CMakeLists.txt b/utils/keeper-bench/CMakeLists.txt index 97d30117d69c..5883b03bbe90 100644 --- a/utils/keeper-bench/CMakeLists.txt +++ b/utils/keeper-bench/CMakeLists.txt @@ -1,2 +1,2 @@ clickhouse_add_executable(keeper-bench Generator.cpp Runner.cpp Stats.cpp main.cpp) -target_link_libraries(keeper-bench PRIVATE clickhouse_common_config_no_zookeeper_log) +target_link_libraries(keeper-bench PRIVATE clickhouse_common_config_no_zookeeper_log ch_contrib::rapidjson) diff --git a/utils/keeper-bench/Generator.cpp b/utils/keeper-bench/Generator.cpp index 12e628ed1e52..2212f7158aef 100644 --- a/utils/keeper-bench/Generator.cpp +++ b/utils/keeper-bench/Generator.cpp @@ -12,6 +12,7 @@ using namespace zkutil; namespace DB::ErrorCodes { extern const int LOGICAL_ERROR; + extern const int BAD_ARGUMENTS; } namespace @@ -308,7 +309,7 @@ RequestGetter RequestGetter::fromConfig(const std::string & key, const Poco::Uti auto weight = request_generator->getWeight(); use_weights |= weight != 1; weight_sum += weight; - + generators.push_back(std::move(request_generator)); } @@ -575,7 +576,7 @@ Coordination::ZooKeeperRequestPtr MultiRequestGenerator::generateImpl(const Coor if (size) { - auto request_count = size->getNumber(); + auto request_count = size->getNumber(); for (size_t i = 0; i < request_count; ++i) ops.push_back(request_getter.getRequestGenerator()->generate(acls)); @@ -604,7 +605,7 @@ Generator::Generator(const Poco::Util::AbstractConfiguration & config) static const std::string generator_key = "generator"; - std::cout << "---- Parsing setup ---- " << std::endl; + std::cerr << "---- Parsing setup ---- " << std::endl; static const std::string setup_key = generator_key + ".setup"; Poco::Util::AbstractConfiguration::Keys keys; config.keys(setup_key, keys); @@ -612,20 +613,34 @@ Generator::Generator(const Poco::Util::AbstractConfiguration & config) { if (key.starts_with("node")) { - const auto & node = root_nodes.emplace_back(parseNode(setup_key + "." + key, config)); + auto node_key = setup_key + "." + key; + auto parsed_root_node = parseNode(node_key, config); + const auto node = root_nodes.emplace_back(parsed_root_node); + + if (config.has(node_key + ".repeat")) + { + if (!node->name.isRandom()) + throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, "Repeating node creation for key {}, but name is not randomly generated", node_key); + + auto repeat_count = config.getUInt64(node_key + ".repeat"); + node->repeat_count = repeat_count; + for (size_t i = 1; i < repeat_count; ++i) + root_nodes.emplace_back(node->clone()); + } + + std::cerr << "Tree to create:" << std::endl; - std::cout << "Tree to create:" << std::endl; node->dumpTree(); - std::cout << std::endl; + std::cerr << std::endl; } } - std::cout << "---- Done parsing data setup ----\n" << std::endl; + std::cerr << "---- Done parsing data setup ----\n" << std::endl; - std::cout << "---- Collecting request generators ----" << std::endl; + std::cerr << "---- Collecting request generators ----" << std::endl; static const std::string requests_key = generator_key + ".requests"; request_getter = RequestGetter::fromConfig(requests_key, config); - std::cout << request_getter.description() << std::endl; - std::cout << "---- Done collecting request generators ----\n" << std::endl; + std::cerr << request_getter.description() << std::endl; + std::cerr << "---- Done collecting request generators ----\n" << std::endl; } std::shared_ptr Generator::parseNode(const std::string & key, const Poco::Util::AbstractConfiguration & config) @@ -654,6 +669,7 @@ std::shared_ptr Generator::parseNode(const std::string & key, c throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, "Repeating node creation for key {}, but name is not randomly generated", node_key_string); auto repeat_count = config.getUInt64(node_key_string + ".repeat"); + child_node->repeat_count = repeat_count; for (size_t i = 1; i < repeat_count; ++i) node->children.push_back(child_node); } @@ -666,10 +682,30 @@ void Generator::Node::dumpTree(int level) const { std::string data_string = data.has_value() ? fmt::format("{}", data->description()) : "no data"; - std::cout << fmt::format("{}name: {}, data: {}", std::string(level, '\t'), name.description(), data_string) << std::endl; - for (const auto & child : children) + std::string repeat_count_string = repeat_count != 0 ? fmt::format(", repeated {} times", repeat_count) : ""; + + std::cerr << fmt::format("{}name: {}, data: {}{}", std::string(level, '\t'), name.description(), data_string, repeat_count_string) << std::endl; + + for (auto it = children.begin(); it != children.end();) + { + const auto & child = *it; child->dumpTree(level + 1); + std::advance(it, child->repeat_count != 0 ? child->repeat_count : 1); + } +} + +std::shared_ptr Generator::Node::clone() const +{ + auto new_node = std::make_shared(); + new_node->name = name; + new_node->data = data; + new_node->repeat_count = repeat_count; + + // don't do deep copy of children because we will do clone only for root nodes + new_node->children = children; + + return new_node; } void Generator::Node::createNode(Coordination::ZooKeeper & zookeeper, const std::string & parent_path, const Coordination::ACLs & acls) const @@ -693,21 +729,21 @@ void Generator::Node::createNode(Coordination::ZooKeeper & zookeeper, const std: void Generator::startup(Coordination::ZooKeeper & zookeeper) { - std::cout << "---- Creating test data ----" << std::endl; + std::cerr << "---- Creating test data ----" << std::endl; for (const auto & node : root_nodes) { auto node_name = node->name.getString(); node->name.setString(node_name); std::string root_path = std::filesystem::path("/") / node_name; - std::cout << "Cleaning up " << root_path << std::endl; + std::cerr << "Cleaning up " << root_path << std::endl; removeRecursive(zookeeper, root_path); node->createNode(zookeeper, "/", default_acls); } - std::cout << "---- Created test data ----\n" << std::endl; + std::cerr << "---- Created test data ----\n" << std::endl; - std::cout << "---- Initializing generators ----" << std::endl; + std::cerr << "---- Initializing generators ----" << std::endl; request_getter.startup(zookeeper); } @@ -719,12 +755,12 @@ Coordination::ZooKeeperRequestPtr Generator::generate() void Generator::cleanup(Coordination::ZooKeeper & zookeeper) { - std::cout << "---- Cleaning up test data ----" << std::endl; + std::cerr << "---- Cleaning up test data ----" << std::endl; for (const auto & node : root_nodes) { auto node_name = node->name.getString(); std::string root_path = std::filesystem::path("/") / node_name; - std::cout << "Cleaning up " << root_path << std::endl; + std::cerr << "Cleaning up " << root_path << std::endl; removeRecursive(zookeeper, root_path); } } diff --git a/utils/keeper-bench/Generator.h b/utils/keeper-bench/Generator.h index 60c4fcb3cc4c..5b4c05b2d8b4 100644 --- a/utils/keeper-bench/Generator.h +++ b/utils/keeper-bench/Generator.h @@ -180,6 +180,9 @@ class Generator StringGetter name; std::optional data; std::vector> children; + size_t repeat_count = 0; + + std::shared_ptr clone() const; void createNode(Coordination::ZooKeeper & zookeeper, const std::string & parent_path, const Coordination::ACLs & acls) const; void dumpTree(int level = 0) const; diff --git a/utils/keeper-bench/Runner.cpp b/utils/keeper-bench/Runner.cpp index 2b645ddc6c3c..4250263f043a 100644 --- a/utils/keeper-bench/Runner.cpp +++ b/utils/keeper-bench/Runner.cpp @@ -5,10 +5,18 @@ #include "Common/ZooKeeper/ZooKeeperConstants.h" #include #include +#include "IO/WriteBufferFromFile.h" + +namespace CurrentMetrics +{ + extern const Metric LocalThread; + extern const Metric LocalThreadActive; +} namespace DB::ErrorCodes { extern const int CANNOT_BLOCK_SIGNAL; + extern const int BAD_ARGUMENTS; } Runner::Runner( @@ -40,41 +48,41 @@ Runner::Runner( parseHostsFromConfig(*config); } - std::cout << "---- Run options ---- " << std::endl; + std::cerr << "---- Run options ---- " << std::endl; static constexpr uint64_t DEFAULT_CONCURRENCY = 1; if (concurrency_) concurrency = *concurrency_; else concurrency = config->getUInt64("concurrency", DEFAULT_CONCURRENCY); - std::cout << "Concurrency: " << concurrency << std::endl; + std::cerr << "Concurrency: " << concurrency << std::endl; static constexpr uint64_t DEFAULT_ITERATIONS = 0; if (max_iterations_) max_iterations = *max_iterations_; else max_iterations = config->getUInt64("iterations", DEFAULT_ITERATIONS); - std::cout << "Iterations: " << max_iterations << std::endl; + std::cerr << "Iterations: " << max_iterations << std::endl; static constexpr double DEFAULT_DELAY = 1.0; if (delay_) delay = *delay_; else delay = config->getDouble("report_delay", DEFAULT_DELAY); - std::cout << "Report delay: " << delay << std::endl; + std::cerr << "Report delay: " << delay << std::endl; - static constexpr double DEFAULT_TIME_LIMIT = 1.0; + static constexpr double DEFAULT_TIME_LIMIT = 0.0; if (max_time_) max_time = *max_time_; else max_time = config->getDouble("timelimit", DEFAULT_TIME_LIMIT); - std::cout << "Time limit: " << max_time << std::endl; + std::cerr << "Time limit: " << max_time << std::endl; if (continue_on_error_) continue_on_error = *continue_on_error_; else continue_on_error = config->getBool("continue_on_error", false); - std::cout << "Continue on error: " << continue_on_error << std::endl; - std::cout << "---- Run options ----\n" << std::endl; + std::cerr << "Continue on error: " << continue_on_error << std::endl; + std::cerr << "---- Run options ----\n" << std::endl; pool.emplace(CurrentMetrics::LocalThread, CurrentMetrics::LocalThreadActive, concurrency); queue.emplace(concurrency); @@ -173,7 +181,7 @@ void Runner::thread(std::vector> zookee else if (response.error == Coordination::Error::ZNONODE) { /// remove can fail with ZNONODE because of different order of execution - /// of generated create and remove requests + /// of generated create and remove requests /// this is okay for concurrent runs if (dynamic_cast(&response)) set_exception = false; @@ -203,14 +211,14 @@ void Runner::thread(std::vector> zookee try { auto response_size = future.get(); - double seconds = watch.elapsedSeconds(); + auto microseconds = watch.elapsedMicroseconds(); std::lock_guard lock(mutex); if (request->isReadRequest()) - info->addRead(seconds, 1, request->bytesSize() + response_size); + info->addRead(microseconds, 1, request->bytesSize() + response_size); else - info->addWrite(seconds, 1, request->bytesSize() + response_size); + info->addWrite(microseconds, 1, request->bytesSize() + response_size); } catch (...) { @@ -268,7 +276,7 @@ bool Runner::tryPushRequestInteractively(Coordination::ZooKeeperRequestPtr && re //if (i % 10000 == 0) //{ // for (const auto & [op_num, count] : counts) - // std::cout << fmt::format("{}: {}", op_num, count) << std::endl; + // std::cerr << fmt::format("{}: {}", op_num, count) << std::endl; //} bool inserted = false; @@ -285,13 +293,13 @@ bool Runner::tryPushRequestInteractively(Coordination::ZooKeeperRequestPtr && re if (max_time > 0 && total_watch.elapsedSeconds() >= max_time) { - std::cout << "Stopping launch of queries. Requested time limit is exhausted.\n"; + std::cerr << "Stopping launch of queries. Requested time limit is exhausted.\n"; return false; } if (interrupt_listener.check()) { - std::cout << "Stopping launch of queries. SIGINT received." << std::endl; + std::cerr << "Stopping launch of queries. SIGINT received." << std::endl; return false; } @@ -300,7 +308,7 @@ bool Runner::tryPushRequestInteractively(Coordination::ZooKeeperRequestPtr && re printNumberOfRequestsExecuted(requests_executed); std::lock_guard lock(mutex); - report(info, concurrency); + info->report(concurrency); delay_watch.restart(); } } @@ -350,18 +358,21 @@ void Runner::runBenchmark() printNumberOfRequestsExecuted(requests_executed); std::lock_guard lock(mutex); - report(info, concurrency); + info->report(concurrency); + + DB::WriteBufferFromFile out("result.json"); + info->writeJSON(out, concurrency); } void Runner::createConnections() { DB::EventNotifier::init(); - std::cout << "---- Creating connections ---- " << std::endl; + std::cerr << "---- Creating connections ---- " << std::endl; for (size_t connection_info_idx = 0; connection_info_idx < connection_infos.size(); ++connection_info_idx) { const auto & connection_info = connection_infos[connection_info_idx]; - std::cout << fmt::format("Creating {} session(s) for:\n" + std::cerr << fmt::format("Creating {} session(s) for:\n" "- host: {}\n" "- secure: {}\n" "- session timeout: {}ms\n" @@ -380,7 +391,7 @@ void Runner::createConnections() connections_to_info_map[connections.size() - 1] = connection_info_idx; } } - std::cout << "---- Done creating connections ----\n" << std::endl; + std::cerr << "---- Done creating connections ----\n" << std::endl; } std::shared_ptr Runner::getConnection(const ConnectionInfo & connection_info) diff --git a/utils/keeper-bench/Runner.h b/utils/keeper-bench/Runner.h index d85dc9e86585..7f41958a45a6 100644 --- a/utils/keeper-bench/Runner.h +++ b/utils/keeper-bench/Runner.h @@ -19,17 +19,6 @@ using Ports = std::vector; using Strings = std::vector; -namespace CurrentMetrics -{ - extern const Metric LocalThread; - extern const Metric LocalThreadActive; -} - -namespace DB::ErrorCodes -{ - extern const int BAD_ARGUMENTS; -} - class Runner { public: diff --git a/utils/keeper-bench/Stats.cpp b/utils/keeper-bench/Stats.cpp index 1f8b02ed09d6..21de72f60473 100644 --- a/utils/keeper-bench/Stats.cpp +++ b/utils/keeper-bench/Stats.cpp @@ -1,67 +1,174 @@ #include "Stats.h" #include -void report(std::shared_ptr & info, size_t concurrency) +#include +#include +#include +#include + +void Stats::StatsCollector::add(uint64_t microseconds, size_t requests_inc, size_t bytes_inc) +{ + work_time += microseconds; + requests += requests_inc; + requests_bytes += bytes_inc; + sampler.insert(microseconds); +} + +void Stats::addRead(uint64_t microseconds, size_t requests_inc, size_t bytes_inc) +{ + read_collector.add(microseconds, requests_inc, bytes_inc); +} + +void Stats::addWrite(uint64_t microseconds, size_t requests_inc, size_t bytes_inc) +{ + write_collector.add(microseconds, requests_inc, bytes_inc); +} + +void Stats::StatsCollector::clear() +{ + requests = 0; + work_time = 0; + requests_bytes = 0; + sampler.clear(); +} + +void Stats::clear() +{ + read_collector.clear(); + write_collector.clear(); +} + +std::pair Stats::StatsCollector::getThroughput(size_t concurrency) +{ + assert(requests != 0); + double seconds = work_time / 1'000'000.0 / concurrency; + + return {requests / seconds, requests_bytes / seconds}; +} + +double Stats::StatsCollector::getPercentile(double percent) +{ + return sampler.quantileNearest(percent / 100.0) / 1000.0; +} + +void Stats::report(size_t concurrency) { std::cerr << "\n"; + const auto & read_requests = read_collector.requests; + const auto & write_requests = write_collector.requests; + /// Avoid zeros, nans or exceptions - if (0 == info->read_requests && 0 == info->write_requests) + if (0 == read_requests && 0 == write_requests) return; - double read_seconds = info->read_work_time / concurrency; - double write_seconds = info->write_work_time / concurrency; + auto [read_rps, read_bps] = read_collector.getThroughput(concurrency); + auto [write_rps, write_bps] = write_collector.getThroughput(concurrency); - std::cerr << "read requests " << info->read_requests << ", write requests " << info->write_requests << ", "; - if (info->errors) - { - std::cerr << "errors " << info->errors << ", "; - } - if (0 != info->read_requests) + std::cerr << "read requests " << read_requests << ", write requests " << write_requests << ", "; + if (errors) + std::cerr << "errors " << errors << ", "; + + if (0 != read_requests) { std::cerr - << "Read RPS: " << (info->read_requests / read_seconds) << ", " - << "Read MiB/s: " << (info->requests_read_bytes / read_seconds / 1048576); - if (0 != info->write_requests) + << "Read RPS: " << read_rps << ", " + << "Read MiB/s: " << read_bps / 1048576; + + if (0 != write_requests) std::cerr << ", "; } - if (0 != info->write_requests) + + if (0 != write_requests) { std::cerr - << "Write RPS: " << (info->write_requests / write_seconds) << ", " - << "Write MiB/s: " << (info->requests_write_bytes / write_seconds / 1048576) << ". " + << "Write RPS: " << write_rps << ", " + << "Write MiB/s: " << write_bps / 1048576 << ". " << "\n"; } std::cerr << "\n"; - auto print_percentile = [&](double percent, Stats::Sampler & sampler) + auto print_percentile = [&](double percent, Stats::StatsCollector & collector) { std::cerr << percent << "%\t\t"; - std::cerr << sampler.quantileNearest(percent / 100.0) << " sec.\t"; + std::cerr << collector.getPercentile(percent) << " msec.\t"; std::cerr << "\n"; }; - if (0 != info->read_requests) + const auto print_all_percentiles = [&](auto & collector) { - std::cerr << "Read sampler:\n"; for (int percent = 0; percent <= 90; percent += 10) - print_percentile(percent, info->read_sampler); + print_percentile(percent, collector); - print_percentile(95, info->read_sampler); - print_percentile(99, info->read_sampler); - print_percentile(99.9, info->read_sampler); - print_percentile(99.99, info->read_sampler); + print_percentile(95, collector); + print_percentile(99, collector); + print_percentile(99.9, collector); + print_percentile(99.99, collector); + }; + + if (0 != read_requests) + { + std::cerr << "Read sampler:\n"; + print_all_percentiles(read_collector); } - if (0 != info->write_requests) + if (0 != write_requests) { std::cerr << "Write sampler:\n"; + print_all_percentiles(write_collector); + } +} + +void Stats::writeJSON(DB::WriteBuffer & out, size_t concurrency) +{ + using namespace rapidjson; + Document results; + auto & allocator = results.GetAllocator(); + results.SetObject(); + + const auto get_results = [&](auto & collector) + { + Value specific_results(kObjectType); + + auto [rps, bps] = collector.getThroughput(concurrency); + specific_results.AddMember("requests_per_second", Value(rps), allocator); + specific_results.AddMember("bytes_per_second", Value(bps), allocator); + + Value percentiles(kArrayType); + + const auto add_percentile = [&](double percent) + { + Value percentile(kObjectType); + percentile.AddMember("percentile", Value(percent), allocator); + percentile.AddMember("value", Value(collector.getPercentile(percent)), allocator); + + percentiles.PushBack(percentile, allocator); + }; + for (int percent = 0; percent <= 90; percent += 10) - print_percentile(percent, info->write_sampler); + add_percentile(percent); - print_percentile(95, info->write_sampler); - print_percentile(99, info->write_sampler); - print_percentile(99.9, info->write_sampler); - print_percentile(99.99, info->write_sampler); - } + add_percentile(95); + add_percentile(99); + add_percentile(99.9); + add_percentile(99.99); + + specific_results.AddMember("percentiles", percentiles, allocator); + + return specific_results; + }; + + if (read_collector.requests != 0) + results.AddMember("read_results", get_results(read_collector), results.GetAllocator()); + + if (write_collector.requests != 0) + results.AddMember("write_results", get_results(write_collector), results.GetAllocator()); + + StringBuffer strbuf; + strbuf.Clear(); + Writer writer(strbuf); + results.Accept(writer); + + const char * output_string = strbuf.GetString(); + out.write(output_string, strlen(output_string)); } diff --git a/utils/keeper-bench/Stats.h b/utils/keeper-bench/Stats.h index 1b9a31bb7349..7afd8ce4f1a7 100644 --- a/utils/keeper-bench/Stats.h +++ b/utils/keeper-bench/Stats.h @@ -5,48 +5,38 @@ #include +#include + struct Stats { - std::atomic read_requests{0}; - std::atomic write_requests{0}; size_t errors = 0; - size_t requests_write_bytes = 0; - size_t requests_read_bytes = 0; - double read_work_time = 0; - double write_work_time = 0; using Sampler = ReservoirSampler; - Sampler read_sampler {1 << 16}; - Sampler write_sampler {1 << 16}; - - void addRead(double seconds, size_t requests_inc, size_t bytes_inc) + struct StatsCollector { - read_work_time += seconds; - read_requests += requests_inc; - requests_read_bytes += bytes_inc; - read_sampler.insert(seconds); - } + std::atomic requests{0}; + uint64_t requests_bytes = 0; + uint64_t work_time = 0; + Sampler sampler; - void addWrite(double seconds, size_t requests_inc, size_t bytes_inc) - { - write_work_time += seconds; - write_requests += requests_inc; - requests_write_bytes += bytes_inc; - write_sampler.insert(seconds); - } + /// requests/second, bytes/second + std::pair getThroughput(size_t concurrency); + double getPercentile(double percent); - void clear() - { - read_requests = 0; - write_requests = 0; - read_work_time = 0; - write_work_time = 0; - requests_read_bytes = 0; - requests_write_bytes = 0; - read_sampler.clear(); - write_sampler.clear(); - } + void add(uint64_t microseconds, size_t requests_inc, size_t bytes_inc); + void clear(); + }; + + StatsCollector read_collector; + StatsCollector write_collector; + + void addRead(uint64_t microseconds, size_t requests_inc, size_t bytes_inc); + void addWrite(uint64_t microseconds, size_t requests_inc, size_t bytes_inc); + + void clear(); + + void report(size_t concurrency); + void writeJSON(DB::WriteBuffer & out, size_t concurrency); }; -void report(std::shared_ptr & info, size_t concurrency); From 3dcc7e2f9eb02c57b1b2d728672fe0b39d740fc6 Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Mon, 17 Apr 2023 11:25:46 +0000 Subject: [PATCH 129/406] Improve outputs --- utils/keeper-bench/Runner.cpp | 71 +++++++++++++++++++++++---------- utils/keeper-bench/Runner.h | 7 +++- utils/keeper-bench/Stats.cpp | 11 +++-- utils/keeper-bench/Stats.h | 2 +- utils/keeper-bench/example.yaml | 6 +++ 5 files changed, 69 insertions(+), 28 deletions(-) diff --git a/utils/keeper-bench/Runner.cpp b/utils/keeper-bench/Runner.cpp index 4250263f043a..bb8ff46a20c7 100644 --- a/utils/keeper-bench/Runner.cpp +++ b/utils/keeper-bench/Runner.cpp @@ -5,7 +5,10 @@ #include "Common/ZooKeeper/ZooKeeperConstants.h" #include #include -#include "IO/WriteBufferFromFile.h" +#include "IO/ReadBufferFromString.h" +#include +#include +#include namespace CurrentMetrics { @@ -82,6 +85,25 @@ Runner::Runner( else continue_on_error = config->getBool("continue_on_error", false); std::cerr << "Continue on error: " << continue_on_error << std::endl; + + static const std::string output_key = "output"; + print_to_stdout = config->getBool(output_key + ".stdout", false); + std::cerr << "Printing output to stdout: " << print_to_stdout << std::endl; + + static const std::string output_file_key = output_key + ".file"; + if (config->has(output_file_key)) + { + if (config->has(output_file_key + ".path")) + { + file_output = config->getString(output_file_key + ".path"); + output_file_with_timestamp = config->getBool(output_file_key + ".with_timestamp"); + } + else + file_output = config->getString(output_file_key); + + std::cerr << "Result file path: " << file_output->string() << std::endl; + } + std::cerr << "---- Run options ----\n" << std::endl; pool.emplace(CurrentMetrics::LocalThread, CurrentMetrics::LocalThreadActive, concurrency); @@ -261,24 +283,6 @@ void Runner::thread(std::vector> zookee bool Runner::tryPushRequestInteractively(Coordination::ZooKeeperRequestPtr && request, DB::InterruptListener & interrupt_listener) { - //static std::unordered_map counts; - //static size_t i = 0; - // - //counts[request->getOpNum()]++; - - //if (request->getOpNum() == Coordination::OpNum::Multi) - //{ - // for (const auto & multi_request : dynamic_cast(*request).requests) - // counts[dynamic_cast(*multi_request).getOpNum()]++; - //} - - //++i; - //if (i % 10000 == 0) - //{ - // for (const auto & [op_num, count] : counts) - // std::cerr << fmt::format("{}: {}", op_num, count) << std::endl; - //} - bool inserted = false; while (!inserted) @@ -324,6 +328,9 @@ void Runner::runBenchmark() std::cerr << "Preparing to run\n"; generator->startup(*connections[0]); std::cerr << "Prepared\n"; + + auto start_timestamp_ms = Poco::Timestamp{}.epochMicroseconds() / 1000; + try { for (size_t i = 0; i < concurrency; ++i) @@ -360,8 +367,30 @@ void Runner::runBenchmark() std::lock_guard lock(mutex); info->report(concurrency); - DB::WriteBufferFromFile out("result.json"); - info->writeJSON(out, concurrency); + DB::WriteBufferFromOwnString out; + info->writeJSON(out, concurrency, start_timestamp_ms); + auto output_string = std::move(out.str()); + + if (print_to_stdout) + std::cout << output_string << std::endl; + + if (file_output) + { + auto path = *file_output; + + if (output_file_with_timestamp) + { + auto filename = file_output->filename(); + filename = fmt::format("{}_{}{}", filename.stem().generic_string(), start_timestamp_ms, filename.extension().generic_string()); + path = file_output->parent_path() / filename; + } + + std::cerr << "Storing output to " << path << std::endl; + + DB::WriteBufferFromFile file_output_buffer(path); + DB::ReadBufferFromString read_buffer(output_string); + DB::copyData(read_buffer, file_output_buffer); + } } diff --git a/utils/keeper-bench/Runner.h b/utils/keeper-bench/Runner.h index 7f41958a45a6..f899f1d538d3 100644 --- a/utils/keeper-bench/Runner.h +++ b/utils/keeper-bench/Runner.h @@ -7,8 +7,6 @@ #include #include #include -#include -#include #include #include @@ -16,6 +14,8 @@ #include #include "Stats.h" +#include + using Ports = std::vector; using Strings = std::vector; @@ -59,6 +59,9 @@ class Runner std::atomic shutdown = false; std::shared_ptr info; + bool print_to_stdout; + std::optional file_output; + bool output_file_with_timestamp; Stopwatch total_watch; Stopwatch delay_watch; diff --git a/utils/keeper-bench/Stats.cpp b/utils/keeper-bench/Stats.cpp index 21de72f60473..f5e5f84ba147 100644 --- a/utils/keeper-bench/Stats.cpp +++ b/utils/keeper-bench/Stats.cpp @@ -119,17 +119,21 @@ void Stats::report(size_t concurrency) } } -void Stats::writeJSON(DB::WriteBuffer & out, size_t concurrency) +void Stats::writeJSON(DB::WriteBuffer & out, size_t concurrency, int64_t start_timestamp) { using namespace rapidjson; Document results; auto & allocator = results.GetAllocator(); results.SetObject(); + results.AddMember("timestamp", Value(start_timestamp), allocator); + const auto get_results = [&](auto & collector) { Value specific_results(kObjectType); + specific_results.AddMember("total_requests", Value(collector.requests), allocator); + auto [rps, bps] = collector.getThroughput(concurrency); specific_results.AddMember("requests_per_second", Value(rps), allocator); specific_results.AddMember("bytes_per_second", Value(bps), allocator); @@ -139,9 +143,8 @@ void Stats::writeJSON(DB::WriteBuffer & out, size_t concurrency) const auto add_percentile = [&](double percent) { Value percentile(kObjectType); - percentile.AddMember("percentile", Value(percent), allocator); - percentile.AddMember("value", Value(collector.getPercentile(percent)), allocator); - + Value percent_key(fmt::format("{:.2f}", percent).c_str(), allocator); + percentile.AddMember(percent_key, Value(collector.getPercentile(percent)), allocator); percentiles.PushBack(percentile, allocator); }; diff --git a/utils/keeper-bench/Stats.h b/utils/keeper-bench/Stats.h index 7afd8ce4f1a7..bc50588e8377 100644 --- a/utils/keeper-bench/Stats.h +++ b/utils/keeper-bench/Stats.h @@ -36,7 +36,7 @@ struct Stats void clear(); void report(size_t concurrency); - void writeJSON(DB::WriteBuffer & out, size_t concurrency); + void writeJSON(DB::WriteBuffer & out, size_t concurrency, int64_t start_timestamp); }; diff --git a/utils/keeper-bench/example.yaml b/utils/keeper-bench/example.yaml index 2febb881634f..e800e9234827 100644 --- a/utils/keeper-bench/example.yaml +++ b/utils/keeper-bench/example.yaml @@ -109,3 +109,9 @@ generator: - "/test4" path: children_of: "/test" + +output: + file: + path: "output.json" + with_timestamp: true + stdout: true From ef5d8b4f37f1e26154aaefe5a2fe63a60586b230 Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Mon, 17 Apr 2023 11:30:59 +0000 Subject: [PATCH 130/406] Add readme --- utils/keeper-bench/README.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 utils/keeper-bench/README.md diff --git a/utils/keeper-bench/README.md b/utils/keeper-bench/README.md new file mode 100644 index 000000000000..0c6d01a0418a --- /dev/null +++ b/utils/keeper-bench/README.md @@ -0,0 +1 @@ +Keeper Bench From 53181c938a5a0dfa8dcdab612a9a80a59645434b Mon Sep 17 00:00:00 2001 From: kssenii Date: Mon, 17 Apr 2023 15:18:10 +0200 Subject: [PATCH 131/406] Fixes --- .../IO/CachedOnDiskReadBufferFromFile.cpp | 14 +-- .../IO/CachedOnDiskWriteBufferFromFile.cpp | 1 - src/Interpreters/Cache/FileCache.cpp | 33 +++--- src/Interpreters/Cache/FileSegment.cpp | 104 ++++++++++-------- src/Interpreters/Cache/FileSegment.h | 13 +-- src/Interpreters/Cache/Metadata.cpp | 22 ++-- src/Interpreters/Cache/Metadata.h | 3 +- 7 files changed, 90 insertions(+), 100 deletions(-) diff --git a/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp b/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp index 7827e537708d..0cee265a11ec 100644 --- a/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp +++ b/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp @@ -598,8 +598,8 @@ void CachedOnDiskReadBufferFromFile::predownload(FileSegment & file_segment) chassert(file_segment.getCurrentWriteOffset(false) == static_cast(implementation_buffer->getPosition())); - bool success = writeCache(implementation_buffer->buffer().begin(), current_predownload_size, current_offset, file_segment); - if (success) + continue_predownload = writeCache(implementation_buffer->buffer().begin(), current_predownload_size, current_offset, file_segment); + if (continue_predownload) { current_offset += current_predownload_size; @@ -609,7 +609,6 @@ void CachedOnDiskReadBufferFromFile::predownload(FileSegment & file_segment) else { LOG_TEST(log, "Bypassing cache because writeCache (in predownload) method failed"); - continue_predownload = false; } } @@ -631,7 +630,7 @@ void CachedOnDiskReadBufferFromFile::predownload(FileSegment & file_segment) /// TODO: allow seek more than once with seek avoiding. bytes_to_predownload = 0; - file_segment.setBroken(); + file_segment.completePartAndResetDownloader(); chassert(file_segment.state() == FileSegment::State::PARTIALLY_DOWNLOADED_NO_CONTINUATION); LOG_TEST(log, "Bypassing cache because for {}", file_segment.getInfoForLog()); @@ -945,13 +944,6 @@ bool CachedOnDiskReadBufferFromFile::nextImplStep() { LOG_TRACE(log, "No space left in cache to reserve {} bytes, will continue without cache download", size); } - - if (!success) - { - file_segment.setBroken(); - read_type = ReadType::REMOTE_FS_READ_BYPASS_CACHE; - download_current_segment = false; - } } /// - If last file segment was read from remote fs, then we read up to segment->range().right, diff --git a/src/Disks/IO/CachedOnDiskWriteBufferFromFile.cpp b/src/Disks/IO/CachedOnDiskWriteBufferFromFile.cpp index 9b351df041bb..d72dcecb4848 100644 --- a/src/Disks/IO/CachedOnDiskWriteBufferFromFile.cpp +++ b/src/Disks/IO/CachedOnDiskWriteBufferFromFile.cpp @@ -88,7 +88,6 @@ bool FileSegmentRangeWriter::write(const char * data, size_t size, size_t offset bool reserved = file_segment->reserve(size_to_write); if (!reserved) { - file_segment->setBroken(); appendFilesystemCacheLog(*file_segment); LOG_DEBUG( diff --git a/src/Interpreters/Cache/FileCache.cpp b/src/Interpreters/Cache/FileCache.cpp index 9db634a81887..d50af057aed7 100644 --- a/src/Interpreters/Cache/FileCache.cpp +++ b/src/Interpreters/Cache/FileCache.cpp @@ -558,7 +558,8 @@ bool FileCache::tryReserve(FileSegment & file_segment, size_t size) auto iterate_func = [&](LockedKey & locked_key, FileSegmentMetadataPtr segment_metadata) { - chassert(segment_metadata->file_segment->getQueueIterator()); + chassert(segment_metadata->file_segment->assertCorrectness()); + const bool is_persistent = allow_persistent_files && segment_metadata->file_segment->isPersistent(); const bool releasable = segment_metadata->releasable() && !is_persistent; @@ -821,13 +822,13 @@ void FileCache::loadMetadata() auto file_segment_metadata_it = addFileSegment( *locked_key, offset, size, FileSegment::State::DOWNLOADED, CreateFileSegmentSettings(segment_kind), &lock); - chassert(file_segment_metadata_it->second->file_segment->getQueueIterator()); - chassert(file_segment_metadata_it->second->size() == size); + const auto & file_segment_metadata = file_segment_metadata_it->second; + chassert(file_segment_metadata->file_segment->assertCorrectness()); total_size += size; queue_entries.emplace_back( - file_segment_metadata_it->second->file_segment->getQueueIterator(), - file_segment_metadata_it->second->file_segment); + file_segment_metadata->getQueueIterator(), + file_segment_metadata->file_segment); } else { @@ -965,22 +966,14 @@ size_t FileCache::getFileSegmentsNum() const void FileCache::assertCacheCorrectness() { - metadata.iterate([&](const LockedKey & locked_key) + auto lock = cache_guard.lock(); + main_priority->iterate([&](LockedKey &, FileSegmentMetadataPtr segment_metadata) { - for (const auto & [offset, file_segment_metadata] : locked_key) - { - const auto & file_segment = *file_segment_metadata->file_segment; - - if (file_segment.key() != locked_key.getKey()) - { - throw Exception( - ErrorCodes::LOGICAL_ERROR, - "Expected {} = {}", file_segment.key(), locked_key.getKey()); - } - - file_segment.assertCorrectness(); - } - }); + const auto & file_segment = *segment_metadata->file_segment; + UNUSED(file_segment); + chassert(file_segment.assertCorrectness()); + return PriorityIterationResult::CONTINUE; + }, lock); } FileCache::QueryContextHolder::QueryContextHolder( diff --git a/src/Interpreters/Cache/FileSegment.cpp b/src/Interpreters/Cache/FileSegment.cpp index b06bc521ac09..7373e371f4c5 100644 --- a/src/Interpreters/Cache/FileSegment.cpp +++ b/src/Interpreters/Cache/FileSegment.cpp @@ -32,7 +32,7 @@ FileSegment::FileSegment( const CreateFileSegmentSettings & settings, FileCache * cache_, std::weak_ptr key_metadata_, - CachePriorityIterator queue_iterator_) + Priority::Iterator queue_iterator_) : file_key(key_) , segment_range(offset_, offset_ + size_ - 1) , segment_kind(settings.kind) @@ -101,13 +101,13 @@ size_t FileSegment::getReservedSize() const return reserved_size; } -FileSegment::CachePriorityIterator FileSegment::getQueueIterator() const +FileSegment::Priority::Iterator FileSegment::getQueueIterator() const { auto lock = segment_guard.lock(); return queue_iterator; } -void FileSegment::setQueueIterator(CachePriorityIterator iterator) +void FileSegment::setQueueIterator(Priority::Iterator iterator) { auto lock = segment_guard.lock(); if (queue_iterator) @@ -355,8 +355,6 @@ void FileSegment::write(const char * from, size_t size, size_t offset) setDownloadFailedUnlocked(lock); - cv.notify_all(); - throw; } @@ -384,7 +382,7 @@ FileSegment::State FileSegment::wait(size_t offset) { return download_state != State::DOWNLOADING || offset < getCurrentWriteOffset(true); }); - chassert(ok); + /// chassert(ok); } return download_state; @@ -457,18 +455,20 @@ bool FileSegment::reserve(size_t size_to_reserve) size_t already_reserved_size = reserved_size - expected_downloaded_size; bool reserved = already_reserved_size >= size_to_reserve; - if (!reserved) - { - size_to_reserve = size_to_reserve - already_reserved_size; + if (reserved) + return reserved; - /// This (resizable file segments) is allowed only for single threaded use of file segment. - /// Currently it is used only for temporary files through cache. - if (is_unbound && is_file_segment_size_exceeded) - segment_range.right = range().left + expected_downloaded_size + size_to_reserve; + size_to_reserve = size_to_reserve - already_reserved_size; - reserved = cache->tryReserve(*this, size_to_reserve); - chassert(assertCorrectness()); - } + /// This (resizable file segments) is allowed only for single threaded use of file segment. + /// Currently it is used only for temporary files through cache. + if (is_unbound && is_file_segment_size_exceeded) + segment_range.right = range().left + expected_downloaded_size + size_to_reserve; + + reserved = cache->tryReserve(*this, size_to_reserve); + + if (!reserved) + setDownloadFailedUnlocked(segment_guard.lock()); return reserved; } @@ -516,27 +516,15 @@ void FileSegment::completePartAndResetDownloader() assertNotDetachedUnlocked(lock); assertIsDownloaderUnlocked("completePartAndResetDownloader", lock); - resetDownloadingStateUnlocked(lock); - resetDownloaderUnlocked(lock); - - LOG_TEST(log, "Complete batch. ({})", getInfoForLogUnlocked(lock)); -} - -void FileSegment::setBroken() -{ - auto lock = segment_guard.lock(); - - SCOPE_EXIT({ cv.notify_all(); }); - - assertNotDetachedUnlocked(lock); - assertIsDownloaderUnlocked("setBroken", lock); + chassert(download_state == State::DOWNLOADING + || download_state == State::PARTIALLY_DOWNLOADED_NO_CONTINUATION); if (download_state == State::DOWNLOADING) resetDownloadingStateUnlocked(lock); - if (download_state != State::DOWNLOADED) - download_state = State::PARTIALLY_DOWNLOADED_NO_CONTINUATION; resetDownloaderUnlocked(lock); + + LOG_TEST(log, "Complete batch. ({})", getInfoForLogUnlocked(lock)); } void FileSegment::complete() @@ -700,24 +688,48 @@ String FileSegment::stateToString(FileSegment::State state) bool FileSegment::assertCorrectness() const { - auto lock = segment_guard.lock(); + return assertCorrectnessUnlocked(segment_guard.lock()); +} - auto current_downloader = getDownloaderUnlocked(lock); - chassert(current_downloader.empty() == (download_state != FileSegment::State::DOWNLOADING)); - chassert(!current_downloader.empty() == (download_state == FileSegment::State::DOWNLOADING)); - chassert(download_state != FileSegment::State::DOWNLOADED || std::filesystem::file_size(getPathInLocalCache()) > 0); +bool FileSegment::assertCorrectnessUnlocked(const FileSegmentGuard::Lock &) const +{ + auto check_iterator = [this](const Priority::Iterator & it) + { + if (!it) + return; - chassert(reserved_size == 0 || queue_iterator); - if (queue_iterator) + const auto entry = it->getEntry(); + UNUSED(entry); + chassert(entry.size == reserved_size); + chassert(entry.key == key()); + chassert(entry.offset == offset()); + }; + + if (download_state == State::DOWNLOADED) { - const auto & entry = queue_iterator->getEntry(); - if (isCompleted(false)) - chassert(reserved_size == entry.size); - else - /// We cannot check == here because reserved_size is not - /// guarded by any mutex, it is just an atomic. - chassert(reserved_size <= entry.size); + chassert(downloader_id.empty()); + chassert(downloaded_size == reserved_size); + chassert(std::filesystem::file_size(getPathInLocalCache()) > 0); + chassert(queue_iterator); + check_iterator(queue_iterator); + } + else + { + if (download_state == State::DOWNLOADED) + { + chassert(!downloader_id.empty()); + } + else if (download_state == State::PARTIALLY_DOWNLOADED + || download_state == State::EMPTY) + { + chassert(downloader_id.empty()); + } + + chassert(reserved_size >= downloaded_size); + chassert((reserved_size == 0) || queue_iterator); + check_iterator(queue_iterator); } + return true; } diff --git a/src/Interpreters/Cache/FileSegment.h b/src/Interpreters/Cache/FileSegment.h index e6aee5827e71..1ac09de23ad1 100644 --- a/src/Interpreters/Cache/FileSegment.h +++ b/src/Interpreters/Cache/FileSegment.h @@ -72,7 +72,7 @@ friend class FileCache; /// Because of reserved_size in tryReserve(). using LocalCacheWriterPtr = std::unique_ptr; using Downloader = std::string; using DownloaderId = std::string; - using CachePriorityIterator = IFileCachePriority::Iterator; + using Priority = IFileCachePriority; enum class State { @@ -116,7 +116,7 @@ friend class FileCache; /// Because of reserved_size in tryReserve(). const CreateFileSegmentSettings & create_settings = {}, FileCache * cache_ = nullptr, std::weak_ptr key_metadata_ = std::weak_ptr(), - CachePriorityIterator queue_iterator_ = CachePriorityIterator{}); + Priority::Iterator queue_iterator_ = Priority::Iterator{}); ~FileSegment() = default; @@ -222,9 +222,9 @@ friend class FileCache; /// Because of reserved_size in tryReserve(). FileSegmentGuard::Lock lock() const { return segment_guard.lock(); } - CachePriorityIterator getQueueIterator() const; + Priority::Iterator getQueueIterator() const; - void setQueueIterator(CachePriorityIterator iterator); + void setQueueIterator(Priority::Iterator iterator); KeyMetadataPtr tryGetKeyMetadata() const; @@ -236,8 +236,6 @@ friend class FileCache; /// Because of reserved_size in tryReserve(). * ========== Methods that must do cv.notify() ================== */ - void setBroken(); - void complete(); void completePartAndResetDownloader(); @@ -285,6 +283,7 @@ friend class FileCache; /// Because of reserved_size in tryReserve(). void assertNotDetached() const; void assertNotDetachedUnlocked(const FileSegmentGuard::Lock &) const; void assertIsDownloaderUnlocked(const std::string & operation, const FileSegmentGuard::Lock &) const; + bool assertCorrectnessUnlocked(const FileSegmentGuard::Lock &) const; LockedKeyPtr lockKeyMetadata(bool assert_exists = true) const; @@ -308,7 +307,7 @@ friend class FileCache; /// Because of reserved_size in tryReserve(). mutable FileSegmentGuard segment_guard; std::weak_ptr key_metadata; - mutable CachePriorityIterator queue_iterator; /// Iterator is put here on first reservation attempt, if successful. + mutable Priority::Iterator queue_iterator; /// Iterator is put here on first reservation attempt, if successful. FileCache * cache; std::condition_variable cv; diff --git a/src/Interpreters/Cache/Metadata.cpp b/src/Interpreters/Cache/Metadata.cpp index 9396b8c6f449..02474c966cd1 100644 --- a/src/Interpreters/Cache/Metadata.cpp +++ b/src/Interpreters/Cache/Metadata.cpp @@ -353,13 +353,10 @@ void LockedKey::shrinkFileSegmentToDownloadedSize( auto metadata = getByOffset(offset); const auto & file_segment = metadata->file_segment; + chassert(file_segment->assertCorrectnessUnlocked(segment_lock)); const size_t downloaded_size = file_segment->getDownloadedSize(false); - const size_t full_size = file_segment->range().size(); - - chassert(downloaded_size <= file_segment->reserved_size); - - if (downloaded_size == full_size) + if (downloaded_size == file_segment->range().size()) { throw Exception( ErrorCodes::LOGICAL_ERROR, @@ -367,20 +364,17 @@ void LockedKey::shrinkFileSegmentToDownloadedSize( file_segment->getInfoForLogUnlocked(segment_lock)); } - CreateFileSegmentSettings create_settings(file_segment->getKind()); - auto queue_iterator = file_segment->queue_iterator; + ssize_t diff = file_segment->reserved_size - downloaded_size; metadata->file_segment = std::make_shared( - getKey(), offset, downloaded_size, FileSegment::State::DOWNLOADED, create_settings, - file_segment->cache, key_metadata, queue_iterator); + getKey(), offset, downloaded_size, FileSegment::State::DOWNLOADED, + CreateFileSegmentSettings(file_segment->getKind()), + file_segment->cache, key_metadata, file_segment->queue_iterator); - chassert(queue_iterator->getEntry().size == file_segment->reserved_size); - ssize_t diff = file_segment->reserved_size - file_segment->downloaded_size; if (diff) - queue_iterator->updateSize(-diff); + metadata->getQueueIterator()->updateSize(-diff); - chassert(file_segment->reserved_size == downloaded_size); - chassert(metadata->size() == queue_iterator->getEntry().size); + chassert(file_segment->assertCorrectnessUnlocked(segment_lock)); } std::shared_ptr LockedKey::getByOffset(size_t offset) const diff --git a/src/Interpreters/Cache/Metadata.h b/src/Interpreters/Cache/Metadata.h index ee6506504886..1348ff31dc7e 100644 --- a/src/Interpreters/Cache/Metadata.h +++ b/src/Interpreters/Cache/Metadata.h @@ -15,7 +15,6 @@ using CleanupQueuePtr = std::shared_ptr; struct FileSegmentMetadata : private boost::noncopyable { using Priority = IFileCachePriority; - using PriorityIterator = IFileCachePriority::Iterator; explicit FileSegmentMetadata(FileSegmentPtr && file_segment_); @@ -25,6 +24,8 @@ struct FileSegmentMetadata : private boost::noncopyable bool valid() const { return !removal_candidate.load(); } + Priority::Iterator getQueueIterator() { return file_segment->getQueueIterator(); } + FileSegmentPtr file_segment; std::atomic removal_candidate{false}; }; From f2f47fc24db69bdff5b498c363d354c72fb6adb3 Mon Sep 17 00:00:00 2001 From: Shane Andrade Date: Mon, 17 Apr 2023 14:22:27 +0000 Subject: [PATCH 132/406] date_trunc function to always return DateTime type --- src/Functions/date_trunc.cpp | 248 ++++++++++++++++++++--------------- 1 file changed, 140 insertions(+), 108 deletions(-) diff --git a/src/Functions/date_trunc.cpp b/src/Functions/date_trunc.cpp index 016b8f4da5e7..e2f22fab757f 100644 --- a/src/Functions/date_trunc.cpp +++ b/src/Functions/date_trunc.cpp @@ -1,6 +1,6 @@ #include -#include #include +#include #include #include #include @@ -20,131 +20,163 @@ namespace ErrorCodes namespace { -class FunctionDateTrunc : public IFunction -{ -public: - static constexpr auto name = "dateTrunc"; + class FunctionDateTrunc : public IFunction + { + public: + static constexpr auto name = "dateTrunc"; - explicit FunctionDateTrunc(ContextPtr context_) : context(context_) {} + explicit FunctionDateTrunc(ContextPtr context_) : context(context_) { } - static FunctionPtr create(ContextPtr context) { return std::make_shared(context); } + static FunctionPtr create(ContextPtr context) { return std::make_shared(context); } - String getName() const override { return name; } + String getName() const override { return name; } - bool isVariadic() const override { return true; } - bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return false; } - size_t getNumberOfArguments() const override { return 0; } + bool isVariadic() const override { return true; } + bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return false; } + size_t getNumberOfArguments() const override { return 0; } - DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override - { - /// The first argument is a constant string with the name of datepart. - - auto result_type_is_date = false; - String datepart_param; - auto check_first_argument = [&] { - const ColumnConst * datepart_column = checkAndGetColumnConst(arguments[0].column.get()); - if (!datepart_column) - throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "First argument for function {} must be constant string: " - "name of datepart", getName()); - - datepart_param = datepart_column->getValue(); - if (datepart_param.empty()) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "First argument (name of datepart) for function {} cannot be empty", - getName()); - - if (!IntervalKind::tryParseString(datepart_param, datepart_kind)) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "{} doesn't look like datepart name in {}", datepart_param, getName()); - - result_type_is_date = (datepart_kind == IntervalKind::Year) - || (datepart_kind == IntervalKind::Quarter) || (datepart_kind == IntervalKind::Month) - || (datepart_kind == IntervalKind::Week); - }; - - bool second_argument_is_date = false; - auto check_second_argument = [&] { - if (!isDate(arguments[1].type) && !isDateTime(arguments[1].type) && !isDateTime64(arguments[1].type)) - throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type {} of 2nd argument of function {}. " - "Should be a date or a date with time", arguments[1].type->getName(), getName()); - - second_argument_is_date = isDate(arguments[1].type); - - if (second_argument_is_date && ((datepart_kind == IntervalKind::Hour) - || (datepart_kind == IntervalKind::Minute) || (datepart_kind == IntervalKind::Second))) - throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type Date of argument for function {}", getName()); - }; - - auto check_timezone_argument = [&] { - if (!WhichDataType(arguments[2].type).isString()) - throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type {} of argument of function {}. " - "This argument is optional and must be a constant string with timezone name", - arguments[2].type->getName(), getName()); - - if (second_argument_is_date && result_type_is_date) - throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, - "The timezone argument of function {} with datepart '{}' " - "is allowed only when the 2nd argument has the type DateTime", - getName(), datepart_param); - }; - - if (arguments.size() == 2) - { - check_first_argument(); - check_second_argument(); - } - else if (arguments.size() == 3) + DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override { - check_first_argument(); - check_second_argument(); - check_timezone_argument(); + /// The first argument is a constant string with the name of datepart. + + result_type_is_date = false; + String datepart_param; + auto check_first_argument = [&] + { + const ColumnConst * datepart_column = checkAndGetColumnConst(arguments[0].column.get()); + if (!datepart_column) + throw Exception( + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "First argument for function {} must be constant string: " + "name of datepart", + getName()); + + datepart_param = datepart_column->getValue(); + if (datepart_param.empty()) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, "First argument (name of datepart) for function {} cannot be empty", getName()); + + if (!IntervalKind::tryParseString(datepart_param, datepart_kind)) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "{} doesn't look like datepart name in {}", datepart_param, getName()); + + result_type_is_date = (datepart_kind == IntervalKind::Year) || (datepart_kind == IntervalKind::Quarter) + || (datepart_kind == IntervalKind::Month) || (datepart_kind == IntervalKind::Week); + }; + + bool second_argument_is_date = false; + auto check_second_argument = [&] + { + if (!isDate(arguments[1].type) && !isDateTime(arguments[1].type) && !isDateTime64(arguments[1].type)) + throw Exception( + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "Illegal type {} of 2nd argument of function {}. " + "Should be a date or a date with time", + arguments[1].type->getName(), + getName()); + + second_argument_is_date = isDate(arguments[1].type); + + if (second_argument_is_date + && ((datepart_kind == IntervalKind::Hour) || (datepart_kind == IntervalKind::Minute) + || (datepart_kind == IntervalKind::Second))) + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type Date of argument for function {}", getName()); + }; + + auto check_timezone_argument = [&] + { + if (!WhichDataType(arguments[2].type).isString()) + throw Exception( + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "Illegal type {} of argument of function {}. " + "This argument is optional and must be a constant string with timezone name", + arguments[2].type->getName(), + getName()); + + if (second_argument_is_date && result_type_is_date) + throw Exception( + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "The timezone argument of function {} with datepart '{}' " + "is allowed only when the 2nd argument has the type DateTime", + getName(), + datepart_param); + }; + + if (arguments.size() == 2) + { + check_first_argument(); + check_second_argument(); + } + else if (arguments.size() == 3) + { + check_first_argument(); + check_second_argument(); + check_timezone_argument(); + } + else + { + throw Exception( + ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, + "Number of arguments for function {} doesn't match: passed {}, should be 2 or 3", + getName(), + arguments.size()); + } + + if (result_type_is_date) + return std::make_shared(); + else + return std::make_shared(extractTimeZoneNameFromFunctionArguments(arguments, 2, 1)); } - else + + bool useDefaultImplementationForConstants() const override { return true; } + ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {0, 2}; } + + ColumnPtr + executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override { - throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, - "Number of arguments for function {} doesn't match: passed {}, should be 2 or 3", - getName(), arguments.size()); - } + ColumnsWithTypeAndName temp_columns(arguments.size()); + temp_columns[0] = arguments[1]; - if (result_type_is_date) - return std::make_shared(); - else - return std::make_shared(extractTimeZoneNameFromFunctionArguments(arguments, 2, 1)); - } + const UInt16 interval_value = 1; + const ColumnPtr interval_column = ColumnConst::create(ColumnInt64::create(1, interval_value), input_rows_count); + temp_columns[1] = {interval_column, std::make_shared(datepart_kind), ""}; - bool useDefaultImplementationForConstants() const override { return true; } - ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {0, 2}; } + auto to_start_of_interval = FunctionFactory::instance().get("toStartOfInterval", context); - ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override - { - ColumnsWithTypeAndName temp_columns(arguments.size()); - temp_columns[0] = arguments[1]; + ColumnPtr truncated_column; + auto date_type = std::make_shared(); - const UInt16 interval_value = 1; - const ColumnPtr interval_column = ColumnConst::create(ColumnInt64::create(1, interval_value), input_rows_count); - temp_columns[1] = {interval_column, std::make_shared(datepart_kind), ""}; + if (arguments.size() == 2) + truncated_column = to_start_of_interval->build(temp_columns) + ->execute(temp_columns, result_type_is_date ? date_type : result_type, input_rows_count); + else + { + temp_columns[2] = arguments[2]; + truncated_column = to_start_of_interval->build(temp_columns) + ->execute(temp_columns, result_type_is_date ? date_type : result_type, input_rows_count); + } - auto to_start_of_interval = FunctionFactory::instance().get("toStartOfInterval", context); + if (!result_type_is_date) + return truncated_column; - if (arguments.size() == 2) - return to_start_of_interval->build(temp_columns)->execute(temp_columns, result_type, input_rows_count); + ColumnsWithTypeAndName temp_truncated_column(1); + temp_truncated_column[0] = {truncated_column, date_type, ""}; - temp_columns[2] = arguments[2]; - return to_start_of_interval->build(temp_columns)->execute(temp_columns, result_type, input_rows_count); - } + auto to_date_time_or_default = FunctionFactory::instance().get("toDateTime", context); + return to_date_time_or_default->build(temp_truncated_column)->execute(temp_truncated_column, result_type, input_rows_count); + } - bool hasInformationAboutMonotonicity() const override - { - return true; - } + bool hasInformationAboutMonotonicity() const override { return true; } - Monotonicity getMonotonicityForRange(const IDataType &, const Field &, const Field &) const override - { - return { .is_monotonic = true, .is_always_monotonic = true }; - } + Monotonicity getMonotonicityForRange(const IDataType &, const Field &, const Field &) const override + { + return {.is_monotonic = true, .is_always_monotonic = true}; + } -private: - ContextPtr context; - mutable IntervalKind::Kind datepart_kind = IntervalKind::Kind::Second; -}; + private: + ContextPtr context; + mutable IntervalKind::Kind datepart_kind = IntervalKind::Kind::Second; + mutable bool result_type_is_date = false; + }; } From 06087b3bf0a0d6c3b3c5e4e8363ca72746be0198 Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Mon, 17 Apr 2023 14:09:54 +0000 Subject: [PATCH 133/406] update README --- utils/keeper-bench/README.md | 318 +++++++++++++++++++++++++++++++++- utils/keeper-bench/Runner.cpp | 2 +- 2 files changed, 318 insertions(+), 2 deletions(-) diff --git a/utils/keeper-bench/README.md b/utils/keeper-bench/README.md index 0c6d01a0418a..8b498228799d 100644 --- a/utils/keeper-bench/README.md +++ b/utils/keeper-bench/README.md @@ -1 +1,317 @@ -Keeper Bench +# Keeper Bench + +Keeper Bench is a tool for benchmarking Keeper or any ZooKeeper compatible systems. + +To run it call following command from the build folder: + +``` +./utils/keeper-bench --config benchmark_config_file.yaml +``` + +## Configuration file + +Keeper Bench runs need to be configured inside a yaml or XML file. +An example of a configuration file can be found in `./utils/keeper-bench/example.yaml` + +### Table of contents +- [Special Types](#special-types) +- [General settings](#general-settings) +- [Connections](#connections) +- [Generator](#generator) +- [Output](#output) + + +## Special types + +### IntegerGetter + +Can be defined with constant integer or as a random value from a range. + +```yaml +key: integer +key: + min_value: integer + max_value: integer +``` + +Example for a constant value: + +```yaml +some_key: 2 +``` + +Example for random value from [10, 20]: + +```yaml +some_key: + min_value: 10 + max_value: 20 +``` + +### StringGetter + +Can be defined with constant string or as a random string of some size. + +```yaml +key: string +key: + random_string: + size: IntegerGetter +``` + +Example for a constant value: +```yaml +some_key: "string" +``` + +Example for a random string with a random size from [10, 20]: +```yaml +some_key: + random_string: + size: + min_value: 10 + max_value: 20 +``` + + +### PathGetter + +If a section contains one or more `path` keys, all `path` keys are collected into a list. \ +Additionally, paths can be defined with key `children_of` which will add all children of some path to the list. + +```yaml +path: string +path: + children_of: string +``` + +Example for defining list of paths (`/path1`, `/path2` and children of `/path3`): + +```yaml +main: + path: + - "/path1" + - "/path2" + path: + children_of: "/path3" +``` + + +## General settings + +```yaml +# number of parallel queries (default: 1) +concurrency: integer + +# amount of queries to be executed, set 0 to disable limit (default: 0) +iterations: integer + +# delay between intermediate reports in seconds, set 0 to disable reports (default: 1.0) +report_delay: double + +# stop launch of queries after specified time limit, set 0 to disable limit (default: 0) +timelimit: double + +# continue testing even if a query fails (default: false) +continue_on_errors: boolean +``` + + +## Connections + +Connection definitions that will be used throughout tests defined under `connections` key. + +Following configurations can be defined under `connections` key or for each specific connection. \ +If it's defined under `connections` key, it will be used by default unless a specific connection overrides it. + +```yaml +secure: boolean +operation_timeout_ms: integer +session_timeout_ms: integer +connection_timeout_ms: integer +``` + +Specific configuration can be defined with a string or with a detailed description. + +```yaml +host: string +connection: + host: string + + # number of sessions to create for host + sessions: integer + # any connection configuration defined above +``` + +Example definition of 3 connections in total, 1 to `localhost:9181` and 2 to `localhost:9182` both will use secure connections: + +```yaml +connections: + secure: true + + host: "localhost:9181" + connection: + host: "localhost:9182" + sessions: 2 +``` + + +## Generator + +Main part of the benchmark is the generator itself which creates necessary nodes and defines how the requests will be generated. \ +It is defined under `generator` key. + +### Setup + +Setup defines nodes that are needed for test, defined under `setup` key. + +Each node is defined with a `node` key in the following format: + +```yaml +node: StringGetter + +node: + name: StringGetter + data: StringGetter + repeat: integer + node: Node +``` + +If only string is defined, a node with that name will be created. \ +Otherwise more detailed definition could be included to set data or the children of the node. \ +If `repeat` key is set, the node definition will be used multiple times. For a `repeat` key to be valid, the name of the node needs to be a random string. + +Example for a setup: + +```yaml +generator: + setup: + node: "node1" + node: + name: + random_string: + size: 20 + data: "somedata" + repeat: 4 + node: + name: + random_string: + size: 10 + repeat: 2 +``` + +We will create node `/node1` with no data and 4 children of random name of size 20 and data set to `somedata`. \ +We will also create 2 nodes with no data and random name of size 10 under `/` node. + +### Requests + +While benchmark is running, we are generating requests. + +Request generator is defined under `requests` key. \ +For each request `weight` (default: 1) can be defined which defines preference for a certain request. + +#### `create` + +```yaml +create: + # parent path for created nodes + path: string + + # length of the name for the create node (default: 5) + name_length: IntegerGetter + + # data for create nodes (default: "") + data: StringGetter + + # value in range [0.0, 1.0> denoting how often a remove request should be generated compared to create request (default: 0) + remove_factor: double +``` + +#### `set` + +```yaml +set: + # paths on which we randomly set data + path: PathGetter + + # data to set + data: StringGetter +``` + +#### `get` + +```yaml +get: + # paths for which we randomly get data + path: PathGetter +``` + +#### `list` + +```yaml +list: + # paths for which we randomly do list request + path: PathGetter +``` + +#### `multi` + +```yaml +multi: + # any request definition defined above can be added + + # optional size for the multi request + size: IntegerGetter +``` + +Multi request definition can contain any other request generator definitions described above. \ +If `size` key is defined, we will randomly pick `size` amount of requests from defined request generators. \ +All request generators can have a higher pick probability by using `weight` key. \ +If `size` is not defined, multi request with same request generators will always be generated. \ +Both write and read multi requests are supported. + +#### Example + +```yaml +generator: + requests: + create: + path: "/test_create" + name_length: + min_value: 10 + max_value: 20 + multi: + weight: 20 + size: 10 + get: + path: + children_of: "/test_get1" + get: + weight: 2 + path: + children_of: "/test_get2" +``` + +We defined a request geneator that will generate either a `create` or a `multi` request. \ +Each `create` request will create a node under `/test_create` with a randomly generated name with size from range `[10, 20]`. \ +`multi` request will be generated 20 times more than `create` request. \ +`multi` request will contain 10 requests and approximately twice as much get requests to children of "/test_get2". + + +## Output + +```yaml +output: + # if defined, JSON output of results will be stored at the defined path + file: string + # or + file: + # if defined, JSON output of results will be stored at the defined path + path: string + + # if set to true, timestamp will be appended to the output file name (default: false) + with_timestamp: boolean + + # if set to true, output will be printed to stdout also (default: false) + stdout: boolean +``` diff --git a/utils/keeper-bench/Runner.cpp b/utils/keeper-bench/Runner.cpp index bb8ff46a20c7..36ffae40ce46 100644 --- a/utils/keeper-bench/Runner.cpp +++ b/utils/keeper-bench/Runner.cpp @@ -329,7 +329,7 @@ void Runner::runBenchmark() generator->startup(*connections[0]); std::cerr << "Prepared\n"; - auto start_timestamp_ms = Poco::Timestamp{}.epochMicroseconds() / 1000; + auto start_timestamp_ms = Poco::Timestamp().epochMicroseconds() / 1000; try { From 0521f58d6f625f24c6249a674113d1db0f2ce319 Mon Sep 17 00:00:00 2001 From: Shane Andrade Date: Mon, 17 Apr 2023 15:06:07 +0000 Subject: [PATCH 134/406] undo automatic indentation --- src/Functions/date_trunc.cpp | 266 +++++++++++++++++------------------ 1 file changed, 133 insertions(+), 133 deletions(-) diff --git a/src/Functions/date_trunc.cpp b/src/Functions/date_trunc.cpp index e2f22fab757f..4cbc605f088d 100644 --- a/src/Functions/date_trunc.cpp +++ b/src/Functions/date_trunc.cpp @@ -20,163 +20,163 @@ namespace ErrorCodes namespace { - class FunctionDateTrunc : public IFunction +class FunctionDateTrunc : public IFunction +{ +public: + static constexpr auto name = "dateTrunc"; + + explicit FunctionDateTrunc(ContextPtr context_) : context(context_) { } + + static FunctionPtr create(ContextPtr context) { return std::make_shared(context); } + + String getName() const override { return name; } + + bool isVariadic() const override { return true; } + bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return false; } + size_t getNumberOfArguments() const override { return 0; } + + DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override { - public: - static constexpr auto name = "dateTrunc"; + /// The first argument is a constant string with the name of datepart. - explicit FunctionDateTrunc(ContextPtr context_) : context(context_) { } + result_type_is_date = false; + String datepart_param; + auto check_first_argument = [&] + { + const ColumnConst * datepart_column = checkAndGetColumnConst(arguments[0].column.get()); + if (!datepart_column) + throw Exception( + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "First argument for function {} must be constant string: " + "name of datepart", + getName()); - static FunctionPtr create(ContextPtr context) { return std::make_shared(context); } + datepart_param = datepart_column->getValue(); + if (datepart_param.empty()) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, "First argument (name of datepart) for function {} cannot be empty", getName()); - String getName() const override { return name; } + if (!IntervalKind::tryParseString(datepart_param, datepart_kind)) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "{} doesn't look like datepart name in {}", datepart_param, getName()); - bool isVariadic() const override { return true; } - bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return false; } - size_t getNumberOfArguments() const override { return 0; } + result_type_is_date = (datepart_kind == IntervalKind::Year) || (datepart_kind == IntervalKind::Quarter) + || (datepart_kind == IntervalKind::Month) || (datepart_kind == IntervalKind::Week); + }; - DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override + bool second_argument_is_date = false; + auto check_second_argument = [&] { - /// The first argument is a constant string with the name of datepart. - - result_type_is_date = false; - String datepart_param; - auto check_first_argument = [&] - { - const ColumnConst * datepart_column = checkAndGetColumnConst(arguments[0].column.get()); - if (!datepart_column) - throw Exception( - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, - "First argument for function {} must be constant string: " - "name of datepart", - getName()); - - datepart_param = datepart_column->getValue(); - if (datepart_param.empty()) - throw Exception( - ErrorCodes::BAD_ARGUMENTS, "First argument (name of datepart) for function {} cannot be empty", getName()); - - if (!IntervalKind::tryParseString(datepart_param, datepart_kind)) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "{} doesn't look like datepart name in {}", datepart_param, getName()); - - result_type_is_date = (datepart_kind == IntervalKind::Year) || (datepart_kind == IntervalKind::Quarter) - || (datepart_kind == IntervalKind::Month) || (datepart_kind == IntervalKind::Week); - }; - - bool second_argument_is_date = false; - auto check_second_argument = [&] - { - if (!isDate(arguments[1].type) && !isDateTime(arguments[1].type) && !isDateTime64(arguments[1].type)) - throw Exception( - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, - "Illegal type {} of 2nd argument of function {}. " - "Should be a date or a date with time", - arguments[1].type->getName(), - getName()); - - second_argument_is_date = isDate(arguments[1].type); - - if (second_argument_is_date - && ((datepart_kind == IntervalKind::Hour) || (datepart_kind == IntervalKind::Minute) - || (datepart_kind == IntervalKind::Second))) - throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type Date of argument for function {}", getName()); - }; - - auto check_timezone_argument = [&] - { - if (!WhichDataType(arguments[2].type).isString()) - throw Exception( - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, - "Illegal type {} of argument of function {}. " - "This argument is optional and must be a constant string with timezone name", - arguments[2].type->getName(), - getName()); - - if (second_argument_is_date && result_type_is_date) - throw Exception( - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, - "The timezone argument of function {} with datepart '{}' " - "is allowed only when the 2nd argument has the type DateTime", - getName(), - datepart_param); - }; - - if (arguments.size() == 2) - { - check_first_argument(); - check_second_argument(); - } - else if (arguments.size() == 3) - { - check_first_argument(); - check_second_argument(); - check_timezone_argument(); - } - else - { + if (!isDate(arguments[1].type) && !isDateTime(arguments[1].type) && !isDateTime64(arguments[1].type)) throw Exception( - ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, - "Number of arguments for function {} doesn't match: passed {}, should be 2 or 3", - getName(), - arguments.size()); - } + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "Illegal type {} of 2nd argument of function {}. " + "Should be a date or a date with time", + arguments[1].type->getName(), + getName()); - if (result_type_is_date) - return std::make_shared(); - else - return std::make_shared(extractTimeZoneNameFromFunctionArguments(arguments, 2, 1)); - } + second_argument_is_date = isDate(arguments[1].type); - bool useDefaultImplementationForConstants() const override { return true; } - ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {0, 2}; } + if (second_argument_is_date + && ((datepart_kind == IntervalKind::Hour) || (datepart_kind == IntervalKind::Minute) + || (datepart_kind == IntervalKind::Second))) + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type Date of argument for function {}", getName()); + }; - ColumnPtr - executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override + auto check_timezone_argument = [&] { - ColumnsWithTypeAndName temp_columns(arguments.size()); - temp_columns[0] = arguments[1]; + if (!WhichDataType(arguments[2].type).isString()) + throw Exception( + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "Illegal type {} of argument of function {}. " + "This argument is optional and must be a constant string with timezone name", + arguments[2].type->getName(), + getName()); - const UInt16 interval_value = 1; - const ColumnPtr interval_column = ColumnConst::create(ColumnInt64::create(1, interval_value), input_rows_count); - temp_columns[1] = {interval_column, std::make_shared(datepart_kind), ""}; + if (second_argument_is_date && result_type_is_date) + throw Exception( + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "The timezone argument of function {} with datepart '{}' " + "is allowed only when the 2nd argument has the type DateTime", + getName(), + datepart_param); + }; - auto to_start_of_interval = FunctionFactory::instance().get("toStartOfInterval", context); + if (arguments.size() == 2) + { + check_first_argument(); + check_second_argument(); + } + else if (arguments.size() == 3) + { + check_first_argument(); + check_second_argument(); + check_timezone_argument(); + } + else + { + throw Exception( + ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, + "Number of arguments for function {} doesn't match: passed {}, should be 2 or 3", + getName(), + arguments.size()); + } - ColumnPtr truncated_column; - auto date_type = std::make_shared(); + if (result_type_is_date) + return std::make_shared(); + else + return std::make_shared(extractTimeZoneNameFromFunctionArguments(arguments, 2, 1)); + } - if (arguments.size() == 2) - truncated_column = to_start_of_interval->build(temp_columns) - ->execute(temp_columns, result_type_is_date ? date_type : result_type, input_rows_count); - else - { - temp_columns[2] = arguments[2]; - truncated_column = to_start_of_interval->build(temp_columns) - ->execute(temp_columns, result_type_is_date ? date_type : result_type, input_rows_count); - } + bool useDefaultImplementationForConstants() const override { return true; } + ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {0, 2}; } - if (!result_type_is_date) - return truncated_column; + ColumnPtr + executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override + { + ColumnsWithTypeAndName temp_columns(arguments.size()); + temp_columns[0] = arguments[1]; - ColumnsWithTypeAndName temp_truncated_column(1); - temp_truncated_column[0] = {truncated_column, date_type, ""}; + const UInt16 interval_value = 1; + const ColumnPtr interval_column = ColumnConst::create(ColumnInt64::create(1, interval_value), input_rows_count); + temp_columns[1] = {interval_column, std::make_shared(datepart_kind), ""}; - auto to_date_time_or_default = FunctionFactory::instance().get("toDateTime", context); - return to_date_time_or_default->build(temp_truncated_column)->execute(temp_truncated_column, result_type, input_rows_count); - } + auto to_start_of_interval = FunctionFactory::instance().get("toStartOfInterval", context); - bool hasInformationAboutMonotonicity() const override { return true; } + ColumnPtr truncated_column; + auto date_type = std::make_shared(); - Monotonicity getMonotonicityForRange(const IDataType &, const Field &, const Field &) const override + if (arguments.size() == 2) + truncated_column = to_start_of_interval->build(temp_columns) + ->execute(temp_columns, result_type_is_date ? date_type : result_type, input_rows_count); + else { - return {.is_monotonic = true, .is_always_monotonic = true}; + temp_columns[2] = arguments[2]; + truncated_column = to_start_of_interval->build(temp_columns) + ->execute(temp_columns, result_type_is_date ? date_type : result_type, input_rows_count); } - private: - ContextPtr context; - mutable IntervalKind::Kind datepart_kind = IntervalKind::Kind::Second; - mutable bool result_type_is_date = false; - }; + if (!result_type_is_date) + return truncated_column; + + ColumnsWithTypeAndName temp_truncated_column(1); + temp_truncated_column[0] = {truncated_column, date_type, ""}; + + auto to_date_time_or_default = FunctionFactory::instance().get("toDateTime", context); + return to_date_time_or_default->build(temp_truncated_column)->execute(temp_truncated_column, result_type, input_rows_count); + } + + bool hasInformationAboutMonotonicity() const override { return true; } + + Monotonicity getMonotonicityForRange(const IDataType &, const Field &, const Field &) const override + { + return {.is_monotonic = true, .is_always_monotonic = true}; + } + +private: + ContextPtr context; + mutable IntervalKind::Kind datepart_kind = IntervalKind::Kind::Second; + mutable bool result_type_is_date = false; +}; } From 10cfc146cb2c5a9ea623bc62022ca2b4476fbe0f Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Mon, 17 Apr 2023 15:09:35 +0000 Subject: [PATCH 135/406] Fix spaces --- utils/keeper-bench/Runner.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/keeper-bench/Runner.cpp b/utils/keeper-bench/Runner.cpp index 36ffae40ce46..f86d2b44dd7a 100644 --- a/utils/keeper-bench/Runner.cpp +++ b/utils/keeper-bench/Runner.cpp @@ -329,7 +329,7 @@ void Runner::runBenchmark() generator->startup(*connections[0]); std::cerr << "Prepared\n"; - auto start_timestamp_ms = Poco::Timestamp().epochMicroseconds() / 1000; + auto start_timestamp_ms = Poco::Timestamp().epochMicroseconds() / 1000; try { From d14cc1691cb0e9efc573856e31423a704b42f6a8 Mon Sep 17 00:00:00 2001 From: Smita Kulkarni Date: Mon, 17 Apr 2023 18:53:26 +0200 Subject: [PATCH 136/406] =?UTF-8?q?Added=20an=20option=20=E2=80=98force?= =?UTF-8?q?=E2=80=99=20to=20clearOldTemporaryDirectories,=20which=20is=20c?= =?UTF-8?q?urrently=20used=20by=20dropAllData=20to=20remove=20blobs=20when?= =?UTF-8?q?=20zero=20copy=20replication=20is=20enabled.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/Storages/MergeTree/MergeTreeData.cpp | 6 +++--- src/Storages/MergeTree/MergeTreeData.h | 3 ++- .../queries/0_stateless/02432_s3_parallel_parts_cleanup.sql | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 45759c449f6c..5c189887e23d 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -1960,7 +1960,7 @@ static bool isOldPartDirectory(const DiskPtr & disk, const String & directory_pa } -size_t MergeTreeData::clearOldTemporaryDirectories(size_t custom_directories_lifetime_seconds, const NameSet & valid_prefixes) +size_t MergeTreeData::clearOldTemporaryDirectories(size_t custom_directories_lifetime_seconds, const NameSet & valid_prefixes, const bool & force) { /// If the method is already called from another thread, then we don't need to do anything. std::unique_lock lock(clear_old_temporary_directories_mutex, std::defer_lock); @@ -2018,7 +2018,7 @@ size_t MergeTreeData::clearOldTemporaryDirectories(size_t custom_directories_lif /// We don't control the amount of refs for temporary parts so we cannot decide can we remove blobs /// or not. So we are not doing it bool keep_shared = false; - if (disk->supportZeroCopyReplication() && settings->allow_remote_fs_zero_copy_replication) + if (disk->supportZeroCopyReplication() && settings->allow_remote_fs_zero_copy_replication && !force) { LOG_WARNING(log, "Since zero-copy replication is enabled we are not going to remove blobs from shared storage for {}", full_path); keep_shared = true; @@ -2724,7 +2724,7 @@ void MergeTreeData::dropAllData() } LOG_INFO(log, "dropAllData: clearing temporary directories"); - clearOldTemporaryDirectories(0, {"tmp_", "delete_tmp_", "tmp-fetch_"}); + clearOldTemporaryDirectories(0, {"tmp_", "delete_tmp_", "tmp-fetch_"}, /* force */ true); column_sizes.clear(); diff --git a/src/Storages/MergeTree/MergeTreeData.h b/src/Storages/MergeTree/MergeTreeData.h index b03b7d4a71ee..3053657e37ba 100644 --- a/src/Storages/MergeTree/MergeTreeData.h +++ b/src/Storages/MergeTree/MergeTreeData.h @@ -667,7 +667,8 @@ class MergeTreeData : public IStorage, public WithMutableContext /// Delete all directories which names begin with "tmp" /// Must be called with locked lockForShare() because it's using relative_data_path. - size_t clearOldTemporaryDirectories(size_t custom_directories_lifetime_seconds, const NameSet & valid_prefixes = {"tmp_", "tmp-fetch_"}); + /// 'force' is used by dropAllData(), this will remove blobs even if zero copy replication is enabled + size_t clearOldTemporaryDirectories(size_t custom_directories_lifetime_seconds, const NameSet & valid_prefixes = {"tmp_", "tmp-fetch_"}, const bool & force = false); size_t clearEmptyParts(); diff --git a/tests/queries/0_stateless/02432_s3_parallel_parts_cleanup.sql b/tests/queries/0_stateless/02432_s3_parallel_parts_cleanup.sql index 3688a649d5ea..0230f30bf058 100644 --- a/tests/queries/0_stateless/02432_s3_parallel_parts_cleanup.sql +++ b/tests/queries/0_stateless/02432_s3_parallel_parts_cleanup.sql @@ -55,7 +55,7 @@ select sleep(3); select count(), sum(n), sum(m) from rmt; select count(), sum(n), sum(m) from rmt2; --- So there will be at least 2 parts (just in case no parts are removed until drop) +-- So there will be at least 2 parts (just in case no parts are removed until drop). insert into rmt(n) values (10); drop table rmt; From c34f277fe74eb5bd024d1e2d9de9be9d590d3eb1 Mon Sep 17 00:00:00 2001 From: kssenii Date: Mon, 17 Apr 2023 20:44:33 +0200 Subject: [PATCH 137/406] Fix conflicts after merge --- src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp | 16 ++++++++-------- src/Interpreters/Cache/FileSegment.cpp | 10 +++++++++- src/Interpreters/Cache/FileSegment.h | 3 ++- 3 files changed, 19 insertions(+), 10 deletions(-) diff --git a/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp b/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp index 67f084b76702..4114ffcc522c 100644 --- a/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp +++ b/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp @@ -201,7 +201,7 @@ CachedOnDiskReadBufferFromFile::getRemoteReadBuffer(FileSegment & file_segment, } else { - chassert(remote_fs_segment_reader->getFileOffsetOfBufferEnd() == file_segment.getCurrentWriteOffset()); + chassert(remote_fs_segment_reader->getFileOffsetOfBufferEnd() == file_segment.getCurrentWriteOffset(false)); } return remote_fs_segment_reader; @@ -403,7 +403,7 @@ CachedOnDiskReadBufferFromFile::getImplementationBuffer(FileSegment & file_segme "Current read type: {}, read offset: {}, impl read range: {}, file segment: {}", toString(read_type), file_offset_of_buffer_end, - read_buffer_for_file_segment->getRemainingReadRange().toString(), + read_buffer_for_file_segment->getFileOffsetOfBufferEnd(), file_segment.getInfoForLog()); read_buffer_for_file_segment->setReadUntilPosition(range.right + 1); /// [..., range.right] @@ -471,7 +471,7 @@ CachedOnDiskReadBufferFromFile::getImplementationBuffer(FileSegment & file_segme current_write_offset, read_buffer_for_file_segment->getPosition(), read_buffer_for_file_segment->getFileOffsetOfBufferEnd(), - file_segment->getInfoForLog()); + file_segment.getInfoForLog()); } break; @@ -544,7 +544,7 @@ void CachedOnDiskReadBufferFromFile::predownload(FileSegment & file_segment) LOG_TEST(log, "Bytes to predownload: {}, caller_id: {}", bytes_to_predownload, FileSegment::getCallerId()); /// chassert(implementation_buffer->getFileOffsetOfBufferEnd() == file_segment.getCurrentWriteOffset(false)); - chassert(static_cast(implementation_buffer->getPosition()) == file_segment->getCurrentWriteOffset(false)); + chassert(static_cast(implementation_buffer->getPosition()) == file_segment.getCurrentWriteOffset(false)); size_t current_offset = file_segment.getCurrentWriteOffset(false); const auto & current_range = file_segment.range(); @@ -814,9 +814,9 @@ bool CachedOnDiskReadBufferFromFile::nextImplStep() if (need_complete_file_segment) { if (!implementation_buffer_can_be_reused) - file_segment->resetRemoteFileReader(); + file_segment.resetRemoteFileReader(); - file_segment->completePartAndResetDownloader(); + file_segment.completePartAndResetDownloader(); } } @@ -926,7 +926,7 @@ bool CachedOnDiskReadBufferFromFile::nextImplStep() log, "Read {} bytes, read type {}, position: {}, offset: {}, segment end: {}", size, toString(read_type), implementation_buffer->getPosition(), - implementation_buffer->getFileOffsetOfBufferEnd(), file_segment->range().right); + implementation_buffer->getFileOffsetOfBufferEnd(), file_segment.range().right); if (read_type == ReadType::CACHED) { @@ -1168,7 +1168,7 @@ void CachedOnDiskReadBufferFromFile::setReadUntilPosition(size_t position) file_offset_of_buffer_end = getPosition(); resetWorkingBuffer(); - file_segments_holder.reset(); + file_segments.reset(); implementation_buffer.reset(); initialized = false; diff --git a/src/Interpreters/Cache/FileSegment.cpp b/src/Interpreters/Cache/FileSegment.cpp index 639160426ea8..a70703ba6718 100644 --- a/src/Interpreters/Cache/FileSegment.cpp +++ b/src/Interpreters/Cache/FileSegment.cpp @@ -226,7 +226,7 @@ void FileSegment::resetDownloaderUnlocked(const FileSegmentGuard::Lock &) void FileSegment::assertIsDownloaderUnlocked(const std::string & operation, const FileSegmentGuard::Lock & lock) const { auto caller = getCallerId(); - auto current_downloader = getDownloaderUnlocked(segment_lock); + auto current_downloader = getDownloaderUnlocked(lock); LOG_TEST(log, "Downloader id: {}, caller id: {}, operation: {}", current_downloader, caller, operation); if (caller != current_downloader) @@ -257,6 +257,13 @@ FileSegment::RemoteFileReaderPtr FileSegment::getRemoteFileReader() return remote_file_reader; } +void FileSegment::resetRemoteFileReader() +{ + auto lock = segment_guard.lock(); + assertIsDownloaderUnlocked("resetRemoteFileReader", lock); + remote_file_reader.reset(); +} + FileSegment::RemoteFileReaderPtr FileSegment::extractRemoteFileReader() { auto locked_key = lockKeyMetadata(false); @@ -699,6 +706,7 @@ bool FileSegment::assertCorrectnessUnlocked(const FileSegmentGuard::Lock &) cons { auto check_iterator = [this](const Priority::Iterator & it) { + UNUSED(this); if (!it) return; diff --git a/src/Interpreters/Cache/FileSegment.h b/src/Interpreters/Cache/FileSegment.h index 3a17442d2318..60883631177e 100644 --- a/src/Interpreters/Cache/FileSegment.h +++ b/src/Interpreters/Cache/FileSegment.h @@ -258,7 +258,6 @@ friend class FileCache; /// Because of reserved_size in tryReserve(). /// Write data into reserved space. void write(const char * from, size_t size, size_t offset); - // Invariant: if state() != DOWNLOADING and remote file reader is present, the reader's // available() == 0, and getFileOffsetOfBufferEnd() == our getCurrentWriteOffset(). // @@ -268,6 +267,8 @@ friend class FileCache; /// Because of reserved_size in tryReserve(). RemoteFileReaderPtr extractRemoteFileReader(); + void resetRemoteFileReader(); + void setRemoteFileReader(RemoteFileReaderPtr remote_file_reader_); void setDownloadedSize(size_t delta); From fb16623d48da107504a6d0c5d6b52fc525a34424 Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Tue, 18 Apr 2023 13:15:23 +0000 Subject: [PATCH 138/406] Add CheckNotExists request to Keeper --- src/Common/ZooKeeper/IKeeper.h | 5 +- src/Common/ZooKeeper/TestKeeper.h | 2 +- src/Common/ZooKeeper/ZooKeeper.cpp | 3 +- src/Common/ZooKeeper/ZooKeeper.h | 18 ++++- src/Common/ZooKeeper/ZooKeeperCommon.cpp | 3 + src/Common/ZooKeeper/ZooKeeperCommon.h | 4 +- src/Common/ZooKeeper/ZooKeeperConstants.cpp | 3 + src/Common/ZooKeeper/ZooKeeperConstants.h | 1 + src/Common/ZooKeeper/ZooKeeperImpl.cpp | 2 +- src/Common/ZooKeeper/ZooKeeperImpl.h | 2 +- .../ZooKeeper/ZooKeeperWithFaultInjection.h | 6 ++ src/Coordination/KeeperConstants.h | 5 +- src/Coordination/KeeperStorage.cpp | 57 +++++++++++---- src/Coordination/tests/gtest_coordination.cpp | 72 +++++++++++++++++++ .../MergeTree/EphemeralLockInZooKeeper.cpp | 18 ++--- src/Storages/StorageReplicatedMergeTree.cpp | 6 +- 16 files changed, 169 insertions(+), 38 deletions(-) diff --git a/src/Common/ZooKeeper/IKeeper.h b/src/Common/ZooKeeper/IKeeper.h index 172714fe04fa..b09f096d761c 100644 --- a/src/Common/ZooKeeper/IKeeper.h +++ b/src/Common/ZooKeeper/IKeeper.h @@ -319,6 +319,9 @@ struct CheckRequest : virtual Request String path; int32_t version = -1; + /// should it check if a node DOES NOT exist + bool not_exists = false; + void addRootPath(const String & root_path) override; String getPath() const override { return path; } @@ -524,7 +527,7 @@ class IKeeper const Requests & requests, MultiCallback callback) = 0; - virtual DB::KeeperApiVersion getApiVersion() = 0; + virtual DB::KeeperApiVersion getApiVersion() const = 0; /// Expire session and finish all pending requests virtual void finalize(const String & reason) = 0; diff --git a/src/Common/ZooKeeper/TestKeeper.h b/src/Common/ZooKeeper/TestKeeper.h index fb4e527e50e1..27405d8d5711 100644 --- a/src/Common/ZooKeeper/TestKeeper.h +++ b/src/Common/ZooKeeper/TestKeeper.h @@ -91,7 +91,7 @@ class TestKeeper final : public IKeeper void finalize(const String & reason) override; - DB::KeeperApiVersion getApiVersion() override + DB::KeeperApiVersion getApiVersion() const override { return KeeperApiVersion::ZOOKEEPER_COMPATIBLE; } diff --git a/src/Common/ZooKeeper/ZooKeeper.cpp b/src/Common/ZooKeeper/ZooKeeper.cpp index a8da0dff0cca..54a2e2dc5193 100644 --- a/src/Common/ZooKeeper/ZooKeeper.cpp +++ b/src/Common/ZooKeeper/ZooKeeper.cpp @@ -821,7 +821,7 @@ bool ZooKeeper::expired() return impl->isExpired(); } -DB::KeeperApiVersion ZooKeeper::getApiVersion() +DB::KeeperApiVersion ZooKeeper::getApiVersion() const { return impl->getApiVersion(); } @@ -1282,7 +1282,6 @@ Coordination::RequestPtr makeExistsRequest(const std::string & path) return request; } - std::string normalizeZooKeeperPath(std::string zookeeper_path, bool check_starts_with_slash, Poco::Logger * log) { if (!zookeeper_path.empty() && zookeeper_path.back() == '/') diff --git a/src/Common/ZooKeeper/ZooKeeper.h b/src/Common/ZooKeeper/ZooKeeper.h index 8e7639b8cc12..b31dbc8da49a 100644 --- a/src/Common/ZooKeeper/ZooKeeper.h +++ b/src/Common/ZooKeeper/ZooKeeper.h @@ -215,7 +215,7 @@ class ZooKeeper /// Returns true, if the session has expired. bool expired(); - DB::KeeperApiVersion getApiVersion(); + DB::KeeperApiVersion getApiVersion() const; /// Create a znode. /// Throw an exception if something went wrong. @@ -674,4 +674,20 @@ bool hasZooKeeperConfig(const Poco::Util::AbstractConfiguration & config); String getZooKeeperConfigName(const Poco::Util::AbstractConfiguration & config); +template +void addCheckNotExistsRequest(Coordination::Requests requests, const Client & client, const std::string & path) +{ + if (client.getApiVersion() >= DB::KeeperApiVersion::WITH_CHECK_NOT_EXISTS) + { + auto request = std::make_shared(); + request->path = path; + request->not_exists = true; + requests.push_back(std::move(request)); + return; + } + + requests.push_back(makeCreateRequest(path, "", zkutil::CreateMode::Persistent)); + requests.push_back(makeRemoveRequest(path, -1)); +} + } diff --git a/src/Common/ZooKeeper/ZooKeeperCommon.cpp b/src/Common/ZooKeeper/ZooKeeperCommon.cpp index 1ee56936889f..03bfafac0c22 100644 --- a/src/Common/ZooKeeper/ZooKeeperCommon.cpp +++ b/src/Common/ZooKeeper/ZooKeeperCommon.cpp @@ -931,6 +931,8 @@ void registerZooKeeperRequest(ZooKeeperRequestFactory & factory) res->operation_type = ZooKeeperMultiRequest::OperationType::Read; else if constexpr (num == OpNum::Multi) res->operation_type = ZooKeeperMultiRequest::OperationType::Write; + else if constexpr (num == OpNum::CheckNotExists) + res->not_exists = true; return res; }); @@ -956,6 +958,7 @@ ZooKeeperRequestFactory::ZooKeeperRequestFactory() registerZooKeeperRequest(*this); registerZooKeeperRequest(*this); registerZooKeeperRequest(*this); + registerZooKeeperRequest(*this); } PathMatchResult matchPath(std::string_view path, std::string_view match_to) diff --git a/src/Common/ZooKeeper/ZooKeeperCommon.h b/src/Common/ZooKeeper/ZooKeeperCommon.h index 1755ebd8cccd..fccccfd20581 100644 --- a/src/Common/ZooKeeper/ZooKeeperCommon.h +++ b/src/Common/ZooKeeper/ZooKeeperCommon.h @@ -390,12 +390,12 @@ struct ZooKeeperSimpleListResponse final : ZooKeeperListResponse size_t bytesSize() const override { return ZooKeeperListResponse::bytesSize() - sizeof(stat); } }; -struct ZooKeeperCheckRequest final : CheckRequest, ZooKeeperRequest +struct ZooKeeperCheckRequest : CheckRequest, ZooKeeperRequest { ZooKeeperCheckRequest() = default; explicit ZooKeeperCheckRequest(const CheckRequest & base) : CheckRequest(base) {} - OpNum getOpNum() const override { return OpNum::Check; } + OpNum getOpNum() const override { return not_exists ? OpNum::CheckNotExists : OpNum::Check; } void writeImpl(WriteBuffer & out) const override; void readImpl(ReadBuffer & in) override; std::string toStringImpl() const override; diff --git a/src/Common/ZooKeeper/ZooKeeperConstants.cpp b/src/Common/ZooKeeper/ZooKeeperConstants.cpp index c2e4c0f5cbd8..86f70ea547a5 100644 --- a/src/Common/ZooKeeper/ZooKeeperConstants.cpp +++ b/src/Common/ZooKeeper/ZooKeeperConstants.cpp @@ -26,6 +26,7 @@ static const std::unordered_set VALID_OPERATIONS = static_cast(OpNum::SetACL), static_cast(OpNum::GetACL), static_cast(OpNum::FilteredList), + static_cast(OpNum::CheckNotExists), }; std::string toString(OpNum op_num) @@ -70,6 +71,8 @@ std::string toString(OpNum op_num) return "GetACL"; case OpNum::FilteredList: return "FilteredList"; + case OpNum::CheckNotExists: + return "CheckNotExists"; } int32_t raw_op = static_cast(op_num); throw Exception("Operation " + std::to_string(raw_op) + " is unknown", Error::ZUNIMPLEMENTED); diff --git a/src/Common/ZooKeeper/ZooKeeperConstants.h b/src/Common/ZooKeeper/ZooKeeperConstants.h index 912e253718b3..6b50c5c5d092 100644 --- a/src/Common/ZooKeeper/ZooKeeperConstants.h +++ b/src/Common/ZooKeeper/ZooKeeperConstants.h @@ -36,6 +36,7 @@ enum class OpNum : int32_t // CH Keeper specific operations FilteredList = 500, + CheckNotExists = 501, SessionID = 997, /// Special internal request }; diff --git a/src/Common/ZooKeeper/ZooKeeperImpl.cpp b/src/Common/ZooKeeper/ZooKeeperImpl.cpp index f97bf292198a..6c79fc4f178b 100644 --- a/src/Common/ZooKeeper/ZooKeeperImpl.cpp +++ b/src/Common/ZooKeeper/ZooKeeperImpl.cpp @@ -1085,7 +1085,7 @@ void ZooKeeper::pushRequest(RequestInfo && info) ProfileEvents::increment(ProfileEvents::ZooKeeperTransactions); } -KeeperApiVersion ZooKeeper::getApiVersion() +KeeperApiVersion ZooKeeper::getApiVersion() const { return keeper_api_version; } diff --git a/src/Common/ZooKeeper/ZooKeeperImpl.h b/src/Common/ZooKeeper/ZooKeeperImpl.h index 9fff12309bd6..c0c57d3f7198 100644 --- a/src/Common/ZooKeeper/ZooKeeperImpl.h +++ b/src/Common/ZooKeeper/ZooKeeperImpl.h @@ -179,7 +179,7 @@ class ZooKeeper final : public IKeeper const Requests & requests, MultiCallback callback) override; - DB::KeeperApiVersion getApiVersion() override; + DB::KeeperApiVersion getApiVersion() const override; /// Without forcefully invalidating (finalizing) ZooKeeper session before /// establishing a new one, there was a possibility that server is using diff --git a/src/Common/ZooKeeper/ZooKeeperWithFaultInjection.h b/src/Common/ZooKeeper/ZooKeeperWithFaultInjection.h index 130590ceb400..214a2eb944af 100644 --- a/src/Common/ZooKeeper/ZooKeeperWithFaultInjection.h +++ b/src/Common/ZooKeeper/ZooKeeperWithFaultInjection.h @@ -6,6 +6,7 @@ #include #include #include +#include "Coordination/KeeperConstants.h" namespace DB { @@ -381,6 +382,11 @@ class ZooKeeperWithFaultInjection ephemeral_nodes.clear(); } + KeeperApiVersion getApiVersion() const + { + return keeper->getApiVersion(); + } + private: void faultInjectionBefore(std::function fault_cleanup) { diff --git a/src/Coordination/KeeperConstants.h b/src/Coordination/KeeperConstants.h index 952689af01fe..4b5a5b54be0e 100644 --- a/src/Coordination/KeeperConstants.h +++ b/src/Coordination/KeeperConstants.h @@ -9,10 +9,11 @@ enum class KeeperApiVersion : uint8_t { ZOOKEEPER_COMPATIBLE = 0, WITH_FILTERED_LIST, - WITH_MULTI_READ + WITH_MULTI_READ, + WITH_CHECK_NOT_EXISTS, }; -inline constexpr auto current_keeper_api_version = KeeperApiVersion::WITH_MULTI_READ; +inline constexpr auto current_keeper_api_version = KeeperApiVersion::WITH_CHECK_NOT_EXISTS; const std::string keeper_system_path = "/keeper"; const std::string keeper_api_version_path = keeper_system_path + "/api_version"; diff --git a/src/Coordination/KeeperStorage.cpp b/src/Coordination/KeeperStorage.cpp index cfc1c2bd12bc..28cb4fba9c9b 100644 --- a/src/Coordination/KeeperStorage.cpp +++ b/src/Coordination/KeeperStorage.cpp @@ -1449,24 +1449,44 @@ struct KeeperStorageListRequestProcessor final : public KeeperStorageRequestProc struct KeeperStorageCheckRequestProcessor final : public KeeperStorageRequestProcessor { + explicit KeeperStorageCheckRequestProcessor(const Coordination::ZooKeeperRequestPtr & zk_request_) + : KeeperStorageRequestProcessor(zk_request_) + { + check_not_exists = zk_request->getOpNum() == Coordination::OpNum::CheckNotExists; + } + bool checkAuth(KeeperStorage & storage, int64_t session_id, bool is_local) const override { - return storage.checkACL(zk_request->getPath(), Coordination::ACL::Read, session_id, is_local); + StringRef path; + if (check_not_exists) + path = parentPath(zk_request->getPath()); + else + path = zk_request->getPath(); + + return storage.checkACL(path, Coordination::ACL::Read, session_id, is_local); } - using KeeperStorageRequestProcessor::KeeperStorageRequestProcessor; std::vector preprocess(KeeperStorage & storage, int64_t zxid, int64_t /*session_id*/, int64_t /*time*/, uint64_t & /*digest*/, const KeeperContext & /*keeper_context*/) const override { ProfileEvents::increment(ProfileEvents::KeeperCheckRequest); - Coordination::ZooKeeperCheckRequest & request = dynamic_cast(*zk_request); - if (!storage.uncommitted_state.getNode(request.path)) - return {KeeperStorage::Delta{zxid, Coordination::Error::ZNONODE}}; + Coordination::ZooKeeperCheckRequest & request = dynamic_cast(*zk_request); auto node = storage.uncommitted_state.getNode(request.path); - if (request.version != -1 && request.version != node->stat.version) - return {KeeperStorage::Delta{zxid, Coordination::Error::ZBADVERSION}}; + if (check_not_exists) + { + if (node && (request.version == -1 || request.version == node->stat.version)) + return {KeeperStorage::Delta{zxid, Coordination::Error::ZNODEEXISTS}}; + } + else + { + if (!node) + return {KeeperStorage::Delta{zxid, Coordination::Error::ZNONODE}}; + + if (request.version != -1 && request.version != node->stat.version) + return {KeeperStorage::Delta{zxid, Coordination::Error::ZBADVERSION}}; + } return {}; } @@ -1497,17 +1517,22 @@ struct KeeperStorageCheckRequestProcessor final : public KeeperStorageRequestPro auto & container = storage.container; auto node_it = container.find(request.path); - if (node_it == container.end()) - { - on_error(Coordination::Error::ZNONODE); - } - else if (request.version != -1 && request.version != node_it->value.stat.version) + + if (check_not_exists) { - on_error(Coordination::Error::ZBADVERSION); + if (node_it != container.end() && (request.version == -1 || request.version == node_it->value.stat.version)) + on_error(Coordination::Error::ZNODEEXISTS); + else + response.error = Coordination::Error::ZOK; } else { - response.error = Coordination::Error::ZOK; + if (node_it == container.end()) + on_error(Coordination::Error::ZNONODE); + else if (request.version != -1 && request.version != node_it->value.stat.version) + on_error(Coordination::Error::ZBADVERSION); + else + response.error = Coordination::Error::ZOK; } return response_ptr; @@ -1523,6 +1548,9 @@ struct KeeperStorageCheckRequestProcessor final : public KeeperStorageRequestPro ProfileEvents::increment(ProfileEvents::KeeperCheckRequest); return processImpl(storage, zxid); } + +private: + bool check_not_exists; }; @@ -1971,6 +1999,7 @@ KeeperStorageRequestProcessorsFactory::KeeperStorageRequestProcessorsFactory() registerKeeperRequestProcessor(*this); registerKeeperRequestProcessor(*this); registerKeeperRequestProcessor(*this); + registerKeeperRequestProcessor(*this); } diff --git a/src/Coordination/tests/gtest_coordination.cpp b/src/Coordination/tests/gtest_coordination.cpp index b1bea8ddf248..62217fb2dd36 100644 --- a/src/Coordination/tests/gtest_coordination.cpp +++ b/src/Coordination/tests/gtest_coordination.cpp @@ -2451,6 +2451,78 @@ TEST_P(CoordinationTest, ChangelogTestMaxLogSize) } +TEST_P(CoordinationTest, TestCheckNotExistsRequest) +{ + using namespace DB; + using namespace Coordination; + + KeeperStorage storage{500, "", keeper_context}; + + int32_t zxid = 0; + + const auto create_path = [&](const auto & path) + { + const auto create_request = std::make_shared(); + int new_zxid = ++zxid; + create_request->path = path; + storage.preprocessRequest(create_request, 1, 0, new_zxid); + auto responses = storage.processRequest(create_request, 1, new_zxid); + + EXPECT_GE(responses.size(), 1); + EXPECT_EQ(responses[0].response->error, Coordination::Error::ZOK) << "Failed to create " << path; + }; + + const auto check_request = std::make_shared(); + check_request->path = "/test_node"; + check_request->not_exists = true; + + { + SCOPED_TRACE("CheckNotExists returns ZOK"); + int new_zxid = ++zxid; + storage.preprocessRequest(check_request, 1, 0, new_zxid); + auto responses = storage.processRequest(check_request, 1, new_zxid); + EXPECT_GE(responses.size(), 1); + auto error = responses[0].response->error; + EXPECT_EQ(error, Coordination::Error::ZOK) << "CheckNotExists returned invalid result: " << errorMessage(error); + } + + create_path("/test_node"); + auto node_it = storage.container.find("/test_node"); + ASSERT_NE(node_it, storage.container.end()); + auto node_version = node_it->value.stat.version; + + { + SCOPED_TRACE("CheckNotExists returns ZNODEEXISTS"); + int new_zxid = ++zxid; + storage.preprocessRequest(check_request, 1, 0, new_zxid); + auto responses = storage.processRequest(check_request, 1, new_zxid); + EXPECT_GE(responses.size(), 1); + auto error = responses[0].response->error; + EXPECT_EQ(error, Coordination::Error::ZNODEEXISTS) << "CheckNotExists returned invalid result: " << errorMessage(error); + } + + { + SCOPED_TRACE("CheckNotExists returns ZNODEEXISTS for same version"); + int new_zxid = ++zxid; + check_request->version = node_version; + storage.preprocessRequest(check_request, 1, 0, new_zxid); + auto responses = storage.processRequest(check_request, 1, new_zxid); + EXPECT_GE(responses.size(), 1); + auto error = responses[0].response->error; + EXPECT_EQ(error, Coordination::Error::ZNODEEXISTS) << "CheckNotExists returned invalid result: " << errorMessage(error); + } + + { + SCOPED_TRACE("CheckNotExists returns ZOK for different version"); + int new_zxid = ++zxid; + check_request->version = node_version + 1; + storage.preprocessRequest(check_request, 1, 0, new_zxid); + auto responses = storage.processRequest(check_request, 1, new_zxid); + EXPECT_GE(responses.size(), 1); + auto error = responses[0].response->error; + EXPECT_EQ(error, Coordination::Error::ZOK) << "CheckNotExists returned invalid result: " << errorMessage(error); + } +} INSTANTIATE_TEST_SUITE_P(CoordinationTestSuite, CoordinationTest, diff --git a/src/Storages/MergeTree/EphemeralLockInZooKeeper.cpp b/src/Storages/MergeTree/EphemeralLockInZooKeeper.cpp index 996d2bc46a5c..5741e11aa224 100644 --- a/src/Storages/MergeTree/EphemeralLockInZooKeeper.cpp +++ b/src/Storages/MergeTree/EphemeralLockInZooKeeper.cpp @@ -24,7 +24,7 @@ template std::optional createEphemeralLockInZooKeeper( const String & path_prefix_, const String & temp_path, const ZooKeeperWithFaultInjectionPtr & zookeeper_, const T & deduplication_path) { - constexpr bool async_insert = std::is_same_v>; + static constexpr bool async_insert = std::is_same_v>; String path; @@ -42,16 +42,15 @@ std::optional createEphemeralLockInZooKeeper( if constexpr (async_insert) { for (const auto & single_dedup_path : deduplication_path) - { - ops.emplace_back(zkutil::makeCreateRequest(single_dedup_path, "", zkutil::CreateMode::Persistent)); - ops.emplace_back(zkutil::makeRemoveRequest(single_dedup_path, -1)); - } + zkutil::addCheckNotExistsRequest(ops, *zookeeper_, single_dedup_path); } else { - ops.emplace_back(zkutil::makeCreateRequest(deduplication_path, "", zkutil::CreateMode::Persistent)); - ops.emplace_back(zkutil::makeRemoveRequest(deduplication_path, -1)); + zkutil::addCheckNotExistsRequest(ops, *zookeeper_, deduplication_path); } + + auto deduplication_path_ops_size = ops.size(); + ops.emplace_back(zkutil::makeCreateRequest(path_prefix_, holder_path, zkutil::CreateMode::EphemeralSequential)); Coordination::Responses responses; Coordination::Error e = zookeeper_->tryMulti(ops, responses); @@ -60,9 +59,10 @@ std::optional createEphemeralLockInZooKeeper( if constexpr (async_insert) { auto failed_idx = zkutil::getFailedOpIndex(Coordination::Error::ZNODEEXISTS, responses); - if (failed_idx < deduplication_path.size() * 2) + + if (failed_idx < deduplication_path_ops_size) { - const String & failed_op_path = deduplication_path[failed_idx / 2]; + const String & failed_op_path = ops[failed_idx]->getPath(); LOG_DEBUG( &Poco::Logger::get("createEphemeralLockInZooKeeper"), "Deduplication path already exists: deduplication_path={}", diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 5cd02c33d554..9aa36f187759 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -2467,8 +2467,7 @@ void StorageReplicatedMergeTree::cloneReplica(const String & source_replica, Coo { /// We check that it was not suddenly upgraded to new version. /// Otherwise it can be upgraded and instantly become lost, but we cannot notice that. - ops.push_back(zkutil::makeCreateRequest(fs::path(source_path) / "is_lost", "0", zkutil::CreateMode::Persistent)); - ops.push_back(zkutil::makeRemoveRequest(fs::path(source_path) / "is_lost", -1)); + zkutil::addCheckNotExistsRequest(ops, *zookeeper, fs::path(source_path) / "is_lost"); } else /// The replica we clone should not suddenly become lost. ops.push_back(zkutil::makeCheckRequest(fs::path(source_path) / "is_lost", source_is_lost_stat.version)); @@ -8869,8 +8868,7 @@ bool StorageReplicatedMergeTree::createEmptyPartInsteadOfLost(zkutil::ZooKeeperP /// We must be sure that this part doesn't exist on other replicas if (!zookeeper->exists(current_part_path)) { - ops.emplace_back(zkutil::makeCreateRequest(current_part_path, "", zkutil::CreateMode::Persistent)); - ops.emplace_back(zkutil::makeRemoveRequest(current_part_path, -1)); + zkutil::addCheckNotExistsRequest(ops, *zookeeper, current_part_path); } else { From 58e9b56fcbb54e3ddfecfa64e9ea015ce25f8107 Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Wed, 19 Apr 2023 09:06:20 +0000 Subject: [PATCH 139/406] Fix CheckNotExists --- src/Common/ZooKeeper/ZooKeeper.h | 2 +- src/Coordination/KeeperStorage.cpp | 10 +++------- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/src/Common/ZooKeeper/ZooKeeper.h b/src/Common/ZooKeeper/ZooKeeper.h index b31dbc8da49a..636c9049af07 100644 --- a/src/Common/ZooKeeper/ZooKeeper.h +++ b/src/Common/ZooKeeper/ZooKeeper.h @@ -675,7 +675,7 @@ bool hasZooKeeperConfig(const Poco::Util::AbstractConfiguration & config); String getZooKeeperConfigName(const Poco::Util::AbstractConfiguration & config); template -void addCheckNotExistsRequest(Coordination::Requests requests, const Client & client, const std::string & path) +void addCheckNotExistsRequest(Coordination::Requests & requests, const Client & client, const std::string & path) { if (client.getApiVersion() >= DB::KeeperApiVersion::WITH_CHECK_NOT_EXISTS) { diff --git a/src/Coordination/KeeperStorage.cpp b/src/Coordination/KeeperStorage.cpp index 28cb4fba9c9b..a838de07ecb8 100644 --- a/src/Coordination/KeeperStorage.cpp +++ b/src/Coordination/KeeperStorage.cpp @@ -1457,13 +1457,8 @@ struct KeeperStorageCheckRequestProcessor final : public KeeperStorageRequestPro bool checkAuth(KeeperStorage & storage, int64_t session_id, bool is_local) const override { - StringRef path; - if (check_not_exists) - path = parentPath(zk_request->getPath()); - else - path = zk_request->getPath(); - - return storage.checkACL(path, Coordination::ACL::Read, session_id, is_local); + auto path = zk_request->getPath(); + return storage.checkACL(check_not_exists ? parentPath(path) : path, Coordination::ACL::Read, session_id, is_local); } std::vector @@ -1744,6 +1739,7 @@ struct KeeperStorageMultiRequestProcessor final : public KeeperStorageRequestPro concrete_requests.push_back(std::make_shared(sub_zk_request)); break; case Coordination::OpNum::Check: + case Coordination::OpNum::CheckNotExists: check_operation_type(OperationType::Write); concrete_requests.push_back(std::make_shared(sub_zk_request)); break; From 3f00d467851db0d41e142e4b4ade31aa2c27d8f2 Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Wed, 19 Apr 2023 14:07:38 +0200 Subject: [PATCH 140/406] Update enum for ZooKeeperLog --- src/Interpreters/ZooKeeperLog.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Interpreters/ZooKeeperLog.cpp b/src/Interpreters/ZooKeeperLog.cpp index faa6d1f9f02b..48f4d510af75 100644 --- a/src/Interpreters/ZooKeeperLog.cpp +++ b/src/Interpreters/ZooKeeperLog.cpp @@ -87,6 +87,7 @@ NamesAndTypesList ZooKeeperLogElement::getNamesAndTypes() {"Auth", static_cast(Coordination::OpNum::Auth)}, {"SessionID", static_cast(Coordination::OpNum::SessionID)}, {"FilteredList", static_cast(Coordination::OpNum::FilteredList)}, + {"CheckNotExists", static_cast(Coordination::OpNum::CheckNotExists)}, }); auto error_enum = getCoordinationErrorCodesEnumType(); From 69de3a3c2dc155b8c32c8043ec56ece06fc04695 Mon Sep 17 00:00:00 2001 From: kssenii Date: Wed, 19 Apr 2023 17:48:56 +0200 Subject: [PATCH 141/406] Fix tsan in test test_temporary_data_in_cache --- src/Interpreters/Cache/FileCache.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Interpreters/Cache/FileCache.cpp b/src/Interpreters/Cache/FileCache.cpp index 95dc9ba016ba..6788ad2d2afb 100644 --- a/src/Interpreters/Cache/FileCache.cpp +++ b/src/Interpreters/Cache/FileCache.cpp @@ -34,8 +34,6 @@ FileCache::FileCache(const FileCacheSettings & settings) if (settings.enable_filesystem_query_cache_limit) query_limit = std::make_unique(); - - cleanup_task = Context::getGlobalContextInstance()->getSchedulePool().createTask("FileCacheCleanup", [this]{ cleanupThreadFunc(); }); } FileCache::Key FileCache::createKeyForPath(const String & path) @@ -99,6 +97,8 @@ void FileCache::initialize() } is_initialized = true; + + cleanup_task = Context::getGlobalContextInstance()->getSchedulePool().createTask("FileCacheCleanup", [this]{ cleanupThreadFunc(); }); cleanup_task->activate(); cleanup_task->scheduleAfter(delayed_cleanup_interval_ms); } From 6867101d8d44f9f6483f12d2b13dad984f7c90da Mon Sep 17 00:00:00 2001 From: Shane Andrade Date: Thu, 20 Apr 2023 00:04:00 -0700 Subject: [PATCH 142/406] Update src/Functions/date_trunc.cpp Co-authored-by: Nikolay Degterinsky <43110995+evillique@users.noreply.github.com> --- src/Functions/date_trunc.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/Functions/date_trunc.cpp b/src/Functions/date_trunc.cpp index 4cbc605f088d..26654b746d44 100644 --- a/src/Functions/date_trunc.cpp +++ b/src/Functions/date_trunc.cpp @@ -130,8 +130,7 @@ class FunctionDateTrunc : public IFunction bool useDefaultImplementationForConstants() const override { return true; } ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {0, 2}; } - ColumnPtr - executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override { ColumnsWithTypeAndName temp_columns(arguments.size()); temp_columns[0] = arguments[1]; From 31548ab17cb439307e64f67601278e7d9da19a76 Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Thu, 20 Apr 2023 12:30:24 +0000 Subject: [PATCH 143/406] Fix result --- src/Common/ZooKeeper/ZooKeeperCommon.cpp | 10 ++++++- src/Common/ZooKeeper/ZooKeeperCommon.h | 8 +++++- .../01158_zookeeper_log_long.reference | 28 ++++++++----------- 3 files changed, 28 insertions(+), 18 deletions(-) diff --git a/src/Common/ZooKeeper/ZooKeeperCommon.cpp b/src/Common/ZooKeeper/ZooKeeperCommon.cpp index 03bfafac0c22..5031af38812a 100644 --- a/src/Common/ZooKeeper/ZooKeeperCommon.cpp +++ b/src/Common/ZooKeeper/ZooKeeperCommon.cpp @@ -666,7 +666,15 @@ ZooKeeperResponsePtr ZooKeeperGetRequest::makeResponse() const { return setTime( ZooKeeperResponsePtr ZooKeeperSetRequest::makeResponse() const { return setTime(std::make_shared()); } ZooKeeperResponsePtr ZooKeeperListRequest::makeResponse() const { return setTime(std::make_shared()); } ZooKeeperResponsePtr ZooKeeperSimpleListRequest::makeResponse() const { return setTime(std::make_shared()); } -ZooKeeperResponsePtr ZooKeeperCheckRequest::makeResponse() const { return setTime(std::make_shared()); } + +ZooKeeperResponsePtr ZooKeeperCheckRequest::makeResponse() const +{ + if (not_exists) + return setTime(std::make_shared()); + + return setTime(std::make_shared()); +} + ZooKeeperResponsePtr ZooKeeperMultiRequest::makeResponse() const { std::shared_ptr response; diff --git a/src/Common/ZooKeeper/ZooKeeperCommon.h b/src/Common/ZooKeeper/ZooKeeperCommon.h index fccccfd20581..5f00698423e5 100644 --- a/src/Common/ZooKeeper/ZooKeeperCommon.h +++ b/src/Common/ZooKeeper/ZooKeeperCommon.h @@ -408,7 +408,7 @@ struct ZooKeeperCheckRequest : CheckRequest, ZooKeeperRequest void createLogElements(LogElements & elems) const override; }; -struct ZooKeeperCheckResponse final : CheckResponse, ZooKeeperResponse +struct ZooKeeperCheckResponse : CheckResponse, ZooKeeperResponse { void readImpl(ReadBuffer &) override {} void writeImpl(WriteBuffer &) const override {} @@ -417,6 +417,12 @@ struct ZooKeeperCheckResponse final : CheckResponse, ZooKeeperResponse size_t bytesSize() const override { return CheckResponse::bytesSize() + sizeof(xid) + sizeof(zxid); } }; +struct ZooKeeperCheckNotExistsResponse : public ZooKeeperCheckResponse +{ + OpNum getOpNum() const override { return OpNum::CheckNotExists; } + using ZooKeeperCheckResponse::ZooKeeperCheckResponse; +}; + /// This response may be received only as an element of responses in MultiResponse. struct ZooKeeperErrorResponse final : ErrorResponse, ZooKeeperResponse { diff --git a/tests/queries/0_stateless/01158_zookeeper_log_long.reference b/tests/queries/0_stateless/01158_zookeeper_log_long.reference index a0088610c9dc..7ec52cb3366b 100644 --- a/tests/queries/0_stateless/01158_zookeeper_log_long.reference +++ b/tests/queries/0_stateless/01158_zookeeper_log_long.reference @@ -18,22 +18,18 @@ Response 0 Create /test/01158/default/rmt/replicas/1/parts/all_0_0_0 0 0 \N 0 4 Request 0 Exists /test/01158/default/rmt/replicas/1/parts/all_0_0_0 0 0 \N 0 0 \N \N \N 0 0 0 0 Response 0 Exists /test/01158/default/rmt/replicas/1/parts/all_0_0_0 0 0 \N 0 0 ZOK \N \N 0 0 96 0 blocks -Request 0 Multi 0 0 \N 3 0 \N \N \N 0 0 0 0 -Request 0 Create /test/01158/default/rmt/blocks/all_6308706741995381342_2495791770474910886 0 0 \N 0 1 \N \N \N 0 0 0 0 -Request 0 Remove /test/01158/default/rmt/blocks/all_6308706741995381342_2495791770474910886 0 0 -1 0 2 \N \N \N 0 0 0 0 -Request 0 Create /test/01158/default/rmt/block_numbers/all/block- 1 1 \N 0 3 \N \N \N 0 0 0 0 -Response 0 Multi 0 0 \N 3 0 ZOK \N \N 0 0 0 0 -Response 0 Create /test/01158/default/rmt/blocks/all_6308706741995381342_2495791770474910886 0 0 \N 0 1 ZOK \N \N /test/01158/default/rmt/blocks/all_6308706741995381342_2495791770474910886 0 0 0 0 -Response 0 Remove /test/01158/default/rmt/blocks/all_6308706741995381342_2495791770474910886 0 0 -1 0 2 ZOK \N \N 0 0 0 0 -Response 0 Create /test/01158/default/rmt/block_numbers/all/block- 1 1 \N 0 3 ZOK \N \N /test/01158/default/rmt/block_numbers/all/block-0000000000 0 0 0 0 -Request 0 Multi 0 0 \N 3 0 \N \N \N 0 0 0 0 -Request 0 Create /test/01158/default/rmt/blocks/all_6308706741995381342_2495791770474910886 0 0 \N 0 1 \N \N \N 0 0 0 0 -Request 0 Remove /test/01158/default/rmt/blocks/all_6308706741995381342_2495791770474910886 0 0 -1 0 2 \N \N \N 0 0 0 0 -Request 0 Create /test/01158/default/rmt/block_numbers/all/block- 1 1 \N 0 3 \N \N \N 0 0 0 0 -Response 0 Multi 0 0 \N 3 0 ZNODEEXISTS \N \N 0 0 0 0 -Response 0 Error /test/01158/default/rmt/blocks/all_6308706741995381342_2495791770474910886 0 0 \N 0 1 ZNODEEXISTS \N \N 0 0 0 0 -Response 0 Error /test/01158/default/rmt/blocks/all_6308706741995381342_2495791770474910886 0 0 -1 0 2 ZRUNTIMEINCONSISTENCY \N \N 0 0 0 0 -Response 0 Error /test/01158/default/rmt/block_numbers/all/block- 1 1 \N 0 3 ZRUNTIMEINCONSISTENCY \N \N 0 0 0 0 +Request 0 Multi 0 0 \N 2 0 \N \N \N 0 0 0 0 +Request 0 CheckNotExists /test/01158/default/rmt/blocks/all_6308706741995381342_2495791770474910886 0 0 -1 0 1 \N \N \N 0 0 0 0 +Request 0 Create /test/01158/default/rmt/block_numbers/all/block- 1 1 \N 0 2 \N \N \N 0 0 0 0 +Response 0 Multi 0 0 \N 2 0 ZOK \N \N 0 0 0 0 +Response 0 CheckNotExists /test/01158/default/rmt/blocks/all_6308706741995381342_2495791770474910886 0 0 -1 0 1 ZOK \N \N 0 0 0 0 +Response 0 Create /test/01158/default/rmt/block_numbers/all/block- 1 1 \N 0 2 ZOK \N \N /test/01158/default/rmt/block_numbers/all/block-0000000000 0 0 0 0 +Request 0 Multi 0 0 \N 2 0 \N \N \N 0 0 0 0 +Request 0 CheckNotExists /test/01158/default/rmt/blocks/all_6308706741995381342_2495791770474910886 0 0 -1 0 1 \N \N \N 0 0 0 0 +Request 0 Create /test/01158/default/rmt/block_numbers/all/block- 1 1 \N 0 2 \N \N \N 0 0 0 0 +Response 0 Multi 0 0 \N 2 0 ZNODEEXISTS \N \N 0 0 0 0 +Response 0 Error /test/01158/default/rmt/blocks/all_6308706741995381342_2495791770474910886 0 0 -1 0 1 ZNODEEXISTS \N \N 0 0 0 0 +Response 0 Error /test/01158/default/rmt/block_numbers/all/block- 1 1 \N 0 2 ZRUNTIMEINCONSISTENCY \N \N 0 0 0 0 Request 0 Get /test/01158/default/rmt/blocks/all_6308706741995381342_2495791770474910886 0 0 \N 0 0 \N \N \N 0 0 0 0 Response 0 Get /test/01158/default/rmt/blocks/all_6308706741995381342_2495791770474910886 0 0 \N 0 0 ZOK \N \N 0 0 9 0 duration_ms From a3c7afc03e4ead57ac2debd2e430ce23ae6217cf Mon Sep 17 00:00:00 2001 From: Smita Kulkarni Date: Thu, 20 Apr 2023 16:05:16 +0200 Subject: [PATCH 144/406] Reverted changes to drop functionality and updated test to sync rmt2 before drop --- src/Storages/MergeTree/MergeTreeData.cpp | 6 +++--- src/Storages/MergeTree/MergeTreeData.h | 2 +- .../queries/0_stateless/02432_s3_parallel_parts_cleanup.sql | 4 +++- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 5c189887e23d..45759c449f6c 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -1960,7 +1960,7 @@ static bool isOldPartDirectory(const DiskPtr & disk, const String & directory_pa } -size_t MergeTreeData::clearOldTemporaryDirectories(size_t custom_directories_lifetime_seconds, const NameSet & valid_prefixes, const bool & force) +size_t MergeTreeData::clearOldTemporaryDirectories(size_t custom_directories_lifetime_seconds, const NameSet & valid_prefixes) { /// If the method is already called from another thread, then we don't need to do anything. std::unique_lock lock(clear_old_temporary_directories_mutex, std::defer_lock); @@ -2018,7 +2018,7 @@ size_t MergeTreeData::clearOldTemporaryDirectories(size_t custom_directories_lif /// We don't control the amount of refs for temporary parts so we cannot decide can we remove blobs /// or not. So we are not doing it bool keep_shared = false; - if (disk->supportZeroCopyReplication() && settings->allow_remote_fs_zero_copy_replication && !force) + if (disk->supportZeroCopyReplication() && settings->allow_remote_fs_zero_copy_replication) { LOG_WARNING(log, "Since zero-copy replication is enabled we are not going to remove blobs from shared storage for {}", full_path); keep_shared = true; @@ -2724,7 +2724,7 @@ void MergeTreeData::dropAllData() } LOG_INFO(log, "dropAllData: clearing temporary directories"); - clearOldTemporaryDirectories(0, {"tmp_", "delete_tmp_", "tmp-fetch_"}, /* force */ true); + clearOldTemporaryDirectories(0, {"tmp_", "delete_tmp_", "tmp-fetch_"}); column_sizes.clear(); diff --git a/src/Storages/MergeTree/MergeTreeData.h b/src/Storages/MergeTree/MergeTreeData.h index 3053657e37ba..119ab2ee1d47 100644 --- a/src/Storages/MergeTree/MergeTreeData.h +++ b/src/Storages/MergeTree/MergeTreeData.h @@ -668,7 +668,7 @@ class MergeTreeData : public IStorage, public WithMutableContext /// Delete all directories which names begin with "tmp" /// Must be called with locked lockForShare() because it's using relative_data_path. /// 'force' is used by dropAllData(), this will remove blobs even if zero copy replication is enabled - size_t clearOldTemporaryDirectories(size_t custom_directories_lifetime_seconds, const NameSet & valid_prefixes = {"tmp_", "tmp-fetch_"}, const bool & force = false); + size_t clearOldTemporaryDirectories(size_t custom_directories_lifetime_seconds, const NameSet & valid_prefixes = {"tmp_", "tmp-fetch_"}); size_t clearEmptyParts(); diff --git a/tests/queries/0_stateless/02432_s3_parallel_parts_cleanup.sql b/tests/queries/0_stateless/02432_s3_parallel_parts_cleanup.sql index 0230f30bf058..522b2481ec7c 100644 --- a/tests/queries/0_stateless/02432_s3_parallel_parts_cleanup.sql +++ b/tests/queries/0_stateless/02432_s3_parallel_parts_cleanup.sql @@ -55,10 +55,12 @@ select sleep(3); select count(), sum(n), sum(m) from rmt; select count(), sum(n), sum(m) from rmt2; --- So there will be at least 2 parts (just in case no parts are removed until drop). +-- So there will be at least 2 parts (just in case no parts are removed until drop) insert into rmt(n) values (10); drop table rmt; + +system sync replica rmt2; drop table rmt2; system flush logs; From c3fe2b9287c782144479c5fd17e3530e2756cfb0 Mon Sep 17 00:00:00 2001 From: Smita Kulkarni Date: Thu, 20 Apr 2023 16:09:21 +0200 Subject: [PATCH 145/406] Removed extra comment --- src/Storages/MergeTree/MergeTreeData.h | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Storages/MergeTree/MergeTreeData.h b/src/Storages/MergeTree/MergeTreeData.h index 119ab2ee1d47..b03b7d4a71ee 100644 --- a/src/Storages/MergeTree/MergeTreeData.h +++ b/src/Storages/MergeTree/MergeTreeData.h @@ -667,7 +667,6 @@ class MergeTreeData : public IStorage, public WithMutableContext /// Delete all directories which names begin with "tmp" /// Must be called with locked lockForShare() because it's using relative_data_path. - /// 'force' is used by dropAllData(), this will remove blobs even if zero copy replication is enabled size_t clearOldTemporaryDirectories(size_t custom_directories_lifetime_seconds, const NameSet & valid_prefixes = {"tmp_", "tmp-fetch_"}); size_t clearEmptyParts(); From 30375d13d2ee821a7bbb5f79e21dc6cdcbc1a2bd Mon Sep 17 00:00:00 2001 From: Smita Kulkarni Date: Fri, 21 Apr 2023 09:35:40 +0200 Subject: [PATCH 146/406] Removed changes from test and updated log level --- tests/queries/0_stateless/02432_s3_parallel_parts_cleanup.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/02432_s3_parallel_parts_cleanup.sql b/tests/queries/0_stateless/02432_s3_parallel_parts_cleanup.sql index 522b2481ec7c..88fb2cdf9b13 100644 --- a/tests/queries/0_stateless/02432_s3_parallel_parts_cleanup.sql +++ b/tests/queries/0_stateless/02432_s3_parallel_parts_cleanup.sql @@ -1,5 +1,7 @@ -- Tags: no-fasttest +SET send_logs_level = 'fatal'; + drop table if exists rmt; drop table if exists rmt2; @@ -59,8 +61,6 @@ select count(), sum(n), sum(m) from rmt2; insert into rmt(n) values (10); drop table rmt; - -system sync replica rmt2; drop table rmt2; system flush logs; From aa9635e35b8651f62b0a5204ab7dec2a55798913 Mon Sep 17 00:00:00 2001 From: Smita Kulkarni Date: Fri, 21 Apr 2023 11:24:02 +0200 Subject: [PATCH 147/406] Removed is_part_outdated flag & usage, updated MergeTreeMarksLoader to hold data_part instead of data_part_storage --- src/Storages/MergeTree/IDataPartStorage.h | 2 -- .../IMergeTreeDataPartInfoForReader.h | 2 ++ .../LoadedMergeTreeDataPartInfoForReader.h | 2 ++ src/Storages/MergeTree/MergeTreeData.cpp | 3 --- .../MergeTree/MergeTreeIndexReader.cpp | 2 +- .../MergeTree/MergeTreeMarksLoader.cpp | 23 +++++-------------- src/Storages/MergeTree/MergeTreeMarksLoader.h | 4 ++-- .../MergeTree/MergeTreeReaderCompact.cpp | 2 +- .../MergeTree/MergeTreeReaderStream.cpp | 6 ++--- .../MergeTree/MergeTreeReaderStream.h | 2 +- .../MergeTree/MergeTreeReaderWide.cpp | 2 +- ...tem_parts_race_condition_drop_zookeeper.sh | 1 - 12 files changed, 19 insertions(+), 32 deletions(-) diff --git a/src/Storages/MergeTree/IDataPartStorage.h b/src/Storages/MergeTree/IDataPartStorage.h index aa473a2ab41f..b0b42b331cdc 100644 --- a/src/Storages/MergeTree/IDataPartStorage.h +++ b/src/Storages/MergeTree/IDataPartStorage.h @@ -285,8 +285,6 @@ class IDataPartStorage : public boost::noncopyable /// It may be flush of buffered data or similar. virtual void precommitTransaction() = 0; virtual bool hasActiveTransaction() const = 0; - - mutable std::atomic is_part_outdated = false; }; using DataPartStoragePtr = std::shared_ptr; diff --git a/src/Storages/MergeTree/IMergeTreeDataPartInfoForReader.h b/src/Storages/MergeTree/IMergeTreeDataPartInfoForReader.h index 648c3cfbb6bb..af3fd7cbef30 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPartInfoForReader.h +++ b/src/Storages/MergeTree/IMergeTreeDataPartInfoForReader.h @@ -40,6 +40,8 @@ class IMergeTreeDataPartInfoForReader : public WithContext virtual DataPartStoragePtr getDataPartStorage() const = 0; + virtual DataPartPtr getDataPart() const = 0; + virtual const NamesAndTypesList & getColumns() const = 0; virtual const ColumnsDescription & getColumnsDescription() const = 0; diff --git a/src/Storages/MergeTree/LoadedMergeTreeDataPartInfoForReader.h b/src/Storages/MergeTree/LoadedMergeTreeDataPartInfoForReader.h index 3363c75dd6ff..a72285d8e3c9 100644 --- a/src/Storages/MergeTree/LoadedMergeTreeDataPartInfoForReader.h +++ b/src/Storages/MergeTree/LoadedMergeTreeDataPartInfoForReader.h @@ -25,6 +25,8 @@ class LoadedMergeTreeDataPartInfoForReader final : public IMergeTreeDataPartInfo DataPartStoragePtr getDataPartStorage() const override { return data_part->getDataPartStoragePtr(); } + DataPartPtr getDataPart() const override { return data_part; } + const NamesAndTypesList & getColumns() const override { return data_part->getColumns(); } const ColumnsDescription & getColumnsDescription() const override { return data_part->getColumnsDescription(); } diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 322558021b7f..f5f126602237 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -3721,8 +3721,6 @@ void MergeTreeData::removePartsFromWorkingSet(MergeTreeTransaction * txn, const if (isInMemoryPart(part) && getSettings()->in_memory_parts_enable_wal) getWriteAheadLog()->dropPart(part->name); - - part->getDataPartStorage().is_part_outdated = true; } if (removed_active_part) @@ -3887,7 +3885,6 @@ void MergeTreeData::restoreAndActivatePart(const DataPartPtr & part, DataPartsLo addPartContributionToColumnAndSecondaryIndexSizes(part); addPartContributionToDataVolume(part); modifyPartState(part, DataPartState::Active); - part->getDataPartStorage().is_part_outdated = false; } diff --git a/src/Storages/MergeTree/MergeTreeIndexReader.cpp b/src/Storages/MergeTree/MergeTreeIndexReader.cpp index 7d7024a8ac24..1ce6d7776440 100644 --- a/src/Storages/MergeTree/MergeTreeIndexReader.cpp +++ b/src/Storages/MergeTree/MergeTreeIndexReader.cpp @@ -20,7 +20,7 @@ std::unique_ptr makeIndexReader( auto * load_marks_threadpool = settings.read_settings.load_marks_asynchronously ? &context->getLoadMarksThreadpool() : nullptr; return std::make_unique( - part->getDataPartStoragePtr(), + part, index->getFileName(), extension, marks_count, all_mark_ranges, std::move(settings), mark_cache, uncompressed_cache, diff --git a/src/Storages/MergeTree/MergeTreeMarksLoader.cpp b/src/Storages/MergeTree/MergeTreeMarksLoader.cpp index 18934fc19b18..300e99c850f6 100644 --- a/src/Storages/MergeTree/MergeTreeMarksLoader.cpp +++ b/src/Storages/MergeTree/MergeTreeMarksLoader.cpp @@ -30,7 +30,7 @@ namespace ErrorCodes } MergeTreeMarksLoader::MergeTreeMarksLoader( - DataPartStoragePtr data_part_storage_, + DataPartPtr data_part_, MarkCache * mark_cache_, const String & mrk_path_, size_t marks_count_, @@ -39,7 +39,7 @@ MergeTreeMarksLoader::MergeTreeMarksLoader( const ReadSettings & read_settings_, ThreadPool * load_marks_threadpool_, size_t columns_in_mark_) - : data_part_storage(std::move(data_part_storage_)) + : data_part(data_part_) , mark_cache(mark_cache_) , mrk_path(mrk_path_) , marks_count(marks_count_) @@ -68,11 +68,6 @@ MarkInCompressedFile MergeTreeMarksLoader::getMark(size_t row_index, size_t colu { if (!marks) { - if (this->data_part_storage->is_part_outdated) - { - throw Exception(ErrorCodes::LOGICAL_ERROR, "Attempting to read from outdated part. path : {}", data_part_storage->getFullPath()); - } - Stopwatch watch(CLOCK_MONOTONIC); if (future.valid()) @@ -103,6 +98,8 @@ MarkCache::MappedPtr MergeTreeMarksLoader::loadMarksImpl() /// Memory for marks must not be accounted as memory usage for query, because they are stored in shared cache. MemoryTrackerBlockerInThread temporarily_disable_memory_tracker; + auto data_part_storage = data_part->getDataPartStoragePtr(); + size_t file_size = data_part_storage->getFileSize(mrk_path); size_t mark_size = index_granularity_info.getMarkSizeInBytes(columns_in_mark); size_t expected_uncompressed_size = mark_size * marks_count; @@ -182,6 +179,8 @@ MarkCache::MappedPtr MergeTreeMarksLoader::loadMarks() { MarkCache::MappedPtr loaded_marks; + auto data_part_storage = data_part->getDataPartStoragePtr(); + if (mark_cache) { auto key = mark_cache->hash(fs::path(data_part_storage->getFullPath()) / mrk_path); @@ -215,16 +214,6 @@ std::future MergeTreeMarksLoader::loadMarksAsync() [this]() -> MarkCache::MappedPtr { ProfileEvents::increment(ProfileEvents::BackgroundLoadingMarksTasks); - if (this->data_part_storage->is_part_outdated) - { - if (mark_cache) - { - auto key = mark_cache->hash(fs::path(data_part_storage->getFullPath()) / mrk_path); - marks.reset(); - mark_cache->remove(key); - } - return nullptr; - } return loadMarks(); }, *load_marks_threadpool, diff --git a/src/Storages/MergeTree/MergeTreeMarksLoader.h b/src/Storages/MergeTree/MergeTreeMarksLoader.h index 17e52939d3f6..816b512d1a78 100644 --- a/src/Storages/MergeTree/MergeTreeMarksLoader.h +++ b/src/Storages/MergeTree/MergeTreeMarksLoader.h @@ -18,7 +18,7 @@ class MergeTreeMarksLoader using MarksPtr = MarkCache::MappedPtr; MergeTreeMarksLoader( - DataPartStoragePtr data_part_storage_, + DataPartPtr data_part_, MarkCache * mark_cache_, const String & mrk_path, size_t marks_count_, @@ -33,7 +33,7 @@ class MergeTreeMarksLoader MarkInCompressedFile getMark(size_t row_index, size_t column_index = 0); private: - DataPartStoragePtr data_part_storage; + DataPartPtr data_part; MarkCache * mark_cache = nullptr; String mrk_path; size_t marks_count; diff --git a/src/Storages/MergeTree/MergeTreeReaderCompact.cpp b/src/Storages/MergeTree/MergeTreeReaderCompact.cpp index d1796dac6ccf..13f8e4852086 100644 --- a/src/Storages/MergeTree/MergeTreeReaderCompact.cpp +++ b/src/Storages/MergeTree/MergeTreeReaderCompact.cpp @@ -36,7 +36,7 @@ MergeTreeReaderCompact::MergeTreeReaderCompact( settings_, avg_value_size_hints_) , marks_loader( - data_part_info_for_read_->getDataPartStorage(), + data_part_info_for_read_->getDataPart(), mark_cache, data_part_info_for_read_->getIndexGranularityInfo().getMarksFilePath(MergeTreeDataPartCompact::DATA_FILE_NAME), data_part_info_for_read_->getMarksCount(), diff --git a/src/Storages/MergeTree/MergeTreeReaderStream.cpp b/src/Storages/MergeTree/MergeTreeReaderStream.cpp index cdca5aa1247d..44cf8f450151 100644 --- a/src/Storages/MergeTree/MergeTreeReaderStream.cpp +++ b/src/Storages/MergeTree/MergeTreeReaderStream.cpp @@ -15,7 +15,7 @@ namespace ErrorCodes } MergeTreeReaderStream::MergeTreeReaderStream( - DataPartStoragePtr data_part_storage_, + DataPartPtr data_part_, const String & path_prefix_, const String & data_file_extension_, size_t marks_count_, @@ -35,7 +35,7 @@ MergeTreeReaderStream::MergeTreeReaderStream( , all_mark_ranges(all_mark_ranges_) , file_size(file_size_) , uncompressed_cache(uncompressed_cache_) - , data_part_storage(std::move(data_part_storage_)) + , data_part_storage(data_part_->getDataPartStoragePtr()) , path_prefix(path_prefix_) , data_file_extension(data_file_extension_) , is_low_cardinality_dictionary(is_low_cardinality_dictionary_) @@ -44,7 +44,7 @@ MergeTreeReaderStream::MergeTreeReaderStream( , save_marks_in_cache(settings.save_marks_in_cache) , index_granularity_info(index_granularity_info_) , marks_loader( - data_part_storage, + data_part_, mark_cache, index_granularity_info->getMarksFilePath(path_prefix), marks_count, diff --git a/src/Storages/MergeTree/MergeTreeReaderStream.h b/src/Storages/MergeTree/MergeTreeReaderStream.h index f3785e175df3..2265de94d076 100644 --- a/src/Storages/MergeTree/MergeTreeReaderStream.h +++ b/src/Storages/MergeTree/MergeTreeReaderStream.h @@ -19,7 +19,7 @@ class MergeTreeReaderStream { public: MergeTreeReaderStream( - DataPartStoragePtr data_part_storage_, + DataPartPtr data_part_, const String & path_prefix_, const String & data_file_extension_, size_t marks_count_, diff --git a/src/Storages/MergeTree/MergeTreeReaderWide.cpp b/src/Storages/MergeTree/MergeTreeReaderWide.cpp index 05af33da20a9..5b90118d9d5f 100644 --- a/src/Storages/MergeTree/MergeTreeReaderWide.cpp +++ b/src/Storages/MergeTree/MergeTreeReaderWide.cpp @@ -242,7 +242,7 @@ void MergeTreeReaderWide::addStreams( auto * load_marks_threadpool = settings.read_settings.load_marks_asynchronously ? &context->getLoadMarksThreadpool() : nullptr; streams.emplace(stream_name, std::make_unique( - data_part_info_for_read->getDataPartStorage(), stream_name, DATA_FILE_EXTENSION, + data_part_info_for_read->getDataPart(), stream_name, DATA_FILE_EXTENSION, data_part_info_for_read->getMarksCount(), all_mark_ranges, settings, mark_cache, uncompressed_cache, data_part_info_for_read->getFileSizeOrZero(stream_name + DATA_FILE_EXTENSION), &data_part_info_for_read->getIndexGranularityInfo(), diff --git a/tests/queries/0_stateless/00993_system_parts_race_condition_drop_zookeeper.sh b/tests/queries/0_stateless/00993_system_parts_race_condition_drop_zookeeper.sh index bceda77c7f81..f4f38ad9c83c 100755 --- a/tests/queries/0_stateless/00993_system_parts_race_condition_drop_zookeeper.sh +++ b/tests/queries/0_stateless/00993_system_parts_race_condition_drop_zookeeper.sh @@ -63,7 +63,6 @@ function thread6() done } - # https://stackoverflow.com/questions/9954794/execute-a-shell-function-with-timeout export -f thread1; export -f thread2; From b599d0bd01d33575e6b66d70f010b7d13df21446 Mon Sep 17 00:00:00 2001 From: Nikolay Degterinsky Date: Fri, 21 Apr 2023 22:03:56 +0000 Subject: [PATCH 148/406] Better --- programs/server/config.xml | 6 ++--- src/Access/AccessControl.cpp | 22 +++++++++++-------- src/Access/AccessControl.h | 1 - src/Access/Common/AuthenticationData.cpp | 10 ++++----- src/Access/Common/AuthenticationData.h | 1 + .../Access/InterpreterCreateUserQuery.cpp | 17 +++++++++----- ...InterpreterShowCreateAccessEntityQuery.cpp | 9 +++----- src/Parsers/Access/ASTAuthenticationData.cpp | 6 ++--- src/Parsers/Access/ASTAuthenticationData.h | 17 +++++++++----- src/Parsers/Access/ParserCreateUserQuery.cpp | 7 ++---- src/Server/TCPHandler.cpp | 2 -- .../configs/default_password_type.xml | 2 +- 12 files changed, 53 insertions(+), 47 deletions(-) diff --git a/programs/server/config.xml b/programs/server/config.xml index a3b6ff937acc..51b1d9e74086 100644 --- a/programs/server/config.xml +++ b/programs/server/config.xml @@ -476,10 +476,10 @@ 1 1 - - sha256 + sha256_password 0 + 20000 s3 @@ -20,6 +21,7 @@ minio123 true + 20000 s3 @@ -32,6 +34,7 @@ 1 1 + 20000 From 3939498ef7ef58f9893c025adda3b40ce2cb6c1a Mon Sep 17 00:00:00 2001 From: kssenii Date: Wed, 26 Apr 2023 15:43:09 +0200 Subject: [PATCH 206/406] Review fixes --- src/Interpreters/Cache/FileCache.cpp | 30 +++++++++++++++++-- src/Interpreters/Cache/FileCache.h | 24 --------------- src/Interpreters/Cache/FileSegment.cpp | 2 +- src/Interpreters/Cache/IFileCachePriority.h | 8 +++-- .../Cache/LRUFileCachePriority.cpp | 4 +-- src/Interpreters/Cache/LRUFileCachePriority.h | 2 +- src/Interpreters/Cache/Metadata.cpp | 2 +- src/Interpreters/Cache/Metadata.h | 2 +- src/Interpreters/Cache/QueryLimit.cpp | 2 +- 9 files changed, 39 insertions(+), 37 deletions(-) diff --git a/src/Interpreters/Cache/FileCache.cpp b/src/Interpreters/Cache/FileCache.cpp index 6788ad2d2afb..9b6e6ca79bcd 100644 --- a/src/Interpreters/Cache/FileCache.cpp +++ b/src/Interpreters/Cache/FileCache.cpp @@ -476,7 +476,7 @@ KeyMetadata::iterator FileCache::addFileSegment( auto & stash_records = stash->records; stash_records.emplace( - stash_key, stash->queue->add(key, offset, 0, locked_key.getKeyMetadata(), *lock)); + stash_key, stash->queue->add(locked_key.getKeyMetadata(), offset, 0, *lock)); if (stash->queue->getElementsCount(*lock) > stash->queue->getElementsLimit()) stash->queue->pop(*lock); @@ -498,7 +498,7 @@ KeyMetadata::iterator FileCache::addFileSegment( PriorityIterator cache_it; if (state == FileSegment::State::DOWNLOADED) { - cache_it = main_priority->add(key, offset, size, locked_key.getKeyMetadata(), *lock); + cache_it = main_priority->add(locked_key.getKeyMetadata(), offset, size, *lock); } try @@ -559,6 +559,30 @@ bool FileCache::tryReserve(FileSegment & file_segment, size_t size) queue_size += 1; size_t removed_size = 0; + + class EvictionCandidates final : public std::vector + { + public: + explicit EvictionCandidates(KeyMetadataPtr key_metadata_) : key_metadata(key_metadata_) {} + + KeyMetadata & getMetadata() { return *key_metadata; } + + void add(FileSegmentMetadataPtr candidate) + { + candidate->removal_candidate = true; + push_back(candidate); + } + + ~EvictionCandidates() + { + for (const auto & candidate : *this) + candidate->removal_candidate = false; + } + + private: + KeyMetadataPtr key_metadata; + }; + std::unordered_map to_delete; auto iterate_func = [&](LockedKey & locked_key, FileSegmentMetadataPtr segment_metadata) @@ -654,7 +678,7 @@ bool FileCache::tryReserve(FileSegment & file_segment, size_t size) /// Space reservation is incremental, so file_segment_metadata is created first (with state empty), /// and getQueueIterator() is assigned on first space reservation attempt. file_segment.setQueueIterator(main_priority->add( - file_segment.key(), file_segment.offset(), size, file_segment.getKeyMetadata(), cache_lock)); + file_segment.getKeyMetadata(), file_segment.offset(), size, cache_lock)); } if (query_context) diff --git a/src/Interpreters/Cache/FileCache.h b/src/Interpreters/Cache/FileCache.h index c21cd811d83a..2ceb6825a54c 100644 --- a/src/Interpreters/Cache/FileCache.h +++ b/src/Interpreters/Cache/FileCache.h @@ -208,30 +208,6 @@ class FileCache : private boost::noncopyable const CacheGuard::Lock *); void cleanupThreadFunc(); - - class EvictionCandidates : public std::vector - { - public: - explicit EvictionCandidates(KeyMetadataPtr key_metadata_) : key_metadata(key_metadata_) {} - - KeyMetadata & getMetadata() { return *key_metadata; } - - void add(FileSegmentMetadataPtr candidate) - { - candidate->removal_candidate = true; - push_back(candidate); - } - - ~EvictionCandidates() - { - for (const auto & candidate : *this) - candidate->removal_candidate = false; - } - - private: - KeyMetadataPtr key_metadata; -}; - }; } diff --git a/src/Interpreters/Cache/FileSegment.cpp b/src/Interpreters/Cache/FileSegment.cpp index b235ea4f0b0f..65bfc8657aaf 100644 --- a/src/Interpreters/Cache/FileSegment.cpp +++ b/src/Interpreters/Cache/FileSegment.cpp @@ -728,7 +728,7 @@ bool FileSegment::assertCorrectnessUnlocked(const FileSegmentGuard::Lock &) cons if (!it) return; - const auto entry = it->getEntry(); + const auto & entry = it->getEntry(); UNUSED(entry); chassert(entry.size == reserved_size); chassert(entry.key == key()); diff --git a/src/Interpreters/Cache/IFileCachePriority.h b/src/Interpreters/Cache/IFileCachePriority.h index 748089fbd1ba..0bd1ed54552b 100644 --- a/src/Interpreters/Cache/IFileCachePriority.h +++ b/src/Interpreters/Cache/IFileCachePriority.h @@ -23,9 +23,12 @@ class IFileCachePriority : private boost::noncopyable Entry(const Key & key_, size_t offset_, size_t size_, KeyMetadataPtr key_metadata_) : key(key_), offset(offset_), size(size_), key_metadata(key_metadata_) {} + Entry(const Entry & other) + : key(other.key), offset(other.offset), size(other.size.load()), hits(other.hits), key_metadata(other.key_metadata) {} + const Key key; const size_t offset; - size_t size; + std::atomic size; size_t hits = 0; const KeyMetadataPtr key_metadata; }; @@ -76,8 +79,7 @@ class IFileCachePriority : private boost::noncopyable virtual size_t getElementsCount(const CacheGuard::Lock &) const = 0; virtual Iterator add( - const Key & key, size_t offset, size_t size, - KeyMetadataPtr key_metadata, const CacheGuard::Lock &) = 0; + KeyMetadataPtr key_metadata, size_t offset, size_t size, const CacheGuard::Lock &) = 0; virtual void pop(const CacheGuard::Lock &) = 0; diff --git a/src/Interpreters/Cache/LRUFileCachePriority.cpp b/src/Interpreters/Cache/LRUFileCachePriority.cpp index 17828013c6c1..febcb4326259 100644 --- a/src/Interpreters/Cache/LRUFileCachePriority.cpp +++ b/src/Interpreters/Cache/LRUFileCachePriority.cpp @@ -19,12 +19,12 @@ namespace ErrorCodes } IFileCachePriority::Iterator LRUFileCachePriority::add( - const Key & key, + KeyMetadataPtr key_metadata, size_t offset, size_t size, - KeyMetadataPtr key_metadata, const CacheGuard::Lock &) { + const auto & key = key_metadata->key; #ifndef NDEBUG for (const auto & entry : queue) { diff --git a/src/Interpreters/Cache/LRUFileCachePriority.h b/src/Interpreters/Cache/LRUFileCachePriority.h index 7d98da4f5991..e2ccd34f775a 100644 --- a/src/Interpreters/Cache/LRUFileCachePriority.h +++ b/src/Interpreters/Cache/LRUFileCachePriority.h @@ -24,7 +24,7 @@ class LRUFileCachePriority : public IFileCachePriority size_t getElementsCount(const CacheGuard::Lock &) const override { return queue.size(); } - Iterator add(const Key & key, size_t offset, size_t size, KeyMetadataPtr key_metadata, const CacheGuard::Lock &) override; + Iterator add(KeyMetadataPtr key_metadata, size_t offset, size_t size, const CacheGuard::Lock &) override; void pop(const CacheGuard::Lock &) override; diff --git a/src/Interpreters/Cache/Metadata.cpp b/src/Interpreters/Cache/Metadata.cpp index 02474c966cd1..1fc19be9d7ef 100644 --- a/src/Interpreters/Cache/Metadata.cpp +++ b/src/Interpreters/Cache/Metadata.cpp @@ -91,7 +91,7 @@ std::string KeyMetadata::getFileSegmentPath(const FileSegment & file_segment) } -struct CleanupQueue +class CleanupQueue { friend struct CacheMetadata; public: diff --git a/src/Interpreters/Cache/Metadata.h b/src/Interpreters/Cache/Metadata.h index 1348ff31dc7e..2a9204630f4e 100644 --- a/src/Interpreters/Cache/Metadata.h +++ b/src/Interpreters/Cache/Metadata.h @@ -8,7 +8,7 @@ namespace DB { -struct CleanupQueue; +class CleanupQueue; using CleanupQueuePtr = std::shared_ptr; diff --git a/src/Interpreters/Cache/QueryLimit.cpp b/src/Interpreters/Cache/QueryLimit.cpp index 3e36129de43f..fc7556f21f5b 100644 --- a/src/Interpreters/Cache/QueryLimit.cpp +++ b/src/Interpreters/Cache/QueryLimit.cpp @@ -72,7 +72,7 @@ void FileCacheQueryLimit::QueryContext::add( const auto offset = file_segment.offset(); auto it = getPriority().add( - key, offset, file_segment.range().size(), file_segment.getKeyMetadata(), lock); + file_segment.getKeyMetadata(), offset, file_segment.range().size(), lock); auto [_, inserted] = records.emplace(FileCacheKeyAndOffset{key, offset}, it); if (!inserted) From c480f81f9454d40c38432f1f8797b3a4158d28c1 Mon Sep 17 00:00:00 2001 From: DanRoscigno Date: Wed, 26 Apr 2023 09:50:23 -0400 Subject: [PATCH 207/406] add: map header, return --- src/Storages/System/StorageSystemFunctions.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Storages/System/StorageSystemFunctions.cpp b/src/Storages/System/StorageSystemFunctions.cpp index a0a406a974c6..8f5f158d9319 100644 --- a/src/Storages/System/StorageSystemFunctions.cpp +++ b/src/Storages/System/StorageSystemFunctions.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -83,6 +84,7 @@ NamesAndTypesList StorageSystemFunctions::getNamesAndTypes() {"create_query", std::make_shared()}, {"origin", std::make_shared(getOriginEnumsAndValues())}, {"description", std::make_shared()}, + {"examples", std::make_shared(std::make_shared(), std::make_shared())}, }; } From 871f9265cc1f053c069f9c4c33daee70565282bd Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Wed, 26 Apr 2023 14:39:59 +0000 Subject: [PATCH 208/406] allow using function 'concat' with Map type --- src/Functions/concat.cpp | 4 ++++ tests/queries/0_stateless/02169_map_functions.reference | 2 ++ tests/queries/0_stateless/02169_map_functions.sql | 2 ++ 3 files changed, 8 insertions(+) diff --git a/src/Functions/concat.cpp b/src/Functions/concat.cpp index 1bdd155aaa15..8fefc2d5b8a0 100644 --- a/src/Functions/concat.cpp +++ b/src/Functions/concat.cpp @@ -205,6 +205,10 @@ class ConcatOverloadResolver : public IFunctionOverloadResolver { return FunctionFactory::instance().getImpl("arrayConcat", context)->build(arguments); } + else if (isMap(arguments.at(0).type)) + { + return FunctionFactory::instance().getImpl("mapConcat", context)->build(arguments); + } else return std::make_unique( FunctionConcat::create(context), collections::map(arguments, [](const auto & elem) { return elem.type; }), return_type); diff --git a/tests/queries/0_stateless/02169_map_functions.reference b/tests/queries/0_stateless/02169_map_functions.reference index bec2eaec5958..10746a70f06c 100644 --- a/tests/queries/0_stateless/02169_map_functions.reference +++ b/tests/queries/0_stateless/02169_map_functions.reference @@ -40,6 +40,8 @@ {'key1':1111,'key2':2222,'key5':500,'key6':600} {'key1':1112,'key2':2224,'key5':500,'key6':600} {'key1':1113,'key2':2226,'key5':500,'key6':600} +{'key5':500,'key6':600} +{'key5':500,'key6':600} 1 1 1 diff --git a/tests/queries/0_stateless/02169_map_functions.sql b/tests/queries/0_stateless/02169_map_functions.sql index 27ceb2520220..febaf2bd9d08 100644 --- a/tests/queries/0_stateless/02169_map_functions.sql +++ b/tests/queries/0_stateless/02169_map_functions.sql @@ -11,6 +11,8 @@ SELECT mapApply((k, v) -> tuple(v + 9223372036854775806), col) FROM table_map; - SELECT mapConcat(col, map('key5', 500), map('key6', 600)) FROM table_map ORDER BY id; SELECT mapConcat(col, materialize(map('key5', 500)), map('key6', 600)) FROM table_map ORDER BY id; +SELECT concat(map('key5', 500), map('key6', 600)); +SELECT map('key5', 500) || map('key6', 600); SELECT mapExists((k, v) -> k LIKE '%3', col) FROM table_map ORDER BY id; SELECT mapExists((k, v) -> k LIKE '%2' AND v < 1000, col) FROM table_map ORDER BY id; From 8ba9ab67c2171962a81cb4d184639e340505c06a Mon Sep 17 00:00:00 2001 From: kssenii Date: Wed, 26 Apr 2023 17:13:15 +0200 Subject: [PATCH 209/406] Fix --- src/Interpreters/Cache/FileCache.cpp | 70 +++++++++++++++------------- 1 file changed, 38 insertions(+), 32 deletions(-) diff --git a/src/Interpreters/Cache/FileCache.cpp b/src/Interpreters/Cache/FileCache.cpp index 046e388430f9..3973ab0d4de0 100644 --- a/src/Interpreters/Cache/FileCache.cpp +++ b/src/Interpreters/Cache/FileCache.cpp @@ -497,7 +497,7 @@ FileCache::FileSegmentCell * FileCache::addCell( /// Create a file segment cell and put it in `files` map by [key][offset]. if (!size) - return nullptr; /// Empty files are not cached. + throw Exception(ErrorCodes::LOGICAL_ERROR, "Zero size files are not allowed"); if (files[key].contains(offset)) throw Exception( @@ -505,54 +505,60 @@ FileCache::FileSegmentCell * FileCache::addCell( "Cache cell already exists for key: `{}`, offset: {}, size: {}.\nCurrent cache structure: {}", key.toString(), offset, size, dumpStructureUnlocked(key, cache_lock)); - auto skip_or_download = [&]() -> FileSegmentPtr + FileSegment::State result_state = state; + if (state == FileSegment::State::EMPTY && enable_cache_hits_threshold) { - FileSegment::State result_state = state; - if (state == FileSegment::State::EMPTY && enable_cache_hits_threshold) + auto record = stash_records.find({key, offset}); + + if (record == stash_records.end()) { - auto record = stash_records.find({key, offset}); + auto priority_iter = stash_priority->add(key, offset, 0, cache_lock); + stash_records.insert({{key, offset}, priority_iter}); - if (record == stash_records.end()) + if (stash_priority->getElementsNum(cache_lock) > max_stash_element_size) { - auto priority_iter = stash_priority->add(key, offset, 0, cache_lock); - stash_records.insert({{key, offset}, priority_iter}); - - if (stash_priority->getElementsNum(cache_lock) > max_stash_element_size) - { - auto remove_priority_iter = stash_priority->getLowestPriorityWriteIterator(cache_lock); - stash_records.erase({remove_priority_iter->key(), remove_priority_iter->offset()}); - remove_priority_iter->removeAndGetNext(cache_lock); - } - - /// For segments that do not reach the download threshold, - /// we do not download them, but directly read them - result_state = FileSegment::State::SKIP_CACHE; + auto remove_priority_iter = stash_priority->getLowestPriorityWriteIterator(cache_lock); + stash_records.erase({remove_priority_iter->key(), remove_priority_iter->offset()}); + remove_priority_iter->removeAndGetNext(cache_lock); } - else - { - auto priority_iter = record->second; - priority_iter->use(cache_lock); - result_state = priority_iter->hits() >= enable_cache_hits_threshold - ? FileSegment::State::EMPTY - : FileSegment::State::SKIP_CACHE; - } + /// For segments that do not reach the download threshold, + /// we do not download them, but directly read them + result_state = FileSegment::State::SKIP_CACHE; } + else + { + auto priority_iter = record->second; + priority_iter->use(cache_lock); - return std::make_shared(offset, size, key, this, result_state, settings); - }; + result_state = priority_iter->hits() >= enable_cache_hits_threshold + ? FileSegment::State::EMPTY + : FileSegment::State::SKIP_CACHE; + } + } - FileSegmentCell cell(skip_or_download(), this, cache_lock); auto & offsets = files[key]; - if (offsets.empty()) { auto key_path = getPathInLocalCache(key); if (!fs::exists(key_path)) - fs::create_directories(key_path); + { + try + { + fs::create_directories(key_path); + } + catch (...) + { + tryLogCurrentException(__PRETTY_FUNCTION__); + result_state = FileSegment::State::SKIP_CACHE; + } + } } + auto file_segment = std::make_shared(offset, size, key, this, result_state, settings); + FileSegmentCell cell(std::move(file_segment), this, cache_lock); + auto [it, inserted] = offsets.insert({offset, std::move(cell)}); if (!inserted) throw Exception( From ddd285cda273dfcb61a4102e2c706bbc733390af Mon Sep 17 00:00:00 2001 From: DanRoscigno Date: Wed, 26 Apr 2023 11:37:10 -0400 Subject: [PATCH 210/406] compiles, but table has map and table is empty --- src/Storages/System/StorageSystemFunctions.cpp | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/Storages/System/StorageSystemFunctions.cpp b/src/Storages/System/StorageSystemFunctions.cpp index 8f5f158d9319..0775488bce83 100644 --- a/src/Storages/System/StorageSystemFunctions.cpp +++ b/src/Storages/System/StorageSystemFunctions.cpp @@ -29,6 +29,7 @@ namespace MutableColumns & res_columns, const String & name, UInt64 is_aggregate, + const String & example, const String & create_query, FunctionOrigin function_origin, const Factory & factory) @@ -62,6 +63,7 @@ namespace } else res_columns[6]->insertDefault(); + res_columns[7]->insert(example); } } @@ -94,14 +96,14 @@ void StorageSystemFunctions::fillData(MutableColumns & res_columns, ContextPtr c const auto & function_names = functions_factory.getAllRegisteredNames(); for (const auto & function_name : function_names) { - fillRow(res_columns, function_name, UInt64(0), "", FunctionOrigin::SYSTEM, functions_factory); + fillRow(res_columns, function_name, UInt64(0), "example here", "", FunctionOrigin::SYSTEM, functions_factory); } const auto & aggregate_functions_factory = AggregateFunctionFactory::instance(); const auto & aggregate_function_names = aggregate_functions_factory.getAllRegisteredNames(); for (const auto & function_name : aggregate_function_names) { - fillRow(res_columns, function_name, UInt64(1), "", FunctionOrigin::SYSTEM, aggregate_functions_factory); + fillRow(res_columns, function_name, UInt64(1), "example here", "", FunctionOrigin::SYSTEM, aggregate_functions_factory); } const auto & user_defined_sql_functions_factory = UserDefinedSQLFunctionFactory::instance(); @@ -109,14 +111,14 @@ void StorageSystemFunctions::fillData(MutableColumns & res_columns, ContextPtr c for (const auto & function_name : user_defined_sql_functions_names) { auto create_query = queryToString(user_defined_sql_functions_factory.get(function_name)); - fillRow(res_columns, function_name, UInt64(0), create_query, FunctionOrigin::SQL_USER_DEFINED, user_defined_sql_functions_factory); + fillRow(res_columns, function_name, UInt64(0), "example here", create_query, FunctionOrigin::SQL_USER_DEFINED, user_defined_sql_functions_factory); } const auto & user_defined_executable_functions_factory = UserDefinedExecutableFunctionFactory::instance(); const auto & user_defined_executable_functions_names = user_defined_executable_functions_factory.getRegisteredNames(context); for (const auto & function_name : user_defined_executable_functions_names) { - fillRow(res_columns, function_name, UInt64(0), "", FunctionOrigin::EXECUTABLE_USER_DEFINED, user_defined_executable_functions_factory); + fillRow(res_columns, function_name, UInt64(0), "example here", "", FunctionOrigin::EXECUTABLE_USER_DEFINED, user_defined_executable_functions_factory); } } From 57164820efeec8533e7dc08dc3f82d58e2cf6f36 Mon Sep 17 00:00:00 2001 From: DanRoscigno Date: Wed, 26 Apr 2023 11:53:50 -0400 Subject: [PATCH 211/406] fixed column to string --- src/Storages/System/StorageSystemFunctions.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Storages/System/StorageSystemFunctions.cpp b/src/Storages/System/StorageSystemFunctions.cpp index 0775488bce83..f8d59cf9fafa 100644 --- a/src/Storages/System/StorageSystemFunctions.cpp +++ b/src/Storages/System/StorageSystemFunctions.cpp @@ -86,7 +86,7 @@ NamesAndTypesList StorageSystemFunctions::getNamesAndTypes() {"create_query", std::make_shared()}, {"origin", std::make_shared(getOriginEnumsAndValues())}, {"description", std::make_shared()}, - {"examples", std::make_shared(std::make_shared(), std::make_shared())}, + {"example", std::make_shared()}, }; } From a34e465abf0e17298c8e4bbaa084726dfc91fc0f Mon Sep 17 00:00:00 2001 From: alesapin Date: Wed, 26 Apr 2023 19:57:18 +0200 Subject: [PATCH 212/406] Fxi --- src/Storages/MergeTree/DataPartsExchange.cpp | 1 - src/Storages/StorageReplicatedMergeTree.cpp | 6 +++--- src/Storages/StorageReplicatedMergeTree.h | 2 +- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/src/Storages/MergeTree/DataPartsExchange.cpp b/src/Storages/MergeTree/DataPartsExchange.cpp index 46c6d09eca47..c6804d260e2c 100644 --- a/src/Storages/MergeTree/DataPartsExchange.cpp +++ b/src/Storages/MergeTree/DataPartsExchange.cpp @@ -938,7 +938,6 @@ MergeTreeData::MutableDataPartPtr Fetcher::downloadPartToDisk( if (to_remote_disk) { - data.lockSharedData(*new_data_part, /* replace_existing_lock = */ true, {}); LOG_DEBUG(log, "Download of part {} unique id {} metadata onto disk {} finished.", part_name, part_id, disk->getName()); } else diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index a2f16da4f4dd..c66c4ef4dfe0 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -1468,7 +1468,7 @@ void StorageReplicatedMergeTree::checkPartChecksumsAndAddCommitOps(const zkutil: } MergeTreeData::DataPartsVector StorageReplicatedMergeTree::checkPartChecksumsAndCommit(Transaction & transaction, - const MutableDataPartPtr & part, std::optional hardlinked_files) + const MutableDataPartPtr & part, std::optional hardlinked_files, bool replace_zero_copy_lock) { auto zookeeper = getZooKeeper(); @@ -1477,7 +1477,7 @@ MergeTreeData::DataPartsVector StorageReplicatedMergeTree::checkPartChecksumsAnd Coordination::Requests ops; NameSet absent_part_paths_on_replicas; - getLockSharedDataOps(*part, std::make_shared(zookeeper), false, hardlinked_files, ops); + getLockSharedDataOps(*part, std::make_shared(zookeeper), replace_zero_copy_lock, hardlinked_files, ops); size_t zero_copy_lock_ops_size = ops.size(); /// Checksums are checked here and `ops` is filled. In fact, the part is added to ZK just below, when executing `multi`. @@ -4162,7 +4162,7 @@ bool StorageReplicatedMergeTree::fetchPart( Transaction transaction(*this, NO_TRANSACTION_RAW); renameTempPartAndReplace(part, transaction); - replaced_parts = checkPartChecksumsAndCommit(transaction, part, hardlinked_files); + replaced_parts = checkPartChecksumsAndCommit(transaction, part, hardlinked_files, !part_to_clone); /** If a quorum is tracked for this part, you must update it. * If you do not have time, in case of losing the session, when you restart the server - see the `ReplicatedMergeTreeRestartingThread::updateQuorumIfWeHavePart` method. diff --git a/src/Storages/StorageReplicatedMergeTree.h b/src/Storages/StorageReplicatedMergeTree.h index 894cf6d12ce7..64511a4a9271 100644 --- a/src/Storages/StorageReplicatedMergeTree.h +++ b/src/Storages/StorageReplicatedMergeTree.h @@ -549,7 +549,7 @@ class StorageReplicatedMergeTree final : public MergeTreeData String getChecksumsForZooKeeper(const MergeTreeDataPartChecksums & checksums) const; /// Accepts a PreActive part, atomically checks its checksums with ones on other replicas and commit the part - DataPartsVector checkPartChecksumsAndCommit(Transaction & transaction, const MutableDataPartPtr & part, std::optional hardlinked_files = {}); + DataPartsVector checkPartChecksumsAndCommit(Transaction & transaction, const MutableDataPartPtr & part, std::optional hardlinked_files = {}, bool replace_zero_copy_lock=false); bool partIsAssignedToBackgroundOperation(const DataPartPtr & part) const override; From 3e1e86cff29b3df7baa1c10455dc65f4d48d5bd0 Mon Sep 17 00:00:00 2001 From: DanRoscigno Date: Wed, 26 Apr 2023 14:26:49 -0400 Subject: [PATCH 213/406] examples added --- .../System/StorageSystemFunctions.cpp | 34 +++++++++++++++---- 1 file changed, 28 insertions(+), 6 deletions(-) diff --git a/src/Storages/System/StorageSystemFunctions.cpp b/src/Storages/System/StorageSystemFunctions.cpp index f8d59cf9fafa..02797e429333 100644 --- a/src/Storages/System/StorageSystemFunctions.cpp +++ b/src/Storages/System/StorageSystemFunctions.cpp @@ -22,6 +22,19 @@ enum class FunctionOrigin : Int8 EXECUTABLE_USER_DEFINED = 2 }; +std::string exampleMapToString( const std::map &example ) + // in the source, examples for functions are stored in a map. To + // create the documentation, we need to convert the map to a string. +{ + std::string stringToReturn; + + for (auto iter = example.begin(); iter != example.end(); ++iter){ + stringToReturn.append(iter->first); + stringToReturn.append(iter->second); + } + return stringToReturn; +} + namespace { template @@ -29,7 +42,6 @@ namespace MutableColumns & res_columns, const String & name, UInt64 is_aggregate, - const String & example, const String & create_query, FunctionOrigin function_origin, const Factory & factory) @@ -41,6 +53,7 @@ namespace { res_columns[2]->insert(false); res_columns[3]->insertDefault(); + res_columns[7]->insert(false); } else { @@ -57,16 +70,25 @@ namespace if constexpr (std::is_same_v) { if (factory.isAlias(name)) + { res_columns[6]->insertDefault(); + res_columns[7]->insert(false); + } else + { res_columns[6]->insert(factory.getDocumentation(name).description); + res_columns[7]->insert(exampleMapToString((factory.getDocumentation(name).examples))); + } } else + { res_columns[6]->insertDefault(); - res_columns[7]->insert(example); + res_columns[7]->insert(false); + } } } + std::vector> getOriginEnumsAndValues() { return std::vector>{ @@ -96,14 +118,14 @@ void StorageSystemFunctions::fillData(MutableColumns & res_columns, ContextPtr c const auto & function_names = functions_factory.getAllRegisteredNames(); for (const auto & function_name : function_names) { - fillRow(res_columns, function_name, UInt64(0), "example here", "", FunctionOrigin::SYSTEM, functions_factory); + fillRow(res_columns, function_name, UInt64(0), "", FunctionOrigin::SYSTEM, functions_factory); } const auto & aggregate_functions_factory = AggregateFunctionFactory::instance(); const auto & aggregate_function_names = aggregate_functions_factory.getAllRegisteredNames(); for (const auto & function_name : aggregate_function_names) { - fillRow(res_columns, function_name, UInt64(1), "example here", "", FunctionOrigin::SYSTEM, aggregate_functions_factory); + fillRow(res_columns, function_name, UInt64(1), "", FunctionOrigin::SYSTEM, aggregate_functions_factory); } const auto & user_defined_sql_functions_factory = UserDefinedSQLFunctionFactory::instance(); @@ -111,14 +133,14 @@ void StorageSystemFunctions::fillData(MutableColumns & res_columns, ContextPtr c for (const auto & function_name : user_defined_sql_functions_names) { auto create_query = queryToString(user_defined_sql_functions_factory.get(function_name)); - fillRow(res_columns, function_name, UInt64(0), "example here", create_query, FunctionOrigin::SQL_USER_DEFINED, user_defined_sql_functions_factory); + fillRow(res_columns, function_name, UInt64(0), create_query, FunctionOrigin::SQL_USER_DEFINED, user_defined_sql_functions_factory); } const auto & user_defined_executable_functions_factory = UserDefinedExecutableFunctionFactory::instance(); const auto & user_defined_executable_functions_names = user_defined_executable_functions_factory.getRegisteredNames(context); for (const auto & function_name : user_defined_executable_functions_names) { - fillRow(res_columns, function_name, UInt64(0), "example here", "", FunctionOrigin::EXECUTABLE_USER_DEFINED, user_defined_executable_functions_factory); + fillRow(res_columns, function_name, UInt64(0), "", FunctionOrigin::EXECUTABLE_USER_DEFINED, user_defined_executable_functions_factory); } } From fb39dc3f19ca7780c12af3f1a664996fef44f30c Mon Sep 17 00:00:00 2001 From: DanRoscigno Date: Wed, 26 Apr 2023 14:36:34 -0400 Subject: [PATCH 214/406] add some markdown --- src/Storages/System/StorageSystemFunctions.cpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/Storages/System/StorageSystemFunctions.cpp b/src/Storages/System/StorageSystemFunctions.cpp index 02797e429333..29d6b81d528f 100644 --- a/src/Storages/System/StorageSystemFunctions.cpp +++ b/src/Storages/System/StorageSystemFunctions.cpp @@ -29,8 +29,15 @@ std::string exampleMapToString( const std::map &exampl std::string stringToReturn; for (auto iter = example.begin(); iter != example.end(); ++iter){ + stringToReturn.append("### "); stringToReturn.append(iter->first); + stringToReturn.append("\n"); + stringToReturn.append("```"); + stringToReturn.append("\n"); stringToReturn.append(iter->second); + stringToReturn.append("\n"); + stringToReturn.append("```"); + stringToReturn.append("\n"); } return stringToReturn; } From d76430fe90132e7dad9b51ddfb2c0a4a2ea83f02 Mon Sep 17 00:00:00 2001 From: MikhailBurdukov Date: Wed, 26 Apr 2023 19:19:10 +0000 Subject: [PATCH 215/406] Added options handling for mongoo dict --- src/Dictionaries/MongoDBDictionarySource.h | 2 + tests/integration/helpers/external_sources.py | 31 +++++- .../configs/ssl_verification.xml | 8 ++ .../test_mongo.py | 104 +++++++++++------- .../test_mongo_uri.py | 83 +++++++------- 5 files changed, 146 insertions(+), 82 deletions(-) create mode 100644 tests/integration/test_dictionaries_all_layouts_separate_sources/configs/ssl_verification.xml diff --git a/src/Dictionaries/MongoDBDictionarySource.h b/src/Dictionaries/MongoDBDictionarySource.h index 4c7ae649f095..fefcb1bff9f2 100644 --- a/src/Dictionaries/MongoDBDictionarySource.h +++ b/src/Dictionaries/MongoDBDictionarySource.h @@ -41,6 +41,7 @@ class MongoDBDictionarySource final : public IDictionarySource const std::string & method_, const std::string & db_, const std::string & collection_, + const std::string & options, const Block & sample_block_); MongoDBDictionarySource(const MongoDBDictionarySource & other); @@ -80,6 +81,7 @@ class MongoDBDictionarySource final : public IDictionarySource const std::string method; std::string db; const std::string collection; + const std::string options; Block sample_block; std::shared_ptr connection; diff --git a/tests/integration/helpers/external_sources.py b/tests/integration/helpers/external_sources.py index fd086fc45266..193107a259d0 100644 --- a/tests/integration/helpers/external_sources.py +++ b/tests/integration/helpers/external_sources.py @@ -161,6 +161,30 @@ def load_data(self, data, table_name): class SourceMongo(ExternalSource): + + def __init__( + self, + name, + internal_hostname, + internal_port, + docker_hostname, + docker_port, + user, + password, + secure = False + ): + ExternalSource.__init__( + self, + name, + internal_hostname, + internal_port, + docker_hostname, + docker_port, + user, + password, + ) + self.secure = secure + def get_source_str(self, table_name): return """ @@ -170,6 +194,7 @@ def get_source_str(self, table_name): {password} test {tbl} + {options} """.format( host=self.docker_hostname, @@ -177,6 +202,7 @@ def get_source_str(self, table_name): user=self.user, password=self.password, tbl=table_name, + options= "ssl=true" if self.secure else "" ) def prepare(self, structure, table_name, cluster): @@ -186,6 +212,8 @@ def prepare(self, structure, table_name, cluster): user=self.user, password=self.password, ) + if self.secure: + connection_str += "/?tls=true&tlsAllowInvalidCertificates=true" self.connection = pymongo.MongoClient(connection_str) self.converters = {} for field in structure.get_all_fields(): @@ -228,7 +256,7 @@ def compatible_with_layout(self, layout): def get_source_str(self, table_name): return """ - mongodb://{user}:{password}@{host}:{port}/test + mongodb://{user}:{password}@{host}:{port}/test{options} {tbl} """.format( @@ -237,6 +265,7 @@ def get_source_str(self, table_name): user=self.user, password=self.password, tbl=table_name, + options = '?ssl=true' if self.secure else "", ) diff --git a/tests/integration/test_dictionaries_all_layouts_separate_sources/configs/ssl_verification.xml b/tests/integration/test_dictionaries_all_layouts_separate_sources/configs/ssl_verification.xml new file mode 100644 index 000000000000..3efe98e70450 --- /dev/null +++ b/tests/integration/test_dictionaries_all_layouts_separate_sources/configs/ssl_verification.xml @@ -0,0 +1,8 @@ + + + + + none + + + diff --git a/tests/integration/test_dictionaries_all_layouts_separate_sources/test_mongo.py b/tests/integration/test_dictionaries_all_layouts_separate_sources/test_mongo.py index 55639877ba0b..2f0bdb7debcc 100644 --- a/tests/integration/test_dictionaries_all_layouts_separate_sources/test_mongo.py +++ b/tests/integration/test_dictionaries_all_layouts_separate_sources/test_mongo.py @@ -17,15 +17,17 @@ test_name = "mongo" -def setup_module(module): - global cluster - global node - global simple_tester - global complex_tester - global ranged_tester - - cluster = ClickHouseCluster(__file__) - SOURCE = SourceMongo( +@pytest.fixture(scope="module") +def secure_connection(request): + return request.param + +@pytest.fixture(scope="module") +def cluster(secure_connection): + return ClickHouseCluster(__file__) + +@pytest.fixture(scope="module") +def source(secure_connection, cluster): + return SourceMongo( "MongoDB", "localhost", cluster.mongo_port, @@ -33,35 +35,58 @@ def setup_module(module): "27017", "root", "clickhouse", + secure = secure_connection ) - simple_tester = SimpleLayoutTester(test_name) - simple_tester.cleanup() - simple_tester.create_dictionaries(SOURCE) +@pytest.fixture(scope="module") +def simple_tester(source): + tester = SimpleLayoutTester(test_name) + tester.cleanup() + tester.create_dictionaries(source) + return tester - complex_tester = ComplexLayoutTester(test_name) - complex_tester.create_dictionaries(SOURCE) +@pytest.fixture(scope="module") +def complex_tester(source): + tester = ComplexLayoutTester(test_name) + tester.create_dictionaries(source) + return tester - ranged_tester = RangedLayoutTester(test_name) - ranged_tester.create_dictionaries(SOURCE) - # Since that all .xml configs were created - main_configs = [] - main_configs.append(os.path.join("configs", "disable_ssl_verification.xml")) +@pytest.fixture(scope="module") +def ranged_tester(source): + tester = RangedLayoutTester(test_name) + tester.create_dictionaries(source) + return tester - dictionaries = simple_tester.list_dictionaries() +@pytest.fixture(scope="module") +def main_config(secure_connection): + main_config = [] + if secure_connection: + main_config.append(os.path.join("configs", "disable_ssl_verification.xml")) + else: + main_config.append(os.path.join("configs", "ssl_verification.xml")) + return main_config + +@pytest.fixture(scope="module") +def started_cluster(secure_connection, cluster, main_config, simple_tester, ranged_tester, complex_tester): + SOURCE = SourceMongo( + "MongoDB", + "localhost", + cluster.mongo_port, + cluster.mongo_host, + "27017", + "root", + "clickhouse", + secure=secure_connection + ) + dictionaries = simple_tester.list_dictionaries() + node = cluster.add_instance( - "node", main_configs=main_configs, dictionaries=dictionaries, with_mongo=True + "node", main_configs = main_config, dictionaries=dictionaries, with_mongo=True, + with_mongo_secure = secure_connection ) - -def teardown_module(module): - simple_tester.cleanup() - - -@pytest.fixture(scope="module") -def started_cluster(): try: cluster.start() @@ -74,17 +99,22 @@ def started_cluster(): finally: cluster.shutdown() - +@pytest.mark.parametrize("secure_connection", [False], indirect=["secure_connection"]) @pytest.mark.parametrize("layout_name", sorted(LAYOUTS_SIMPLE)) -def test_simple(started_cluster, layout_name): - simple_tester.execute(layout_name, node) - +def test_simple(secure_connection, started_cluster, layout_name, simple_tester): + simple_tester.execute(layout_name, started_cluster.instances["node"]) +@pytest.mark.parametrize("secure_connection", [False], indirect=["secure_connection"]) @pytest.mark.parametrize("layout_name", sorted(LAYOUTS_COMPLEX)) -def test_complex(started_cluster, layout_name): - complex_tester.execute(layout_name, node) - +def test_complex(secure_connection, started_cluster, layout_name, complex_tester): + complex_tester.execute(layout_name, started_cluster.instances["node"]) +@pytest.mark.parametrize("secure_connection", [False], indirect=["secure_connection"]) @pytest.mark.parametrize("layout_name", sorted(LAYOUTS_RANGED)) -def test_ranged(started_cluster, layout_name): - ranged_tester.execute(layout_name, node) +def test_ranged(secure_connection, started_cluster, layout_name, ranged_tester): + ranged_tester.execute(layout_name, started_cluster.instances["node"]) + +@pytest.mark.parametrize("secure_connection", [True], indirect=["secure_connection"]) +@pytest.mark.parametrize("layout_name", sorted(LAYOUTS_SIMPLE)) +def test_simple_ssl(secure_connection, started_cluster, layout_name, simple_tester): + simple_tester.execute(layout_name, started_cluster.instances["node"]) diff --git a/tests/integration/test_dictionaries_all_layouts_separate_sources/test_mongo_uri.py b/tests/integration/test_dictionaries_all_layouts_separate_sources/test_mongo_uri.py index 84c547b7a6b3..13ab980b61b7 100644 --- a/tests/integration/test_dictionaries_all_layouts_separate_sources/test_mongo_uri.py +++ b/tests/integration/test_dictionaries_all_layouts_separate_sources/test_mongo_uri.py @@ -8,25 +8,19 @@ from helpers.dictionary import Field, Row, Dictionary, DictionaryStructure, Layout from helpers.external_sources import SourceMongoURI -SOURCE = None -cluster = None -node = None -simple_tester = None -complex_tester = None -ranged_tester = None test_name = "mongo_uri" +@pytest.fixture(scope="module") +def secure_connection(request): + return request.param -def setup_module(module): - global cluster - global node - global simple_tester - global complex_tester - global ranged_tester - - cluster = ClickHouseCluster(__file__) +@pytest.fixture(scope="module") +def cluster(secure_connection): + return ClickHouseCluster(__file__) - SOURCE = SourceMongoURI( +@pytest.fixture(scope="module") +def source(secure_connection, cluster): + return SourceMongoURI( "MongoDB", "localhost", cluster.mongo_port, @@ -34,52 +28,53 @@ def setup_module(module): "27017", "root", "clickhouse", + secure = secure_connection ) - simple_tester = SimpleLayoutTester(test_name) - simple_tester.cleanup() - simple_tester.create_dictionaries(SOURCE) - - complex_tester = ComplexLayoutTester(test_name) - complex_tester.create_dictionaries(SOURCE) - - ranged_tester = RangedLayoutTester(test_name) - ranged_tester.create_dictionaries(SOURCE) - # Since that all .xml configs were created - - main_configs = [] - main_configs.append(os.path.join("configs", "disable_ssl_verification.xml")) +@pytest.fixture(scope="module") +def simple_tester(source): + tester = SimpleLayoutTester(test_name) + tester.cleanup() + tester.create_dictionaries(source) + return tester +@pytest.fixture(scope="module") +def main_config(secure_connection): + main_config = [] + if secure_connection: + main_config.append(os.path.join("configs", "disable_ssl_verification.xml")) + else: + main_config.append(os.path.join("configs", "ssl_verification.xml")) + return main_config + +@pytest.fixture(scope="module") +def started_cluster(secure_connection, cluster, main_config, simple_tester): + dictionaries = simple_tester.list_dictionaries() node = cluster.add_instance( "uri_node", - main_configs=main_configs, + main_configs=main_config, dictionaries=dictionaries, with_mongo=True, + with_mongo_secure = secure_connection ) - - -def teardown_module(module): - simple_tester.cleanup() - - -@pytest.fixture(scope="module") -def started_cluster(): try: cluster.start() - simple_tester.prepare(cluster) - complex_tester.prepare(cluster) - ranged_tester.prepare(cluster) - yield cluster - finally: cluster.shutdown() # See comment in SourceMongoURI +@pytest.mark.parametrize("secure_connection", [False], indirect=["secure_connection"]) +@pytest.mark.parametrize("layout_name", ["flat"]) +def test_simple(secure_connection ,started_cluster, simple_tester,layout_name): + simple_tester.execute(layout_name, started_cluster.instances["uri_node"]) + + +@pytest.mark.parametrize("secure_connection", [True], indirect=["secure_connection"]) @pytest.mark.parametrize("layout_name", ["flat"]) -def test_simple(started_cluster, layout_name): - simple_tester.execute(layout_name, node) +def test_simple_ssl(secure_connection ,started_cluster, simple_tester,layout_name): + simple_tester.execute(layout_name, started_cluster.instances["uri_node"]) From baaee66e8557bb942190f6ee9c7485f938078f54 Mon Sep 17 00:00:00 2001 From: MikhailBurdukov Date: Wed, 26 Apr 2023 19:29:29 +0000 Subject: [PATCH 216/406] Missing files --- docs/en/sql-reference/dictionaries/index.md | 3 +++ src/Dictionaries/MongoDBDictionarySource.cpp | 16 +++++++++++----- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/docs/en/sql-reference/dictionaries/index.md b/docs/en/sql-reference/dictionaries/index.md index f697b1ecdcfe..1d5ffd49e97a 100644 --- a/docs/en/sql-reference/dictionaries/index.md +++ b/docs/en/sql-reference/dictionaries/index.md @@ -1658,6 +1658,7 @@ Example of settings: test dictionary_source + ssl=true ``` @@ -1672,6 +1673,7 @@ SOURCE(MONGODB( password '' db 'test' collection 'dictionary_source' + options 'ssl=true' )) ``` @@ -1683,6 +1685,7 @@ Setting fields: - `password` – Password of the MongoDB user. - `db` – Name of the database. - `collection` – Name of the collection. +- `options` - MongoDB connection string options (optional parameter). ### Redis diff --git a/src/Dictionaries/MongoDBDictionarySource.cpp b/src/Dictionaries/MongoDBDictionarySource.cpp index 922e1e71bbb7..4434280fbfec 100644 --- a/src/Dictionaries/MongoDBDictionarySource.cpp +++ b/src/Dictionaries/MongoDBDictionarySource.cpp @@ -3,13 +3,13 @@ #include "DictionaryStructure.h" #include "registerDictionaries.h" #include - +#include namespace DB { static const std::unordered_set dictionary_allowed_keys = { - "host", "port", "user", "password", "db", "database", "uri", "collection", "name", "method"}; + "host", "port", "user", "password", "db", "database", "uri", "collection", "name", "method", "options"}; void registerDictionarySourceMongoDB(DictionarySourceFactory & factory) { @@ -51,6 +51,7 @@ void registerDictionarySourceMongoDB(DictionarySourceFactory & factory) config.getString(config_prefix + ".method", ""), configuration.database, config.getString(config_prefix + ".collection"), + config.getString(config_prefix + ".options", ""), sample_block); }; @@ -98,6 +99,7 @@ MongoDBDictionarySource::MongoDBDictionarySource( const std::string & method_, const std::string & db_, const std::string & collection_, + const std::string & options_, const Block & sample_block_) : dict_struct{dict_struct_} , uri{uri_} @@ -108,13 +110,15 @@ MongoDBDictionarySource::MongoDBDictionarySource( , method{method_} , db{db_} , collection{collection_} + , options(options_) , sample_block{sample_block_} , connection{std::make_shared()} { + + StorageMongoDBSocketFactory socket_factory; if (!uri.empty()) { // Connect with URI. - Poco::MongoDB::Connection::SocketFactory socket_factory; connection->connect(uri, socket_factory); Poco::URI poco_uri(connection->uri()); @@ -141,7 +145,9 @@ MongoDBDictionarySource::MongoDBDictionarySource( else { // Connect with host/port/user/etc. - connection->connect(host, port); + std::string uri_constructed("mongodb://" + host + ":" + std::to_string(port) + "/" + db + "?" + options); + connection->connect(uri_constructed, socket_factory); + if (!user.empty()) { Poco::MongoDB::Database poco_db(db); @@ -154,7 +160,7 @@ MongoDBDictionarySource::MongoDBDictionarySource( MongoDBDictionarySource::MongoDBDictionarySource(const MongoDBDictionarySource & other) : MongoDBDictionarySource{ - other.dict_struct, other.uri, other.host, other.port, other.user, other.password, other.method, other.db, other.collection, other.sample_block} + other.dict_struct, other.uri, other.host, other.port, other.user, other.password, other.method, other.db, other.collection,other.options, other.sample_block} { } From 389c0af92277882249257b6cc96b4cf5bcc4dd87 Mon Sep 17 00:00:00 2001 From: MikhailBurdukov Date: Wed, 26 Apr 2023 19:36:34 +0000 Subject: [PATCH 217/406] Fix style --- src/Dictionaries/MongoDBDictionarySource.cpp | 6 ++- .../test_mongo.py | 37 ++++++++++++++----- .../test_mongo_uri.py | 17 ++++++--- 3 files changed, 43 insertions(+), 17 deletions(-) diff --git a/src/Dictionaries/MongoDBDictionarySource.cpp b/src/Dictionaries/MongoDBDictionarySource.cpp index 4434280fbfec..28a67f3d7c5e 100644 --- a/src/Dictionaries/MongoDBDictionarySource.cpp +++ b/src/Dictionaries/MongoDBDictionarySource.cpp @@ -144,7 +144,7 @@ MongoDBDictionarySource::MongoDBDictionarySource( } else { - // Connect with host/port/user/etc. + // Connect with host/port/user/etc through constructing std::string uri_constructed("mongodb://" + host + ":" + std::to_string(port) + "/" + db + "?" + options); connection->connect(uri_constructed, socket_factory); @@ -160,7 +160,9 @@ MongoDBDictionarySource::MongoDBDictionarySource( MongoDBDictionarySource::MongoDBDictionarySource(const MongoDBDictionarySource & other) : MongoDBDictionarySource{ - other.dict_struct, other.uri, other.host, other.port, other.user, other.password, other.method, other.db, other.collection,other.options, other.sample_block} + other.dict_struct, other.uri, other.host, other.port, other.user, other.password, other.method, other.db, + other.collection, other.options, other.sample_block + } { } diff --git a/tests/integration/test_dictionaries_all_layouts_separate_sources/test_mongo.py b/tests/integration/test_dictionaries_all_layouts_separate_sources/test_mongo.py index 2f0bdb7debcc..973dbfc04299 100644 --- a/tests/integration/test_dictionaries_all_layouts_separate_sources/test_mongo.py +++ b/tests/integration/test_dictionaries_all_layouts_separate_sources/test_mongo.py @@ -21,10 +21,12 @@ def secure_connection(request): return request.param + @pytest.fixture(scope="module") def cluster(secure_connection): return ClickHouseCluster(__file__) + @pytest.fixture(scope="module") def source(secure_connection, cluster): return SourceMongo( @@ -35,9 +37,10 @@ def source(secure_connection, cluster): "27017", "root", "clickhouse", - secure = secure_connection + secure=secure_connection, ) + @pytest.fixture(scope="module") def simple_tester(source): tester = SimpleLayoutTester(test_name) @@ -45,11 +48,12 @@ def simple_tester(source): tester.create_dictionaries(source) return tester + @pytest.fixture(scope="module") def complex_tester(source): tester = ComplexLayoutTester(test_name) tester.create_dictionaries(source) - return tester + return tester @pytest.fixture(scope="module") @@ -58,6 +62,7 @@ def ranged_tester(source): tester.create_dictionaries(source) return tester + @pytest.fixture(scope="module") def main_config(secure_connection): main_config = [] @@ -66,10 +71,17 @@ def main_config(secure_connection): else: main_config.append(os.path.join("configs", "ssl_verification.xml")) return main_config - -@pytest.fixture(scope="module") -def started_cluster(secure_connection, cluster, main_config, simple_tester, ranged_tester, complex_tester): + +@pytest.fixture(scope="module") +def started_cluster( + secure_connection, + cluster, + main_config, + simple_tester, + ranged_tester, + complex_tester, +): SOURCE = SourceMongo( "MongoDB", "localhost", @@ -78,13 +90,16 @@ def started_cluster(secure_connection, cluster, main_config, simple_tester, rang "27017", "root", "clickhouse", - secure=secure_connection + secure=secure_connection, ) dictionaries = simple_tester.list_dictionaries() - + node = cluster.add_instance( - "node", main_configs = main_config, dictionaries=dictionaries, with_mongo=True, - with_mongo_secure = secure_connection + "node", + main_configs=main_config, + dictionaries=dictionaries, + with_mongo=True, + with_mongo_secure=secure_connection, ) try: @@ -99,21 +114,25 @@ def started_cluster(secure_connection, cluster, main_config, simple_tester, rang finally: cluster.shutdown() + @pytest.mark.parametrize("secure_connection", [False], indirect=["secure_connection"]) @pytest.mark.parametrize("layout_name", sorted(LAYOUTS_SIMPLE)) def test_simple(secure_connection, started_cluster, layout_name, simple_tester): simple_tester.execute(layout_name, started_cluster.instances["node"]) + @pytest.mark.parametrize("secure_connection", [False], indirect=["secure_connection"]) @pytest.mark.parametrize("layout_name", sorted(LAYOUTS_COMPLEX)) def test_complex(secure_connection, started_cluster, layout_name, complex_tester): complex_tester.execute(layout_name, started_cluster.instances["node"]) + @pytest.mark.parametrize("secure_connection", [False], indirect=["secure_connection"]) @pytest.mark.parametrize("layout_name", sorted(LAYOUTS_RANGED)) def test_ranged(secure_connection, started_cluster, layout_name, ranged_tester): ranged_tester.execute(layout_name, started_cluster.instances["node"]) + @pytest.mark.parametrize("secure_connection", [True], indirect=["secure_connection"]) @pytest.mark.parametrize("layout_name", sorted(LAYOUTS_SIMPLE)) def test_simple_ssl(secure_connection, started_cluster, layout_name, simple_tester): diff --git a/tests/integration/test_dictionaries_all_layouts_separate_sources/test_mongo_uri.py b/tests/integration/test_dictionaries_all_layouts_separate_sources/test_mongo_uri.py index 13ab980b61b7..225414322591 100644 --- a/tests/integration/test_dictionaries_all_layouts_separate_sources/test_mongo_uri.py +++ b/tests/integration/test_dictionaries_all_layouts_separate_sources/test_mongo_uri.py @@ -10,14 +10,17 @@ test_name = "mongo_uri" + @pytest.fixture(scope="module") def secure_connection(request): return request.param + @pytest.fixture(scope="module") def cluster(secure_connection): return ClickHouseCluster(__file__) + @pytest.fixture(scope="module") def source(secure_connection, cluster): return SourceMongoURI( @@ -28,9 +31,10 @@ def source(secure_connection, cluster): "27017", "root", "clickhouse", - secure = secure_connection + secure=secure_connection, ) + @pytest.fixture(scope="module") def simple_tester(source): tester = SimpleLayoutTester(test_name) @@ -38,6 +42,7 @@ def simple_tester(source): tester.create_dictionaries(source) return tester + @pytest.fixture(scope="module") def main_config(secure_connection): main_config = [] @@ -46,10 +51,10 @@ def main_config(secure_connection): else: main_config.append(os.path.join("configs", "ssl_verification.xml")) return main_config - + + @pytest.fixture(scope="module") def started_cluster(secure_connection, cluster, main_config, simple_tester): - dictionaries = simple_tester.list_dictionaries() node = cluster.add_instance( @@ -57,7 +62,7 @@ def started_cluster(secure_connection, cluster, main_config, simple_tester): main_configs=main_config, dictionaries=dictionaries, with_mongo=True, - with_mongo_secure = secure_connection + with_mongo_secure=secure_connection, ) try: cluster.start() @@ -70,11 +75,11 @@ def started_cluster(secure_connection, cluster, main_config, simple_tester): # See comment in SourceMongoURI @pytest.mark.parametrize("secure_connection", [False], indirect=["secure_connection"]) @pytest.mark.parametrize("layout_name", ["flat"]) -def test_simple(secure_connection ,started_cluster, simple_tester,layout_name): +def test_simple(secure_connection, started_cluster, simple_tester, layout_name): simple_tester.execute(layout_name, started_cluster.instances["uri_node"]) @pytest.mark.parametrize("secure_connection", [True], indirect=["secure_connection"]) @pytest.mark.parametrize("layout_name", ["flat"]) -def test_simple_ssl(secure_connection ,started_cluster, simple_tester,layout_name): +def test_simple_ssl(secure_connection, started_cluster, simple_tester, layout_name): simple_tester.execute(layout_name, started_cluster.instances["uri_node"]) From 150ea22a123361bcc697606817e0361faf90b680 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Tue, 25 Apr 2023 12:25:14 +0000 Subject: [PATCH 218/406] CMake: Remove dead code paths --- cmake/fuzzer.cmake | 4 ---- cmake/sanitize.cmake | 37 +++---------------------------------- 2 files changed, 3 insertions(+), 38 deletions(-) diff --git a/cmake/fuzzer.cmake b/cmake/fuzzer.cmake index 578a97572701..52f301ab8ad4 100644 --- a/cmake/fuzzer.cmake +++ b/cmake/fuzzer.cmake @@ -7,10 +7,6 @@ if (FUZZER) set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SAN_FLAGS} -fsanitize=fuzzer-no-link") set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${SAN_FLAGS} -fsanitize=fuzzer-no-link") - if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") - set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=fuzzer-no-link") - endif() - # NOTE: oss-fuzz can change LIB_FUZZING_ENGINE variable if (NOT LIB_FUZZING_ENGINE) set (LIB_FUZZING_ENGINE "-fsanitize=fuzzer") diff --git a/cmake/sanitize.cmake b/cmake/sanitize.cmake index fc9793d8f356..bf5eddf09f5d 100644 --- a/cmake/sanitize.cmake +++ b/cmake/sanitize.cmake @@ -16,49 +16,24 @@ if (SANITIZE) set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SAN_FLAGS} ${ASAN_FLAGS}") set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${SAN_FLAGS} ${ASAN_FLAGS}") - if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") - set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${ASAN_FLAGS}") - endif() - if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") - set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -static-libasan") - endif () - elseif (SANITIZE STREQUAL "memory") # MemorySanitizer flags are set according to the official documentation: # https://clang.llvm.org/docs/MemorySanitizer.html#usage - # - # For now, it compiles with `cmake -DSANITIZE=memory -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCMAKE_CXX_FLAGS_ADD="-O1" -DCMAKE_C_FLAGS_ADD="-O1"` - # Compiling with -DCMAKE_BUILD_TYPE=Debug leads to ld.lld failures because - # of large files (was not tested with ld.gold). This is why we compile with - # RelWithDebInfo, and downgrade optimizations to -O1 but not to -Og, to - # keep the binary size down. - # TODO: try compiling with -Og and with ld.gold. + + # Linking can fail due to relocation overflows (see #49145), caused by too big object files / libraries. + # Work around this with position-independent builds (-fPIC and -fpie), this is slightly slower than non-PIC/PIE but that's okay. set (MSAN_FLAGS "-fsanitize=memory -fsanitize-memory-use-after-dtor -fsanitize-memory-track-origins -fno-optimize-sibling-calls -fPIC -fpie -fsanitize-blacklist=${CMAKE_SOURCE_DIR}/tests/msan_suppressions.txt") set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SAN_FLAGS} ${MSAN_FLAGS}") set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${SAN_FLAGS} ${MSAN_FLAGS}") - if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") - set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=memory") - endif() - if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") - set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -static-libmsan") - endif () - elseif (SANITIZE STREQUAL "thread") set (TSAN_FLAGS "-fsanitize=thread") if (COMPILER_CLANG) set (TSAN_FLAGS "${TSAN_FLAGS} -fsanitize-blacklist=${CMAKE_SOURCE_DIR}/tests/tsan_suppressions.txt") endif() - set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SAN_FLAGS} ${TSAN_FLAGS}") set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${SAN_FLAGS} ${TSAN_FLAGS}") - if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") - set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=thread") - endif() - if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") - set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -static-libtsan") - endif () elseif (SANITIZE STREQUAL "undefined") set (UBSAN_FLAGS "-fsanitize=undefined -fno-sanitize-recover=all -fno-sanitize=float-divide-by-zero") @@ -77,12 +52,6 @@ if (SANITIZE) set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SAN_FLAGS} ${UBSAN_FLAGS}") set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${SAN_FLAGS} ${UBSAN_FLAGS}") - if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") - set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=undefined") - endif() - if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") - set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -static-libubsan") - endif () # llvm-tblgen, that is used during LLVM build, doesn't work with UBSan. set (ENABLE_EMBEDDED_COMPILER 0 CACHE BOOL "") From 7764168bd5552080a1ca52e318dc3460dc9e96af Mon Sep 17 00:00:00 2001 From: MikhailBurdukov Date: Wed, 26 Apr 2023 19:50:58 +0000 Subject: [PATCH 219/406] Resove conflict --- docs/en/sql-reference/dictionaries/index.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/en/sql-reference/dictionaries/index.md b/docs/en/sql-reference/dictionaries/index.md index 1d5ffd49e97a..648d4f5beb79 100644 --- a/docs/en/sql-reference/dictionaries/index.md +++ b/docs/en/sql-reference/dictionaries/index.md @@ -1679,13 +1679,13 @@ SOURCE(MONGODB( Setting fields: -- `host` – The MongoDB host. -- `port` – The port on the MongoDB server. -- `user` – Name of the MongoDB user. -- `password` – Password of the MongoDB user. -- `db` – Name of the database. -- `collection` – Name of the collection. -- `options` - MongoDB connection string options (optional parameter). +- `host` – The MongoDB host. +- `port` – The port on the MongoDB server. +- `user` – Name of the MongoDB user. +- `password` – Password of the MongoDB user. +- `db` – Name of the database. +- `collection` – Name of the collection. +- `options` - MongoDB connection string options (optional parameter). ### Redis From ca68cd85d4a7bd3588e108034c7913b400fb9cf7 Mon Sep 17 00:00:00 2001 From: MikhailBurdukov Date: Wed, 26 Apr 2023 20:10:10 +0000 Subject: [PATCH 220/406] Fix style --- tests/integration/helpers/external_sources.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/integration/helpers/external_sources.py b/tests/integration/helpers/external_sources.py index 193107a259d0..afb91083d57b 100644 --- a/tests/integration/helpers/external_sources.py +++ b/tests/integration/helpers/external_sources.py @@ -161,7 +161,6 @@ def load_data(self, data, table_name): class SourceMongo(ExternalSource): - def __init__( self, name, @@ -171,7 +170,7 @@ def __init__( docker_port, user, password, - secure = False + secure=False, ): ExternalSource.__init__( self, @@ -202,7 +201,7 @@ def get_source_str(self, table_name): user=self.user, password=self.password, tbl=table_name, - options= "ssl=true" if self.secure else "" + options="ssl=true" if self.secure else "", ) def prepare(self, structure, table_name, cluster): @@ -265,7 +264,7 @@ def get_source_str(self, table_name): user=self.user, password=self.password, tbl=table_name, - options = '?ssl=true' if self.secure else "", + options="?ssl=true" if self.secure else "", ) From d72030b8762572e9f0e18851b0242c35b2fc1f43 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Wed, 26 Apr 2023 20:33:52 +0000 Subject: [PATCH 221/406] Various fixes --- src/Common/Documentation.h | 42 ++++++++++++++- .../System/StorageSystemFunctions.cpp | 51 +++++++++---------- 2 files changed, 64 insertions(+), 29 deletions(-) diff --git a/src/Common/Documentation.h b/src/Common/Documentation.h index 0b0eacbeccd8..dea7f28bfc19 100644 --- a/src/Common/Documentation.h +++ b/src/Common/Documentation.h @@ -42,27 +42,65 @@ namespace DB * * Documentation does not support multiple languages. * The only available language is English. + * + * TODO: Allow to specify Syntax, Argument(s) and a Returned Value. + * TODO: Organize Examples as a struct of ExampleName, ExampleQuery and ExampleResult. */ struct Documentation { using Description = std::string; + + using Syntax = std::string; + + using Argument = std::string; + using Arguments = std::vector; + + using ReturnedValue = std::string; + using ExampleName = std::string; using ExampleQuery = std::string; using Examples = std::map; + using Category = std::string; using Categories = std::vector; + using Related = std::string; + Description description; Examples examples; Categories categories; - Documentation(Description description_) : description(std::move(description_)) {} + Documentation(Description description_) : description(std::move(description_)) {} /// NOLINT Documentation(Description description_, Examples examples_) : description(std::move(description_)), examples(std::move(examples_)) {} Documentation(Description description_, Examples examples_, Categories categories_) : description(std::move(description_)), examples(std::move(examples_)), categories(std::move(categories_)) {} /// TODO: Please remove this constructor. Documentation should always be non-empty. - Documentation() {} + Documentation() = default; + + std::string examplesAsString() const + { + std::string res; + for (const auto & [example_name, example_query] : examples) + { + res += example_name + ":\n\n"; + res += "```sql\n"; + res += example_query + "\n"; + res += "```\n"; + } + return res; + } + + std::string categoriesAsString() const + { + if (categories.empty()) + return ""; + + std::string res = categories[0]; + for (const auto & category : categories) + res += ", " + category; + return res; + } }; } diff --git a/src/Storages/System/StorageSystemFunctions.cpp b/src/Storages/System/StorageSystemFunctions.cpp index 29d6b81d528f..a4773b0be78a 100644 --- a/src/Storages/System/StorageSystemFunctions.cpp +++ b/src/Storages/System/StorageSystemFunctions.cpp @@ -22,26 +22,6 @@ enum class FunctionOrigin : Int8 EXECUTABLE_USER_DEFINED = 2 }; -std::string exampleMapToString( const std::map &example ) - // in the source, examples for functions are stored in a map. To - // create the documentation, we need to convert the map to a string. -{ - std::string stringToReturn; - - for (auto iter = example.begin(); iter != example.end(); ++iter){ - stringToReturn.append("### "); - stringToReturn.append(iter->first); - stringToReturn.append("\n"); - stringToReturn.append("```"); - stringToReturn.append("\n"); - stringToReturn.append(iter->second); - stringToReturn.append("\n"); - stringToReturn.append("```"); - stringToReturn.append("\n"); - } - return stringToReturn; -} - namespace { template @@ -79,19 +59,32 @@ namespace if (factory.isAlias(name)) { res_columns[6]->insertDefault(); - res_columns[7]->insert(false); + res_columns[7]->insertDefault(); + res_columns[8]->insertDefault(); + res_columns[9]->insertDefault(); + res_columns[10]->insertDefault(); + res_columns[11]->insertDefault(); } else { - res_columns[6]->insert(factory.getDocumentation(name).description); - res_columns[7]->insert(exampleMapToString((factory.getDocumentation(name).examples))); + auto documentation = factory.getDocumentation(name); + res_columns[6]->insert(documentation.description); + res_columns[7]->insertDefault(); + res_columns[8]->insertDefault(); + res_columns[9]->insertDefault(); + res_columns[10]->insert(documentation.examplesAsString()); + res_columns[11]->insert(documentation.categoriesAsString()); } } else - { + { res_columns[6]->insertDefault(); - res_columns[7]->insert(false); - } + res_columns[7]->insertDefault(); + res_columns[8]->insertDefault(); + res_columns[9]->insertDefault(); + res_columns[10]->insertDefault(); + res_columns[11]->insertDefault(); + } } } @@ -115,7 +108,11 @@ NamesAndTypesList StorageSystemFunctions::getNamesAndTypes() {"create_query", std::make_shared()}, {"origin", std::make_shared(getOriginEnumsAndValues())}, {"description", std::make_shared()}, - {"example", std::make_shared()}, + {"syntax", std::make_shared()}, + {"arguments", std::make_shared()}, + {"returned_value", std::make_shared()}, + {"examples", std::make_shared()}, + {"categories", std::make_shared()} }; } From 40ad8499a069e011124d28f1f305a4eb8e4b0c0b Mon Sep 17 00:00:00 2001 From: MikhailBurdukov Date: Wed, 26 Apr 2023 21:03:27 +0000 Subject: [PATCH 222/406] fix --- src/Dictionaries/MongoDBDictionarySource.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Dictionaries/MongoDBDictionarySource.cpp b/src/Dictionaries/MongoDBDictionarySource.cpp index 28a67f3d7c5e..0ac6e337a5b4 100644 --- a/src/Dictionaries/MongoDBDictionarySource.cpp +++ b/src/Dictionaries/MongoDBDictionarySource.cpp @@ -144,7 +144,7 @@ MongoDBDictionarySource::MongoDBDictionarySource( } else { - // Connect with host/port/user/etc through constructing + // Connect with host/port/user/etc through constructing std::string uri_constructed("mongodb://" + host + ":" + std::to_string(port) + "/" + db + "?" + options); connection->connect(uri_constructed, socket_factory); From 739bfc9b8dc7f13d0fd36b5941054229113a3f54 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Wed, 26 Apr 2023 21:23:26 +0000 Subject: [PATCH 223/406] Fix categories --- src/Common/Documentation.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Common/Documentation.h b/src/Common/Documentation.h index dea7f28bfc19..66ff2a58b5ce 100644 --- a/src/Common/Documentation.h +++ b/src/Common/Documentation.h @@ -97,8 +97,8 @@ struct Documentation return ""; std::string res = categories[0]; - for (const auto & category : categories) - res += ", " + category; + for (size_t i = 1; i < categories.size(); ++i) + res += ", " + categories[i]; return res; } }; From 1bb685a081675649c0f3454945688672d2c8892e Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 27 Apr 2023 01:32:54 +0300 Subject: [PATCH 224/406] Fix error. --- src/Common/AsynchronousMetrics.cpp | 12 ++++++------ src/Common/AsynchronousMetrics.h | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/Common/AsynchronousMetrics.cpp b/src/Common/AsynchronousMetrics.cpp index b53e2e81c0bd..49e1339e4529 100644 --- a/src/Common/AsynchronousMetrics.cpp +++ b/src/Common/AsynchronousMetrics.cpp @@ -906,7 +906,7 @@ void AsynchronousMetrics::update(TimePoint update_time) } if (cgroupmem_limit_in_bytes && cgroupmem_usage_in_bytes) - updateCgroupMemoryMetrics(cgroupmem_limit_in_bytes, cgroupmem_usage_in_bytes); + updateCgroupMemoryMetrics(*cgroupmem_limit_in_bytes, *cgroupmem_usage_in_bytes); if (meminfo) { @@ -1478,18 +1478,18 @@ void AsynchronousMetrics::update(TimePoint update_time) values = new_values; } -void AsynchronousMetrics::updateCgroupMemoryMetrics(std::optional memory_limit_in, std::optional memory_usage_in) +void AsynchronousMetrics::updateCgroupMemoryMetrics(ReadBufferFromFilePRead & memory_limit_in, ReadBufferFromFilePRead & memory_usage_in) { try { - memory_limit_in->rewind(); - memory_usage_in->rewind(); + memory_limit_in.rewind(); + memory_usage_in.rewind(); uint64_t cgroup_mem_limit_in_bytes = 0; uint64_t cgroup_mem_usage_in_bytes = 0; - tryReadText(cgroup_mem_limit_in_bytes, *memory_limit_in); - tryReadText(cgroup_mem_usage_in_bytes, *memory_usage_in); + tryReadText(cgroup_mem_limit_in_bytes, memory_limit_in); + tryReadText(cgroup_mem_usage_in_bytes, memory_usage_in); new_values["CgroupMemoryTotal"] = { cgroup_mem_limit_in_bytes, "The total amount of memory in cgroup, in bytes. If stated zero, the limit is the same as OSMemoryTotal." }; new_values["CgroupMemoryUsed"] = { cgroup_mem_usage_in_bytes, "The amount of memory used in cgroup, in bytes." }; diff --git a/src/Common/AsynchronousMetrics.h b/src/Common/AsynchronousMetrics.h index 54295de591be..4f61569d7a6a 100644 --- a/src/Common/AsynchronousMetrics.h +++ b/src/Common/AsynchronousMetrics.h @@ -202,7 +202,7 @@ class AsynchronousMetrics void openBlockDevices(); void openSensorsChips(); void openEDAC(); - void updateCgroupMemoryMetrics(std::optional memoryLimitReadBuffer, std::optional memoryUsageReadBuffer); + void updateCgroupMemoryMetrics(ReadBufferFromFilePRead & memory_limit_in, ReadBufferFromFilePRead & memory_usage_in); #endif std::unique_ptr thread; From d61214b4ed744c451fc98c19a618891e37da9638 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 27 Apr 2023 02:06:19 +0200 Subject: [PATCH 225/406] Merge #24050 --- .../02724_jit_logical_functions.reference | 18 ++++++++++++++++ .../02724_jit_logical_functions.sql | 21 +++++++++++++++++++ 2 files changed, 39 insertions(+) create mode 100644 tests/queries/0_stateless/02724_jit_logical_functions.reference create mode 100644 tests/queries/0_stateless/02724_jit_logical_functions.sql diff --git a/tests/queries/0_stateless/02724_jit_logical_functions.reference b/tests/queries/0_stateless/02724_jit_logical_functions.reference new file mode 100644 index 000000000000..673ffe02613d --- /dev/null +++ b/tests/queries/0_stateless/02724_jit_logical_functions.reference @@ -0,0 +1,18 @@ +Logical functions not null +0 0 0 0 0 +0 1 0 1 1 +1 0 0 1 1 +1 1 1 1 0 +Logical functions nullable +0 0 0 0 0 +0 1 0 1 1 +1 0 0 1 1 +1 1 1 1 0 +0 \N 0 \N \N +1 \N \N 1 \N +0 0 0 +1 1 0 +0 0 0 +1 1 0 +\N \N \N +\N \N \N diff --git a/tests/queries/0_stateless/02724_jit_logical_functions.sql b/tests/queries/0_stateless/02724_jit_logical_functions.sql new file mode 100644 index 000000000000..fe6646337d04 --- /dev/null +++ b/tests/queries/0_stateless/02724_jit_logical_functions.sql @@ -0,0 +1,21 @@ +SET compile_expressions = 1; +SET min_count_to_compile_expression = 0; + +DROP TABLE IF EXISTS test_table; +CREATE TABLE test_table (a UInt8, b UInt8) ENGINE = TinyLog; +INSERT INTO test_table VALUES (0, 0), (0, 1), (1, 0), (1, 1); + +SELECT 'Logical functions not null'; +SELECT a, b, and(a, b), or(a, b), xor(a, b) FROM test_table; + +DROP TABLE test_table; + +DROP TABLE IF EXISTS test_table_nullable; +CREATE TABLE test_table_nullable (a UInt8, b Nullable(UInt8)) ENGINE = TinyLog; +INSERT INTO test_table_nullable VALUES (0, 0), (0, 1), (1, 0), (1, 1), (0, NULL), (1, NULL); + +SELECT 'Logical functions nullable'; +SELECT a, b, and(a, b), or(a, b), xor(a, b) FROM test_table_nullable; +SELECT and(b, b), or(b, b), xor(b, b) FROM test_table_nullable; + +DROP TABLE test_table_nullable; From 111fb4b8a9cf92d35010d8f2f14eb8a2d8977a8f Mon Sep 17 00:00:00 2001 From: Nikolay Degterinsky Date: Thu, 27 Apr 2023 02:36:50 +0000 Subject: [PATCH 226/406] Add file name to exception raised during decompression --- src/IO/BrotliReadBuffer.cpp | 16 +++++++++++++--- src/IO/Bzip2ReadBuffer.cpp | 11 ++++++++--- src/IO/HadoopSnappyReadBuffer.cpp | 14 ++++++++++++-- src/IO/LZMAInflatingReadBuffer.cpp | 11 +++++++---- src/IO/Lz4InflatingReadBuffer.cpp | 6 ++++-- src/IO/WithFileName.cpp | 10 ++++++++++ src/IO/WithFileName.h | 1 + src/IO/ZlibInflatingReadBuffer.cpp | 14 +++++++++++--- src/IO/ZstdInflatingReadBuffer.cpp | 6 ++++-- 9 files changed, 70 insertions(+), 19 deletions(-) diff --git a/src/IO/BrotliReadBuffer.cpp b/src/IO/BrotliReadBuffer.cpp index 56ef2b5446bb..1863cef8a39e 100644 --- a/src/IO/BrotliReadBuffer.cpp +++ b/src/IO/BrotliReadBuffer.cpp @@ -3,6 +3,7 @@ #if USE_BROTLI # include # include "BrotliReadBuffer.h" +# include namespace DB { @@ -60,7 +61,10 @@ bool BrotliReadBuffer::nextImpl() if (brotli->result == BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT && (!in_available || in->eof())) { - throw Exception(ErrorCodes::BROTLI_READ_FAILED, "brotli decode error"); + throw Exception( + ErrorCodes::BROTLI_READ_FAILED, + "brotli decode error{}", + getExceptionEntryWithFileName(*in)); } out_capacity = internal_buffer.size(); @@ -83,13 +87,19 @@ bool BrotliReadBuffer::nextImpl() } else { - throw Exception(ErrorCodes::BROTLI_READ_FAILED, "brotli decode error"); + throw Exception( + ErrorCodes::BROTLI_READ_FAILED, + "brotli decode error{}", + getExceptionEntryWithFileName(*in)); } } if (brotli->result == BROTLI_DECODER_RESULT_ERROR) { - throw Exception(ErrorCodes::BROTLI_READ_FAILED, "brotli decode error"); + throw Exception( + ErrorCodes::BROTLI_READ_FAILED, + "brotli decode error{}", + getExceptionEntryWithFileName(*in)); } return true; diff --git a/src/IO/Bzip2ReadBuffer.cpp b/src/IO/Bzip2ReadBuffer.cpp index 9970edcbcf3a..45ce8f452324 100644 --- a/src/IO/Bzip2ReadBuffer.cpp +++ b/src/IO/Bzip2ReadBuffer.cpp @@ -3,6 +3,7 @@ #if USE_BZIP2 # include # include +# include namespace DB { @@ -118,13 +119,17 @@ bool Bzip2ReadBuffer::nextImpl() if (ret != BZ_OK) throw Exception( ErrorCodes::BZIP2_STREAM_DECODER_FAILED, - "bzip2 stream decoder failed: error code: {}", - ret); + "bzip2 stream decoder failed: error code: {}{}", + ret, + getExceptionEntryWithFileName(*in)); if (in->eof()) { eof_flag = true; - throw Exception(ErrorCodes::UNEXPECTED_END_OF_FILE, "Unexpected end of bzip2 archive"); + throw Exception( + ErrorCodes::UNEXPECTED_END_OF_FILE, + "Unexpected end of bzip2 archive{}", + getExceptionEntryWithFileName(*in)); } return true; diff --git a/src/IO/HadoopSnappyReadBuffer.cpp b/src/IO/HadoopSnappyReadBuffer.cpp index 6ba31997b37f..577367e56075 100644 --- a/src/IO/HadoopSnappyReadBuffer.cpp +++ b/src/IO/HadoopSnappyReadBuffer.cpp @@ -11,6 +11,8 @@ #include "HadoopSnappyReadBuffer.h" +#include + namespace DB { namespace ErrorCodes @@ -196,7 +198,11 @@ bool HadoopSnappyReadBuffer::nextImpl() if (decoder->result == Status::NEEDS_MORE_INPUT && (!in_available || in->eof())) { - throw Exception(ErrorCodes::SNAPPY_UNCOMPRESS_FAILED, "hadoop snappy decode error: {}", statusToString(decoder->result)); + throw Exception( + ErrorCodes::SNAPPY_UNCOMPRESS_FAILED, + "hadoop snappy decode error: {}{}", + statusToString(decoder->result), + getExceptionEntryWithFileName(*in)); } out_capacity = internal_buffer.size(); @@ -221,7 +227,11 @@ bool HadoopSnappyReadBuffer::nextImpl() } else if (decoder->result == Status::INVALID_INPUT || decoder->result == Status::BUFFER_TOO_SMALL) { - throw Exception(ErrorCodes::SNAPPY_UNCOMPRESS_FAILED, "hadoop snappy decode error: {}", statusToString(decoder->result)); + throw Exception( + ErrorCodes::SNAPPY_UNCOMPRESS_FAILED, + "hadoop snappy decode error: {}{}", + statusToString(decoder->result), + getExceptionEntryWithFileName(*in)); } return true; } diff --git a/src/IO/LZMAInflatingReadBuffer.cpp b/src/IO/LZMAInflatingReadBuffer.cpp index 6d40dafd517a..a6f3c74ae73b 100644 --- a/src/IO/LZMAInflatingReadBuffer.cpp +++ b/src/IO/LZMAInflatingReadBuffer.cpp @@ -1,4 +1,5 @@ #include +#include namespace DB { @@ -78,18 +79,20 @@ bool LZMAInflatingReadBuffer::nextImpl() { throw Exception( ErrorCodes::LZMA_STREAM_DECODER_FAILED, - "lzma decoder finished, but input stream has not exceeded: error code: {}; lzma version: {}", + "lzma decoder finished, but input stream has not exceeded: error code: {}; lzma version: {}{}", ret, - LZMA_VERSION_STRING); + LZMA_VERSION_STRING, + getExceptionEntryWithFileName(*in)); } } if (ret != LZMA_OK) throw Exception( ErrorCodes::LZMA_STREAM_DECODER_FAILED, - "lzma_stream_decoder failed: error code: error codeL {}; lzma version: {}", + "lzma_stream_decoder failed: error code: error code {}; lzma version: {}{}", ret, - LZMA_VERSION_STRING); + LZMA_VERSION_STRING, + getExceptionEntryWithFileName(*in)); return true; } diff --git a/src/IO/Lz4InflatingReadBuffer.cpp b/src/IO/Lz4InflatingReadBuffer.cpp index 049f3a4d15a6..eaa71048e703 100644 --- a/src/IO/Lz4InflatingReadBuffer.cpp +++ b/src/IO/Lz4InflatingReadBuffer.cpp @@ -1,4 +1,5 @@ #include +#include namespace DB { @@ -72,9 +73,10 @@ bool Lz4InflatingReadBuffer::nextImpl() if (LZ4F_isError(ret)) throw Exception( ErrorCodes::LZ4_DECODER_FAILED, - "LZ4 decompression failed. LZ4F version: {}. Error: {}", + "LZ4 decompression failed. LZ4F version: {}. Error: {}{}", LZ4F_VERSION, - LZ4F_getErrorName(ret)); + LZ4F_getErrorName(ret), + getExceptionEntryWithFileName(*in)); if (in->eof()) { diff --git a/src/IO/WithFileName.cpp b/src/IO/WithFileName.cpp index 6ecb3671ca04..0ec9ed5dd539 100644 --- a/src/IO/WithFileName.cpp +++ b/src/IO/WithFileName.cpp @@ -26,4 +26,14 @@ String getFileNameFromReadBuffer(const ReadBuffer & in) return getFileName(in); } +String getExceptionEntryWithFileName(const ReadBuffer & in) +{ + auto filename = getFileNameFromReadBuffer(in); + + if (filename.empty()) + return ""; + + return "; While reading from: " + filename; +} + } diff --git a/src/IO/WithFileName.h b/src/IO/WithFileName.h index d770634e7388..595f1a768c59 100644 --- a/src/IO/WithFileName.h +++ b/src/IO/WithFileName.h @@ -14,5 +14,6 @@ class WithFileName }; String getFileNameFromReadBuffer(const ReadBuffer & in); +String getExceptionEntryWithFileName(const ReadBuffer & in); } diff --git a/src/IO/ZlibInflatingReadBuffer.cpp b/src/IO/ZlibInflatingReadBuffer.cpp index 09e4fce7c4c8..b43dda1bfccb 100644 --- a/src/IO/ZlibInflatingReadBuffer.cpp +++ b/src/IO/ZlibInflatingReadBuffer.cpp @@ -1,5 +1,5 @@ #include - +#include namespace DB { @@ -99,14 +99,22 @@ bool ZlibInflatingReadBuffer::nextImpl() { rc = inflateReset(&zstr); if (rc != Z_OK) - throw Exception(ErrorCodes::ZLIB_INFLATE_FAILED, "inflateReset failed: {}", zError(rc)); + throw Exception( + ErrorCodes::ZLIB_INFLATE_FAILED, + "inflateReset failed: {}{}", + zError(rc), + getExceptionEntryWithFileName(*in)); return true; } } /// If it is not end and not OK, something went wrong, throw exception if (rc != Z_OK) - throw Exception(ErrorCodes::ZLIB_INFLATE_FAILED, "inflate failed: {}", zError(rc)); + throw Exception( + ErrorCodes::ZLIB_INFLATE_FAILED, + "inflate failed: {}{}", + zError(rc), + getExceptionEntryWithFileName(*in)); } while (working_buffer.empty()); diff --git a/src/IO/ZstdInflatingReadBuffer.cpp b/src/IO/ZstdInflatingReadBuffer.cpp index 6f5c8b4dc715..2b663ec71452 100644 --- a/src/IO/ZstdInflatingReadBuffer.cpp +++ b/src/IO/ZstdInflatingReadBuffer.cpp @@ -1,4 +1,5 @@ #include +#include #include @@ -61,12 +62,13 @@ bool ZstdInflatingReadBuffer::nextImpl() { throw Exception( ErrorCodes::ZSTD_DECODER_FAILED, - "ZSTD stream decoding failed: error '{}'{}; ZSTD version: {}", + "ZSTD stream decoding failed: error '{}'{}; ZSTD version: {}{}", ZSTD_getErrorName(ret), ZSTD_error_frameParameter_windowTooLarge == ret ? ". You can increase the maximum window size with the 'zstd_window_log_max' setting in ClickHouse. Example: 'SET zstd_window_log_max = 31'" : "", - ZSTD_VERSION_STRING); + ZSTD_VERSION_STRING, + getExceptionEntryWithFileName(*in)); } /// Check that something has changed after decompress (input or output position) From 5b4e014b8e383fc13eeb56eec1f1ddd302e62e04 Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Thu, 27 Apr 2023 15:18:26 +0800 Subject: [PATCH 227/406] Don't choose io_uring for macOS --- tests/clickhouse-test | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/clickhouse-test b/tests/clickhouse-test index 7c492a9b4673..72e0f276bb56 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -11,6 +11,7 @@ import shutil import sys import os import os.path +import platform import signal import re import copy @@ -542,7 +543,8 @@ class SettingsRandomizer: 0.2, 0.5, 1, 10 * 1024 * 1024 * 1024 ), "local_filesystem_read_method": lambda: random.choice( - ["read", "pread", "mmap", "pread_threadpool", "io_uring"] + # io_uring is not supported on macOS, don't choose it + ["read", "pread", "mmap", "pread_threadpool", "io_uring"] if platform.system().lower() != "darwin" else ["read", "pread", "mmap", "pread_threadpool"] ), "remote_filesystem_read_method": lambda: random.choice(["read", "threadpool"]), "local_filesystem_read_prefetch": lambda: random.randint(0, 1), From 3373ed5fc84e2a8680862d6477995edceb4a8833 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Thu, 27 Apr 2023 11:22:11 +0200 Subject: [PATCH 228/406] Release 23.4 is not LTS --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e2505856d0c8..1ccd4f9846d9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,7 @@ # 2023 Changelog -### ClickHouse release 23.4 LTS, 2023-04-26 +### ClickHouse release 23.4, 2023-04-26 #### Backward Incompatible Change * Formatter '%M' in function formatDateTime() now prints the month name instead of the minutes. This makes the behavior consistent with MySQL. The previous behavior can be restored using setting "formatdatetime_parsedatetime_m_is_month_name = 0". [#47246](https://github.com/ClickHouse/ClickHouse/pull/47246) ([Robert Schulze](https://github.com/rschu1ze)). From 096667be45b9a7e52f5fc5267e419d7b009322ae Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Thu, 27 Apr 2023 09:33:52 +0000 Subject: [PATCH 229/406] More fixes --- src/Common/Documentation.cpp | 30 +++++++++++++++++++ src/Common/Documentation.h | 25 ++-------------- .../System/StorageSystemFunctions.cpp | 1 - 3 files changed, 32 insertions(+), 24 deletions(-) create mode 100644 src/Common/Documentation.cpp diff --git a/src/Common/Documentation.cpp b/src/Common/Documentation.cpp new file mode 100644 index 000000000000..862ecb6054c6 --- /dev/null +++ b/src/Common/Documentation.cpp @@ -0,0 +1,30 @@ +#include + +namespace DB +{ + +std::string Documentation::examplesAsString() const +{ + std::string res; + for (const auto & [example_name, example_query] : examples) + { + res += example_name + ":\n\n"; + res += "```sql\n"; + res += example_query + "\n"; + res += "```\n"; + } + return res; +} + +std::string Documentation::categoriesAsString() const +{ + if (categories.empty()) + return ""; + + std::string res = categories[0]; + for (size_t i = 1; i < categories.size(); ++i) + res += ", " + categories[i]; + return res; +} + +} diff --git a/src/Common/Documentation.h b/src/Common/Documentation.h index 66ff2a58b5ce..c71aa8772eda 100644 --- a/src/Common/Documentation.h +++ b/src/Common/Documentation.h @@ -78,29 +78,8 @@ struct Documentation /// TODO: Please remove this constructor. Documentation should always be non-empty. Documentation() = default; - std::string examplesAsString() const - { - std::string res; - for (const auto & [example_name, example_query] : examples) - { - res += example_name + ":\n\n"; - res += "```sql\n"; - res += example_query + "\n"; - res += "```\n"; - } - return res; - } - - std::string categoriesAsString() const - { - if (categories.empty()) - return ""; - - std::string res = categories[0]; - for (size_t i = 1; i < categories.size(); ++i) - res += ", " + categories[i]; - return res; - } + std::string examplesAsString() const; + std::string categoriesAsString() const; }; } diff --git a/src/Storages/System/StorageSystemFunctions.cpp b/src/Storages/System/StorageSystemFunctions.cpp index a4773b0be78a..f3a297a11d15 100644 --- a/src/Storages/System/StorageSystemFunctions.cpp +++ b/src/Storages/System/StorageSystemFunctions.cpp @@ -40,7 +40,6 @@ namespace { res_columns[2]->insert(false); res_columns[3]->insertDefault(); - res_columns[7]->insert(false); } else { From be702ee5841592f5dc9cae1f5d4702ccec12d481 Mon Sep 17 00:00:00 2001 From: Jordi Date: Thu, 27 Apr 2023 12:08:14 +0200 Subject: [PATCH 230/406] Disable ISA-L on aarch64 architectures --- CMakeLists.txt | 6 ++++++ contrib/CMakeLists.txt | 4 +++- contrib/libhdfs3-cmake/CMakeLists.txt | 6 ++++-- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 0554403cce51..c6ca18773a77 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -176,6 +176,12 @@ if (OS_DARWIN) set (ENABLE_CURL_BUILD OFF) endif () +option(ENABLE_ISAL_LIBRARY "Enable ISA-L library ON by default except on aarch64." ON) +if (ARCH_AARCH64) + # Disable ISA-L libray on aarch64. + set (ENABLE_ISAL_LIBRARY OFF) +endif () + if (NOT CMAKE_BUILD_TYPE_UC STREQUAL "RELEASE") # Can be lld or ld-lld or lld-13 or /path/to/lld. if (LINKER_NAME MATCHES "lld") diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 0ff8b550a982..f88323f022b2 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -191,7 +191,9 @@ add_contrib (google-benchmark-cmake google-benchmark) add_contrib (ulid-c-cmake ulid-c) -add_contrib (isa-l-cmake isa-l) +if (ENABLE_ISAL_LIBRARY) + add_contrib (isa-l-cmake isa-l) +endif() # Put all targets defined here and in subdirectories under "contrib/" folders in GUI-based IDEs. # Some of third-party projects may override CMAKE_FOLDER or FOLDER property of their targets, so they would not appear diff --git a/contrib/libhdfs3-cmake/CMakeLists.txt b/contrib/libhdfs3-cmake/CMakeLists.txt index d9f7009c1bd4..ecc0627354a8 100644 --- a/contrib/libhdfs3-cmake/CMakeLists.txt +++ b/contrib/libhdfs3-cmake/CMakeLists.txt @@ -172,8 +172,10 @@ if (TARGET OpenSSL::SSL) target_link_libraries(_hdfs3 PRIVATE OpenSSL::Crypto OpenSSL::SSL) endif() -target_link_libraries(_hdfs3 PRIVATE ch_contrib::isal) -add_definitions(-DHADOOP_ISAL_LIBRARY) +if (ENABLE_ISAL_LIBRARY) + target_link_libraries(_hdfs3 PRIVATE ch_contrib::isal) + add_definitions(-DHADOOP_ISAL_LIBRARY) +endif() add_library(ch_contrib::hdfs ALIAS _hdfs3) From 93257878844dd0d06b7df575a887784a629904fa Mon Sep 17 00:00:00 2001 From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com> Date: Thu, 27 Apr 2023 13:03:41 +0200 Subject: [PATCH 231/406] Update FileCache.cpp --- src/Interpreters/Cache/FileCache.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/Interpreters/Cache/FileCache.cpp b/src/Interpreters/Cache/FileCache.cpp index cad018e772a1..39399c9ce09b 100644 --- a/src/Interpreters/Cache/FileCache.cpp +++ b/src/Interpreters/Cache/FileCache.cpp @@ -550,6 +550,9 @@ FileCache::FileSegmentCell * FileCache::addCell( } catch (...) { + /// Avoid errors like + /// std::__1::__fs::filesystem::filesystem_error: filesystem error: in create_directories: No space left on device + /// and mark file segment with SKIP_CACHE state tryLogCurrentException(__PRETTY_FUNCTION__); result_state = FileSegment::State::SKIP_CACHE; } From fab84345fd9baba7c299d374beb4382aac132fe0 Mon Sep 17 00:00:00 2001 From: kssenii Date: Thu, 27 Apr 2023 13:18:32 +0200 Subject: [PATCH 232/406] Corrections after merge with master --- src/Interpreters/Cache/FileCache.cpp | 4 +++- src/Interpreters/Cache/LRUFileCachePriority.cpp | 4 +++- src/Interpreters/Cache/Metadata.cpp | 16 +++++++++++++--- src/Interpreters/Cache/Metadata.h | 2 +- 4 files changed, 20 insertions(+), 6 deletions(-) diff --git a/src/Interpreters/Cache/FileCache.cpp b/src/Interpreters/Cache/FileCache.cpp index 9b6e6ca79bcd..ab00db789783 100644 --- a/src/Interpreters/Cache/FileCache.cpp +++ b/src/Interpreters/Cache/FileCache.cpp @@ -648,6 +648,9 @@ bool FileCache::tryReserve(FileSegment & file_segment, size_t size) if (is_main_priority_overflow()) return false; + if (!file_segment.getKeyMetadata()->createBaseDirectory()) + return false; + for (auto & [current_key, deletion_info] : to_delete) { auto locked_key = deletion_info.getMetadata().lock(); @@ -693,7 +696,6 @@ bool FileCache::tryReserve(FileSegment & file_segment, size_t size) if (main_priority->getSize(cache_lock) > (1ull << 63)) throw Exception(ErrorCodes::LOGICAL_ERROR, "Cache became inconsistent. There must be a bug"); - file_segment.getKeyMetadata()->createBaseDirectory(); file_segment.reserved_size += size; return true; } diff --git a/src/Interpreters/Cache/LRUFileCachePriority.cpp b/src/Interpreters/Cache/LRUFileCachePriority.cpp index febcb4326259..f1798cd626c1 100644 --- a/src/Interpreters/Cache/LRUFileCachePriority.cpp +++ b/src/Interpreters/Cache/LRUFileCachePriority.cpp @@ -148,13 +148,15 @@ void LRUFileCachePriority::LRUFileCacheIterator::annul() queue_iter->size = 0; } -void LRUFileCachePriority::LRUFileCacheIterator::updateSize(ssize_t size) +void LRUFileCachePriority::LRUFileCacheIterator::updateSize(int64_t size) { cache_priority->current_size += size; + if (size > 0) CurrentMetrics::add(CurrentMetrics::FilesystemCacheSize, size); else CurrentMetrics::sub(CurrentMetrics::FilesystemCacheSize, size); + queue_iter->size += size; chassert(cache_priority->current_size >= 0); diff --git a/src/Interpreters/Cache/Metadata.cpp b/src/Interpreters/Cache/Metadata.cpp index 1fc19be9d7ef..37b5330b26c3 100644 --- a/src/Interpreters/Cache/Metadata.cpp +++ b/src/Interpreters/Cache/Metadata.cpp @@ -76,12 +76,22 @@ LockedKeyPtr KeyMetadata::tryLock() return nullptr; } -void KeyMetadata::createBaseDirectory() +bool KeyMetadata::createBaseDirectory() { if (!created_base_directory.exchange(true)) { - fs::create_directories(key_path); + try + { + fs::create_directories(key_path); + } + catch (...) + { + tryLogCurrentException(__PRETTY_FUNCTION__); + created_base_directory = false; + return false; + } } + return true; } std::string KeyMetadata::getFileSegmentPath(const FileSegment & file_segment) @@ -364,7 +374,7 @@ void LockedKey::shrinkFileSegmentToDownloadedSize( file_segment->getInfoForLogUnlocked(segment_lock)); } - ssize_t diff = file_segment->reserved_size - downloaded_size; + int64_t diff = file_segment->reserved_size - downloaded_size; metadata->file_segment = std::make_shared( getKey(), offset, downloaded_size, FileSegment::State::DOWNLOADED, diff --git a/src/Interpreters/Cache/Metadata.h b/src/Interpreters/Cache/Metadata.h index 2a9204630f4e..a0effc26516d 100644 --- a/src/Interpreters/Cache/Metadata.h +++ b/src/Interpreters/Cache/Metadata.h @@ -60,7 +60,7 @@ struct KeyMetadata : public std::map, LockedKeyPtr tryLock(); - void createBaseDirectory(); + bool createBaseDirectory(); std::string getFileSegmentPath(const FileSegment & file_segment); From 0a192d1160005bbac70bd15a6423ad96c34032b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Thu, 27 Apr 2023 13:24:20 +0200 Subject: [PATCH 233/406] Fix GCS s3 compat disks (#49194) --- src/IO/S3/PocoHTTPClient.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/IO/S3/PocoHTTPClient.cpp b/src/IO/S3/PocoHTTPClient.cpp index 9aabc73fc660..ee14e251cb5a 100644 --- a/src/IO/S3/PocoHTTPClient.cpp +++ b/src/IO/S3/PocoHTTPClient.cpp @@ -260,6 +260,7 @@ void PocoHTTPClient::makeRequestInternal( Poco::Logger * log = &Poco::Logger::get("AWSClient"); auto uri = request.GetUri().GetURIString(); +#if 0 auto provider_type = getProviderTypeFromURL(uri); if (provider_type == ProviderType::GCS) @@ -269,6 +270,7 @@ void PocoHTTPClient::makeRequestInternal( request.DeleteHeader("amz-sdk-invocation-id"); request.DeleteHeader("amz-sdk-request"); } +#endif if (enable_s3_requests_logging) LOG_TEST(log, "Make request to: {}", uri); From 5b179ecd02b572439cbe4c0293b6deab4aef1223 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Thu, 27 Apr 2023 11:47:35 +0000 Subject: [PATCH 234/406] Fix 02117_show_create_table_system --- .../0_stateless/02117_show_create_table_system.reference | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/02117_show_create_table_system.reference b/tests/queries/0_stateless/02117_show_create_table_system.reference index f2524cac1151..4332b8bcd0d8 100644 --- a/tests/queries/0_stateless/02117_show_create_table_system.reference +++ b/tests/queries/0_stateless/02117_show_create_table_system.reference @@ -281,7 +281,12 @@ CREATE TABLE system.functions `alias_to` String, `create_query` String, `origin` Enum8('System' = 0, 'SQLUserDefined' = 1, 'ExecutableUserDefined' = 2), - `description` String + `description` String, + `syntax` String, + `arguments` String, + `returned_value` String, + `examples` String, + `categories` String ) ENGINE = SystemFunctions COMMENT 'SYSTEM TABLE is built on the fly.' From 8eae3ed3a5d8af6f5a1468e105b05514f9646857 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 27 Apr 2023 13:48:23 +0200 Subject: [PATCH 235/406] Fix garbage #48719 --- docker/test/util/process_functional_tests_result.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/test/util/process_functional_tests_result.py b/docker/test/util/process_functional_tests_result.py index 3c1c6e2a795a..b7a56d817aa1 100755 --- a/docker/test/util/process_functional_tests_result.py +++ b/docker/test/util/process_functional_tests_result.py @@ -80,10 +80,10 @@ def process_test_log(log_path, broken_tests): test_results.append( ( test_name, - "FAIL", + "SKIPPED", test_time, [ - "Test is expected to fail! Please, update broken_tests.txt!\n" + "This test passed. Update broken_tests.txt.\n" ], ) ) From fc9b2fbac8a750ce0d8932b0ad561823eef47c49 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Thu, 27 Apr 2023 14:52:09 +0300 Subject: [PATCH 236/406] Fix race on Outdated parts loading (#49223) --- src/Storages/MergeTree/MergeTreeData.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index afde1cd2fcad..9bab18acd02c 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -3948,6 +3948,9 @@ void MergeTreeData::forcefullyMovePartToDetachedAndRemoveFromMemory(const MergeT else LOG_INFO(log, "Renaming {} to {}_{} and forgetting it.", part_to_detach->getDataPartStorage().getPartDirectory(), prefix, part_to_detach->name); + if (restore_covered) + waitForOutdatedPartsToBeLoaded(); + auto lock = lockParts(); bool removed_active_part = false; bool restored_active_part = false; From c5793f9684f1623bbe42ecc3670371d4b3a54a84 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 27 Apr 2023 14:56:51 +0300 Subject: [PATCH 237/406] Fix build --- src/Common/AsynchronousMetrics.cpp | 42 ++++++++++++++---------------- src/Common/AsynchronousMetrics.h | 1 - 2 files changed, 19 insertions(+), 24 deletions(-) diff --git a/src/Common/AsynchronousMetrics.cpp b/src/Common/AsynchronousMetrics.cpp index 49e1339e4529..ef144598648c 100644 --- a/src/Common/AsynchronousMetrics.cpp +++ b/src/Common/AsynchronousMetrics.cpp @@ -906,8 +906,26 @@ void AsynchronousMetrics::update(TimePoint update_time) } if (cgroupmem_limit_in_bytes && cgroupmem_usage_in_bytes) - updateCgroupMemoryMetrics(*cgroupmem_limit_in_bytes, *cgroupmem_usage_in_bytes); + { + try + { + memory_limit_in.rewind(); + memory_usage_in.rewind(); + + uint64_t cgroup_mem_limit_in_bytes = 0; + uint64_t cgroup_mem_usage_in_bytes = 0; + + tryReadText(cgroup_mem_limit_in_bytes, memory_limit_in); + tryReadText(cgroup_mem_usage_in_bytes, memory_usage_in); + new_values["CgroupMemoryTotal"] = { cgroup_mem_limit_in_bytes, "The total amount of memory in cgroup, in bytes. If stated zero, the limit is the same as OSMemoryTotal." }; + new_values["CgroupMemoryUsed"] = { cgroup_mem_usage_in_bytes, "The amount of memory used in cgroup, in bytes." }; + } + catch (...) + { + tryLogCurrentException(__PRETTY_FUNCTION__); + } + } if (meminfo) { try @@ -1478,26 +1496,4 @@ void AsynchronousMetrics::update(TimePoint update_time) values = new_values; } -void AsynchronousMetrics::updateCgroupMemoryMetrics(ReadBufferFromFilePRead & memory_limit_in, ReadBufferFromFilePRead & memory_usage_in) -{ - try - { - memory_limit_in.rewind(); - memory_usage_in.rewind(); - - uint64_t cgroup_mem_limit_in_bytes = 0; - uint64_t cgroup_mem_usage_in_bytes = 0; - - tryReadText(cgroup_mem_limit_in_bytes, memory_limit_in); - tryReadText(cgroup_mem_usage_in_bytes, memory_usage_in); - - new_values["CgroupMemoryTotal"] = { cgroup_mem_limit_in_bytes, "The total amount of memory in cgroup, in bytes. If stated zero, the limit is the same as OSMemoryTotal." }; - new_values["CgroupMemoryUsed"] = { cgroup_mem_usage_in_bytes, "The amount of memory used in cgroup, in bytes." }; - } - catch (...) - { - tryLogCurrentException(__PRETTY_FUNCTION__); - } -} - } diff --git a/src/Common/AsynchronousMetrics.h b/src/Common/AsynchronousMetrics.h index 4f61569d7a6a..d104b872f52b 100644 --- a/src/Common/AsynchronousMetrics.h +++ b/src/Common/AsynchronousMetrics.h @@ -202,7 +202,6 @@ class AsynchronousMetrics void openBlockDevices(); void openSensorsChips(); void openEDAC(); - void updateCgroupMemoryMetrics(ReadBufferFromFilePRead & memory_limit_in, ReadBufferFromFilePRead & memory_usage_in); #endif std::unique_ptr thread; From 3448b112d8432f4749fcb94139ee80867823dec4 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 27 Apr 2023 12:31:00 +0000 Subject: [PATCH 238/406] Automatic style fix --- docker/test/util/process_functional_tests_result.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/docker/test/util/process_functional_tests_result.py b/docker/test/util/process_functional_tests_result.py index b7a56d817aa1..470eb61b3fad 100755 --- a/docker/test/util/process_functional_tests_result.py +++ b/docker/test/util/process_functional_tests_result.py @@ -82,9 +82,7 @@ def process_test_log(log_path, broken_tests): test_name, "SKIPPED", test_time, - [ - "This test passed. Update broken_tests.txt.\n" - ], + ["This test passed. Update broken_tests.txt.\n"], ) ) else: From 33443351c3b5a1e49cc2acf294c23c85980b68b1 Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Thu, 27 Apr 2023 21:03:14 +0800 Subject: [PATCH 239/406] Resolve review comments Signed-off-by: Frank Chen --- tests/clickhouse-test | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/clickhouse-test b/tests/clickhouse-test index 72e0f276bb56..e7bf3668ac11 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -543,8 +543,8 @@ class SettingsRandomizer: 0.2, 0.5, 1, 10 * 1024 * 1024 * 1024 ), "local_filesystem_read_method": lambda: random.choice( - # io_uring is not supported on macOS, don't choose it - ["read", "pread", "mmap", "pread_threadpool", "io_uring"] if platform.system().lower() != "darwin" else ["read", "pread", "mmap", "pread_threadpool"] + # Allow to use uring only when running on Linux + ["read", "pread", "mmap", "pread_threadpool", "io_uring"] if platform.system().lower() == "linux" else ["read", "pread", "mmap", "pread_threadpool"] ), "remote_filesystem_read_method": lambda: random.choice(["read", "threadpool"]), "local_filesystem_read_prefetch": lambda: random.randint(0, 1), From c0126ce67b45f8aecaebb32a90172279d195954c Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Thu, 27 Apr 2023 17:33:45 +0200 Subject: [PATCH 240/406] Added english version of the documentation --- .../reference/kolmogorovsmirnovtest.md | 119 ++++++++++++++++++ 1 file changed, 119 insertions(+) create mode 100644 docs/en/sql-reference/aggregate-functions/reference/kolmogorovsmirnovtest.md diff --git a/docs/en/sql-reference/aggregate-functions/reference/kolmogorovsmirnovtest.md b/docs/en/sql-reference/aggregate-functions/reference/kolmogorovsmirnovtest.md new file mode 100644 index 000000000000..e5a577ea365b --- /dev/null +++ b/docs/en/sql-reference/aggregate-functions/reference/kolmogorovsmirnovtest.md @@ -0,0 +1,119 @@ +--- +slug: /en/sql-reference/aggregate-functions/reference/kolmogorovsmirnovtest +sidebar_position: 300 +sidebar_label: kolmogorovSmirnovTest +--- + +# kolmogorovSmirnovTest + +Applies Kolmogorov-Smirnov's test to samples from two populations. + +**Syntax** + +``` sql +kolmogorovSmirnovTest([alternative, computation_method])(sample_data, sample_index) +``` + +Values of both samples are in the `sample_data` column. If `sample_index` equals to 0 then the value in that row belongs to the sample from the first population. Otherwise it belongs to the sample from the second population. +Samples must belong to continuous, one-dimensional probability distributions. + +**Arguments** + +- `sample_data` — Sample data. [Integer](../../../sql-reference/data-types/int-uint.md), [Float](../../../sql-reference/data-types/float.md) or [Decimal](../../../sql-reference/data-types/decimal.md). +- `sample_index` — Sample index. [Integer](../../../sql-reference/data-types/int-uint.md). + +**Parameters** + +- `alternative` — alternative hypothesis. (Optional, default: `'two-sided'`.) [String](../../../sql-reference/data-types/string.md). + Let F(x) and G(x) be the CDFs of the first and second distributions respectively. + - `'two-sided'` + The null hypothesis is that samples come from the same distribution, e.g. F(x) = G(x) for all x. + And the alternative is that the distributions are not identical. + - `'greater'` + The null hypothesis is that values in the first sample are *stohastically smaller* than those in the second one, + e.g. the CDF of first distribution lies above and hence to the left of that for the second one. + Which in fact means that F(x) >= G(x) for all x. And the alternative in this case is that F(x) < G(x) for at least one x. + - `'less'`. + The null hypothesis is that values in the first sample are *stohastically greater* than those in the second one, + e.g. the CDF of first distribution lies below and hence to the right of that for the second one. + Which in fact means that F(x) <= G(x) for all x. And the alternative in this case is that F(x) > G(x) for at least one x. +- `computation_method` — the method used to compute p-value. (Optional, default: `'auto'`.) [String](../../../sql-reference/data-types/string.md). + - `'exact'` - calculation is performed using precise probability distribution of the test statistics. Compute intensive and wasteful except for small samples. + - `'asymp'` - calculation is performed using an approximation. For large sample sizes, the exact and asymptotic p-values are very similar. + - `'auto'` - the `'exact'` method is used when a maximum number of samples is less than 10'000. + + +**Returned values** + +[Tuple](../../../sql-reference/data-types/tuple.md) with two elements: + +- calculated statistic. [Float64](../../../sql-reference/data-types/float.md). +- calculated p-value. [Float64](../../../sql-reference/data-types/float.md). + + +**Example** + +Query: + +``` sql +SELECT kolmogorovSmirnovTest('less', 'exact')(value, num) +FROM +( + SELECT + randNormal(0, 10) AS value, + 0 AS num + FROM numbers(10000) + UNION ALL + SELECT + randNormal(0, 10) AS value, + 1 AS num + FROM numbers(10000) +) +``` + +Result: + +``` text +┌─kolmogorovSmirnovTest('less', 'exact')(value, num)─┐ +│ (0.009899999999999996,0.37528595205132287) │ +└────────────────────────────────────────────────────┘ +``` + +Note: +P-value is bigger than 0.05 (for confidence level of 95%), so null hypothesis is not rejected. + + +Query: + +``` sql +SELECT kolmogorovSmirnovTest('two-sided', 'exact')(value, num) +FROM +( + SELECT + randStudentT(10) AS value, + 0 AS num + FROM numbers(100) + UNION ALL + SELECT + randNormal(0, 10) AS value, + 1 AS num + FROM numbers(100) +) +``` + +Result: + +``` text +┌─kolmogorovSmirnovTest('two-sided', 'exact')(value, num)─┐ +│ (0.4100000000000002,6.61735760482795e-8) │ +└─────────────────────────────────────────────────────────┘ +``` + +Note: +P-value is less than 0.05 (for confidence level of 95%), so null hypothesis is rejected. + + +**See Also** + +- [Student's t-test](https://en.wikipedia.org/wiki/Student%27s_t-test) +- [welchTTest function](welchttest.md#welchttest) From fe9cc7ad907970bcd89942494f0a49e8fbe89c16 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Thu, 27 Apr 2023 17:38:15 +0200 Subject: [PATCH 241/406] Better --- .../aggregate-functions/reference/kolmogorovsmirnovtest.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/en/sql-reference/aggregate-functions/reference/kolmogorovsmirnovtest.md b/docs/en/sql-reference/aggregate-functions/reference/kolmogorovsmirnovtest.md index e5a577ea365b..6f2e70704d47 100644 --- a/docs/en/sql-reference/aggregate-functions/reference/kolmogorovsmirnovtest.md +++ b/docs/en/sql-reference/aggregate-functions/reference/kolmogorovsmirnovtest.md @@ -115,5 +115,4 @@ P-value is less than 0.05 (for confidence level of 95%), so null hypothesis is r **See Also** -- [Student's t-test](https://en.wikipedia.org/wiki/Student%27s_t-test) -- [welchTTest function](welchttest.md#welchttest) +- [Kolmogorov-Smirnov'test](https://en.wikipedia.org/wiki/Kolmogorov%E2%80%93Smirnov_test) From f74367d74d64fd440c5ae0bf6e8afb2038ddda9d Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Fri, 28 Apr 2023 00:08:26 +0800 Subject: [PATCH 242/406] Fix style check --- tests/clickhouse-test | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/clickhouse-test b/tests/clickhouse-test index e7bf3668ac11..9470f87ff756 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -544,7 +544,9 @@ class SettingsRandomizer: ), "local_filesystem_read_method": lambda: random.choice( # Allow to use uring only when running on Linux - ["read", "pread", "mmap", "pread_threadpool", "io_uring"] if platform.system().lower() == "linux" else ["read", "pread", "mmap", "pread_threadpool"] + ["read", "pread", "mmap", "pread_threadpool", "io_uring"] + if platform.system().lower() == "linux" + else ["read", "pread", "mmap", "pread_threadpool"] ), "remote_filesystem_read_method": lambda: random.choice(["read", "threadpool"]), "local_filesystem_read_prefetch": lambda: random.randint(0, 1), From da72eb630efbefd6620f0b9d0b5d92a63eab67ee Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Thu, 27 Apr 2023 18:14:46 +0200 Subject: [PATCH 243/406] Done --- .../reference/kolmogorovsmirnovtest.md | 2 +- .../reference/kolmogorovsmirnovtest.md | 117 ++++++++++++++++++ .../AggregateFunctionKolmogorovSmirnovTest.h | 10 +- 3 files changed, 123 insertions(+), 6 deletions(-) create mode 100644 docs/ru/sql-reference/aggregate-functions/reference/kolmogorovsmirnovtest.md diff --git a/docs/en/sql-reference/aggregate-functions/reference/kolmogorovsmirnovtest.md b/docs/en/sql-reference/aggregate-functions/reference/kolmogorovsmirnovtest.md index 6f2e70704d47..3da9645181ee 100644 --- a/docs/en/sql-reference/aggregate-functions/reference/kolmogorovsmirnovtest.md +++ b/docs/en/sql-reference/aggregate-functions/reference/kolmogorovsmirnovtest.md @@ -39,7 +39,7 @@ Samples must belong to continuous, one-dimensional probability distributions. Which in fact means that F(x) <= G(x) for all x. And the alternative in this case is that F(x) > G(x) for at least one x. - `computation_method` — the method used to compute p-value. (Optional, default: `'auto'`.) [String](../../../sql-reference/data-types/string.md). - `'exact'` - calculation is performed using precise probability distribution of the test statistics. Compute intensive and wasteful except for small samples. - - `'asymp'` - calculation is performed using an approximation. For large sample sizes, the exact and asymptotic p-values are very similar. + - `'asymp'` (`'asymptotic'`) - calculation is performed using an approximation. For large sample sizes, the exact and asymptotic p-values are very similar. - `'auto'` - the `'exact'` method is used when a maximum number of samples is less than 10'000. diff --git a/docs/ru/sql-reference/aggregate-functions/reference/kolmogorovsmirnovtest.md b/docs/ru/sql-reference/aggregate-functions/reference/kolmogorovsmirnovtest.md new file mode 100644 index 000000000000..2f8c6bb6760c --- /dev/null +++ b/docs/ru/sql-reference/aggregate-functions/reference/kolmogorovsmirnovtest.md @@ -0,0 +1,117 @@ +--- +slug: /ru/sql-reference/aggregate-functions/reference/kolmogorovsmirnovtest +sidebar_position: 300 +sidebar_label: kolmogorovSmirnovTest +--- + +# kolmogorovSmirnovTest {#kolmogorovSmirnovTest} + +Проводит статистический тест Колмогорова-Смирнова для двух независимых выборок. + +**Синтаксис** + +``` sql +kolmogorovSmirnovTest([alternative, computation_method])(sample_data, sample_index) +``` + +Значения выборок берутся из столбца `sample_data`. Если `sample_index` равно 0, то значение из этой строки принадлежит первой выборке. Во всех остальных случаях значение принадлежит второй выборке. +Выборки должны принадлежать непрерывным одномерным распределениям. + +**Аргументы** + +- `sample_data` — данные выборок. [Integer](../../../sql-reference/data-types/int-uint.md), [Float](../../../sql-reference/data-types/float.md) or [Decimal](../../../sql-reference/data-types/decimal.md). +- `sample_index` — индексы выборок. [Integer](../../../sql-reference/data-types/int-uint.md). + +**Параметры** + +- `alternative` — альтернативная гипотеза (Необязательный параметр, по умолчанию: `'two-sided'`.) [String](../../../sql-reference/data-types/string.md). + Пусть F(x) и G(x) - функции распределения первой и второй выборки соотвественно. + - `'two-sided'` + Нулевая гипотеза состоит в том, что выборки происходит из одного и того же распределение, то есть F(x) = G(x) для любого x. + Альтернатива - выборки принадлежат разным распределениям. + - `'greater'` + Нулевая гипотеза состоит в том, что элементы первой выборки в асимптотически почти наверное меньше элементов из второй выборки, + то есть функция распределения первой выборки лежит выше и соотвественно левее, чем функция распределения второй выборки. + Таким образом это означает, что F(x) >= G(x) for любого x, а альтернатива в этом случае состоит в том, что F(x) < G(x) хотя бы для одного x. + - `'less'`. + Нулевая гипотеза состоит в том, что элементы первой выборки в асимптотически почти наверное больше элементов из второй выборки, + то есть функция распределения первой выборки лежит ниже и соотвественно правее, чем функция распределения второй выборки. + Таким образом это означает, что F(x) <= G(x) for любого x, а альтернатива в этом случае состоит в том, что F(x) > G(x) хотя бы для одного x. +- `computation_method` — метод, используемый для вычисления p-value. (Необязательный параметр, по умолчанию: `'auto'`.) [String](../../../sql-reference/data-types/string.md). + - `'exact'` - вычисление производится с помощью вычисления точного распределения статистики. Требует большого количества вычислительных ресурсов и расточительно для больших выборок. + - `'asymp'`(`'asymptotic'`) - используется приближенное вычисление. Для больших выборок приближенный результат и точный почти идентичны. + - `'auto'` - значение вычисляется точно (с помощью метода `'exact'`), если максимальный размер двух выборок не превышает 10'000. + +**Возвращаемые значения** + +[Кортеж](../../../sql-reference/data-types/tuple.md) с двумя элементами: + +- вычисленное статистики. [Float64](../../../sql-reference/data-types/float.md). +- вычисленное p-value. [Float64](../../../sql-reference/data-types/float.md). + + +**Пример** + +Запрос: + +``` sql +SELECT kolmogorovSmirnovTest('less', 'exact')(value, num) +FROM +( + SELECT + randNormal(0, 10) AS value, + 0 AS num + FROM numbers(10000) + UNION ALL + SELECT + randNormal(0, 10) AS value, + 1 AS num + FROM numbers(10000) +) +``` + +Результат: + +``` text +┌─kolmogorovSmirnovTest('less', 'exact')(value, num)─┐ +│ (0.009899999999999996,0.37528595205132287) │ +└────────────────────────────────────────────────────┘ +``` + +Заметки: +P-value больше чем 0.05 (для уровня значимости 95%), то есть нулевая гипотеза не отвергается. + + +Запрос: + +``` sql +SELECT kolmogorovSmirnovTest('two-sided', 'exact')(value, num) +FROM +( + SELECT + randStudentT(10) AS value, + 0 AS num + FROM numbers(100) + UNION ALL + SELECT + randNormal(0, 10) AS value, + 1 AS num + FROM numbers(100) +) +``` + +Результат: + +``` text +┌─kolmogorovSmirnovTest('two-sided', 'exact')(value, num)─┐ +│ (0.4100000000000002,6.61735760482795e-8) │ +└─────────────────────────────────────────────────────────┘ +``` + +Заметки: +P-value меньше чем 0.05 (для уровня значимости 95%), то есть нулевая гипотеза отвергается. + + +**Смотрите также** + +- [Критерий согласия Колмогорова-Смирнова](https://ru.wikipedia.org/wiki/%D0%9A%D1%80%D0%B8%D1%82%D0%B5%D1%80%D0%B8%D0%B9_%D1%81%D0%BE%D0%B3%D0%BB%D0%B0%D1%81%D0%B8%D1%8F_%D0%9A%D0%BE%D0%BB%D0%BC%D0%BE%D0%B3%D0%BE%D1%80%D0%BE%D0%B2%D0%B0) diff --git a/src/AggregateFunctions/AggregateFunctionKolmogorovSmirnovTest.h b/src/AggregateFunctions/AggregateFunctionKolmogorovSmirnovTest.h index 737f37b1fba8..33a9966ee2c7 100644 --- a/src/AggregateFunctions/AggregateFunctionKolmogorovSmirnovTest.h +++ b/src/AggregateFunctions/AggregateFunctionKolmogorovSmirnovTest.h @@ -91,9 +91,9 @@ struct KolmogorovSmirnov : public StatisticalSample UInt64 ny_g = n2 / g; if (method == "auto") - method = std::max(n1, n2) <= 10000 ? "exact" : "asymp"; + method = std::max(n1, n2) <= 10000 ? "exact" : "asymptotic"; else if (method == "exact" && nx_g >= std::numeric_limits::max() / ny_g) - method = "asymp"; + method = "asymptotic"; Float64 p_value = std::numeric_limits::infinity(); @@ -143,7 +143,7 @@ struct KolmogorovSmirnov : public StatisticalSample } p_value = c[n1]; } - else if (method == "asymp") + else if (method == "asymp" || method == "asymptotic") { Float64 n = std::min(n1, n2); Float64 m = std::max(n1, n2); @@ -242,9 +242,9 @@ class AggregateFunctionKolmogorovSmirnov final: throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Aggregate function {} require second parameter to be a String", getName()); method = params[1].get(); - if (method != "auto" && method != "exact" && method != "asymp") + if (method != "auto" && method != "exact" && method != "asymp" && method != "asymptotic") throw Exception(ErrorCodes::BAD_ARGUMENTS, "Unknown method in aggregate function {}. " - "It must be one of: 'auto', 'exact', 'asymp'", getName()); + "It must be one of: 'auto', 'exact', 'asymp' (or 'asymptotic')", getName()); } String getName() const override From fcf0bb2aa07d029ef54c4e18f031eeec76b3451b Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Thu, 27 Apr 2023 17:20:53 +0000 Subject: [PATCH 244/406] Update build guide for nasm, #49254 --- docs/en/development/build.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/en/development/build.md b/docs/en/development/build.md index e3a63da6a3e1..eaae8cceb462 100644 --- a/docs/en/development/build.md +++ b/docs/en/development/build.md @@ -22,7 +22,7 @@ The minimum recommended Ubuntu version for development is 22.04 LTS. ### Install Prerequisites {#install-prerequisites} ``` bash -sudo apt-get install git cmake ccache python3 ninja-build yasm gawk +sudo apt-get install git cmake ccache python3 ninja-build nasm yasm gawk ``` ### Install and Use the Clang compiler @@ -92,7 +92,7 @@ If all the components are installed, you may build in the same way as the steps Example for OpenSUSE Tumbleweed: ``` bash -sudo zypper install git cmake ninja clang-c++ python lld yasm gawk +sudo zypper install git cmake ninja clang-c++ python lld nasm yasm gawk git clone --recursive https://github.com/ClickHouse/ClickHouse.git mkdir build cmake -S . -B build @@ -103,7 +103,7 @@ Example for Fedora Rawhide: ``` bash sudo yum update -sudo yum --nogpg install git cmake make clang python3 ccache yasm gawk +sudo yum --nogpg install git cmake make clang python3 ccache nasm yasm gawk git clone --recursive https://github.com/ClickHouse/ClickHouse.git mkdir build cmake -S . -B build From 742a93f168af6aaf4d530a0ebc865e5e6df3a7a6 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Thu, 27 Apr 2023 17:37:45 +0000 Subject: [PATCH 245/406] Check NASM exists --- contrib/isa-l-cmake/CMakeLists.txt | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/contrib/isa-l-cmake/CMakeLists.txt b/contrib/isa-l-cmake/CMakeLists.txt index fd0218a7b801..0cb8e226fc26 100644 --- a/contrib/isa-l-cmake/CMakeLists.txt +++ b/contrib/isa-l-cmake/CMakeLists.txt @@ -1,6 +1,12 @@ set(ISAL_SOURCE_DIR "${ClickHouse_SOURCE_DIR}/contrib/isa-l") -# check nasm compiler +# The YASM and NASM assembers are somewhat mutually compatible. ISAL specifically needs NASM. If only YASM is installed, then check_language(ASM_NASM) +# below happily finds YASM, leading to weird errors at build time. Therefore, do an explicit check for # NASM here. +find_program(NASM_PATH NAMES nasm) +if (NOT NASM_PATH) + message(FATAL_ERROR "Please install NASM from 'https://www.nasm.us/' because NASM compiler can not be found!") +endif () + include(CheckLanguage) check_language(ASM_NASM) if(NOT CMAKE_ASM_NASM_COMPILER) From 744acc943ca2ad273c971a52b946591c8e2d33b3 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Thu, 27 Apr 2023 19:43:51 +0200 Subject: [PATCH 246/406] Update CMakeLists.txt --- contrib/isa-l-cmake/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/isa-l-cmake/CMakeLists.txt b/contrib/isa-l-cmake/CMakeLists.txt index 0cb8e226fc26..5c549cbba02a 100644 --- a/contrib/isa-l-cmake/CMakeLists.txt +++ b/contrib/isa-l-cmake/CMakeLists.txt @@ -1,7 +1,7 @@ set(ISAL_SOURCE_DIR "${ClickHouse_SOURCE_DIR}/contrib/isa-l") # The YASM and NASM assembers are somewhat mutually compatible. ISAL specifically needs NASM. If only YASM is installed, then check_language(ASM_NASM) -# below happily finds YASM, leading to weird errors at build time. Therefore, do an explicit check for # NASM here. +# below happily finds YASM, leading to weird errors at build time. Therefore, do an explicit check for NASM here. find_program(NASM_PATH NAMES nasm) if (NOT NASM_PATH) message(FATAL_ERROR "Please install NASM from 'https://www.nasm.us/' because NASM compiler can not be found!") From f7404d7e724956629e9146b48d03e685ee6e2732 Mon Sep 17 00:00:00 2001 From: kssenii Date: Thu, 27 Apr 2023 21:40:55 +0200 Subject: [PATCH 247/406] Fix --- src/Interpreters/Cache/FileCache.cpp | 8 +++++++- src/Interpreters/Cache/FileSegment.cpp | 15 ++++++--------- src/Interpreters/Cache/Metadata.h | 1 + 3 files changed, 14 insertions(+), 10 deletions(-) diff --git a/src/Interpreters/Cache/FileCache.cpp b/src/Interpreters/Cache/FileCache.cpp index ab00db789783..ad77a03c2339 100644 --- a/src/Interpreters/Cache/FileCache.cpp +++ b/src/Interpreters/Cache/FileCache.cpp @@ -653,7 +653,13 @@ bool FileCache::tryReserve(FileSegment & file_segment, size_t size) for (auto & [current_key, deletion_info] : to_delete) { - auto locked_key = deletion_info.getMetadata().lock(); + auto locked_key = deletion_info.getMetadata().tryLock(); + if (!locked_key) + { + /// key could become invalid after we released the key lock above, just skip it. + chassert(locked_key->getKeyState() != KeyMetadata::KeyState::ACTIVE); + continue; + } for (auto it = deletion_info.begin(); it != deletion_info.end();) { chassert((*it)->releasable()); diff --git a/src/Interpreters/Cache/FileSegment.cpp b/src/Interpreters/Cache/FileSegment.cpp index 65bfc8657aaf..b05847a1ce75 100644 --- a/src/Interpreters/Cache/FileSegment.cpp +++ b/src/Interpreters/Cache/FileSegment.cpp @@ -431,16 +431,13 @@ KeyMetadataPtr FileSegment::tryGetKeyMetadata() const LockedKeyPtr FileSegment::lockKeyMetadata(bool assert_exists) const { - KeyMetadataPtr metadata; if (assert_exists) - metadata = getKeyMetadata(); - else - { - metadata = tryGetKeyMetadata(); - if (!metadata) - return nullptr; - } - return metadata->lock(); + return getKeyMetadata()->lock(); + + auto metadata = tryGetKeyMetadata(); + if (!metadata) + return nullptr; + return metadata->tryLock(); } bool FileSegment::reserve(size_t size_to_reserve) diff --git a/src/Interpreters/Cache/Metadata.h b/src/Interpreters/Cache/Metadata.h index a0effc26516d..586c7e5c2a89 100644 --- a/src/Interpreters/Cache/Metadata.h +++ b/src/Interpreters/Cache/Metadata.h @@ -58,6 +58,7 @@ struct KeyMetadata : public std::map, LockedKeyPtr lock(); + /// Return nullptr if key has non-ACTIVE state. LockedKeyPtr tryLock(); bool createBaseDirectory(); From 4dccd5f9b0e493a0699d71cd8c9de7fe450d12e4 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 27 Apr 2023 23:20:38 +0300 Subject: [PATCH 248/406] Update AsynchronousMetrics.cpp --- src/Common/AsynchronousMetrics.cpp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/Common/AsynchronousMetrics.cpp b/src/Common/AsynchronousMetrics.cpp index ef144598648c..540015229bf5 100644 --- a/src/Common/AsynchronousMetrics.cpp +++ b/src/Common/AsynchronousMetrics.cpp @@ -909,17 +909,17 @@ void AsynchronousMetrics::update(TimePoint update_time) { try { - memory_limit_in.rewind(); - memory_usage_in.rewind(); + cgroupmem_limit_in_bytes.rewind(); + cgroupmem_usage_in_bytes.rewind(); - uint64_t cgroup_mem_limit_in_bytes = 0; - uint64_t cgroup_mem_usage_in_bytes = 0; + uint64_t limit = 0; + uint64_t usage = 0; - tryReadText(cgroup_mem_limit_in_bytes, memory_limit_in); - tryReadText(cgroup_mem_usage_in_bytes, memory_usage_in); + tryReadText(limit, cgroupmem_limit_in_bytes); + tryReadText(usage, cgroupmem_usage_in_bytes); - new_values["CgroupMemoryTotal"] = { cgroup_mem_limit_in_bytes, "The total amount of memory in cgroup, in bytes. If stated zero, the limit is the same as OSMemoryTotal." }; - new_values["CgroupMemoryUsed"] = { cgroup_mem_usage_in_bytes, "The amount of memory used in cgroup, in bytes." }; + new_values["CGroupMemoryTotal"] = { limit, "The total amount of memory in cgroup, in bytes. If stated zero, the limit is the same as OSMemoryTotal." }; + new_values["CGroupMemoryUsed"] = { usage, "The amount of memory used in cgroup, in bytes." }; } catch (...) { From 2e14b91b9250507808356e3c888317702ecfb2bc Mon Sep 17 00:00:00 2001 From: Feng Kaiyu Date: Fri, 28 Apr 2023 16:53:19 +0800 Subject: [PATCH 249/406] docs: update build.md, add missing argument value. `--build` need an option (`build`). --- docs/en/development/build.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/development/build.md b/docs/en/development/build.md index eaae8cceb462..a55d44bdf939 100644 --- a/docs/en/development/build.md +++ b/docs/en/development/build.md @@ -72,7 +72,7 @@ cmake -S . -B build cmake --build build # or: `cd build; ninja` ``` -To create an executable, run `cmake --build --target clickhouse` (or: `cd build; ninja clickhouse`). +To create an executable, run `cmake --build build --target clickhouse` (or: `cd build; ninja clickhouse`). This will create executable `build/programs/clickhouse` which can be used with `client` or `server` arguments. ## Building on Any Linux {#how-to-build-clickhouse-on-any-linux} From c8ec152f573e252da1fca94ded56e6c5ba6d867b Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Wed, 5 Apr 2023 22:21:19 +0200 Subject: [PATCH 250/406] Add ability to use strict parts size for S3 (compatibility with R2) The are some implementations that does not allow to use parts of different size for multi-part upload: - CouldFlare R2 [1] and also got help on discord - AWS S3 Glacier [2] - and I also know of one bug in Google Cloud Storage (GCS), but it should be fixed in a couple of months. [1]: https://developers.cloudflare.com/r2/reference/changelog/#2022-10-28 > Multipart upload part sizes are always expected to be of the > same size, but this enforcement is now done when you complete > an upload instead of being done very time you upload a part [2]: https://docs.aws.amazon.com/amazonglacier/latest/dev/api-multipart-initiate-upload.html On top of this I'm going to add auto detection of this issue for disks, but not for S3 function, since those checks are quite costly, and they need to write at least 5MB*3, which may be questionable. v2: mark test as long [1] [1]: https://s3.amazonaws.com/clickhouse-test-reports/48492/d4a90880e947ab3a17b3eddeea274afed8ebceb4/stateless_tests_flaky_check__asan_.html Signed-off-by: Azat Khuzhin --- src/Core/Settings.h | 1 + src/IO/WriteBufferFromS3.cpp | 87 +++++++++++++++---- src/IO/WriteBufferFromS3.h | 8 +- src/Storages/StorageS3Settings.cpp | 13 ++- src/Storages/StorageS3Settings.h | 1 + ...02720_s3_strict_upload_part_size.reference | 4 + .../02720_s3_strict_upload_part_size.sh | 25 ++++++ 7 files changed, 121 insertions(+), 18 deletions(-) create mode 100644 tests/queries/0_stateless/02720_s3_strict_upload_part_size.reference create mode 100755 tests/queries/0_stateless/02720_s3_strict_upload_part_size.sh diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 7f1fe838b802..aa5e69ce571f 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -71,6 +71,7 @@ class IColumn; M(UInt64, idle_connection_timeout, 3600, "Close idle TCP connections after specified number of seconds.", 0) \ M(UInt64, distributed_connections_pool_size, 1024, "Maximum number of connections with one remote server in the pool.", 0) \ M(UInt64, connections_with_failover_max_tries, DBMS_CONNECTION_POOL_WITH_FAILOVER_DEFAULT_MAX_TRIES, "The maximum number of attempts to connect to replicas.", 0) \ + M(UInt64, s3_strict_upload_part_size, 0, "The exact size of part to upload during multipart upload to S3 (some implementations does not supports variable size parts).", 0) \ M(UInt64, s3_min_upload_part_size, 16*1024*1024, "The minimum size of part to upload during multipart upload to S3.", 0) \ M(UInt64, s3_max_upload_part_size, 5ull*1024*1024*1024, "The maximum size of part to upload during multipart upload to S3.", 0) \ M(UInt64, s3_upload_part_size_multiply_factor, 2, "Multiply s3_min_upload_part_size by this factor each time s3_multiply_parts_count_threshold parts were uploaded from a single write to S3.", 0) \ diff --git a/src/IO/WriteBufferFromS3.cpp b/src/IO/WriteBufferFromS3.cpp index 4c1b1b65d19c..19f25b1617ff 100644 --- a/src/IO/WriteBufferFromS3.cpp +++ b/src/IO/WriteBufferFromS3.cpp @@ -85,7 +85,8 @@ WriteBufferFromS3::WriteBufferFromS3( , upload_settings(request_settings.getUploadSettings()) , client_ptr(std::move(client_ptr_)) , object_metadata(std::move(object_metadata_)) - , upload_part_size(upload_settings.min_upload_part_size) + , strict_upload_part_size(upload_settings.strict_upload_part_size) + , current_upload_part_size(upload_settings.min_upload_part_size) , schedule(std::move(schedule_)) , write_settings(write_settings_) { @@ -100,28 +101,79 @@ void WriteBufferFromS3::nextImpl() /// Buffer in a bad state after exception if (temporary_buffer->tellp() == -1) allocateBuffer(); + else + chassert(temporary_buffer->tellp() == static_cast(last_part_size)); + + if (strict_upload_part_size) + processWithStrictParts(); + else + processWithDynamicParts(); + + waitForReadyBackGroundTasks(); +} + +void WriteBufferFromS3::processWithStrictParts() +{ + chassert(strict_upload_part_size > 0); + + size_t buffer_size = offset(); + size_t left_in_buffer = buffer_size; + size_t new_size = last_part_size + buffer_size; + size_t buffer_offset = 0; + + if (new_size > strict_upload_part_size) + { + /// Data size will exceed fixed part size threshold for multipart upload, need to use multipart upload. + if (multipart_upload_id.empty()) + createMultipartUpload(); + + while (new_size > strict_upload_part_size) + { + size_t to_write = strict_upload_part_size - last_part_size; + temporary_buffer->write(working_buffer.begin() + buffer_offset, to_write); + buffer_offset += to_write; + + writePart(); + allocateBuffer(); + + new_size -= strict_upload_part_size; + left_in_buffer -= to_write; + } + } + + if (left_in_buffer) + { + temporary_buffer->write(working_buffer.begin() + buffer_offset, left_in_buffer); + last_part_size += left_in_buffer; + } + + ProfileEvents::increment(ProfileEvents::WriteBufferFromS3Bytes, buffer_size); + + if (write_settings.remote_throttler) + write_settings.remote_throttler->add(buffer_size, ProfileEvents::RemoteWriteThrottlerBytes, ProfileEvents::RemoteWriteThrottlerSleepMicroseconds); +} + +void WriteBufferFromS3::processWithDynamicParts() +{ + chassert(current_upload_part_size > 0); size_t size = offset(); temporary_buffer->write(working_buffer.begin(), size); + ProfileEvents::increment(ProfileEvents::WriteBufferFromS3Bytes, size); + last_part_size += size; - ProfileEvents::increment(ProfileEvents::WriteBufferFromS3Bytes, offset()); - last_part_size += offset(); if (write_settings.remote_throttler) - write_settings.remote_throttler->add(offset(), ProfileEvents::RemoteWriteThrottlerBytes, ProfileEvents::RemoteWriteThrottlerSleepMicroseconds); + write_settings.remote_throttler->add(size, ProfileEvents::RemoteWriteThrottlerBytes, ProfileEvents::RemoteWriteThrottlerSleepMicroseconds); /// Data size exceeds singlepart upload threshold, need to use multipart upload. if (multipart_upload_id.empty() && last_part_size > upload_settings.max_single_part_upload_size) createMultipartUpload(); - chassert(upload_part_size > 0); - if (!multipart_upload_id.empty() && last_part_size > upload_part_size) + if (!multipart_upload_id.empty() && last_part_size > current_upload_part_size) { writePart(); - allocateBuffer(); } - - waitForReadyBackGroundTasks(); } void WriteBufferFromS3::allocateBuffer() @@ -335,14 +387,17 @@ void WriteBufferFromS3::fillUploadRequest(S3::UploadPartRequest & req) /// If we don't do it, AWS SDK can mistakenly set it to application/xml, see https://github.com/aws/aws-sdk-cpp/issues/1840 req.SetContentType("binary/octet-stream"); - /// Maybe increase `upload_part_size` (we need to increase it sometimes to keep `part_number` less or equal than `max_part_number`). - auto threshold = upload_settings.upload_part_size_multiply_parts_count_threshold; - if (!multipart_upload_id.empty() && (part_number % threshold == 0)) + if (!strict_upload_part_size) { - auto max_upload_part_size = upload_settings.max_upload_part_size; - auto upload_part_size_multiply_factor = upload_settings.upload_part_size_multiply_factor; - upload_part_size *= upload_part_size_multiply_factor; - upload_part_size = std::min(upload_part_size, max_upload_part_size); + /// Maybe increase `current_upload_part_size` (we need to increase it sometimes to keep `part_number` less or equal than `max_part_number`). + auto threshold = upload_settings.upload_part_size_multiply_parts_count_threshold; + if (!multipart_upload_id.empty() && (part_number % threshold == 0)) + { + auto max_upload_part_size = upload_settings.max_upload_part_size; + auto upload_part_size_multiply_factor = upload_settings.upload_part_size_multiply_factor; + current_upload_part_size *= upload_part_size_multiply_factor; + current_upload_part_size = std::min(current_upload_part_size, max_upload_part_size); + } } } diff --git a/src/IO/WriteBufferFromS3.h b/src/IO/WriteBufferFromS3.h index 5fa934b886e1..e57b8c159a24 100644 --- a/src/IO/WriteBufferFromS3.h +++ b/src/IO/WriteBufferFromS3.h @@ -58,6 +58,9 @@ class WriteBufferFromS3 final : public BufferWithOwnMemory private: void allocateBuffer(); + void processWithStrictParts(); + void processWithDynamicParts(); + void createMultipartUpload(); void writePart(); void completeMultipartUpload(); @@ -86,7 +89,10 @@ class WriteBufferFromS3 final : public BufferWithOwnMemory const std::shared_ptr client_ptr; const std::optional> object_metadata; - size_t upload_part_size = 0; + /// Strict/static Part size, no adjustments will be done on fly. + size_t strict_upload_part_size = 0; + /// Part size will be adjusted on fly (for bigger uploads) + size_t current_upload_part_size = 0; std::shared_ptr temporary_buffer; /// Buffer to accumulate data. size_t last_part_size = 0; size_t part_number = 0; diff --git a/src/Storages/StorageS3Settings.cpp b/src/Storages/StorageS3Settings.cpp index 17a11ba98480..4db44b7b4f78 100644 --- a/src/Storages/StorageS3Settings.cpp +++ b/src/Storages/StorageS3Settings.cpp @@ -32,6 +32,7 @@ S3Settings::RequestSettings::PartUploadSettings::PartUploadSettings( : PartUploadSettings(settings) { String key = config_prefix + "." + setting_name_prefix; + strict_upload_part_size = config.getUInt64(key + "strict_upload_part_size", strict_upload_part_size); min_upload_part_size = config.getUInt64(key + "min_upload_part_size", min_upload_part_size); max_upload_part_size = config.getUInt64(key + "max_upload_part_size", max_upload_part_size); upload_part_size_multiply_factor = config.getUInt64(key + "upload_part_size_multiply_factor", upload_part_size_multiply_factor); @@ -49,10 +50,11 @@ S3Settings::RequestSettings::PartUploadSettings::PartUploadSettings( S3Settings::RequestSettings::PartUploadSettings::PartUploadSettings(const NamedCollection & collection) { + strict_upload_part_size = collection.getOrDefault("strict_upload_part_size", strict_upload_part_size); min_upload_part_size = collection.getOrDefault("min_upload_part_size", min_upload_part_size); + max_single_part_upload_size = collection.getOrDefault("max_single_part_upload_size", max_single_part_upload_size); upload_part_size_multiply_factor = collection.getOrDefault("upload_part_size_multiply_factor", upload_part_size_multiply_factor); upload_part_size_multiply_parts_count_threshold = collection.getOrDefault("upload_part_size_multiply_parts_count_threshold", upload_part_size_multiply_parts_count_threshold); - max_single_part_upload_size = collection.getOrDefault("max_single_part_upload_size", max_single_part_upload_size); /// This configuration is only applicable to s3. Other types of object storage are not applicable or have different meanings. storage_class_name = collection.getOrDefault("s3_storage_class", storage_class_name); @@ -63,6 +65,9 @@ S3Settings::RequestSettings::PartUploadSettings::PartUploadSettings(const NamedC void S3Settings::RequestSettings::PartUploadSettings::updateFromSettingsImpl(const Settings & settings, bool if_changed) { + if (!if_changed || settings.s3_strict_upload_part_size.changed) + strict_upload_part_size = settings.s3_strict_upload_part_size; + if (!if_changed || settings.s3_min_upload_part_size.changed) min_upload_part_size = settings.s3_min_upload_part_size; @@ -82,6 +87,12 @@ void S3Settings::RequestSettings::PartUploadSettings::updateFromSettingsImpl(con void S3Settings::RequestSettings::PartUploadSettings::validate() { static constexpr size_t min_upload_part_size_limit = 5 * 1024 * 1024; + if (strict_upload_part_size && strict_upload_part_size < min_upload_part_size_limit) + throw Exception( + ErrorCodes::INVALID_SETTING_VALUE, + "Setting strict_upload_part_size has invalid value {} which is less than the s3 API limit {}", + ReadableSize(strict_upload_part_size), ReadableSize(min_upload_part_size_limit)); + if (min_upload_part_size < min_upload_part_size_limit) throw Exception( ErrorCodes::INVALID_SETTING_VALUE, diff --git a/src/Storages/StorageS3Settings.h b/src/Storages/StorageS3Settings.h index 49cb481626d3..cd5be1626b54 100644 --- a/src/Storages/StorageS3Settings.h +++ b/src/Storages/StorageS3Settings.h @@ -28,6 +28,7 @@ struct S3Settings { struct PartUploadSettings { + size_t strict_upload_part_size = 0; size_t min_upload_part_size = 16 * 1024 * 1024; size_t max_upload_part_size = 5ULL * 1024 * 1024 * 1024; size_t upload_part_size_multiply_factor = 2; diff --git a/tests/queries/0_stateless/02720_s3_strict_upload_part_size.reference b/tests/queries/0_stateless/02720_s3_strict_upload_part_size.reference new file mode 100644 index 000000000000..360b484bf28d --- /dev/null +++ b/tests/queries/0_stateless/02720_s3_strict_upload_part_size.reference @@ -0,0 +1,4 @@ +Size: 6000001 +Size: 6000001 +Size: 6000001 +Size: 2971517 diff --git a/tests/queries/0_stateless/02720_s3_strict_upload_part_size.sh b/tests/queries/0_stateless/02720_s3_strict_upload_part_size.sh new file mode 100755 index 000000000000..69e2f7349144 --- /dev/null +++ b/tests/queries/0_stateless/02720_s3_strict_upload_part_size.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, long +# Tag no-fasttest: requires S3 + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +in="$CUR_DIR/$CLICKHOUSE_TEST_UNIQUE_NAME.in" +out="$CUR_DIR/$CLICKHOUSE_TEST_UNIQUE_NAME.out" +log="$CUR_DIR/$CLICKHOUSE_TEST_UNIQUE_NAME.log" + +set -e +trap 'rm -f "${out:?}" "${in:?}" "${log:?}"' EXIT + +# Generate a file of 20MiB in size, with our part size it will have 4 parts +# NOTE: 1 byte is for new line, so 1023 not 1024 +$CLICKHOUSE_LOCAL -q "SELECT randomPrintableASCII(1023) FROM numbers(20*1024) FORMAT LineAsString" > "$in" + +$CLICKHOUSE_CLIENT --send_logs_level=trace --server_logs_file="$log" -q "INSERT INTO FUNCTION s3(s3_conn, filename='$CLICKHOUSE_TEST_UNIQUE_NAME', format='LineAsString', structure='line String') FORMAT LineAsString" --s3_strict_upload_part_size=6000001 < "$in" +grep -F '' "$log" || : +grep -o 'WriteBufferFromS3: Writing part.*Size: .*' "$log" | grep -o 'Size: .*' +$CLICKHOUSE_CLIENT -q "SELECT * FROM s3(s3_conn, filename='$CLICKHOUSE_TEST_UNIQUE_NAME', format='LineAsString', structure='line String') FORMAT LineAsString" > "$out" + +diff -q "$in" "$out" From bb4a7f65de5ccb9250dcf1d6eb6ee9d20c9da8ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Fri, 28 Apr 2023 13:25:53 +0200 Subject: [PATCH 251/406] Slight improvements to coordinator logging (#49204) --- .../ParallelReplicasReadingCoordinator.cpp | 10 ++++---- src/Storages/MergeTree/RangesInDataPart.cpp | 25 ++++++++++++------- src/Storages/MergeTree/RequestResponse.cpp | 5 +--- 3 files changed, 22 insertions(+), 18 deletions(-) diff --git a/src/Storages/MergeTree/ParallelReplicasReadingCoordinator.cpp b/src/Storages/MergeTree/ParallelReplicasReadingCoordinator.cpp index 6ad0628eac45..77c280d4710b 100644 --- a/src/Storages/MergeTree/ParallelReplicasReadingCoordinator.cpp +++ b/src/Storages/MergeTree/ParallelReplicasReadingCoordinator.cpp @@ -131,7 +131,7 @@ class DefaultCoordinator : public ParallelReplicasReadingCoordinator::ImplInterf DefaultCoordinator::~DefaultCoordinator() { - LOG_INFO(log, "Coordination done: {}", toString(stats)); + LOG_DEBUG(log, "Coordination done: {}", toString(stats)); } void DefaultCoordinator::updateReadingState(const InitialAllRangesAnnouncement & announcement) @@ -214,7 +214,7 @@ void DefaultCoordinator::finalizeReadingState() description += fmt::format("Replicas: ({}) --- ", fmt::join(part.replicas, ",")); } - LOG_INFO(log, "Reading state is fully initialized: {}", description); + LOG_DEBUG(log, "Reading state is fully initialized: {}", description); } @@ -228,7 +228,7 @@ void DefaultCoordinator::handleInitialAllRangesAnnouncement(InitialAllRangesAnno stats[announcement.replica_num].number_of_requests +=1; ++sent_initial_requests; - LOG_INFO(log, "{} {}", sent_initial_requests, replicas_count); + LOG_DEBUG(log, "Sent initial requests: {} Replicas count: {}", sent_initial_requests, replicas_count); if (sent_initial_requests == replicas_count) finalizeReadingState(); } @@ -334,7 +334,7 @@ class InOrderCoordinator : public ParallelReplicasReadingCoordinator::ImplInterf {} ~InOrderCoordinator() override { - LOG_INFO(log, "Coordination done: {}", toString(stats)); + LOG_DEBUG(log, "Coordination done: {}", toString(stats)); } ParallelReadResponse handleRequest([[ maybe_unused ]] ParallelReadRequest request) override; @@ -349,7 +349,7 @@ class InOrderCoordinator : public ParallelReplicasReadingCoordinator::ImplInterf template void InOrderCoordinator::handleInitialAllRangesAnnouncement(InitialAllRangesAnnouncement announcement) { - LOG_TRACE(log, "Received an announecement {}", announcement.describe()); + LOG_TRACE(log, "Received an announcement {}", announcement.describe()); /// To get rid of duplicates for (const auto & part: announcement.description) diff --git a/src/Storages/MergeTree/RangesInDataPart.cpp b/src/Storages/MergeTree/RangesInDataPart.cpp index ab76611a5073..6203f9f74832 100644 --- a/src/Storages/MergeTree/RangesInDataPart.cpp +++ b/src/Storages/MergeTree/RangesInDataPart.cpp @@ -1,12 +1,23 @@ #include -#include - -#include "IO/VarInt.h" +#include #include #include +#include +#include "IO/VarInt.h" +template <> +struct fmt::formatter +{ + static constexpr auto parse(format_parse_context & ctx) { return ctx.begin(); } + + template + auto format(const DB::RangesInDataPartDescription & range, FormatContext & ctx) + { + return format_to(ctx.out(), "{}", range.describe()); + } +}; namespace DB { @@ -26,8 +37,7 @@ void RangesInDataPartDescription::serialize(WriteBuffer & out) const String RangesInDataPartDescription::describe() const { String result; - result += fmt::format("Part: {}, ", info.getPartNameV1()); - result += fmt::format("Ranges: [{}], ", fmt::join(ranges, ",")); + result += fmt::format("part {} with ranges [{}]", info.getPartNameV1(), fmt::join(ranges, ",")); return result; } @@ -46,10 +56,7 @@ void RangesInDataPartsDescription::serialize(WriteBuffer & out) const String RangesInDataPartsDescription::describe() const { - String result; - for (const auto & desc : *this) - result += desc.describe() + ","; - return result; + return fmt::format("{} parts: [{}]", this->size(), fmt::join(*this, ", ")); } void RangesInDataPartsDescription::deserialize(ReadBuffer & in) diff --git a/src/Storages/MergeTree/RequestResponse.cpp b/src/Storages/MergeTree/RequestResponse.cpp index 945477c5a34d..05930d5a4c41 100644 --- a/src/Storages/MergeTree/RequestResponse.cpp +++ b/src/Storages/MergeTree/RequestResponse.cpp @@ -88,10 +88,7 @@ void ParallelReadResponse::serialize(WriteBuffer & out) const String ParallelReadResponse::describe() const { - String result; - result += fmt::format("finish: {} \n", finish); - result += description.describe(); - return result; + return fmt::format("{}. Finish: {}", description.describe(), finish); } void ParallelReadResponse::deserialize(ReadBuffer & in) From 7f91105e7d67faa2ad44062cf40bfb2aafe2d7f6 Mon Sep 17 00:00:00 2001 From: Boris Kuschel Date: Thu, 13 Apr 2023 14:17:03 -0700 Subject: [PATCH 252/406] Update curl to 8.0.1 --- contrib/curl | 2 +- contrib/curl-cmake/CMakeLists.txt | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/contrib/curl b/contrib/curl index c12fb3ddaf48..b16d1fa8ee56 160000 --- a/contrib/curl +++ b/contrib/curl @@ -1 +1 @@ -Subproject commit c12fb3ddaf48e709a7a4deaa55ec485e4df163ee +Subproject commit b16d1fa8ee567b52c09a0f89940b07d8491b881d diff --git a/contrib/curl-cmake/CMakeLists.txt b/contrib/curl-cmake/CMakeLists.txt index 8a570bd267c7..bd0efa115d5b 100644 --- a/contrib/curl-cmake/CMakeLists.txt +++ b/contrib/curl-cmake/CMakeLists.txt @@ -38,7 +38,7 @@ set (SRCS "${LIBRARY_DIR}/lib/easy.c" "${LIBRARY_DIR}/lib/curl_fnmatch.c" "${LIBRARY_DIR}/lib/fileinfo.c" - "${LIBRARY_DIR}/lib/wildcard.c" +# "${LIBRARY_DIR}/lib/wildcard.c" "${LIBRARY_DIR}/lib/krb5.c" "${LIBRARY_DIR}/lib/memdebug.c" "${LIBRARY_DIR}/lib/http_chunks.c" @@ -147,9 +147,9 @@ set (SRCS "${LIBRARY_DIR}/lib/vtls/keylog.c" "${LIBRARY_DIR}/lib/vtls/x509asn1.c" "${LIBRARY_DIR}/lib/vtls/hostcheck.c" - "${LIBRARY_DIR}/lib/vquic/ngtcp2.c" - "${LIBRARY_DIR}/lib/vquic/quiche.c" - "${LIBRARY_DIR}/lib/vquic/msh3.c" +# "${LIBRARY_DIR}/lib/vquic/ngtcp2.c" +# "${LIBRARY_DIR}/lib/vquic/quiche.c" +# "${LIBRARY_DIR}/lib/vquic/msh3.c" "${LIBRARY_DIR}/lib/vssh/libssh2.c" "${LIBRARY_DIR}/lib/vssh/libssh.c" ) From 2246306370a9ff4fc4f476106d821dcda1d3ce20 Mon Sep 17 00:00:00 2001 From: Boris Kuschel Date: Thu, 13 Apr 2023 14:21:56 -0700 Subject: [PATCH 253/406] remove commented lines --- contrib/curl-cmake/CMakeLists.txt | 4 ---- 1 file changed, 4 deletions(-) diff --git a/contrib/curl-cmake/CMakeLists.txt b/contrib/curl-cmake/CMakeLists.txt index bd0efa115d5b..e3c64266d5d3 100644 --- a/contrib/curl-cmake/CMakeLists.txt +++ b/contrib/curl-cmake/CMakeLists.txt @@ -38,7 +38,6 @@ set (SRCS "${LIBRARY_DIR}/lib/easy.c" "${LIBRARY_DIR}/lib/curl_fnmatch.c" "${LIBRARY_DIR}/lib/fileinfo.c" -# "${LIBRARY_DIR}/lib/wildcard.c" "${LIBRARY_DIR}/lib/krb5.c" "${LIBRARY_DIR}/lib/memdebug.c" "${LIBRARY_DIR}/lib/http_chunks.c" @@ -147,9 +146,6 @@ set (SRCS "${LIBRARY_DIR}/lib/vtls/keylog.c" "${LIBRARY_DIR}/lib/vtls/x509asn1.c" "${LIBRARY_DIR}/lib/vtls/hostcheck.c" -# "${LIBRARY_DIR}/lib/vquic/ngtcp2.c" -# "${LIBRARY_DIR}/lib/vquic/quiche.c" -# "${LIBRARY_DIR}/lib/vquic/msh3.c" "${LIBRARY_DIR}/lib/vssh/libssh2.c" "${LIBRARY_DIR}/lib/vssh/libssh.c" ) From 449f63a2efa443e7e9a628653574ba9377f1e998 Mon Sep 17 00:00:00 2001 From: Boris Kuschel Date: Fri, 14 Apr 2023 13:04:48 -0700 Subject: [PATCH 254/406] Add missing files --- contrib/curl-cmake/CMakeLists.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/contrib/curl-cmake/CMakeLists.txt b/contrib/curl-cmake/CMakeLists.txt index e3c64266d5d3..0451614193cc 100644 --- a/contrib/curl-cmake/CMakeLists.txt +++ b/contrib/curl-cmake/CMakeLists.txt @@ -12,6 +12,8 @@ set (SRCS "${LIBRARY_DIR}/lib/noproxy.c" "${LIBRARY_DIR}/lib/idn.c" "${LIBRARY_DIR}/lib/cfilters.c" + "${LIBRARY_DIR}/lib/cf-socket.c" + "${LIBRARY_DIR}/lib/cf-https-connect.c" "${LIBRARY_DIR}/lib/file.c" "${LIBRARY_DIR}/lib/timeval.c" "${LIBRARY_DIR}/lib/base64.c" @@ -37,6 +39,7 @@ set (SRCS "${LIBRARY_DIR}/lib/strcase.c" "${LIBRARY_DIR}/lib/easy.c" "${LIBRARY_DIR}/lib/curl_fnmatch.c" + "${LIBRARY_DIR}/lib/curl_log.c" "${LIBRARY_DIR}/lib/fileinfo.c" "${LIBRARY_DIR}/lib/krb5.c" "${LIBRARY_DIR}/lib/memdebug.c" From dcfee036a8b91c126ea6281a8180a22b12a39f7a Mon Sep 17 00:00:00 2001 From: Boris Kuschel Date: Fri, 14 Apr 2023 14:24:39 -0700 Subject: [PATCH 255/406] One more file --- contrib/curl-cmake/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/contrib/curl-cmake/CMakeLists.txt b/contrib/curl-cmake/CMakeLists.txt index 0451614193cc..757ecdaba4f7 100644 --- a/contrib/curl-cmake/CMakeLists.txt +++ b/contrib/curl-cmake/CMakeLists.txt @@ -135,6 +135,7 @@ set (SRCS "${LIBRARY_DIR}/lib/vauth/oauth2.c" "${LIBRARY_DIR}/lib/vauth/spnego_gssapi.c" "${LIBRARY_DIR}/lib/vauth/spnego_sspi.c" + "${LIBRARY_DIR}/lib/vquic/vquic.c" "${LIBRARY_DIR}/lib/vtls/openssl.c" "${LIBRARY_DIR}/lib/vtls/gtls.c" "${LIBRARY_DIR}/lib/vtls/vtls.c" From e640d79765c50b01306d161bea61acb50c7d1791 Mon Sep 17 00:00:00 2001 From: Boris Kuschel Date: Fri, 28 Apr 2023 04:54:42 -0700 Subject: [PATCH 256/406] Update Curl to latest --- contrib/curl | 2 +- contrib/curl-cmake/CMakeLists.txt | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/contrib/curl b/contrib/curl index b16d1fa8ee56..b0edf0b7dae4 160000 --- a/contrib/curl +++ b/contrib/curl @@ -1 +1 @@ -Subproject commit b16d1fa8ee567b52c09a0f89940b07d8491b881d +Subproject commit b0edf0b7dae44d9e66f270a257cf654b35d5263d diff --git a/contrib/curl-cmake/CMakeLists.txt b/contrib/curl-cmake/CMakeLists.txt index 757ecdaba4f7..70d9c2816dc1 100644 --- a/contrib/curl-cmake/CMakeLists.txt +++ b/contrib/curl-cmake/CMakeLists.txt @@ -13,6 +13,7 @@ set (SRCS "${LIBRARY_DIR}/lib/idn.c" "${LIBRARY_DIR}/lib/cfilters.c" "${LIBRARY_DIR}/lib/cf-socket.c" + "${LIBRARY_DIR}/lib/cf-haproxy.c" "${LIBRARY_DIR}/lib/cf-https-connect.c" "${LIBRARY_DIR}/lib/file.c" "${LIBRARY_DIR}/lib/timeval.c" @@ -98,6 +99,7 @@ set (SRCS "${LIBRARY_DIR}/lib/rand.c" "${LIBRARY_DIR}/lib/curl_multibyte.c" "${LIBRARY_DIR}/lib/conncache.c" + "${LIBRARY_DIR}/lib/cf-h1-proxy.c" "${LIBRARY_DIR}/lib/http2.c" "${LIBRARY_DIR}/lib/smb.c" "${LIBRARY_DIR}/lib/curl_endian.c" @@ -115,12 +117,13 @@ set (SRCS "${LIBRARY_DIR}/lib/altsvc.c" "${LIBRARY_DIR}/lib/socketpair.c" "${LIBRARY_DIR}/lib/bufref.c" + "${LIBRARY_DIR}/lib/bufq.c" "${LIBRARY_DIR}/lib/dynbuf.c" + "${LIBRARY_DIR}/lib/dynhds.c" "${LIBRARY_DIR}/lib/hsts.c" "${LIBRARY_DIR}/lib/http_aws_sigv4.c" "${LIBRARY_DIR}/lib/mqtt.c" "${LIBRARY_DIR}/lib/rename.c" - "${LIBRARY_DIR}/lib/h2h3.c" "${LIBRARY_DIR}/lib/headers.c" "${LIBRARY_DIR}/lib/timediff.c" "${LIBRARY_DIR}/lib/vauth/vauth.c" From 5c9959af49889f86fddcae7a751ad6dbd9e793be Mon Sep 17 00:00:00 2001 From: MikhailBurdukov Date: Fri, 28 Apr 2023 12:40:47 +0000 Subject: [PATCH 257/406] Resolve conservation --- src/Dictionaries/MongoDBDictionarySource.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Dictionaries/MongoDBDictionarySource.cpp b/src/Dictionaries/MongoDBDictionarySource.cpp index 0ac6e337a5b4..b7e342f3c809 100644 --- a/src/Dictionaries/MongoDBDictionarySource.cpp +++ b/src/Dictionaries/MongoDBDictionarySource.cpp @@ -144,8 +144,8 @@ MongoDBDictionarySource::MongoDBDictionarySource( } else { - // Connect with host/port/user/etc through constructing - std::string uri_constructed("mongodb://" + host + ":" + std::to_string(port) + "/" + db + "?" + options); + // Connect with host/port/user/etc through constructing the uri + std::string uri_constructed("mongodb://" + host + ":" + std::to_string(port) + "/" + db + (options.empty() ? "" : "?" + options)); connection->connect(uri_constructed, socket_factory); if (!user.empty()) From c86d3daee55e332192170a9e8c0604ea1cbeecab Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 28 Apr 2023 16:21:37 +0300 Subject: [PATCH 258/406] Update AsynchronousMetrics.cpp --- src/Common/AsynchronousMetrics.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Common/AsynchronousMetrics.cpp b/src/Common/AsynchronousMetrics.cpp index 540015229bf5..ac2180103c5b 100644 --- a/src/Common/AsynchronousMetrics.cpp +++ b/src/Common/AsynchronousMetrics.cpp @@ -909,14 +909,14 @@ void AsynchronousMetrics::update(TimePoint update_time) { try { - cgroupmem_limit_in_bytes.rewind(); - cgroupmem_usage_in_bytes.rewind(); + cgroupmem_limit_in_bytes->rewind(); + cgroupmem_usage_in_bytes->rewind(); uint64_t limit = 0; uint64_t usage = 0; - tryReadText(limit, cgroupmem_limit_in_bytes); - tryReadText(usage, cgroupmem_usage_in_bytes); + tryReadText(limit, *cgroupmem_limit_in_bytes); + tryReadText(usage, *cgroupmem_usage_in_bytes); new_values["CGroupMemoryTotal"] = { limit, "The total amount of memory in cgroup, in bytes. If stated zero, the limit is the same as OSMemoryTotal." }; new_values["CGroupMemoryUsed"] = { usage, "The amount of memory used in cgroup, in bytes." }; From 8c60b60916529a9f1a32f4d646821d0474e0f64b Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Fri, 28 Apr 2023 16:01:53 +0200 Subject: [PATCH 259/406] Receive the best robot token only once --- tests/ci/get_robot_token.py | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/tests/ci/get_robot_token.py b/tests/ci/get_robot_token.py index 6ecaf468ed15..b41eba49cc32 100644 --- a/tests/ci/get_robot_token.py +++ b/tests/ci/get_robot_token.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 import logging from dataclasses import dataclass +from typing import Optional import boto3 # type: ignore from github import Github @@ -20,7 +21,13 @@ def get_parameter_from_ssm(name, decrypt=True, client=None): return client.get_parameter(Name=name, WithDecryption=decrypt)["Parameter"]["Value"] +ROBOT_TOKEN = None # type: Optional[Token] + + def get_best_robot_token(token_prefix_env_name="github_robot_token_"): + global ROBOT_TOKEN + if ROBOT_TOKEN is not None: + return ROBOT_TOKEN.value client = boto3.client("ssm", region_name="us-east-1") parameters = client.describe_parameters( ParameterFilters=[ @@ -28,7 +35,6 @@ def get_best_robot_token(token_prefix_env_name="github_robot_token_"): ] )["Parameters"] assert parameters - token = None for token_name in [p["Name"] for p in parameters]: value = get_parameter_from_ssm(token_name, True, client) @@ -38,15 +44,17 @@ def get_best_robot_token(token_prefix_env_name="github_robot_token_"): user = gh.get_user() rest, _ = gh.rate_limiting logging.info("Get token with %s remaining requests", rest) - if token is None: - token = Token(user, value, rest) + if ROBOT_TOKEN is None: + ROBOT_TOKEN = Token(user, value, rest) continue - if token.rest < rest: - token.user, token.value, token.rest = user, value, rest + if ROBOT_TOKEN.rest < rest: + ROBOT_TOKEN.user, ROBOT_TOKEN.value, ROBOT_TOKEN.rest = user, value, rest - assert token + assert ROBOT_TOKEN logging.info( - "User %s with %s remaining requests is used", token.user.login, token.rest + "User %s with %s remaining requests is used", + ROBOT_TOKEN.user.login, + ROBOT_TOKEN.rest, ) - return token.value + return ROBOT_TOKEN.value From 691fdbf6a6da5213783c277354b30aab18ebf1af Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Fri, 28 Apr 2023 17:21:03 +0300 Subject: [PATCH 260/406] Update StorageReplicatedMergeTree.cpp (#49302) --- src/Storages/StorageReplicatedMergeTree.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 470bb181c026..1e8c2c583a4f 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -7617,7 +7617,7 @@ bool StorageReplicatedMergeTree::waitForProcessingQueue(UInt64 max_wait_millisec background_operations_assignee.trigger(); std::unordered_set wait_for_ids; - bool was_interrupted = false; + std::atomic_bool was_interrupted = false; Poco::Event target_entry_event; auto callback = [this, &target_entry_event, &wait_for_ids, &was_interrupted, sync_mode] From b2aa2aa244ca32f6f09ddd3e4f09a1685c329ecb Mon Sep 17 00:00:00 2001 From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com> Date: Fri, 28 Apr 2023 16:45:00 +0200 Subject: [PATCH 261/406] Fix --- src/Interpreters/Cache/FileCache.cpp | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/Interpreters/Cache/FileCache.cpp b/src/Interpreters/Cache/FileCache.cpp index ad77a03c2339..a96e0ef7222c 100644 --- a/src/Interpreters/Cache/FileCache.cpp +++ b/src/Interpreters/Cache/FileCache.cpp @@ -655,11 +655,8 @@ bool FileCache::tryReserve(FileSegment & file_segment, size_t size) { auto locked_key = deletion_info.getMetadata().tryLock(); if (!locked_key) - { - /// key could become invalid after we released the key lock above, just skip it. - chassert(locked_key->getKeyState() != KeyMetadata::KeyState::ACTIVE); continue; - } + for (auto it = deletion_info.begin(); it != deletion_info.end();) { chassert((*it)->releasable()); From a04025623c617e048f2978e7022c8de0ac336519 Mon Sep 17 00:00:00 2001 From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com> Date: Fri, 28 Apr 2023 16:47:15 +0200 Subject: [PATCH 262/406] Update FileCache.cpp --- src/Interpreters/Cache/FileCache.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Interpreters/Cache/FileCache.cpp b/src/Interpreters/Cache/FileCache.cpp index a96e0ef7222c..2ba44f64d1f7 100644 --- a/src/Interpreters/Cache/FileCache.cpp +++ b/src/Interpreters/Cache/FileCache.cpp @@ -655,7 +655,7 @@ bool FileCache::tryReserve(FileSegment & file_segment, size_t size) { auto locked_key = deletion_info.getMetadata().tryLock(); if (!locked_key) - continue; + continue; /// key could become invalid after we released the key lock above, just skip it. for (auto it = deletion_info.begin(); it != deletion_info.end();) { From 2d156d09ab4a838613b83511a9291892c5f9fdfd Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Fri, 28 Apr 2023 14:47:30 +0000 Subject: [PATCH 263/406] Fix 02516_join_with_totals_and_subquery_bug with new analyzer --- tests/broken_tests.txt | 1 - ...oin_with_totals_and_subquery_bug.reference | 4 ++++ ...2516_join_with_totals_and_subquery_bug.sql | 23 ++++++++++++++++++- 3 files changed, 26 insertions(+), 2 deletions(-) diff --git a/tests/broken_tests.txt b/tests/broken_tests.txt index 4a504e15dd85..b0a8e0fe5350 100644 --- a/tests/broken_tests.txt +++ b/tests/broken_tests.txt @@ -124,7 +124,6 @@ 02707_skip_index_with_in 02707_complex_query_fails_analyzer 02680_mysql_ast_logical_err -02516_join_with_totals_and_subquery_bug 02324_map_combinator_bug 02241_join_rocksdb_bs 02003_WithMergeableStateAfterAggregationAndLimit_LIMIT_BY_LIMIT_OFFSET diff --git a/tests/queries/0_stateless/02516_join_with_totals_and_subquery_bug.reference b/tests/queries/0_stateless/02516_join_with_totals_and_subquery_bug.reference index fd0b223f8e57..19da8828c30e 100644 --- a/tests/queries/0_stateless/02516_join_with_totals_and_subquery_bug.reference +++ b/tests/queries/0_stateless/02516_join_with_totals_and_subquery_bug.reference @@ -2,6 +2,10 @@ 1 0 +1 +1 + +1 \N 100000000000000000000 diff --git a/tests/queries/0_stateless/02516_join_with_totals_and_subquery_bug.sql b/tests/queries/0_stateless/02516_join_with_totals_and_subquery_bug.sql index b6e60aa2e1f1..6b58d737a3ec 100644 --- a/tests/queries/0_stateless/02516_join_with_totals_and_subquery_bug.sql +++ b/tests/queries/0_stateless/02516_join_with_totals_and_subquery_bug.sql @@ -1,3 +1,5 @@ +SET allow_experimental_analyzer = 1; + SELECT * FROM ( @@ -12,7 +14,26 @@ INNER JOIN SELECT 1 GROUP BY 1 WITH TOTALS -) AS t2 USING (a); +) AS t2 USING (a) +SETTINGS allow_experimental_analyzer=0; + +SELECT * +FROM +( + SELECT 1 AS a +) AS t1 +INNER JOIN +( + SELECT 1 AS a + GROUP BY 1 + WITH TOTALS + UNION ALL + SELECT 1 + GROUP BY 1 + WITH TOTALS +) AS t2 USING (a) +SETTINGS allow_experimental_analyzer=1; + SELECT a FROM From 54c8f63a33fd5c0082c775cbb3a57cb2c98ef24a Mon Sep 17 00:00:00 2001 From: Nikolay Degterinsky Date: Fri, 28 Apr 2023 15:12:06 +0000 Subject: [PATCH 264/406] Fixes --- programs/client/Client.cpp | 1 - src/Access/AccessControl.cpp | 4 +- src/Access/AccessControl.h | 2 +- src/Access/AuthenticationData.cpp | 48 +++++++++++++------- src/Access/AuthenticationData.h | 1 + src/Parsers/Access/ASTAuthenticationData.cpp | 9 ++++ 6 files changed, 46 insertions(+), 19 deletions(-) diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp index cc8721f3ddeb..df0abceb8c6c 100644 --- a/programs/client/Client.cpp +++ b/programs/client/Client.cpp @@ -350,7 +350,6 @@ try /// Set user password complexity rules auto & access_control = global_context->getAccessControl(); access_control.setPasswordComplexityRules(connection->getPasswordComplexityRules()); - access_control.setBcryptWorkfactor(connection->getBcryptWorkfactor()); if (is_interactive && !delayed_interactive) { diff --git a/src/Access/AccessControl.cpp b/src/Access/AccessControl.cpp index fa5571d45373..6179c823b56c 100644 --- a/src/Access/AccessControl.cpp +++ b/src/Access/AccessControl.cpp @@ -701,11 +701,13 @@ void AccessControl::setBcryptWorkfactor(int workfactor_) { if (workfactor_ < 4) bcrypt_workfactor = 4; + else if (workfactor_ > 31) + bcrypt_workfactor = 31; else bcrypt_workfactor = workfactor_; } -int AccessControl::getBcryptWorkfactor() +int AccessControl::getBcryptWorkfactor() const { return bcrypt_workfactor; } diff --git a/src/Access/AccessControl.h b/src/Access/AccessControl.h index cd2a2252edd6..2a8293a49e77 100644 --- a/src/Access/AccessControl.h +++ b/src/Access/AccessControl.h @@ -160,7 +160,7 @@ class AccessControl : public MultipleAccessStorage /// Workfactor for bcrypt encoded passwords void setBcryptWorkfactor(int workfactor_); - int getBcryptWorkfactor(); + int getBcryptWorkfactor() const; /// Enables logic that users without permissive row policies can still read rows using a SELECT query. /// For example, if there two users A, B and a row policy is defined only for A, then diff --git a/src/Access/AuthenticationData.cpp b/src/Access/AuthenticationData.cpp index 6073a79852b9..dfec9bd1cd5e 100644 --- a/src/Access/AuthenticationData.cpp +++ b/src/Access/AuthenticationData.cpp @@ -44,7 +44,7 @@ AuthenticationData::Digest AuthenticationData::Util::encodeSHA256(std::string_vi ::DB::encodeSHA256(text, hash.data()); return hash; #else - throw DB::Exception(DB::ErrorCodes::SUPPORT_IS_DISABLED, "SHA256 passwords support is disabled, because ClickHouse was built without SSL library"); + throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "SHA256 passwords support is disabled, because ClickHouse was built without SSL library"); #endif } @@ -60,9 +60,9 @@ AuthenticationData::Digest AuthenticationData::Util::encodeBcrypt(std::string_vi { #if USE_BCRYPT if (text.size() > 72) - throw DB::Exception( - "bcrypt does not support passwords with a length of more than 72 bytes", - DB::ErrorCodes::BAD_ARGUMENTS); + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "bcrypt does not support passwords with a length of more than 72 bytes"); char salt[BCRYPT_HASHSIZE]; Digest hash; @@ -70,17 +70,17 @@ AuthenticationData::Digest AuthenticationData::Util::encodeBcrypt(std::string_vi int ret = bcrypt_gensalt(workfactor, salt); if (ret != 0) - throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "BCrypt library failed: bcrypt_gensalt returned {}", ret); + throw Exception(ErrorCodes::LOGICAL_ERROR, "BCrypt library failed: bcrypt_gensalt returned {}", ret); ret = bcrypt_hashpw(text.data(), salt, reinterpret_cast(hash.data())); if (ret != 0) - throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "BCrypt library failed: bcrypt_hashpw returned {}", ret); + throw Exception(ErrorCodes::LOGICAL_ERROR, "BCrypt library failed: bcrypt_hashpw returned {}", ret); return hash; #else - throw DB::Exception( - "bcrypt passwords support is disabled, because ClickHouse was built without bcrypt library", - DB::ErrorCodes::SUPPORT_IS_DISABLED); + throw Exception( + ErrorCodes::SUPPORT_IS_DISABLED, + "bcrypt passwords support is disabled, because ClickHouse was built without bcrypt library"); #endif } @@ -89,12 +89,12 @@ bool AuthenticationData::Util::checkPasswordBcrypt(std::string_view password [[m #if USE_BCRYPT int ret = bcrypt_checkpw(password.data(), reinterpret_cast(password_bcrypt.data())); if (ret == -1) - throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "BCrypt library failed: bcrypt_checkpw returned {}", ret); + throw Exception(ErrorCodes::LOGICAL_ERROR, "BCrypt library failed: bcrypt_checkpw returned {}", ret); return (ret == 0); #else - throw DB::Exception( - "bcrypt passwords support is disabled, because ClickHouse was built without bcrypt library", - DB::ErrorCodes::SUPPORT_IS_DISABLED); + throw Exception( + ErrorCodes::SUPPORT_IS_DISABLED, + "bcrypt passwords support is disabled, because ClickHouse was built without bcrypt library"); #endif } @@ -210,8 +210,15 @@ void AuthenticationData::setPasswordHashBinary(const Digest & hash) case AuthenticationType::BCRYPT_PASSWORD: { - /// TODO: check size + /// Depending on the workfactor the resulting hash can be 59 or 60 characters long. + /// However the library we use to encode it requires hash string to be 64 characters long, + /// so we also allow the hash of this length. + if (hash.size() != 59 && hash.size() != 60 && hash.size() != 64) + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "Password hash for the 'BCRYPT_PASSWORD' authentication type has length {} " + "but must be 59 or 60 bytes.", hash.size()); password_hash = hash; + password_hash.resize(64); return; } @@ -278,7 +285,7 @@ std::shared_ptr AuthenticationData::toAST() const case AuthenticationType::BCRYPT_PASSWORD: { node->contains_hash = true; - node->children.push_back(std::make_shared(getPasswordHashHex())); + node->children.push_back(std::make_shared(AuthenticationData::Util::digestToString(getPasswordHashBinary()))); break; } case AuthenticationType::LDAP: @@ -391,7 +398,16 @@ AuthenticationData AuthenticationData::fromAST(const ASTAuthenticationData & que if (query.contains_hash) { String value = checkAndGetLiteralArgument(args[0], "hash"); - auth_data.setPasswordHashHex(value); + + if (query.type == AuthenticationType::BCRYPT_PASSWORD) + { + auth_data.setPasswordHashBinary(AuthenticationData::Util::stringToDigest(value)); + return auth_data; + } + else + { + auth_data.setPasswordHashHex(value); + } if (query.type == AuthenticationType::SHA256_PASSWORD && args_size == 2) { diff --git a/src/Access/AuthenticationData.h b/src/Access/AuthenticationData.h index ff5cc1f9618d..5ebef7d44f2f 100644 --- a/src/Access/AuthenticationData.h +++ b/src/Access/AuthenticationData.h @@ -65,6 +65,7 @@ class AuthenticationData struct Util { + static String digestToString(const Digest & text) { return String(text.data(), text.data() + text.size()); } static Digest stringToDigest(std::string_view text) { return Digest(text.data(), text.data() + text.size()); } static Digest encodeSHA256(std::string_view text); static Digest encodeSHA1(std::string_view text); diff --git a/src/Parsers/Access/ASTAuthenticationData.cpp b/src/Parsers/Access/ASTAuthenticationData.cpp index 4e0cabfdec7a..3dd05d831ee5 100644 --- a/src/Parsers/Access/ASTAuthenticationData.cpp +++ b/src/Parsers/Access/ASTAuthenticationData.cpp @@ -87,6 +87,15 @@ void ASTAuthenticationData::formatImpl(const FormatSettings & settings, FormatSt password = true; break; } + case AuthenticationType::BCRYPT_PASSWORD: + { + if (contains_hash) + auth_type_name = "bcrypt_hash"; + + prefix = "BY"; + password = true; + break; + } case AuthenticationType::LDAP: { prefix = "SERVER"; From 69d8d41197d797fcb289507e19a0d2a774e13fe3 Mon Sep 17 00:00:00 2001 From: Nikolay Degterinsky Date: Fri, 28 Apr 2023 15:12:48 +0000 Subject: [PATCH 265/406] Add docs, better tests --- docs/en/operations/system-tables/users.md | 2 +- .../sql-reference/statements/create/user.md | 32 ++++++++++++------- .../queries/0_stateless/01292_create_user.sql | 2 +- .../02713_create_user_substitutions.reference | 10 ++++-- .../02713_create_user_substitutions.sh | 30 +++++++++++++---- 5 files changed, 52 insertions(+), 24 deletions(-) diff --git a/docs/en/operations/system-tables/users.md b/docs/en/operations/system-tables/users.md index a90fa01a45db..28b1602caac8 100644 --- a/docs/en/operations/system-tables/users.md +++ b/docs/en/operations/system-tables/users.md @@ -12,7 +12,7 @@ Columns: - `storage` ([String](../../sql-reference/data-types/string.md)) — Path to the storage of users. Configured in the `access_control_path` parameter. -- `auth_type` ([Enum8](../../sql-reference/data-types/enum.md)('no_password' = 0,'plaintext_password' = 1, 'sha256_password' = 2, 'double_sha1_password' = 3, 'ldap' = 4, 'kerberos' = 5, 'ssl_certificate' = 6)) — Shows the authentication type. There are multiple ways of user identification: with no password, with plain text password, with [SHA256](https://ru.wikipedia.org/wiki/SHA-2)-encoded password or with [double SHA-1](https://ru.wikipedia.org/wiki/SHA-1)-encoded password. +- `auth_type` ([Enum8](../../sql-reference/data-types/enum.md)('no_password' = 0, 'plaintext_password' = 1, 'sha256_password' = 2, 'double_sha1_password' = 3, 'bcrypt_password' = 4, 'ldap' = 5, 'kerberos' = 6, 'ssl_certificate' = 7)) — Shows the authentication type. There are multiple ways of user identification: with no password, with plain text password, with [SHA256](https://ru.wikipedia.org/wiki/SHA-2)-encoded password, with [double SHA-1](https://ru.wikipedia.org/wiki/SHA-1)-encoded password or with [bcrypt](https://en.wikipedia.org/wiki/Bcrypt)-encoded password. - `auth_params` ([String](../../sql-reference/data-types/string.md)) — Authentication parameters in the JSON format depending on the `auth_type`. diff --git a/docs/en/sql-reference/statements/create/user.md b/docs/en/sql-reference/statements/create/user.md index 223220621289..d168be63c36c 100644 --- a/docs/en/sql-reference/statements/create/user.md +++ b/docs/en/sql-reference/statements/create/user.md @@ -32,9 +32,12 @@ There are multiple ways of user identification: - `IDENTIFIED WITH sha256_hash BY 'hash'` or `IDENTIFIED WITH sha256_hash BY 'hash' SALT 'salt'` - `IDENTIFIED WITH double_sha1_password BY 'qwerty'` - `IDENTIFIED WITH double_sha1_hash BY 'hash'` +- `IDENTIFIED WITH bcrypt_password BY 'qwerty'` +- `IDENTIFIED WITH bcrypt_hash BY 'hash'` - `IDENTIFIED WITH ldap SERVER 'server_name'` - `IDENTIFIED WITH kerberos` or `IDENTIFIED WITH kerberos REALM 'realm'` - `IDENTIFIED WITH ssl_certificate CN 'mysite.com:user'` +- `IDENTIFIED BY 'qwerty'` ## Examples @@ -54,21 +57,12 @@ There are multiple ways of user identification: The password is stored in a SQL text file in `/var/lib/clickhouse/access`, so it's not a good idea to use `plaintext_password`. Try `sha256_password` instead, as demonstrated next... ::: -3. The best option is to use a password that is hashed using SHA-256. ClickHouse will hash the password for you when you specify `IDENTIFIED WITH sha256_password`. For example: +3. The most common option is to use a password that is hashed using SHA-256. ClickHouse will hash the password for you when you specify `IDENTIFIED WITH sha256_password`. For example: ```sql CREATE USER name3 IDENTIFIED WITH sha256_password BY 'my_password' ``` - Notice ClickHouse generates and runs the following command for you: - - ```response - CREATE USER name3 - IDENTIFIED WITH sha256_hash - BY '8B3404953FCAA509540617F082DB13B3E0734F90FF6365C19300CC6A6EA818D6' - SALT 'D6489D8B5692D82FF944EA6415785A8A8A1AF33825456AFC554487725A74A609' - ``` - The `name3` user can now login using `my_password`, but the password is stored as the hashed value above. THe following SQL file was created in `/var/lib/clickhouse/access` and gets executed at server startup: ```bash @@ -92,10 +86,24 @@ There are multiple ways of user identification: CREATE USER name4 IDENTIFIED WITH double_sha1_hash BY 'CCD3A959D6A004B9C3807B728BC2E55B67E10518' ``` -5. The type of the password can also be omitted: +5. The `bcrypt_password` is the most secure option for storing passwords. It uses the [bcrypt](https://en.wikipedia.org/wiki/Bcrypt) algorithm, which is resilient against brute force attacks even if the password hash is compromised. + + ```sql + CREATE USER name5 IDENTIFIED WITH bcrypt_password BY 'my_password' + ``` + + The length of the password is limited to 72 characters with this method. The bcrypt work factor parameter, which defines the amount of computations and time needed to compute the hash and verify the password, can be modified in the server configuration: + + ```xml + 12 + ``` + + The work factor must be between 4 and 31, with a default value of 12. + +6. The type of the password can also be omitted: ```sql - CREATE USER name4 IDENTIFIED BY 'my_password' + CREATE USER name6 IDENTIFIED BY 'my_password' ``` In this case, ClickHouse will use the default password type specified in the server configuration: diff --git a/tests/queries/0_stateless/01292_create_user.sql b/tests/queries/0_stateless/01292_create_user.sql index 2e49248e4633..56f7091772fe 100644 --- a/tests/queries/0_stateless/01292_create_user.sql +++ b/tests/queries/0_stateless/01292_create_user.sql @@ -32,7 +32,7 @@ CREATE USER u5_01292 IDENTIFIED WITH sha256_hash BY '18138372FAD4B94533CD4881F03 CREATE USER u6_01292 IDENTIFIED WITH double_sha1_password BY 'qwe123'; CREATE USER u7_01292 IDENTIFIED WITH double_sha1_hash BY '8DCDD69CE7D121DE8013062AEAEB2A148910D50E'; CREATE USER u8_01292 IDENTIFIED WITH bcrypt_password BY 'qwe123'; -CREATE USER u9_01292 IDENTIFIED WITH bcrypt_hash BY '2432612431322459345A4F6C786659746C7167594A59484C434678776537366F51506232764C71533070795135394E5744784763456D5A703278346500000000'; +CREATE USER u9_01292 IDENTIFIED WITH bcrypt_hash BY '$2a$12$rz5iy2LhuwBezsM88ZzWiemOVUeJ94xHTzwAlLMDhTzwUxOHaY64q'; SHOW CREATE USER u1_01292; SHOW CREATE USER u2_01292; SHOW CREATE USER u3_01292; diff --git a/tests/queries/0_stateless/02713_create_user_substitutions.reference b/tests/queries/0_stateless/02713_create_user_substitutions.reference index 02f7d1d87934..f9b5cc495b5b 100644 --- a/tests/queries/0_stateless/02713_create_user_substitutions.reference +++ b/tests/queries/0_stateless/02713_create_user_substitutions.reference @@ -2,6 +2,10 @@ 2 3 4 -CREATE USER user5_02713 IDENTIFIED WITH ldap SERVER \'qwerty5\' -CREATE USER user6_02713 IDENTIFIED WITH kerberos REALM \'qwerty6\' -CREATE USER user7_02713 IDENTIFIED WITH ssl_certificate CN \'qwerty7\', \'qwerty8\' +5 +6 +7 +8 +CREATE USER user9_02713 IDENTIFIED WITH ldap SERVER \'qwerty9\' +CREATE USER user10_02713 IDENTIFIED WITH kerberos REALM \'qwerty10\' +CREATE USER user11_02713 IDENTIFIED WITH ssl_certificate CN \'qwerty11\', \'qwerty12\' diff --git a/tests/queries/0_stateless/02713_create_user_substitutions.sh b/tests/queries/0_stateless/02713_create_user_substitutions.sh index 2d7fef56a21e..42926335acbb 100755 --- a/tests/queries/0_stateless/02713_create_user_substitutions.sh +++ b/tests/queries/0_stateless/02713_create_user_substitutions.sh @@ -11,17 +11,33 @@ $CLICKHOUSE_CLIENT --param_password=qwerty1 -q "CREATE USER user1_02713 IDENTIFI $CLICKHOUSE_CLIENT --param_password=qwerty2 -q "CREATE USER user2_02713 IDENTIFIED WITH PLAINTEXT_PASSWORD BY {password:String}"; $CLICKHOUSE_CLIENT --param_password=qwerty3 -q "CREATE USER user3_02713 IDENTIFIED WITH SHA256_PASSWORD BY {password:String}"; $CLICKHOUSE_CLIENT --param_password=qwerty4 -q "CREATE USER user4_02713 IDENTIFIED WITH DOUBLE_SHA1_PASSWORD BY {password:String}"; -$CLICKHOUSE_CLIENT --param_server=qwerty5 -q "CREATE USER user5_02713 IDENTIFIED WITH LDAP SERVER {server:String}"; -$CLICKHOUSE_CLIENT --param_realm=qwerty6 -q "CREATE USER user6_02713 IDENTIFIED WITH KERBEROS REALM {realm:String}"; -$CLICKHOUSE_CLIENT --param_cert1=qwerty7 --param_cert2=qwerty8 -q "CREATE USER user7_02713 IDENTIFIED WITH SSL_CERTIFICATE CN {cert1:String}, {cert2:String}"; +$CLICKHOUSE_CLIENT --param_password=qwerty5 -q "CREATE USER user5_02713 IDENTIFIED WITH BCRYPT_PASSWORD BY {password:String}"; + +# Generated online +$CLICKHOUSE_CLIENT --param_hash=310cef2caff72c0224f38ca8e2141ca6012cd4da550c692573c25a917d9a75e6 \ + -q "CREATE USER user6_02713 IDENTIFIED WITH SHA256_HASH BY {hash:String}"; +# Generated with ClickHouse +$CLICKHOUSE_CLIENT --param_hash=5886A74C452575627522F3A80D8B9E239FD8955F \ + -q "CREATE USER user7_02713 IDENTIFIED WITH DOUBLE_SHA1_HASH BY {hash:String}"; +# Generated online +$CLICKHOUSE_CLIENT --param_hash=\$2a\$12\$wuohz0HFSBBNE8huN0Yx6.kmWrefiYVKeMp4gsuNoO1rOWwF2FXXC \ + -q "CREATE USER user8_02713 IDENTIFIED WITH BCRYPT_HASH BY {hash:String}"; + +$CLICKHOUSE_CLIENT --param_server=qwerty9 -q "CREATE USER user9_02713 IDENTIFIED WITH LDAP SERVER {server:String}"; +$CLICKHOUSE_CLIENT --param_realm=qwerty10 -q "CREATE USER user10_02713 IDENTIFIED WITH KERBEROS REALM {realm:String}"; +$CLICKHOUSE_CLIENT --param_cert1=qwerty11 --param_cert2=qwerty12 -q "CREATE USER user11_02713 IDENTIFIED WITH SSL_CERTIFICATE CN {cert1:String}, {cert2:String}"; $CLICKHOUSE_CLIENT --user=user1_02713 --password=qwerty1 -q "SELECT 1"; $CLICKHOUSE_CLIENT --user=user2_02713 --password=qwerty2 -q "SELECT 2"; $CLICKHOUSE_CLIENT --user=user3_02713 --password=qwerty3 -q "SELECT 3"; $CLICKHOUSE_CLIENT --user=user4_02713 --password=qwerty4 -q "SELECT 4"; +$CLICKHOUSE_CLIENT --user=user5_02713 --password=qwerty5 -q "SELECT 5"; +$CLICKHOUSE_CLIENT --user=user6_02713 --password=qwerty6 -q "SELECT 6"; +$CLICKHOUSE_CLIENT --user=user7_02713 --password=qwerty7 -q "SELECT 7"; +$CLICKHOUSE_CLIENT --user=user8_02713 --password=qwerty8 -q "SELECT 8"; -$CLICKHOUSE_CLIENT -q "SHOW CREATE USER user5_02713"; -$CLICKHOUSE_CLIENT -q "SHOW CREATE USER user6_02713"; -$CLICKHOUSE_CLIENT -q "SHOW CREATE USER user7_02713"; +$CLICKHOUSE_CLIENT -q "SHOW CREATE USER user9_02713"; +$CLICKHOUSE_CLIENT -q "SHOW CREATE USER user10_02713"; +$CLICKHOUSE_CLIENT -q "SHOW CREATE USER user11_02713"; -$CLICKHOUSE_CLIENT -q "DROP USER user1_02713, user2_02713, user3_02713, user4_02713, user5_02713, user6_02713, user7_02713"; +$CLICKHOUSE_CLIENT -q "DROP USER user1_02713, user2_02713, user3_02713, user4_02713, user5_02713, user6_02713, user7_02713, user8_02713, user9_02713, user10_02713, user11_02713"; From 70ee02a3eb55a292f0c3150365fba9a3764a3537 Mon Sep 17 00:00:00 2001 From: alesapin Date: Fri, 28 Apr 2023 17:39:32 +0200 Subject: [PATCH 266/406] Add node holder --- src/Common/ZooKeeper/ZooKeeper.h | 2 -- src/Storages/MergeTree/DataPartsExchange.cpp | 5 ++++- src/Storages/StorageReplicatedMergeTree.cpp | 23 ++++++++++---------- src/Storages/StorageReplicatedMergeTree.h | 2 +- 4 files changed, 16 insertions(+), 16 deletions(-) diff --git a/src/Common/ZooKeeper/ZooKeeper.h b/src/Common/ZooKeeper/ZooKeeper.h index 2fd94dfbb6b1..42ffccc678e0 100644 --- a/src/Common/ZooKeeper/ZooKeeper.h +++ b/src/Common/ZooKeeper/ZooKeeper.h @@ -524,8 +524,6 @@ class ZooKeeper void setServerCompletelyStarted(); private: - friend class EphemeralNodeHolder; - void init(ZooKeeperArgs args_); /// The following methods don't any throw exceptions but return error codes. diff --git a/src/Storages/MergeTree/DataPartsExchange.cpp b/src/Storages/MergeTree/DataPartsExchange.cpp index c6804d260e2c..f0dadcbd89f1 100644 --- a/src/Storages/MergeTree/DataPartsExchange.cpp +++ b/src/Storages/MergeTree/DataPartsExchange.cpp @@ -821,6 +821,7 @@ MergeTreeData::MutableDataPartPtr Fetcher::downloadPartToDisk( const auto data_settings = data.getSettings(); MergeTreeData::DataPart::Checksums data_checksums; + zkutil::EphemeralNodeHolderPtr zero_copy_temporary_lock_holder; if (to_remote_disk) { readStringBinary(part_id, in); @@ -829,7 +830,7 @@ MergeTreeData::MutableDataPartPtr Fetcher::downloadPartToDisk( throw Exception(ErrorCodes::ZERO_COPY_REPLICATION_ERROR, "Part {} unique id {} doesn't exist on {} (with type {}).", part_name, part_id, disk->getName(), toString(disk->getDataSourceDescription().type)); LOG_DEBUG(log, "Downloading part {} unique id {} metadata onto disk {}.", part_name, part_id, disk->getName()); - data.lockSharedDataTemporary(part_name, part_id, disk); + zero_copy_temporary_lock_holder = data.lockSharedDataTemporary(part_name, part_id, disk); } else { @@ -946,6 +947,8 @@ MergeTreeData::MutableDataPartPtr Fetcher::downloadPartToDisk( new_data_part->checksums.checkEqual(data_checksums, false); LOG_DEBUG(log, "Download of part {} onto disk {} finished.", part_name, disk->getName()); } + if (zero_copy_temporary_lock_holder) + zero_copy_temporary_lock_holder->setAlreadyRemoved(); return new_data_part; } diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 88fa602f59b4..49a5b2ea881e 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -8113,31 +8113,30 @@ std::optional StorageReplicatedMergeTree::tryGetTableSharedIDFromCreateQ } -void StorageReplicatedMergeTree::lockSharedDataTemporary(const String & part_name, const String & part_id, const DiskPtr & disk) const +zkutil::EphemeralNodeHolderPtr StorageReplicatedMergeTree::lockSharedDataTemporary(const String & part_name, const String & part_id, const DiskPtr & disk) const { auto settings = getSettings(); if (!disk || !disk->supportZeroCopyReplication() || !settings->allow_remote_fs_zero_copy_replication) - return; + return {}; zkutil::ZooKeeperPtr zookeeper = tryGetZooKeeper(); if (!zookeeper) - return; + return {}; String id = part_id; boost::replace_all(id, "/", "_"); - Strings zc_zookeeper_paths = getZeroCopyPartPath(*getSettings(), toString(disk->getDataSourceDescription().type), getTableSharedID(), - part_name, zookeeper_path); + String zc_zookeeper_path = getZeroCopyPartPath(*getSettings(), toString(disk->getDataSourceDescription().type), getTableSharedID(), + part_name, zookeeper_path)[0]; - for (const auto & zc_zookeeper_path : zc_zookeeper_paths) - { - String zookeeper_node = fs::path(zc_zookeeper_path) / id / replica_name; + String zookeeper_node = fs::path(zc_zookeeper_path) / id / replica_name; - LOG_TRACE(log, "Set zookeeper temporary ephemeral lock {}", zookeeper_node); - createZeroCopyLockNode( - std::make_shared(zookeeper), zookeeper_node, zkutil::CreateMode::Ephemeral, false); - } + LOG_TRACE(log, "Set zookeeper temporary ephemeral lock {}", zookeeper_node); + createZeroCopyLockNode( + std::make_shared(zookeeper), zookeeper_node, zkutil::CreateMode::Ephemeral, false); + + return zkutil::EphemeralNodeHolder::existing(zookeeper_node, *zookeeper); } void StorageReplicatedMergeTree::lockSharedData( diff --git a/src/Storages/StorageReplicatedMergeTree.h b/src/Storages/StorageReplicatedMergeTree.h index 64511a4a9271..4e4f383894a4 100644 --- a/src/Storages/StorageReplicatedMergeTree.h +++ b/src/Storages/StorageReplicatedMergeTree.h @@ -259,7 +259,7 @@ class StorageReplicatedMergeTree final : public MergeTreeData std::optional hardlinked_files, Coordination::Requests & requests) const; - void lockSharedDataTemporary(const String & part_name, const String & part_id, const DiskPtr & disk) const; + zkutil::EphemeralNodeHolderPtr lockSharedDataTemporary(const String & part_name, const String & part_id, const DiskPtr & disk) const; /// Unlock shared data part in zookeeper /// Return true if data unlocked From 01e70c64c2500230e2ddf18bacc8c72878e2cf74 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Fri, 28 Apr 2023 18:26:50 +0200 Subject: [PATCH 267/406] Add a fallback to authenticated requests to GH API --- tests/ci/build_download_helper.py | 53 +++++++++++++++++++++++++++---- tests/ci/env_helper.py | 4 +-- tests/ci/pr_info.py | 12 +++---- 3 files changed, 55 insertions(+), 14 deletions(-) diff --git a/tests/ci/build_download_helper.py b/tests/ci/build_download_helper.py index c61360153160..76f81343b54d 100644 --- a/tests/ci/build_download_helper.py +++ b/tests/ci/build_download_helper.py @@ -6,11 +6,12 @@ import sys import time from pathlib import Path -from typing import Any, Callable, List, Optional +from typing import Any, Callable, List import requests # type: ignore from ci_config import CI_CONFIG +from get_robot_token import ROBOT_TOKEN, get_best_robot_token DOWNLOAD_RETRIES_COUNT = 5 @@ -24,22 +25,62 @@ def get_with_retries( logging.info( "Getting URL with %i tries and sleep %i in between: %s", retries, sleep, url ) - exc = None # type: Optional[Exception] + exc = Exception("A placeholder to satisfy typing and avoid nesting") for i in range(retries): try: response = requests.get(url, **kwargs) response.raise_for_status() - break + return response except Exception as e: if i + 1 < retries: logging.info("Exception '%s' while getting, retry %i", e, i + 1) time.sleep(sleep) exc = e - else: - raise Exception(exc) - return response + raise exc + + +def get_gh_api( + url: str, + retries: int = DOWNLOAD_RETRIES_COUNT, + sleep: int = 3, + **kwargs: Any, +) -> requests.Response: + """It's a wrapper around get_with_retries that requests GH api w/o auth by + default, and falls back to the get_best_robot_token in case of receiving + "403 rate limit exceeded" error + It sets auth automatically when ROBOT_TOKEN is already set by get_best_robot_token + """ + + def set_auth_header(): + if "headers" in kwargs: + if "Authorization" not in kwargs["headers"]: + kwargs["headers"]["Authorization"] = f"Bearer {get_best_robot_token()}" + else: + kwargs["headers"] = {"Authorization": f"Bearer {get_best_robot_token()}"} + + if ROBOT_TOKEN is not None: + set_auth_header() + + for _ in range(retries): + try: + response = get_with_retries(url, 1, sleep, **kwargs) + response.raise_for_status() + return response + except requests.HTTPError as exc: + if ( + exc.response.status_code == 403 + and b"rate limit exceeded" + in exc.response._content # pylint:disable=protected-access + ): + logging.warning( + "Received rate limit exception, setting the auth header and retry" + ) + set_auth_header() + break + + return get_with_retries(url, retries, sleep, **kwargs) def get_build_name_for_check(check_name: str) -> str: diff --git a/tests/ci/env_helper.py b/tests/ci/env_helper.py index a5a4913be0b4..5c2139ae0bc2 100644 --- a/tests/ci/env_helper.py +++ b/tests/ci/env_helper.py @@ -1,7 +1,7 @@ import os from os import path as p -from build_download_helper import get_with_retries +from build_download_helper import get_gh_api module_dir = p.abspath(p.dirname(__file__)) git_root = p.abspath(p.join(module_dir, "..", "..")) @@ -46,7 +46,7 @@ def GITHUB_JOB_ID() -> str: jobs = [] page = 1 while not _GITHUB_JOB_ID: - response = get_with_retries( + response = get_gh_api( f"https://api.github.com/repos/{GITHUB_REPOSITORY}/" f"actions/runs/{GITHUB_RUN_ID}/jobs?per_page=100&page={page}" ) diff --git a/tests/ci/pr_info.py b/tests/ci/pr_info.py index ddeb070b2b94..86d4985c6b27 100644 --- a/tests/ci/pr_info.py +++ b/tests/ci/pr_info.py @@ -6,7 +6,7 @@ from unidiff import PatchSet # type: ignore -from build_download_helper import get_with_retries +from build_download_helper import get_gh_api from env_helper import ( GITHUB_REPOSITORY, GITHUB_SERVER_URL, @@ -45,7 +45,7 @@ def get_pr_for_commit(sha, ref): f"https://api.github.com/repos/{GITHUB_REPOSITORY}/commits/{sha}/pulls" ) try: - response = get_with_retries(try_get_pr_url, sleep=RETRY_SLEEP) + response = get_gh_api(try_get_pr_url, sleep=RETRY_SLEEP) data = response.json() our_prs = [] # type: List[Dict] if len(data) > 1: @@ -105,7 +105,7 @@ def __init__( # workflow completed event, used for PRs only if "action" in github_event and github_event["action"] == "completed": self.sha = github_event["workflow_run"]["head_sha"] - prs_for_sha = get_with_retries( + prs_for_sha = get_gh_api( f"https://api.github.com/repos/{GITHUB_REPOSITORY}/commits/{self.sha}" "/pulls", sleep=RETRY_SLEEP, @@ -117,7 +117,7 @@ def __init__( self.number = github_event["pull_request"]["number"] if pr_event_from_api: try: - response = get_with_retries( + response = get_gh_api( f"https://api.github.com/repos/{GITHUB_REPOSITORY}" f"/pulls/{self.number}", sleep=RETRY_SLEEP, @@ -159,7 +159,7 @@ def __init__( self.user_login = github_event["pull_request"]["user"]["login"] self.user_orgs = set([]) if need_orgs: - user_orgs_response = get_with_retries( + user_orgs_response = get_gh_api( github_event["pull_request"]["user"]["organizations_url"], sleep=RETRY_SLEEP, ) @@ -255,7 +255,7 @@ def fetch_changed_files(self): raise TypeError("The event does not have diff URLs") for diff_url in self.diff_urls: - response = get_with_retries( + response = get_gh_api( diff_url, sleep=RETRY_SLEEP, ) From 434aa3e4e097af0b3329b3a7ad60e2ed22d82a68 Mon Sep 17 00:00:00 2001 From: Nikolay Degterinsky Date: Fri, 28 Apr 2023 18:54:18 +0000 Subject: [PATCH 268/406] Fix build --- base/harmful/harmful.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/base/harmful/harmful.c b/base/harmful/harmful.c index 6112f9a339c0..78796ca0c054 100644 --- a/base/harmful/harmful.c +++ b/base/harmful/harmful.c @@ -31,7 +31,8 @@ TRAP(argp_state_help) TRAP(argp_usage) TRAP(asctime) TRAP(clearenv) -TRAP(crypt) +// Redefined at contrib/libbcrypt/crypt_blowfish/wrapper.c:186 +// TRAP(crypt) TRAP(ctime) TRAP(cuserid) TRAP(drand48) From 3b536165cb51a792c19e81e5a6afe6f98d12e4a5 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Fri, 28 Apr 2023 19:44:53 +0000 Subject: [PATCH 269/406] Update tests. --- src/Core/Settings.h | 2 +- .../0_stateless/01600_parts_states_metrics_long.sh | 12 +++++++++++- .../0_stateless/01600_parts_types_metrics_long.sh | 12 +++++++++++- 3 files changed, 23 insertions(+), 3 deletions(-) diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 26409e98763e..21c3c1c4dbf8 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -316,7 +316,7 @@ class IColumn; M(Float, opentelemetry_start_trace_probability, 0., "Probability to start an OpenTelemetry trace for an incoming query.", 0) \ M(Bool, opentelemetry_trace_processors, false, "Collect OpenTelemetry spans for processors.", 0) \ M(Bool, prefer_column_name_to_alias, false, "Prefer using column names instead of aliases if possible.", 0) \ - M(Bool, allow_experimental_analyzer, false, "Allow experimental analyzer", 0) \ + M(Bool, allow_experimental_analyzer, true, "Allow experimental analyzer", 0) \ M(Bool, prefer_global_in_and_join, false, "If enabled, all IN/JOIN operators will be rewritten as GLOBAL IN/JOIN. It's useful when the to-be-joined tables are only available on the initiator and we need to always scatter their data on-the-fly during distributed processing with the GLOBAL keyword. It's also useful to reduce the need to access the external sources joining external tables.", 0) \ \ \ diff --git a/tests/queries/0_stateless/01600_parts_states_metrics_long.sh b/tests/queries/0_stateless/01600_parts_states_metrics_long.sh index f47d0863e698..50abd6ade907 100755 --- a/tests/queries/0_stateless/01600_parts_states_metrics_long.sh +++ b/tests/queries/0_stateless/01600_parts_states_metrics_long.sh @@ -15,10 +15,20 @@ verify_sql="SELECT # In case of test failure, this code will do infinite loop and timeout. verify() { - while true + for i in $(seq 1 3001) do result=$( $CLICKHOUSE_CLIENT -m --query="$verify_sql" ) [ "$result" = "1" ] && break + + if [ "$i" = "3000" ]; then + echo "=======" + $CLICKHOUSE_CLIENT --query="SELECT * FROM system.parts format TSVWithNames" + echo "=======" + $CLICKHOUSE_CLIENT --query="SELECT * FROM system.metrics format TSVWithNames" + echo "=======" + return + fi + sleep 0.1 done echo 1 diff --git a/tests/queries/0_stateless/01600_parts_types_metrics_long.sh b/tests/queries/0_stateless/01600_parts_types_metrics_long.sh index 05edf02f7edc..dcac6dcab399 100755 --- a/tests/queries/0_stateless/01600_parts_types_metrics_long.sh +++ b/tests/queries/0_stateless/01600_parts_types_metrics_long.sh @@ -20,12 +20,22 @@ verify_sql="SELECT # In case of test failure, this code will do infinite loop and timeout. verify() { - while true; do + for i in $(seq 1 3001); do result=$( $CLICKHOUSE_CLIENT -m --query="$verify_sql" ) if [ "$result" = "1" ]; then echo 1 return fi + + if [ "$i" = "3000" ]; then + echo "=======" + $CLICKHOUSE_CLIENT --query="SELECT * FROM system.parts format TSVWithNames" + echo "=======" + $CLICKHOUSE_CLIENT --query="SELECT * FROM system.metrics format TSVWithNames" + echo "=======" + return + fi + sleep 0.1 done } From af558e282c59cff653031ac0c4a9d424c1cee34c Mon Sep 17 00:00:00 2001 From: Nikita Taranov Date: Fri, 28 Apr 2023 20:48:37 +0000 Subject: [PATCH 270/406] impl --- src/IO/ReadSettings.h | 2 +- src/Interpreters/Context.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/IO/ReadSettings.h b/src/IO/ReadSettings.h index fc229ada59c1..50ef5ec2988d 100644 --- a/src/IO/ReadSettings.h +++ b/src/IO/ReadSettings.h @@ -68,7 +68,7 @@ struct ReadSettings /// Method to use reading from remote filesystem. RemoteFSReadMethod remote_fs_method = RemoteFSReadMethod::threadpool; - size_t local_fs_buffer_size = DBMS_DEFAULT_BUFFER_SIZE; + size_t local_fs_buffer_size = 128 * 1024; size_t remote_fs_buffer_size = DBMS_DEFAULT_BUFFER_SIZE; size_t prefetch_buffer_size = DBMS_DEFAULT_BUFFER_SIZE; diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index afeddbbe1706..c78d2e8eb7bd 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -4288,7 +4288,7 @@ ReadSettings Context::getReadSettings() const "Invalid value '{}' for max_read_buffer_size", settings.max_read_buffer_size); } - res.local_fs_buffer_size = settings.max_read_buffer_size; + /* res.local_fs_buffer_size = settings.max_read_buffer_size; */ res.remote_fs_buffer_size = settings.max_read_buffer_size; res.prefetch_buffer_size = settings.prefetch_buffer_size; res.direct_io_threshold = settings.min_bytes_to_use_direct_io; From e670c51680433b66780b1bb5890c84d55e4d6ef6 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Tue, 4 Apr 2023 16:09:34 +0200 Subject: [PATCH 271/406] Unify commit.create_status and post_commit_status --- tests/ci/docs_check.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tests/ci/docs_check.py b/tests/ci/docs_check.py index ed2743ca9655..f868986ae98d 100644 --- a/tests/ci/docs_check.py +++ b/tests/ci/docs_check.py @@ -61,10 +61,7 @@ def main(): if not pr_info.has_changes_in_documentation() and not args.force: logging.info("No changes in documentation") - commit = get_commit(gh, pr_info.sha) - commit.create_status( - context=NAME, description="No changes in docs", state="success" - ) + post_commit_status(gh, pr_info.sha, NAME, "No changes in docs", "success", "") sys.exit(0) if pr_info.has_changes_in_documentation(): From 144ebd3d36813950900262636058d34db2be1b0d Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Thu, 6 Apr 2023 11:03:32 +0200 Subject: [PATCH 272/406] Implement main() in run_check.py --- tests/ci/run_check.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/ci/run_check.py b/tests/ci/run_check.py index 44e1e4132c82..b0fe27d188c3 100644 --- a/tests/ci/run_check.py +++ b/tests/ci/run_check.py @@ -203,7 +203,7 @@ def check_pr_description(pr_info: PRInfo) -> Tuple[str, str]: return description_error, category -if __name__ == "__main__": +def main(): logging.basicConfig(level=logging.INFO) pr_info = PRInfo(need_orgs=True, pr_event_from_api=True, need_changed_files=True) @@ -295,3 +295,7 @@ def check_pr_description(pr_info: PRInfo) -> Tuple[str, str]: commit.create_status( context=NAME, description=description, state="pending", target_url=url ) + + +if __name__ == "__main__": + main() From ccd4fd7e64ec639909575b9dca3e6bbb27a84b99 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Thu, 6 Apr 2023 11:40:32 +0200 Subject: [PATCH 273/406] Use commit_status_helper.post_commit_status everywhere --- tests/ci/build_check.py | 15 ++++++++++++--- tests/ci/build_report_check.py | 15 ++++++++------- tests/ci/docs_check.py | 2 +- tests/ci/finish_check.py | 18 ++++++++++++------ tests/ci/functional_test_check.py | 10 +++------- tests/ci/mark_release_ready.py | 11 +++-------- tests/ci/run_check.py | 27 ++++++++++++--------------- 7 files changed, 51 insertions(+), 47 deletions(-) diff --git a/tests/ci/build_check.py b/tests/ci/build_check.py index a829069985db..a62340461c36 100644 --- a/tests/ci/build_check.py +++ b/tests/ci/build_check.py @@ -9,7 +9,11 @@ from typing import List, Tuple from ci_config import CI_CONFIG, BuildConfig -from commit_status_helper import get_commit_filtered_statuses, get_commit +from commit_status_helper import ( + get_commit_filtered_statuses, + get_commit, + post_commit_status, +) from docker_pull_helper import get_image_with_version from env_helper import ( GITHUB_JOB, @@ -248,8 +252,13 @@ def mark_failed_reports_pending(build_name: str, sha: str) -> None: "Commit already have failed status for '%s', setting it to 'pending'", report_status, ) - commit.create_status( - "pending", status.url, "Set to pending on rerun", report_status + post_commit_status( + gh, + sha, + report_status, + "Set to pending on rerun", + "pending", + status.url, ) except: # we do not care about any exception here logging.info("Failed to get or mark the reports status as pending, continue") diff --git a/tests/ci/build_report_check.py b/tests/ci/build_report_check.py index 32cbaf08f071..be96182ec267 100644 --- a/tests/ci/build_report_check.py +++ b/tests/ci/build_report_check.py @@ -22,7 +22,7 @@ from get_robot_token import get_best_robot_token from pr_info import NeedsDataType, PRInfo from commit_status_helper import ( - get_commit, + post_commit_status, update_mergeable_check, ) from ci_config import CI_CONFIG @@ -274,12 +274,13 @@ def main(): description = f"{ok_groups}/{total_groups} artifact groups are OK {addition}" - commit = get_commit(gh, pr_info.sha) - commit.create_status( - context=build_check_name, - description=description, - state=summary_status, - target_url=url, + post_commit_status( + gh, + pr_info.sha, + build_check_name, + description, + summary_status, + url, ) if summary_status == "error": diff --git a/tests/ci/docs_check.py b/tests/ci/docs_check.py index f868986ae98d..d8302ae4153f 100644 --- a/tests/ci/docs_check.py +++ b/tests/ci/docs_check.py @@ -9,7 +9,7 @@ from github import Github from clickhouse_helper import ClickHouseHelper, prepare_tests_results_for_clickhouse -from commit_status_helper import post_commit_status, get_commit, update_mergeable_check +from commit_status_helper import post_commit_status, update_mergeable_check from docker_pull_helper import get_image_with_version from env_helper import TEMP_PATH, REPO_COPY from get_robot_token import get_best_robot_token diff --git a/tests/ci/finish_check.py b/tests/ci/finish_check.py index ea2f5eb3136b..27d1249db679 100644 --- a/tests/ci/finish_check.py +++ b/tests/ci/finish_check.py @@ -5,7 +5,11 @@ from env_helper import GITHUB_RUN_URL from pr_info import PRInfo from get_robot_token import get_best_robot_token -from commit_status_helper import get_commit, get_commit_filtered_statuses +from commit_status_helper import ( + get_commit, + get_commit_filtered_statuses, + post_commit_status, +) NAME = "Run Check" @@ -25,9 +29,11 @@ if status.context == NAME and status.state == "pending" ) if pending_status: - commit.create_status( - context=NAME, - description="All checks finished", - state="success", - target_url=url, + post_commit_status( + gh, + pr_info.sha, + NAME, + "All checks finished", + "success", + url, ) diff --git a/tests/ci/functional_test_check.py b/tests/ci/functional_test_check.py index 813386bc0db6..573cfef1ae67 100644 --- a/tests/ci/functional_test_check.py +++ b/tests/ci/functional_test_check.py @@ -20,9 +20,8 @@ prepare_tests_results_for_clickhouse, ) from commit_status_helper import ( - post_commit_status, - get_commit, override_status, + post_commit_status, post_commit_status_to_file, update_mergeable_check, ) @@ -283,13 +282,10 @@ def main(): if run_changed_tests: tests_to_run = get_tests_to_run(pr_info) if not tests_to_run: - commit = get_commit(gh, pr_info.sha) state = override_status("success", check_name, validate_bugfix_check) if args.post_commit_status == "commit_status": - commit.create_status( - context=check_name_with_group, - description=NO_CHANGES_MSG, - state=state, + post_commit_status( + gh, pr_info.sha, check_name_with_group, NO_CHANGES_MSG, state, "" ) elif args.post_commit_status == "file": post_commit_status_to_file( diff --git a/tests/ci/mark_release_ready.py b/tests/ci/mark_release_ready.py index b103dd053bbc..22b1ce648028 100755 --- a/tests/ci/mark_release_ready.py +++ b/tests/ci/mark_release_ready.py @@ -4,7 +4,7 @@ import logging import os -from commit_status_helper import get_commit +from commit_status_helper import post_commit_status from env_helper import GITHUB_JOB_URL from get_robot_token import get_best_robot_token from github_helper import GitHub @@ -46,13 +46,8 @@ def main(): gh = GitHub(args.token, create_cache_dir=False) # Get the rate limits for a quick fail gh.get_rate_limit() - commit = get_commit(gh, args.commit) - - commit.create_status( - context=RELEASE_READY_STATUS, - description=description, - state="success", - target_url=url, + post_commit_status( + gh, args.commit, RELEASE_READY_STATUS, description, "success", url ) diff --git a/tests/ci/run_check.py b/tests/ci/run_check.py index b0fe27d188c3..c5deb9e971c1 100644 --- a/tests/ci/run_check.py +++ b/tests/ci/run_check.py @@ -9,6 +9,7 @@ from commit_status_helper import ( format_description, get_commit, + post_commit_status, post_labels, remove_labels, set_mergeable_check, @@ -253,10 +254,13 @@ def main(): if FEATURE_LABEL in pr_info.labels: print(f"The '{FEATURE_LABEL}' in the labels, expect the 'Docs Check' status") - commit.create_status( - context=DOCS_NAME, - description=f"expect adding docs for {FEATURE_LABEL}", - state="pending", + post_commit_status( + gh, + pr_info.sha, + DOCS_NAME, + f"expect adding docs for {FEATURE_LABEL}", + "pending", + "", ) else: set_mergeable_check(commit, "skipped") @@ -275,26 +279,19 @@ def main(): f"{GITHUB_SERVER_URL}/{GITHUB_REPOSITORY}/" "blob/master/.github/PULL_REQUEST_TEMPLATE.md?plain=1" ) - commit.create_status( - context=NAME, - description=format_description(description_error), - state="failure", - target_url=url, + post_commit_status( + gh, pr_info.sha, NAME, format_description(description_error), "failure", url ) sys.exit(1) url = GITHUB_RUN_URL if not can_run: print("::notice ::Cannot run") - commit.create_status( - context=NAME, description=description, state=labels_state, target_url=url - ) + post_commit_status(gh, pr_info.sha, NAME, description, labels_state, url) sys.exit(1) else: print("::notice ::Can run") - commit.create_status( - context=NAME, description=description, state="pending", target_url=url - ) + post_commit_status(gh, pr_info.sha, NAME, description, "pending", url) if __name__ == "__main__": From eb11fb02fa30869868004858e18a65e96b507731 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Thu, 6 Apr 2023 12:46:43 +0200 Subject: [PATCH 274/406] Clean up ast_fuzzer_check.py --- tests/ci/ast_fuzzer_check.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/ci/ast_fuzzer_check.py b/tests/ci/ast_fuzzer_check.py index 3c2fa05016fe..acd0d0d78bf9 100644 --- a/tests/ci/ast_fuzzer_check.py +++ b/tests/ci/ast_fuzzer_check.py @@ -15,7 +15,6 @@ GITHUB_REPOSITORY, GITHUB_RUN_URL, REPORTS_PATH, - REPO_COPY, TEMP_PATH, ) from get_robot_token import get_best_robot_token @@ -47,13 +46,12 @@ def get_commit(gh, commit_sha): return commit -if __name__ == "__main__": +def main(): logging.basicConfig(level=logging.INFO) stopwatch = Stopwatch() temp_path = TEMP_PATH - repo_path = REPO_COPY reports_path = REPORTS_PATH check_name = sys.argv[1] @@ -173,3 +171,7 @@ def get_commit(gh, commit_sha): logging.info("Result: '%s', '%s', '%s'", status, description, report_url) print(f"::notice ::Report url: {report_url}") post_commit_status(gh, pr_info.sha, check_name, description, status, report_url) + + +if __name__ == "__main__": + main() From 19fd5f9c3caea884f7c688a0ae55d7001cf0fdd8 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Wed, 12 Apr 2023 21:07:12 +0200 Subject: [PATCH 275/406] Add descriptions for the status comment --- tests/ci/ci_config.py | 157 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 156 insertions(+), 1 deletion(-) diff --git a/tests/ci/ci_config.py b/tests/ci/ci_config.py index bda04853d81a..56acbf7bc655 100644 --- a/tests/ci/ci_config.py +++ b/tests/ci/ci_config.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 -from typing import Dict, TypeVar +from dataclasses import dataclass +from typing import Callable, Dict, TypeVar ConfValue = TypeVar("ConfValue", str, bool) BuildConfig = Dict[str, ConfValue] @@ -399,3 +400,157 @@ "Unit tests (tsan)", "Unit tests (ubsan)", ] + + +@dataclass +class CheckDescription: + name: str + description: str # the check descriptions, will be put into the status table + match_func: Callable[[str], bool] # the function to check vs the commit status + + +CHECK_DESCRIPTIONS = [ + CheckDescription( + "AST fuzzer", + "Runs randomly generated queries to catch program errors. " + "The build type is optionally given in parenthesis. " + "If it fails, ask a maintainer for help.", + lambda x: x.startswith("AST fuzzer"), + ), + CheckDescription( + "Bugfix validate check", + "Checks that either a new test (functional or integration) or there " + "some changed tests that fail with the binary built on master branch", + lambda x: x == "Bugfix validate check", + ), + CheckDescription( + "ClickHouse build check", + "Builds ClickHouse in various configurations for use in further steps. " + "You have to fix the builds that fail. Build logs often has enough " + "information to fix the error, but you might have to reproduce the failure " + "locally. The `cmake` options can be found in the build log, grepping for " + "`cmake`. Use these options and follow the [general build process]" + "(https://clickhouse.com/docs/en/development/build).", + lambda x: x.startswith("ClickHouse") and x.endswith("build check"), + ), + CheckDescription( + "Compatibility check", + "Checks that `clickhouse` binary runs on distributions with old libc " + "versions. If it fails, ask a maintainer for help.", + lambda x: x.startswith("Compatibility check"), + ), + CheckDescription( + "Docker image for servers", + "The check to build and optionally push the mentioned image to docker hub", + lambda x: x.startswith("Docker image") + and (x.endswith("building check") or x.endswith("build and push")), + ), + CheckDescription( + "Docs Check", "Builds and tests the documentation", lambda x: x == "Docs Check" + ), + CheckDescription( + "Fast test", + "Normally this is the first check that is ran for a PR. It builds ClickHouse " + "and runs most of [stateless functional tests]" + "(https://clickhouse.com/docs/en/development/tests#functional-tests), " + "omitting some. If it fails, further checks are not started until it is fixed. " + "Look at the report to see which tests fail, then reproduce the failure " + "locally as described [here]" + "(https://clickhouse.com/docs/en/development/tests#functional-test-locally).", + lambda x: x == "Fast test", + ), + CheckDescription( + "Flaky tests", + "Runs a flaky tests from master multiple times to identify if they are stable.", + lambda x: "tests flaky check" in x, + ), + CheckDescription( + "Install packages", + "Checks that the built packages are installable in a clear environment", + lambda x: x.startswith("Install packages ("), + ), + CheckDescription( + "Integration tests", + "The integration tests report. In parenthesis the package type is given, " + "and in square brackets are the optional part/total tests", + lambda x: x.startswith("Integration tests ("), + ), + CheckDescription( + "Mergeable Check", + "Checks if all other necessary checks are successful", + lambda x: x == "Mergeable Check", + ), + CheckDescription( + "Performance Comparison", + "Measure changes in query performance. The performance test report is " + "described in detail [here](https://github.com/ClickHouse/ClickHouse/tree/" + "master/docker/test/performance-comparison#how-to-read-the-report). " + "In square brackets are the optional part/total tests", + lambda x: x.startswith("Performance Comparison"), + ), + CheckDescription( + "Push to Dockerhub", + "The check for building and pushing the CI related docker images to docker hub", + lambda x: x.startswith("Push") and "to Dockerhub" in x, + ), + CheckDescription( + "Run Check", + "A meta-check that indicates the running CI. Normally, it's in `success` or " + "`pending` state. The failed status indicates some problems with the PR", + lambda x: x == "Run Check", + ), + CheckDescription( + "Sqllogic", + "Run clickhouse on the [sqllogic](https://www.sqlite.org/sqllogictest) " + "test set against sqlite and checks that all statements are passed.", + lambda x: x.startswith("Sqllogic test"), + ), + CheckDescription( + "SQLancer", + "Fuzzing tests that detect logical bugs with " + "[SQLancer](https://github.com/sqlancer/sqlancer) tool.", + lambda x: x.startswith("SQLancer"), + ), + CheckDescription( + "Stateful tests", + "Runs stateful functional tests for ClickHouse binaries built in various " + "configurations -- release, debug, with sanitizers, etc.", + lambda x: x.startswith("Stateful tests ("), + ), + CheckDescription( + "Stateless tests", + "Runs stateless functional tests for ClickHouse binaries built in various " + "configurations -- release, debug, with sanitizers, etc.", + lambda x: x.startswith("Stateless tests ("), + ), + CheckDescription( + "Stress test", + "Runs stateless functional tests concurrently from several clients to detect " + "concurrency-related errors.", + lambda x: x.startswith("Stress test ("), + ), + CheckDescription( + "Style Check", + "Runs a set of checks to keep the code style clean. If some of tests failed, " + "see the related log from the report.", + lambda x: x == "Style Check", + ), + CheckDescription( + "Unit tests", + "Runs the unit tests for different release types", + lambda x: x.startswith("Unit tests ("), + ), + CheckDescription( + "Upgrade check", + "Runs stress tests on server version from last release and then tries to " + "upgrade it to the version from the PR. It checks if the new server can " + "successfully startup without any errors, crashes or sanitizer asserts.", + lambda x: x.startswith("Upgrade check ("), + ), + CheckDescription( + "Falback for unknown", + "There's no description for the check yet, please add it to " + "tests/ci/ci_config.py:CHECK_DESCRIPTIONS", + lambda x: True, + ), +] From 4405459925ea066514edbef26633bf3f5802225e Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Tue, 18 Apr 2023 16:58:17 +0200 Subject: [PATCH 276/406] Refactor RerunHelper to reduce API calls --- tests/ci/ast_fuzzer_check.py | 18 ++++++------ tests/ci/build_report_check.py | 6 ++-- tests/ci/commit_status_helper.py | 25 +++++++++++++++- tests/ci/compatibility_check.py | 6 ++-- tests/ci/docs_check.py | 11 ++++++-- tests/ci/fast_test_check.py | 6 ++-- tests/ci/functional_test_check.py | 6 ++-- tests/ci/install_check.py | 6 ++-- tests/ci/integration_test_check.py | 8 ++++-- tests/ci/jepsen_check.py | 6 ++-- tests/ci/performance_comparison_check.py | 5 ++-- tests/ci/rerun_helper.py | 36 ------------------------ tests/ci/sqlancer_check.py | 18 ++++++------ tests/ci/sqllogic_test.py | 11 ++++++-- tests/ci/stress_check.py | 6 ++-- tests/ci/style_check.py | 11 ++++++-- tests/ci/unit_tests_check.py | 11 ++++++-- 17 files changed, 104 insertions(+), 92 deletions(-) delete mode 100644 tests/ci/rerun_helper.py diff --git a/tests/ci/ast_fuzzer_check.py b/tests/ci/ast_fuzzer_check.py index acd0d0d78bf9..9ea2d862094e 100644 --- a/tests/ci/ast_fuzzer_check.py +++ b/tests/ci/ast_fuzzer_check.py @@ -9,10 +9,14 @@ from build_download_helper import get_build_name_for_check, read_build_urls from clickhouse_helper import ClickHouseHelper, prepare_tests_results_for_clickhouse -from commit_status_helper import format_description, post_commit_status +from commit_status_helper import ( + RerunHelper, + format_description, + get_commit, + post_commit_status, +) from docker_pull_helper import get_image_with_version from env_helper import ( - GITHUB_REPOSITORY, GITHUB_RUN_URL, REPORTS_PATH, TEMP_PATH, @@ -20,7 +24,6 @@ from get_robot_token import get_best_robot_token from pr_info import PRInfo from report import TestResult -from rerun_helper import RerunHelper from s3_helper import S3Helper from stopwatch import Stopwatch @@ -40,12 +43,6 @@ def get_run_command(pr_number, sha, download_url, workspace_path, image): ) -def get_commit(gh, commit_sha): - repo = gh.get_repo(GITHUB_REPOSITORY) - commit = repo.get_commit(commit_sha) - return commit - - def main(): logging.basicConfig(level=logging.INFO) @@ -62,8 +59,9 @@ def main(): pr_info = PRInfo() gh = Github(get_best_robot_token(), per_page=100) + commit = get_commit(gh, pr_info.sha) - rerun_helper = RerunHelper(gh, pr_info, check_name) + rerun_helper = RerunHelper(commit, check_name) if rerun_helper.is_already_finished_by_status(): logging.info("Check is already finished according to github status, exiting") sys.exit(0) diff --git a/tests/ci/build_report_check.py b/tests/ci/build_report_check.py index be96182ec267..2d4f87500b6a 100644 --- a/tests/ci/build_report_check.py +++ b/tests/ci/build_report_check.py @@ -22,11 +22,12 @@ from get_robot_token import get_best_robot_token from pr_info import NeedsDataType, PRInfo from commit_status_helper import ( + RerunHelper, + get_commit, post_commit_status, update_mergeable_check, ) from ci_config import CI_CONFIG -from rerun_helper import RerunHelper NEEDS_DATA_PATH = os.getenv("NEEDS_DATA_PATH", "") @@ -136,10 +137,11 @@ def main(): gh = Github(get_best_robot_token(), per_page=100) pr_info = PRInfo() + commit = get_commit(gh, pr_info.sha) atexit.register(update_mergeable_check, gh, pr_info, build_check_name) - rerun_helper = RerunHelper(gh, pr_info, build_check_name) + rerun_helper = RerunHelper(commit, build_check_name) if rerun_helper.is_already_finished_by_status(): logging.info("Check is already finished according to github status, exiting") sys.exit(0) diff --git a/tests/ci/commit_status_helper.py b/tests/ci/commit_status_helper.py index 6260abac1eb7..bef388de02f6 100644 --- a/tests/ci/commit_status_helper.py +++ b/tests/ci/commit_status_helper.py @@ -3,7 +3,7 @@ import csv import os import time -from typing import List, Literal +from typing import List, Literal, Optional import logging from github import Github @@ -19,6 +19,29 @@ MERGEABLE_NAME = "Mergeable Check" +class RerunHelper: + def __init__(self, commit: Commit, check_name: str): + self.check_name = check_name + self.commit = commit + self.statuses = get_commit_filtered_statuses(commit) + + def is_already_finished_by_status(self) -> bool: + # currently we agree even for failed statuses + for status in self.statuses: + if self.check_name in status.context and status.state in ( + "success", + "failure", + ): + return True + return False + + def get_finished_status(self) -> Optional[CommitStatus]: + for status in self.statuses: + if self.check_name in status.context: + return status + return None + + def override_status(status: str, check_name: str, invert: bool = False) -> str: if CI_CONFIG["tests_config"].get(check_name, {}).get("force_tests", False): return "success" diff --git a/tests/ci/compatibility_check.py b/tests/ci/compatibility_check.py index 432e9ec7c019..9490138d191b 100644 --- a/tests/ci/compatibility_check.py +++ b/tests/ci/compatibility_check.py @@ -16,13 +16,12 @@ mark_flaky_tests, prepare_tests_results_for_clickhouse, ) -from commit_status_helper import post_commit_status +from commit_status_helper import RerunHelper, get_commit, post_commit_status from docker_pull_helper import get_images_with_versions from env_helper import TEMP_PATH, REPORTS_PATH from get_robot_token import get_best_robot_token from pr_info import PRInfo from report import TestResults, TestResult -from rerun_helper import RerunHelper from s3_helper import S3Helper from stopwatch import Stopwatch from upload_result_helper import upload_results @@ -150,8 +149,9 @@ def main(): pr_info = PRInfo() gh = Github(get_best_robot_token(), per_page=100) + commit = get_commit(gh, pr_info.sha) - rerun_helper = RerunHelper(gh, pr_info, args.check_name) + rerun_helper = RerunHelper(commit, args.check_name) if rerun_helper.is_already_finished_by_status(): logging.info("Check is already finished according to github status, exiting") sys.exit(0) diff --git a/tests/ci/docs_check.py b/tests/ci/docs_check.py index d8302ae4153f..47ad0f8202fc 100644 --- a/tests/ci/docs_check.py +++ b/tests/ci/docs_check.py @@ -9,13 +9,17 @@ from github import Github from clickhouse_helper import ClickHouseHelper, prepare_tests_results_for_clickhouse -from commit_status_helper import post_commit_status, update_mergeable_check +from commit_status_helper import ( + RerunHelper, + get_commit, + post_commit_status, + update_mergeable_check, +) from docker_pull_helper import get_image_with_version from env_helper import TEMP_PATH, REPO_COPY from get_robot_token import get_best_robot_token from pr_info import PRInfo from report import TestResults, TestResult -from rerun_helper import RerunHelper from s3_helper import S3Helper from stopwatch import Stopwatch from tee_popen import TeePopen @@ -52,8 +56,9 @@ def main(): pr_info = PRInfo(need_changed_files=True) gh = Github(get_best_robot_token(), per_page=100) + commit = get_commit(gh, pr_info.sha) - rerun_helper = RerunHelper(gh, pr_info, NAME) + rerun_helper = RerunHelper(commit, NAME) if rerun_helper.is_already_finished_by_status(): logging.info("Check is already finished according to github status, exiting") sys.exit(0) diff --git a/tests/ci/fast_test_check.py b/tests/ci/fast_test_check.py index f13b40996572..688568bed6f8 100644 --- a/tests/ci/fast_test_check.py +++ b/tests/ci/fast_test_check.py @@ -17,6 +17,8 @@ prepare_tests_results_for_clickhouse, ) from commit_status_helper import ( + RerunHelper, + get_commit, post_commit_status, update_mergeable_check, ) @@ -25,7 +27,6 @@ from get_robot_token import get_best_robot_token from pr_info import FORCE_TESTS_LABEL, PRInfo from report import TestResults, read_test_results -from rerun_helper import RerunHelper from s3_helper import S3Helper from stopwatch import Stopwatch from tee_popen import TeePopen @@ -106,10 +107,11 @@ def main(): pr_info = PRInfo() gh = Github(get_best_robot_token(), per_page=100) + commit = get_commit(gh, pr_info.sha) atexit.register(update_mergeable_check, gh, pr_info, NAME) - rerun_helper = RerunHelper(gh, pr_info, NAME) + rerun_helper = RerunHelper(commit, NAME) if rerun_helper.is_already_finished_by_status(): logging.info("Check is already finished according to github status, exiting") status = rerun_helper.get_finished_status() diff --git a/tests/ci/functional_test_check.py b/tests/ci/functional_test_check.py index 573cfef1ae67..b89f03dbab55 100644 --- a/tests/ci/functional_test_check.py +++ b/tests/ci/functional_test_check.py @@ -20,6 +20,8 @@ prepare_tests_results_for_clickhouse, ) from commit_status_helper import ( + RerunHelper, + get_commit, override_status, post_commit_status, post_commit_status_to_file, @@ -31,7 +33,6 @@ from get_robot_token import get_best_robot_token from pr_info import FORCE_TESTS_LABEL, PRInfo from report import TestResults, read_test_results -from rerun_helper import RerunHelper from s3_helper import S3Helper from stopwatch import Stopwatch from tee_popen import TeePopen @@ -246,6 +247,7 @@ def main(): need_changed_files=run_changed_tests, pr_event_from_api=validate_bugfix_check ) + commit = get_commit(gh, pr_info.sha) atexit.register(update_mergeable_check, gh, pr_info, check_name) if not os.path.exists(temp_path): @@ -273,7 +275,7 @@ def main(): run_by_hash_total = 0 check_name_with_group = check_name - rerun_helper = RerunHelper(gh, pr_info, check_name_with_group) + rerun_helper = RerunHelper(commit, check_name_with_group) if rerun_helper.is_already_finished_by_status(): logging.info("Check is already finished according to github status, exiting") sys.exit(0) diff --git a/tests/ci/install_check.py b/tests/ci/install_check.py index 54245670b26c..2a8e6fb53b06 100644 --- a/tests/ci/install_check.py +++ b/tests/ci/install_check.py @@ -19,7 +19,9 @@ prepare_tests_results_for_clickhouse, ) from commit_status_helper import ( + RerunHelper, format_description, + get_commit, post_commit_status, update_mergeable_check, ) @@ -29,7 +31,6 @@ from get_robot_token import get_best_robot_token from pr_info import PRInfo from report import TestResults, TestResult -from rerun_helper import RerunHelper from s3_helper import S3Helper from stopwatch import Stopwatch from tee_popen import TeePopen @@ -268,9 +269,10 @@ def main(): if CI: gh = Github(get_best_robot_token(), per_page=100) + commit = get_commit(gh, pr_info.sha) atexit.register(update_mergeable_check, gh, pr_info, args.check_name) - rerun_helper = RerunHelper(gh, pr_info, args.check_name) + rerun_helper = RerunHelper(commit, args.check_name) if rerun_helper.is_already_finished_by_status(): logging.info( "Check is already finished according to github status, exiting" diff --git a/tests/ci/integration_test_check.py b/tests/ci/integration_test_check.py index f864751e8304..d16b2a545d32 100644 --- a/tests/ci/integration_test_check.py +++ b/tests/ci/integration_test_check.py @@ -19,8 +19,10 @@ prepare_tests_results_for_clickhouse, ) from commit_status_helper import ( - post_commit_status, + RerunHelper, + get_commit, override_status, + post_commit_status, post_commit_status_to_file, ) from docker_pull_helper import get_images_with_versions @@ -29,7 +31,6 @@ from get_robot_token import get_best_robot_token from pr_info import PRInfo from report import TestResults, read_test_results -from rerun_helper import RerunHelper from s3_helper import S3Helper from stopwatch import Stopwatch from tee_popen import TeePopen @@ -198,8 +199,9 @@ def main(): sys.exit(0) gh = Github(get_best_robot_token(), per_page=100) + commit = get_commit(gh, pr_info.sha) - rerun_helper = RerunHelper(gh, pr_info, check_name_with_group) + rerun_helper = RerunHelper(commit, check_name_with_group) if rerun_helper.is_already_finished_by_status(): logging.info("Check is already finished according to github status, exiting") sys.exit(0) diff --git a/tests/ci/jepsen_check.py b/tests/ci/jepsen_check.py index ffa9e45373f4..3679d00e592b 100644 --- a/tests/ci/jepsen_check.py +++ b/tests/ci/jepsen_check.py @@ -13,13 +13,12 @@ from build_download_helper import get_build_name_for_check from clickhouse_helper import ClickHouseHelper, prepare_tests_results_for_clickhouse -from commit_status_helper import post_commit_status +from commit_status_helper import RerunHelper, get_commit, post_commit_status from compress_files import compress_fast from env_helper import REPO_COPY, TEMP_PATH, S3_BUILDS_BUCKET, S3_DOWNLOAD from get_robot_token import get_best_robot_token, get_parameter_from_ssm from pr_info import PRInfo from report import TestResults, TestResult -from rerun_helper import RerunHelper from s3_helper import S3Helper from ssh import SSHKey from stopwatch import Stopwatch @@ -181,10 +180,11 @@ def get_run_command( sys.exit(0) gh = Github(get_best_robot_token(), per_page=100) + commit = get_commit(gh, pr_info.sha) check_name = KEEPER_CHECK_NAME if args.program == "keeper" else SERVER_CHECK_NAME - rerun_helper = RerunHelper(gh, pr_info, check_name) + rerun_helper = RerunHelper(commit, check_name) if rerun_helper.is_already_finished_by_status(): logging.info("Check is already finished according to github status, exiting") sys.exit(0) diff --git a/tests/ci/performance_comparison_check.py b/tests/ci/performance_comparison_check.py index 0da41e0ae82f..dd4896a89a62 100644 --- a/tests/ci/performance_comparison_check.py +++ b/tests/ci/performance_comparison_check.py @@ -12,13 +12,12 @@ from github import Github -from commit_status_helper import get_commit, post_commit_status +from commit_status_helper import RerunHelper, get_commit, post_commit_status from ci_config import CI_CONFIG from docker_pull_helper import get_image_with_version from env_helper import GITHUB_EVENT_PATH, GITHUB_RUN_URL, S3_BUILDS_BUCKET, S3_DOWNLOAD from get_robot_token import get_best_robot_token, get_parameter_from_ssm from pr_info import PRInfo -from rerun_helper import RerunHelper from s3_helper import S3Helper from tee_popen import TeePopen @@ -131,7 +130,7 @@ def __exit__(self, exc_type, exc_val, exc_tb): "Fill fliter our performance tests by grep -v %s", test_grep_exclude_filter ) - rerun_helper = RerunHelper(gh, pr_info, check_name_with_group) + rerun_helper = RerunHelper(commit, check_name_with_group) if rerun_helper.is_already_finished_by_status(): logging.info("Check is already finished according to github status, exiting") sys.exit(0) diff --git a/tests/ci/rerun_helper.py b/tests/ci/rerun_helper.py deleted file mode 100644 index fa73256d759f..000000000000 --- a/tests/ci/rerun_helper.py +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env python3 -from typing import Optional - -from commit_status_helper import get_commit, get_commit_filtered_statuses -from github import Github -from github.CommitStatus import CommitStatus -from pr_info import PRInfo - - -# TODO: move it to commit_status_helper -class RerunHelper: - def __init__(self, gh: Github, pr_info: PRInfo, check_name: str): - self.gh = gh - self.pr_info = pr_info - self.check_name = check_name - commit = get_commit(gh, self.pr_info.sha) - if commit is None: - raise ValueError(f"unable to receive commit for {pr_info.sha}") - self.pygh_commit = commit - self.statuses = get_commit_filtered_statuses(commit) - - def is_already_finished_by_status(self) -> bool: - # currently we agree even for failed statuses - for status in self.statuses: - if self.check_name in status.context and status.state in ( - "success", - "failure", - ): - return True - return False - - def get_finished_status(self) -> Optional[CommitStatus]: - for status in self.statuses: - if self.check_name in status.context: - return status - return None diff --git a/tests/ci/sqlancer_check.py b/tests/ci/sqlancer_check.py index 1a6c4d146162..04a664add0c8 100644 --- a/tests/ci/sqlancer_check.py +++ b/tests/ci/sqlancer_check.py @@ -10,10 +10,14 @@ from build_download_helper import get_build_name_for_check, read_build_urls from clickhouse_helper import ClickHouseHelper, prepare_tests_results_for_clickhouse -from commit_status_helper import format_description, post_commit_status +from commit_status_helper import ( + RerunHelper, + format_description, + get_commit, + post_commit_status, +) from docker_pull_helper import get_image_with_version from env_helper import ( - GITHUB_REPOSITORY, GITHUB_RUN_URL, REPORTS_PATH, TEMP_PATH, @@ -21,7 +25,6 @@ from get_robot_token import get_best_robot_token from pr_info import PRInfo from report import TestResults, TestResult -from rerun_helper import RerunHelper from s3_helper import S3Helper from stopwatch import Stopwatch from upload_result_helper import upload_results @@ -46,12 +49,6 @@ def get_run_command(download_url, workspace_path, image): ) -def get_commit(gh, commit_sha): - repo = gh.get_repo(GITHUB_REPOSITORY) - commit = repo.get_commit(commit_sha) - return commit - - def main(): logging.basicConfig(level=logging.INFO) @@ -68,8 +65,9 @@ def main(): pr_info = PRInfo() gh = Github(get_best_robot_token(), per_page=100) + commit = get_commit(gh, pr_info.sha) - rerun_helper = RerunHelper(gh, pr_info, check_name) + rerun_helper = RerunHelper(commit, check_name) if rerun_helper.is_already_finished_by_status(): logging.info("Check is already finished according to github status, exiting") sys.exit(0) diff --git a/tests/ci/sqllogic_test.py b/tests/ci/sqllogic_test.py index 9b41ff4680f6..ee550344a106 100755 --- a/tests/ci/sqllogic_test.py +++ b/tests/ci/sqllogic_test.py @@ -17,11 +17,15 @@ from build_download_helper import download_all_deb_packages from upload_result_helper import upload_results from docker_pull_helper import get_image_with_version -from commit_status_helper import override_status, post_commit_status +from commit_status_helper import ( + RerunHelper, + get_commit, + override_status, + post_commit_status, +) from report import TestResults, read_test_results from stopwatch import Stopwatch -from rerun_helper import RerunHelper from tee_popen import TeePopen @@ -103,8 +107,9 @@ def parse_args(): pr_info = PRInfo() gh = Github(get_best_robot_token(), per_page=100) + commit = get_commit(gh, pr_info.sha) - rerun_helper = RerunHelper(gh, pr_info, check_name) + rerun_helper = RerunHelper(commit, check_name) if rerun_helper.is_already_finished_by_status(): logging.info("Check is already finished according to github status, exiting") sys.exit(0) diff --git a/tests/ci/stress_check.py b/tests/ci/stress_check.py index 7596a81ebc9c..d2df766a8c24 100644 --- a/tests/ci/stress_check.py +++ b/tests/ci/stress_check.py @@ -16,13 +16,12 @@ mark_flaky_tests, prepare_tests_results_for_clickhouse, ) -from commit_status_helper import post_commit_status +from commit_status_helper import RerunHelper, get_commit, post_commit_status from docker_pull_helper import get_image_with_version from env_helper import TEMP_PATH, REPO_COPY, REPORTS_PATH from get_robot_token import get_best_robot_token from pr_info import PRInfo from report import TestResults, read_test_results -from rerun_helper import RerunHelper from s3_helper import S3Helper from stopwatch import Stopwatch from tee_popen import TeePopen @@ -125,8 +124,9 @@ def run_stress_test(docker_image_name): pr_info = PRInfo() gh = Github(get_best_robot_token(), per_page=100) + commit = get_commit(gh, pr_info.sha) - rerun_helper = RerunHelper(gh, pr_info, check_name) + rerun_helper = RerunHelper(commit, check_name) if rerun_helper.is_already_finished_by_status(): logging.info("Check is already finished according to github status, exiting") sys.exit(0) diff --git a/tests/ci/style_check.py b/tests/ci/style_check.py index 89878990c2c3..4b58c240c204 100644 --- a/tests/ci/style_check.py +++ b/tests/ci/style_check.py @@ -15,7 +15,12 @@ mark_flaky_tests, prepare_tests_results_for_clickhouse, ) -from commit_status_helper import post_commit_status, update_mergeable_check +from commit_status_helper import ( + RerunHelper, + get_commit, + post_commit_status, + update_mergeable_check, +) from docker_pull_helper import get_image_with_version from env_helper import GITHUB_WORKSPACE, RUNNER_TEMP from get_robot_token import get_best_robot_token @@ -23,7 +28,6 @@ from git_helper import git_runner from pr_info import PRInfo from report import TestResults, read_test_results -from rerun_helper import RerunHelper from s3_helper import S3Helper from ssh import SSHKey from stopwatch import Stopwatch @@ -149,10 +153,11 @@ def main(): checkout_head(pr_info) gh = GitHub(get_best_robot_token(), create_cache_dir=False) + commit = get_commit(gh, pr_info.sha) atexit.register(update_mergeable_check, gh, pr_info, NAME) - rerun_helper = RerunHelper(gh, pr_info, NAME) + rerun_helper = RerunHelper(commit, NAME) if rerun_helper.is_already_finished_by_status(): logging.info("Check is already finished according to github status, exiting") # Finish with the same code as previous diff --git a/tests/ci/unit_tests_check.py b/tests/ci/unit_tests_check.py index edc096908f41..75d28ee7fc33 100644 --- a/tests/ci/unit_tests_check.py +++ b/tests/ci/unit_tests_check.py @@ -15,13 +15,17 @@ mark_flaky_tests, prepare_tests_results_for_clickhouse, ) -from commit_status_helper import post_commit_status, update_mergeable_check +from commit_status_helper import ( + RerunHelper, + get_commit, + post_commit_status, + update_mergeable_check, +) from docker_pull_helper import get_image_with_version from env_helper import TEMP_PATH, REPORTS_PATH from get_robot_token import get_best_robot_token from pr_info import PRInfo from report import TestResults, TestResult -from rerun_helper import RerunHelper from s3_helper import S3Helper from stopwatch import Stopwatch from tee_popen import TeePopen @@ -116,10 +120,11 @@ def main(): pr_info = PRInfo() gh = Github(get_best_robot_token(), per_page=100) + commit = get_commit(gh, pr_info.sha) atexit.register(update_mergeable_check, gh, pr_info, check_name) - rerun_helper = RerunHelper(gh, pr_info, check_name) + rerun_helper = RerunHelper(commit, check_name) if rerun_helper.is_already_finished_by_status(): logging.info("Check is already finished according to github status, exiting") sys.exit(0) From d58abd3c3794543bfaf5ff93575ff301af0eb5d4 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Tue, 18 Apr 2023 21:32:22 +0200 Subject: [PATCH 277/406] Add a function get_repo with global state --- tests/ci/commit_status_helper.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/tests/ci/commit_status_helper.py b/tests/ci/commit_status_helper.py index bef388de02f6..3b72b6b590ac 100644 --- a/tests/ci/commit_status_helper.py +++ b/tests/ci/commit_status_helper.py @@ -9,6 +9,7 @@ from github import Github from github.Commit import Commit from github.CommitStatus import CommitStatus +from github.Repository import Repository from ci_config import CI_CONFIG, REQUIRED_CHECKS from env_helper import GITHUB_REPOSITORY, GITHUB_RUN_URL @@ -17,6 +18,7 @@ RETRY = 5 CommitStatuses = List[CommitStatus] MERGEABLE_NAME = "Mergeable Check" +GH_REPO = None # type: Optional[Repository] class RerunHelper: @@ -57,7 +59,7 @@ def override_status(status: str, check_name: str, invert: bool = False) -> str: def get_commit(gh: Github, commit_sha: str, retry_count: int = RETRY) -> Commit: for i in range(retry_count): try: - repo = gh.get_repo(GITHUB_REPOSITORY) + repo = get_repo(gh) commit = repo.get_commit(commit_sha) break except Exception as ex: @@ -113,8 +115,16 @@ def get_commit_filtered_statuses(commit: Commit) -> CommitStatuses: return list(filtered.values()) +def get_repo(gh: Github) -> Repository: + global GH_REPO + if GH_REPO is not None: + return GH_REPO + GH_REPO = gh.get_repo(GITHUB_REPOSITORY) + return GH_REPO + + def remove_labels(gh: Github, pr_info: PRInfo, labels_names: List[str]) -> None: - repo = gh.get_repo(GITHUB_REPOSITORY) + repo = get_repo(gh) pull_request = repo.get_pull(pr_info.number) for label in labels_names: pull_request.remove_from_labels(label) @@ -122,7 +132,7 @@ def remove_labels(gh: Github, pr_info: PRInfo, labels_names: List[str]) -> None: def post_labels(gh: Github, pr_info: PRInfo, labels_names: List[str]) -> None: - repo = gh.get_repo(GITHUB_REPOSITORY) + repo = get_repo(gh) pull_request = repo.get_pull(pr_info.number) for label in labels_names: pull_request.add_to_labels(label) From df33b99907fb549e5edeaeb645f452b844f884ca Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Wed, 19 Apr 2023 01:03:48 +0200 Subject: [PATCH 278/406] Rework post_commit_status, add set_status_comment there --- tests/ci/ast_fuzzer_check.py | 2 +- tests/ci/bugfix_validate_check.py | 11 +-- tests/ci/build_check.py | 14 ++-- tests/ci/build_report_check.py | 7 +- tests/ci/codebrowser_check.py | 5 +- tests/ci/commit_status_helper.py | 87 ++++++++++++++++++++++-- tests/ci/compatibility_check.py | 2 +- tests/ci/docker_images_check.py | 5 +- tests/ci/docker_manifests_merge.py | 5 +- tests/ci/docker_server.py | 5 +- tests/ci/docs_check.py | 4 +- tests/ci/fast_test_check.py | 2 +- tests/ci/finish_check.py | 9 +-- tests/ci/functional_test_check.py | 12 ++-- tests/ci/install_check.py | 2 +- tests/ci/integration_test_check.py | 9 +-- tests/ci/jepsen_check.py | 2 +- tests/ci/mark_release_ready.py | 6 +- tests/ci/performance_comparison_check.py | 4 +- tests/ci/run_check.py | 15 ++-- tests/ci/sqlancer_check.py | 8 +-- tests/ci/sqllogic_test.py | 2 +- tests/ci/stress_check.py | 2 +- tests/ci/style_check.py | 2 +- tests/ci/unit_tests_check.py | 2 +- 25 files changed, 142 insertions(+), 82 deletions(-) diff --git a/tests/ci/ast_fuzzer_check.py b/tests/ci/ast_fuzzer_check.py index 9ea2d862094e..514aaf7e2ac5 100644 --- a/tests/ci/ast_fuzzer_check.py +++ b/tests/ci/ast_fuzzer_check.py @@ -168,7 +168,7 @@ def main(): logging.info("Result: '%s', '%s', '%s'", status, description, report_url) print(f"::notice ::Report url: {report_url}") - post_commit_status(gh, pr_info.sha, check_name, description, status, report_url) + post_commit_status(commit, status, report_url, description, check_name, pr_info) if __name__ == "__main__": diff --git a/tests/ci/bugfix_validate_check.py b/tests/ci/bugfix_validate_check.py index 14ea58500bcf..e5ce655bdde3 100644 --- a/tests/ci/bugfix_validate_check.py +++ b/tests/ci/bugfix_validate_check.py @@ -8,7 +8,7 @@ from github import Github -from commit_status_helper import post_commit_status +from commit_status_helper import get_commit, post_commit_status from get_robot_token import get_best_robot_token from pr_info import PRInfo from report import TestResults, TestResult @@ -81,13 +81,14 @@ def main(args): ) gh = Github(get_best_robot_token(), per_page=100) + commit = get_commit(gh, pr_info.sha) post_commit_status( - gh, - pr_info.sha, - check_name_with_group, - "" if is_ok else "Changed tests don't reproduce the bug", + commit, "success" if is_ok else "error", report_url, + "" if is_ok else "Changed tests don't reproduce the bug", + check_name_with_group, + pr_info, ) diff --git a/tests/ci/build_check.py b/tests/ci/build_check.py index a62340461c36..1286701d6c28 100644 --- a/tests/ci/build_check.py +++ b/tests/ci/build_check.py @@ -236,10 +236,10 @@ def upload_master_static_binaries( print(f"::notice ::Binary static URL: {url}") -def mark_failed_reports_pending(build_name: str, sha: str) -> None: +def mark_failed_reports_pending(build_name: str, pr_info: PRInfo) -> None: try: gh = GitHub(get_best_robot_token()) - commit = get_commit(gh, sha) + commit = get_commit(gh, pr_info.sha) statuses = get_commit_filtered_statuses(commit) report_status = [ name @@ -253,12 +253,12 @@ def mark_failed_reports_pending(build_name: str, sha: str) -> None: report_status, ) post_commit_status( - gh, - sha, - report_status, - "Set to pending on rerun", + commit, "pending", status.url, + "Set to pending on rerun", + report_status, + pr_info, ) except: # we do not care about any exception here logging.info("Failed to get or mark the reports status as pending, continue") @@ -294,7 +294,7 @@ def main(): check_for_success_run(s3_helper, s3_path_prefix, build_name, build_config) # If it's a latter running, we need to mark possible failed status - mark_failed_reports_pending(build_name, pr_info.sha) + mark_failed_reports_pending(build_name, pr_info) docker_image = get_image_with_version(IMAGES_PATH, IMAGE_NAME) image_version = docker_image.version diff --git a/tests/ci/build_report_check.py b/tests/ci/build_report_check.py index 2d4f87500b6a..13257eabb717 100644 --- a/tests/ci/build_report_check.py +++ b/tests/ci/build_report_check.py @@ -277,12 +277,7 @@ def main(): description = f"{ok_groups}/{total_groups} artifact groups are OK {addition}" post_commit_status( - gh, - pr_info.sha, - build_check_name, - description, - summary_status, - url, + commit, summary_status, url, description, build_check_name, pr_info ) if summary_status == "error": diff --git a/tests/ci/codebrowser_check.py b/tests/ci/codebrowser_check.py index 9fa202a357c3..0eb4921e3fef 100644 --- a/tests/ci/codebrowser_check.py +++ b/tests/ci/codebrowser_check.py @@ -7,7 +7,7 @@ from github import Github -from commit_status_helper import post_commit_status +from commit_status_helper import get_commit, post_commit_status from docker_pull_helper import get_image_with_version from env_helper import ( IMAGES_PATH, @@ -43,6 +43,7 @@ def get_run_command(repo_path, output_path, image): gh = Github(get_best_robot_token(), per_page=100) pr_info = PRInfo() + commit = get_commit(gh, pr_info.sha) if not os.path.exists(TEMP_PATH): os.makedirs(TEMP_PATH) @@ -87,4 +88,4 @@ def get_run_command(repo_path, output_path, image): print(f"::notice ::Report url: {report_url}") - post_commit_status(gh, pr_info.sha, NAME, "Report built", "success", report_url) + post_commit_status(commit, "success", report_url, "Report built", NAME, pr_info) diff --git a/tests/ci/commit_status_helper.py b/tests/ci/commit_status_helper.py index 3b72b6b590ac..cec46c901c77 100644 --- a/tests/ci/commit_status_helper.py +++ b/tests/ci/commit_status_helper.py @@ -9,9 +9,10 @@ from github import Github from github.Commit import Commit from github.CommitStatus import CommitStatus +from github.IssueComment import IssueComment from github.Repository import Repository -from ci_config import CI_CONFIG, REQUIRED_CHECKS +from ci_config import CI_CONFIG, REQUIRED_CHECKS, CHECK_DESCRIPTIONS from env_helper import GITHUB_REPOSITORY, GITHUB_RUN_URL from pr_info import PRInfo, SKIP_MERGEABLE_CHECK_LABEL @@ -71,22 +72,98 @@ def get_commit(gh: Github, commit_sha: str, retry_count: int = RETRY) -> Commit: def post_commit_status( - gh: Github, sha: str, check_name: str, description: str, state: str, report_url: str + commit: Commit, + state: str, + report_url: str, + description: str, + check_name: str, + pr_info: Optional[PRInfo] = None, ) -> None: + """The parameters are given in the same order as for commit.create_status, + if an optional parameter `pr_info` is given, the `set_status_comment` functions + is invoked to add or update the comment with statuses overview""" for i in range(RETRY): try: - commit = get_commit(gh, sha, 1) commit.create_status( - context=check_name, - description=description, state=state, target_url=report_url, + description=description, + context=check_name, ) break except Exception as ex: if i == RETRY - 1: raise ex time.sleep(i) + if pr_info: + set_status_comment(commit, pr_info) + + +def set_status_comment(commit: Commit, pr_info: PRInfo) -> None: + """It adds or updates the comment status to all Pull Requests but for release + one, so the method does nothing for simple pushes and pull requests with + `release`/`release-lts` labels""" + if pr_info.number == 0 or pr_info.labels.intersection({"release", "release-lts"}): + return + # to reduce number of parameters, the Github is constructed on the fly + gh = Github() + gh.__requester = commit._requester # type:ignore #pylint:disable=protected-access + repo = get_repo(gh) + pr = repo.get_pull(pr_info.number) + comment_header = f"\n" + comment = None # type: Optional[IssueComment] + for ic in pr.get_issue_comments(): + if ic.body.startswith(comment_header): + comment = ic + break + + statuses_limit = 25 # the arbitrary limit to use
html tag + statuses = sorted(get_commit_filtered_statuses(commit), key=lambda x: x.context) + details = "" + details_ending = "" + if statuses_limit < len(statuses): + details = "
Show the statuses\n\n" + details_ending = "\n\n
" + comment_body = ( + f"This is an automated comment for commit `{commit.sha}` with " + f"description of existing statuses\n\n{details}" + "" + "\n" + ) + table_rows = [] # type: List[str] + for status in statuses: + description = "" # it's impossible to have nothing, but safer to define + for cd in CHECK_DESCRIPTIONS: + if cd.match_func(status.context): + description = cd.description + break + + if status.state == "success": + state = "🟢 " + elif status.state == "pending": + state = "🟡 " + elif status.state in ["error", "failure"]: + state = "🔴 " + else: + state = "" + + if status.target_url: + link = f'Report' + else: + link = "" + + table_rows.append( + f"" + f"" + f"\n" + ) + + comment_footer = f"
Check nameDescriptionStatusStatus messageOptional report URL
{status.context}{description}{state}{status.state}{status.description}{link}
{details_ending}" + comment_body = "".join([comment_header, comment_body, *table_rows, comment_footer]) + if comment is not None: + comment.edit(comment_body) + return + pr.create_issue_comment(comment_body) def post_commit_status_to_file( diff --git a/tests/ci/compatibility_check.py b/tests/ci/compatibility_check.py index 9490138d191b..04203617dca7 100644 --- a/tests/ci/compatibility_check.py +++ b/tests/ci/compatibility_check.py @@ -242,7 +242,7 @@ def url_filter(url): args.check_name, ) print(f"::notice ::Report url: {report_url}") - post_commit_status(gh, pr_info.sha, args.check_name, description, state, report_url) + post_commit_status(commit, state, report_url, description, args.check_name, pr_info) prepared_events = prepare_tests_results_for_clickhouse( pr_info, diff --git a/tests/ci/docker_images_check.py b/tests/ci/docker_images_check.py index f2b1105b3b0f..16a58a90dcf6 100644 --- a/tests/ci/docker_images_check.py +++ b/tests/ci/docker_images_check.py @@ -14,7 +14,7 @@ from github import Github from clickhouse_helper import ClickHouseHelper, prepare_tests_results_for_clickhouse -from commit_status_helper import format_description, post_commit_status +from commit_status_helper import format_description, get_commit, post_commit_status from env_helper import GITHUB_WORKSPACE, RUNNER_TEMP, GITHUB_RUN_URL from get_robot_token import get_best_robot_token, get_parameter_from_ssm from pr_info import PRInfo @@ -474,7 +474,8 @@ def main(): return gh = Github(get_best_robot_token(), per_page=100) - post_commit_status(gh, pr_info.sha, NAME, description, status, url) + commit = get_commit(gh, pr_info.sha) + post_commit_status(commit, status, url, description, NAME, pr_info) prepared_events = prepare_tests_results_for_clickhouse( pr_info, diff --git a/tests/ci/docker_manifests_merge.py b/tests/ci/docker_manifests_merge.py index 0484ea8f6416..d89708b9277e 100644 --- a/tests/ci/docker_manifests_merge.py +++ b/tests/ci/docker_manifests_merge.py @@ -10,7 +10,7 @@ from github import Github from clickhouse_helper import ClickHouseHelper, prepare_tests_results_for_clickhouse -from commit_status_helper import format_description, post_commit_status +from commit_status_helper import format_description, get_commit, post_commit_status from env_helper import RUNNER_TEMP from get_robot_token import get_best_robot_token, get_parameter_from_ssm from pr_info import PRInfo @@ -221,7 +221,8 @@ def main(): description = format_description(description) gh = Github(get_best_robot_token(), per_page=100) - post_commit_status(gh, pr_info.sha, NAME, description, status, url) + commit = get_commit(gh, pr_info.sha) + post_commit_status(commit, status, url, description, NAME, pr_info) prepared_events = prepare_tests_results_for_clickhouse( pr_info, diff --git a/tests/ci/docker_server.py b/tests/ci/docker_server.py index c6854c5aa788..a434d3cc8411 100644 --- a/tests/ci/docker_server.py +++ b/tests/ci/docker_server.py @@ -15,7 +15,7 @@ from build_check import get_release_or_pr from clickhouse_helper import ClickHouseHelper, prepare_tests_results_for_clickhouse -from commit_status_helper import format_description, post_commit_status +from commit_status_helper import format_description, get_commit, post_commit_status from docker_images_check import DockerImage from env_helper import CI, GITHUB_RUN_URL, RUNNER_TEMP, S3_BUILDS_BUCKET, S3_DOWNLOAD from get_robot_token import get_best_robot_token, get_parameter_from_ssm @@ -372,7 +372,8 @@ def main(): description = format_description(description) gh = Github(get_best_robot_token(), per_page=100) - post_commit_status(gh, pr_info.sha, NAME, description, status, url) + commit = get_commit(gh, pr_info.sha) + post_commit_status(commit, status, url, description, NAME, pr_info) prepared_events = prepare_tests_results_for_clickhouse( pr_info, diff --git a/tests/ci/docs_check.py b/tests/ci/docs_check.py index 47ad0f8202fc..0b632d20acbe 100644 --- a/tests/ci/docs_check.py +++ b/tests/ci/docs_check.py @@ -66,7 +66,7 @@ def main(): if not pr_info.has_changes_in_documentation() and not args.force: logging.info("No changes in documentation") - post_commit_status(gh, pr_info.sha, NAME, "No changes in docs", "success", "") + post_commit_status(commit, "success", "", "No changes in docs", NAME, pr_info) sys.exit(0) if pr_info.has_changes_in_documentation(): @@ -134,7 +134,7 @@ def main(): s3_helper, pr_info.number, pr_info.sha, test_results, additional_files, NAME ) print("::notice ::Report url: {report_url}") - post_commit_status(gh, pr_info.sha, NAME, description, status, report_url) + post_commit_status(commit, status, report_url, description, NAME, pr_info) prepared_events = prepare_tests_results_for_clickhouse( pr_info, diff --git a/tests/ci/fast_test_check.py b/tests/ci/fast_test_check.py index 688568bed6f8..89066ade2cb3 100644 --- a/tests/ci/fast_test_check.py +++ b/tests/ci/fast_test_check.py @@ -199,7 +199,7 @@ def main(): NAME, ) print(f"::notice ::Report url: {report_url}") - post_commit_status(gh, pr_info.sha, NAME, description, state, report_url) + post_commit_status(commit, state, report_url, description, NAME, pr_info) prepared_events = prepare_tests_results_for_clickhouse( pr_info, diff --git a/tests/ci/finish_check.py b/tests/ci/finish_check.py index 27d1249db679..ae8a60bb1e8a 100644 --- a/tests/ci/finish_check.py +++ b/tests/ci/finish_check.py @@ -29,11 +29,4 @@ if status.context == NAME and status.state == "pending" ) if pending_status: - post_commit_status( - gh, - pr_info.sha, - NAME, - "All checks finished", - "success", - url, - ) + post_commit_status(commit, "success", url, "All checks finished", NAME, pr_info) diff --git a/tests/ci/functional_test_check.py b/tests/ci/functional_test_check.py index b89f03dbab55..9d1f3056d669 100644 --- a/tests/ci/functional_test_check.py +++ b/tests/ci/functional_test_check.py @@ -287,7 +287,7 @@ def main(): state = override_status("success", check_name, validate_bugfix_check) if args.post_commit_status == "commit_status": post_commit_status( - gh, pr_info.sha, check_name_with_group, NO_CHANGES_MSG, state, "" + commit, state, "", NO_CHANGES_MSG, check_name_with_group, pr_info ) elif args.post_commit_status == "file": post_commit_status_to_file( @@ -374,16 +374,16 @@ def main(): if args.post_commit_status == "commit_status": if "parallelreplicas" in check_name.lower(): post_commit_status( - gh, - pr_info.sha, - check_name_with_group, - description, + commit, "success", report_url, + description, + check_name_with_group, + pr_info, ) else: post_commit_status( - gh, pr_info.sha, check_name_with_group, description, state, report_url + commit, state, report_url, description, check_name_with_group, pr_info ) elif args.post_commit_status == "file": if "parallelreplicas" in check_name.lower(): diff --git a/tests/ci/install_check.py b/tests/ci/install_check.py index 2a8e6fb53b06..d619ce96ceec 100644 --- a/tests/ci/install_check.py +++ b/tests/ci/install_check.py @@ -349,7 +349,7 @@ def filter_artifacts(path: str) -> bool: description = format_description(description) - post_commit_status(gh, pr_info.sha, args.check_name, description, state, report_url) + post_commit_status(commit, state, report_url, description, args.check_name, pr_info) prepared_events = prepare_tests_results_for_clickhouse( pr_info, diff --git a/tests/ci/integration_test_check.py b/tests/ci/integration_test_check.py index d16b2a545d32..8ef6244a1c55 100644 --- a/tests/ci/integration_test_check.py +++ b/tests/ci/integration_test_check.py @@ -286,15 +286,10 @@ def main(): print(f"::notice:: {check_name} Report url: {report_url}") if args.post_commit_status == "commit_status": post_commit_status( - gh, pr_info.sha, check_name_with_group, description, state, report_url + commit, state, report_url, description, check_name_with_group, pr_info ) elif args.post_commit_status == "file": - post_commit_status_to_file( - post_commit_path, - description, - state, - report_url, - ) + post_commit_status_to_file(post_commit_path, description, state, report_url) else: raise Exception( f'Unknown post_commit_status option "{args.post_commit_status}"' diff --git a/tests/ci/jepsen_check.py b/tests/ci/jepsen_check.py index 3679d00e592b..9d35d2d6e352 100644 --- a/tests/ci/jepsen_check.py +++ b/tests/ci/jepsen_check.py @@ -293,7 +293,7 @@ def get_run_command( ) print(f"::notice ::Report url: {report_url}") - post_commit_status(gh, pr_info.sha, check_name, description, status, report_url) + post_commit_status(commit, status, report_url, description, check_name, pr_info) ch_helper = ClickHouseHelper() prepared_events = prepare_tests_results_for_clickhouse( diff --git a/tests/ci/mark_release_ready.py b/tests/ci/mark_release_ready.py index 22b1ce648028..99037a1749a7 100755 --- a/tests/ci/mark_release_ready.py +++ b/tests/ci/mark_release_ready.py @@ -4,7 +4,7 @@ import logging import os -from commit_status_helper import post_commit_status +from commit_status_helper import get_commit, post_commit_status from env_helper import GITHUB_JOB_URL from get_robot_token import get_best_robot_token from github_helper import GitHub @@ -34,6 +34,7 @@ def main(): args = parser.parse_args() url = "" description = "the release can be created from the commit, manually set" + pr_info = None if not args.commit: pr_info = PRInfo() if pr_info.event == pr_info.default_event: @@ -45,9 +46,10 @@ def main(): gh = GitHub(args.token, create_cache_dir=False) # Get the rate limits for a quick fail + commit = get_commit(gh, args.commit) gh.get_rate_limit() post_commit_status( - gh, args.commit, RELEASE_READY_STATUS, description, "success", url + commit, "success", url, description, RELEASE_READY_STATUS, pr_info ) diff --git a/tests/ci/performance_comparison_check.py b/tests/ci/performance_comparison_check.py index dd4896a89a62..bf5704f31bd2 100644 --- a/tests/ci/performance_comparison_check.py +++ b/tests/ci/performance_comparison_check.py @@ -117,7 +117,7 @@ def __exit__(self, exc_type, exc_val, exc_tb): message = "Skipped, not labeled with 'pr-performance'" report_url = GITHUB_RUN_URL post_commit_status( - gh, pr_info.sha, check_name_with_group, message, status, report_url + commit, status, report_url, message, check_name_with_group, pr_info ) sys.exit(0) @@ -266,7 +266,7 @@ def __exit__(self, exc_type, exc_val, exc_tb): report_url = uploaded["report.html"] post_commit_status( - gh, pr_info.sha, check_name_with_group, message, status, report_url + commit, status, report_url, message, check_name_with_group, pr_info ) if status == "error": diff --git a/tests/ci/run_check.py b/tests/ci/run_check.py index c5deb9e971c1..ced46ebd3756 100644 --- a/tests/ci/run_check.py +++ b/tests/ci/run_check.py @@ -254,13 +254,12 @@ def main(): if FEATURE_LABEL in pr_info.labels: print(f"The '{FEATURE_LABEL}' in the labels, expect the 'Docs Check' status") - post_commit_status( - gh, - pr_info.sha, - DOCS_NAME, - f"expect adding docs for {FEATURE_LABEL}", + post_commit_status( # do not pass pr_info here intentionally + commit, "pending", "", + f"expect adding docs for {FEATURE_LABEL}", + DOCS_NAME, ) else: set_mergeable_check(commit, "skipped") @@ -280,18 +279,18 @@ def main(): "blob/master/.github/PULL_REQUEST_TEMPLATE.md?plain=1" ) post_commit_status( - gh, pr_info.sha, NAME, format_description(description_error), "failure", url + commit, "failure", url, format_description(description_error), NAME, pr_info ) sys.exit(1) url = GITHUB_RUN_URL if not can_run: print("::notice ::Cannot run") - post_commit_status(gh, pr_info.sha, NAME, description, labels_state, url) + post_commit_status(commit, labels_state, url, description, NAME, pr_info) sys.exit(1) else: print("::notice ::Can run") - post_commit_status(gh, pr_info.sha, NAME, description, "pending", url) + post_commit_status(commit, "pending", url, description, NAME, pr_info) if __name__ == "__main__": diff --git a/tests/ci/sqlancer_check.py b/tests/ci/sqlancer_check.py index 04a664add0c8..144dea54133f 100644 --- a/tests/ci/sqlancer_check.py +++ b/tests/ci/sqlancer_check.py @@ -185,12 +185,10 @@ def main(): check_name, ) - post_commit_status(gh, pr_info.sha, check_name, description, status, report_url) - + post_commit_status(commit, status, report_url, description, check_name, pr_info) print(f"::notice:: {check_name} Report url: {report_url}") ch_helper = ClickHouseHelper() - prepared_events = prepare_tests_results_for_clickhouse( pr_info, test_results, @@ -200,12 +198,8 @@ def main(): report_url, check_name, ) - ch_helper.insert_events_into(db="default", table="checks", events=prepared_events) - print(f"::notice Result: '{status}', '{description}', '{report_url}'") - post_commit_status(gh, pr_info.sha, check_name, description, status, report_url) - if __name__ == "__main__": main() diff --git a/tests/ci/sqllogic_test.py b/tests/ci/sqllogic_test.py index ee550344a106..942c9c60ee8c 100755 --- a/tests/ci/sqllogic_test.py +++ b/tests/ci/sqllogic_test.py @@ -208,7 +208,7 @@ def parse_args(): # Until it pass all tests, do not block CI, report "success" assert description is not None - post_commit_status(gh, pr_info.sha, check_name, description, "success", report_url) + post_commit_status(commit, "success", report_url, description, check_name, pr_info) if status != "success": if FORCE_TESTS_LABEL in pr_info.labels: diff --git a/tests/ci/stress_check.py b/tests/ci/stress_check.py index d2df766a8c24..ac280916a2f7 100644 --- a/tests/ci/stress_check.py +++ b/tests/ci/stress_check.py @@ -180,7 +180,7 @@ def run_stress_test(docker_image_name): ) print(f"::notice ::Report url: {report_url}") - post_commit_status(gh, pr_info.sha, check_name, description, state, report_url) + post_commit_status(commit, state, report_url, description, check_name, pr_info) prepared_events = prepare_tests_results_for_clickhouse( pr_info, diff --git a/tests/ci/style_check.py b/tests/ci/style_check.py index 4b58c240c204..33a5cd21f39b 100644 --- a/tests/ci/style_check.py +++ b/tests/ci/style_check.py @@ -195,7 +195,7 @@ def main(): s3_helper, pr_info.number, pr_info.sha, test_results, additional_files, NAME ) print(f"::notice ::Report url: {report_url}") - post_commit_status(gh, pr_info.sha, NAME, description, state, report_url) + post_commit_status(commit, state, report_url, description, NAME, pr_info) prepared_events = prepare_tests_results_for_clickhouse( pr_info, diff --git a/tests/ci/unit_tests_check.py b/tests/ci/unit_tests_check.py index 75d28ee7fc33..5279ccde492e 100644 --- a/tests/ci/unit_tests_check.py +++ b/tests/ci/unit_tests_check.py @@ -170,7 +170,7 @@ def main(): check_name, ) print(f"::notice ::Report url: {report_url}") - post_commit_status(gh, pr_info.sha, check_name, description, state, report_url) + post_commit_status(commit, state, report_url, description, check_name, pr_info) prepared_events = prepare_tests_results_for_clickhouse( pr_info, From f411be8c37c0a56eb1b70a80dfa69195c4c080bf Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Thu, 20 Apr 2023 13:55:33 +0200 Subject: [PATCH 279/406] Rename `Run Check` to `CI running` --- tests/ci/ci_config.py | 12 +++++----- tests/ci/commit_status_helper.py | 1 + tests/ci/finish_check.py | 41 ++++++++++++++++++++------------ tests/ci/run_check.py | 22 ++++++++++------- 4 files changed, 47 insertions(+), 29 deletions(-) diff --git a/tests/ci/ci_config.py b/tests/ci/ci_config.py index 56acbf7bc655..3f172e120a4e 100644 --- a/tests/ci/ci_config.py +++ b/tests/ci/ci_config.py @@ -423,6 +423,12 @@ class CheckDescription: "some changed tests that fail with the binary built on master branch", lambda x: x == "Bugfix validate check", ), + CheckDescription( + "CI running", + "A meta-check that indicates the running CI. Normally, it's in `success` or " + "`pending` state. The failed status indicates some problems with the PR", + lambda x: x == "CI running", + ), CheckDescription( "ClickHouse build check", "Builds ClickHouse in various configurations for use in further steps. " @@ -493,12 +499,6 @@ class CheckDescription: "The check for building and pushing the CI related docker images to docker hub", lambda x: x.startswith("Push") and "to Dockerhub" in x, ), - CheckDescription( - "Run Check", - "A meta-check that indicates the running CI. Normally, it's in `success` or " - "`pending` state. The failed status indicates some problems with the PR", - lambda x: x == "Run Check", - ), CheckDescription( "Sqllogic", "Run clickhouse on the [sqllogic](https://www.sqlite.org/sqllogictest) " diff --git a/tests/ci/commit_status_helper.py b/tests/ci/commit_status_helper.py index cec46c901c77..dbfea7d79c57 100644 --- a/tests/ci/commit_status_helper.py +++ b/tests/ci/commit_status_helper.py @@ -20,6 +20,7 @@ CommitStatuses = List[CommitStatus] MERGEABLE_NAME = "Mergeable Check" GH_REPO = None # type: Optional[Repository] +CI_STATUS_NAME = "CI running" class RerunHelper: diff --git a/tests/ci/finish_check.py b/tests/ci/finish_check.py index ae8a60bb1e8a..93e3d2c18e71 100644 --- a/tests/ci/finish_check.py +++ b/tests/ci/finish_check.py @@ -2,31 +2,42 @@ import logging from github import Github -from env_helper import GITHUB_RUN_URL -from pr_info import PRInfo -from get_robot_token import get_best_robot_token from commit_status_helper import ( + CI_STATUS_NAME, get_commit, get_commit_filtered_statuses, post_commit_status, ) - -NAME = "Run Check" +from env_helper import GITHUB_RUN_URL +from get_robot_token import get_best_robot_token +from pr_info import PRInfo -if __name__ == "__main__": +def main(): logging.basicConfig(level=logging.INFO) pr_info = PRInfo(need_orgs=True) gh = Github(get_best_robot_token(), per_page=100) commit = get_commit(gh, pr_info.sha) - url = GITHUB_RUN_URL - statuses = get_commit_filtered_statuses(commit) - pending_status = any( # find NAME status in pending state - True - for status in statuses - if status.context == NAME and status.state == "pending" - ) - if pending_status: - post_commit_status(commit, "success", url, "All checks finished", NAME, pr_info) + statuses = [ + status + for status in get_commit_filtered_statuses(commit) + if status.context == CI_STATUS_NAME + ] + if not statuses: + return + status = statuses[0] + if status.state == "pending": + post_commit_status( + commit, + "success", + status.target_url or "", + "All checks finished", + CI_STATUS_NAME, + pr_info, + ) + + +if __name__ == "__main__": + main() diff --git a/tests/ci/run_check.py b/tests/ci/run_check.py index ced46ebd3756..681466d5b9ad 100644 --- a/tests/ci/run_check.py +++ b/tests/ci/run_check.py @@ -7,6 +7,7 @@ from github import Github from commit_status_helper import ( + CI_STATUS_NAME, format_description, get_commit, post_commit_status, @@ -20,8 +21,6 @@ from pr_info import FORCE_TESTS_LABEL, PRInfo from workflow_approve_rerun_lambda.app import TRUSTED_CONTRIBUTORS -NAME = "Run Check" - TRUSTED_ORG_IDS = { 54801242, # clickhouse } @@ -90,7 +89,7 @@ def pr_is_by_trusted_user(pr_user_login, pr_user_orgs): # Returns whether we should look into individual checks for this PR. If not, it # can be skipped entirely. # Returns can_run, description, labels_state -def should_run_checks_for_pr(pr_info: PRInfo) -> Tuple[bool, str, str]: +def should_run_ci_for_pr(pr_info: PRInfo) -> Tuple[bool, str, str]: # Consider the labels and whether the user is trusted. print("Got labels", pr_info.labels) if FORCE_TESTS_LABEL in pr_info.labels: @@ -214,7 +213,7 @@ def main(): print("::notice ::Cannot run, no PR exists for the commit") sys.exit(1) - can_run, description, labels_state = should_run_checks_for_pr(pr_info) + can_run, description, labels_state = should_run_ci_for_pr(pr_info) if can_run and OK_SKIP_LABELS.intersection(pr_info.labels): print("::notice :: Early finish the check, running in a special PR") sys.exit(0) @@ -270,7 +269,7 @@ def main(): f"{description_error}" ) logging.info( - "PR body doesn't match the template: (start)\n%s\n(end)\n" "Reason: %s", + "PR body doesn't match the template: (start)\n%s\n(end)\nReason: %s", pr_info.body, description_error, ) @@ -279,18 +278,25 @@ def main(): "blob/master/.github/PULL_REQUEST_TEMPLATE.md?plain=1" ) post_commit_status( - commit, "failure", url, format_description(description_error), NAME, pr_info + commit, + "failure", + url, + format_description(description_error), + CI_STATUS_NAME, + pr_info, ) sys.exit(1) url = GITHUB_RUN_URL if not can_run: print("::notice ::Cannot run") - post_commit_status(commit, labels_state, url, description, NAME, pr_info) + post_commit_status( + commit, labels_state, url, description, CI_STATUS_NAME, pr_info + ) sys.exit(1) else: print("::notice ::Can run") - post_commit_status(commit, "pending", url, description, NAME, pr_info) + post_commit_status(commit, "pending", url, description, CI_STATUS_NAME, pr_info) if __name__ == "__main__": From c66e50f2835a0345d1a2f1fdabb5e6eced2eae6f Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Thu, 20 Apr 2023 14:20:14 +0200 Subject: [PATCH 280/406] Remove options from S3Helper, they are global --- tests/ci/s3_helper.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/ci/s3_helper.py b/tests/ci/s3_helper.py index fbe9f33b49bb..2af02d572c89 100644 --- a/tests/ci/s3_helper.py +++ b/tests/ci/s3_helper.py @@ -40,11 +40,11 @@ def _flatten_list(lst): class S3Helper: - def __init__(self, host=S3_URL, download_host=S3_DOWNLOAD): + def __init__(self): self.session = boto3.session.Session(region_name="us-east-1") - self.client = self.session.client("s3", endpoint_url=host) - self.host = host - self.download_host = download_host + self.client = self.session.client("s3", endpoint_url=S3_URL) + self.host = S3_URL + self.download_host = S3_DOWNLOAD def _upload_file_to_s3(self, bucket_name: str, file_path: str, s3_path: str) -> str: logging.debug( From e8cf417350335e7ebd9575d53925b61725d4c302 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Mon, 24 Apr 2023 15:41:43 +0200 Subject: [PATCH 281/406] Use NotSet in post_commit_status for optional arguments --- tests/ci/build_check.py | 3 ++- tests/ci/commit_status_helper.py | 9 +++++---- tests/ci/docs_check.py | 5 ++++- tests/ci/finish_check.py | 4 ++-- tests/ci/functional_test_check.py | 8 +++++++- tests/ci/mark_release_ready.py | 4 ++-- tests/ci/run_check.py | 12 +++++++----- 7 files changed, 29 insertions(+), 16 deletions(-) diff --git a/tests/ci/build_check.py b/tests/ci/build_check.py index 1286701d6c28..4bc61c79fc03 100644 --- a/tests/ci/build_check.py +++ b/tests/ci/build_check.py @@ -10,6 +10,7 @@ from ci_config import CI_CONFIG, BuildConfig from commit_status_helper import ( + NotSet, get_commit_filtered_statuses, get_commit, post_commit_status, @@ -255,7 +256,7 @@ def mark_failed_reports_pending(build_name: str, pr_info: PRInfo) -> None: post_commit_status( commit, "pending", - status.url, + status.target_url or NotSet, "Set to pending on rerun", report_status, pr_info, diff --git a/tests/ci/commit_status_helper.py b/tests/ci/commit_status_helper.py index dbfea7d79c57..28ca9d5655e5 100644 --- a/tests/ci/commit_status_helper.py +++ b/tests/ci/commit_status_helper.py @@ -3,10 +3,11 @@ import csv import os import time -from typing import List, Literal, Optional +from typing import List, Literal, Optional, Union import logging from github import Github +from github.GithubObject import _NotSetType, NotSet as NotSet # type: ignore from github.Commit import Commit from github.CommitStatus import CommitStatus from github.IssueComment import IssueComment @@ -75,9 +76,9 @@ def get_commit(gh: Github, commit_sha: str, retry_count: int = RETRY) -> Commit: def post_commit_status( commit: Commit, state: str, - report_url: str, - description: str, - check_name: str, + report_url: Union[_NotSetType, str] = NotSet, + description: Union[_NotSetType, str] = NotSet, + check_name: Union[_NotSetType, str] = NotSet, pr_info: Optional[PRInfo] = None, ) -> None: """The parameters are given in the same order as for commit.create_status, diff --git a/tests/ci/docs_check.py b/tests/ci/docs_check.py index 0b632d20acbe..e3930a20bd98 100644 --- a/tests/ci/docs_check.py +++ b/tests/ci/docs_check.py @@ -10,6 +10,7 @@ from clickhouse_helper import ClickHouseHelper, prepare_tests_results_for_clickhouse from commit_status_helper import ( + NotSet, RerunHelper, get_commit, post_commit_status, @@ -66,7 +67,9 @@ def main(): if not pr_info.has_changes_in_documentation() and not args.force: logging.info("No changes in documentation") - post_commit_status(commit, "success", "", "No changes in docs", NAME, pr_info) + post_commit_status( + commit, "success", NotSet, "No changes in docs", NAME, pr_info + ) sys.exit(0) if pr_info.has_changes_in_documentation(): diff --git a/tests/ci/finish_check.py b/tests/ci/finish_check.py index 93e3d2c18e71..aa8a0cf9553f 100644 --- a/tests/ci/finish_check.py +++ b/tests/ci/finish_check.py @@ -4,11 +4,11 @@ from commit_status_helper import ( CI_STATUS_NAME, + NotSet, get_commit, get_commit_filtered_statuses, post_commit_status, ) -from env_helper import GITHUB_RUN_URL from get_robot_token import get_best_robot_token from pr_info import PRInfo @@ -32,7 +32,7 @@ def main(): post_commit_status( commit, "success", - status.target_url or "", + status.target_url or NotSet, "All checks finished", CI_STATUS_NAME, pr_info, diff --git a/tests/ci/functional_test_check.py b/tests/ci/functional_test_check.py index 9d1f3056d669..037bb13f1f87 100644 --- a/tests/ci/functional_test_check.py +++ b/tests/ci/functional_test_check.py @@ -20,6 +20,7 @@ prepare_tests_results_for_clickhouse, ) from commit_status_helper import ( + NotSet, RerunHelper, get_commit, override_status, @@ -287,7 +288,12 @@ def main(): state = override_status("success", check_name, validate_bugfix_check) if args.post_commit_status == "commit_status": post_commit_status( - commit, state, "", NO_CHANGES_MSG, check_name_with_group, pr_info + commit, + state, + NotSet, + NO_CHANGES_MSG, + check_name_with_group, + pr_info, ) elif args.post_commit_status == "file": post_commit_status_to_file( diff --git a/tests/ci/mark_release_ready.py b/tests/ci/mark_release_ready.py index 99037a1749a7..4501d40e4d32 100755 --- a/tests/ci/mark_release_ready.py +++ b/tests/ci/mark_release_ready.py @@ -4,7 +4,7 @@ import logging import os -from commit_status_helper import get_commit, post_commit_status +from commit_status_helper import NotSet, get_commit, post_commit_status from env_helper import GITHUB_JOB_URL from get_robot_token import get_best_robot_token from github_helper import GitHub @@ -49,7 +49,7 @@ def main(): commit = get_commit(gh, args.commit) gh.get_rate_limit() post_commit_status( - commit, "success", url, description, RELEASE_READY_STATUS, pr_info + commit, "success", url or NotSet, description, RELEASE_READY_STATUS, pr_info ) diff --git a/tests/ci/run_check.py b/tests/ci/run_check.py index 681466d5b9ad..344ca208883d 100644 --- a/tests/ci/run_check.py +++ b/tests/ci/run_check.py @@ -8,6 +8,7 @@ from commit_status_helper import ( CI_STATUS_NAME, + NotSet, format_description, get_commit, post_commit_status, @@ -16,7 +17,7 @@ set_mergeable_check, ) from docs_check import NAME as DOCS_NAME -from env_helper import GITHUB_RUN_URL, GITHUB_REPOSITORY, GITHUB_SERVER_URL +from env_helper import GITHUB_REPOSITORY, GITHUB_SERVER_URL from get_robot_token import get_best_robot_token from pr_info import FORCE_TESTS_LABEL, PRInfo from workflow_approve_rerun_lambda.app import TRUSTED_CONTRIBUTORS @@ -256,7 +257,7 @@ def main(): post_commit_status( # do not pass pr_info here intentionally commit, "pending", - "", + NotSet, f"expect adding docs for {FEATURE_LABEL}", DOCS_NAME, ) @@ -287,16 +288,17 @@ def main(): ) sys.exit(1) - url = GITHUB_RUN_URL if not can_run: print("::notice ::Cannot run") post_commit_status( - commit, labels_state, url, description, CI_STATUS_NAME, pr_info + commit, labels_state, NotSet, description, CI_STATUS_NAME, pr_info ) sys.exit(1) else: print("::notice ::Can run") - post_commit_status(commit, "pending", url, description, CI_STATUS_NAME, pr_info) + post_commit_status( + commit, "pending", NotSet, description, CI_STATUS_NAME, pr_info + ) if __name__ == "__main__": From 61c6c6163850d3021a123eec3335a31bb70a54f8 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Tue, 25 Apr 2023 17:02:41 +0200 Subject: [PATCH 282/406] Rework the status comment to be a digest with the full report available by link --- tests/ci/ci_config.py | 3 + tests/ci/commit_status_helper.py | 155 ++++++++++++++++++++++--------- tests/ci/run_check.py | 6 +- 3 files changed, 119 insertions(+), 45 deletions(-) diff --git a/tests/ci/ci_config.py b/tests/ci/ci_config.py index 3f172e120a4e..ee7f08c319a8 100644 --- a/tests/ci/ci_config.py +++ b/tests/ci/ci_config.py @@ -408,6 +408,9 @@ class CheckDescription: description: str # the check descriptions, will be put into the status table match_func: Callable[[str], bool] # the function to check vs the commit status + def __hash__(self) -> int: + return hash(self.name + self.description) + CHECK_DESCRIPTIONS = [ CheckDescription( diff --git a/tests/ci/commit_status_helper.py b/tests/ci/commit_status_helper.py index 28ca9d5655e5..a8e89ac14659 100644 --- a/tests/ci/commit_status_helper.py +++ b/tests/ci/commit_status_helper.py @@ -3,7 +3,7 @@ import csv import os import time -from typing import List, Literal, Optional, Union +from typing import Dict, List, Literal, Optional, Union import logging from github import Github @@ -13,9 +13,12 @@ from github.IssueComment import IssueComment from github.Repository import Repository -from ci_config import CI_CONFIG, REQUIRED_CHECKS, CHECK_DESCRIPTIONS +from ci_config import CI_CONFIG, REQUIRED_CHECKS, CHECK_DESCRIPTIONS, CheckDescription from env_helper import GITHUB_REPOSITORY, GITHUB_RUN_URL from pr_info import PRInfo, SKIP_MERGEABLE_CHECK_LABEL +from report import TestResult, TestResults +from s3_helper import S3Helper +from upload_result_helper import upload_results RETRY = 5 CommitStatuses = List[CommitStatus] @@ -105,67 +108,133 @@ def set_status_comment(commit: Commit, pr_info: PRInfo) -> None: """It adds or updates the comment status to all Pull Requests but for release one, so the method does nothing for simple pushes and pull requests with `release`/`release-lts` labels""" - if pr_info.number == 0 or pr_info.labels.intersection({"release", "release-lts"}): - return # to reduce number of parameters, the Github is constructed on the fly gh = Github() gh.__requester = commit._requester # type:ignore #pylint:disable=protected-access repo = get_repo(gh) - pr = repo.get_pull(pr_info.number) - comment_header = f"\n" + statuses = sorted(get_commit_filtered_statuses(commit), key=lambda x: x.context) + if not statuses: + return + + # We update the report in generate_status_comment function, so do it each + # run, even in the release PRs and normal pushes + comment_body = generate_status_comment(pr_info, statuses) + # We post the comment only to normal and backport PRs + if pr_info.number == 0 or pr_info.labels.intersection({"release", "release-lts"}): + return + + comment_service_header = comment_body.split("\n", 1)[0] comment = None # type: Optional[IssueComment] + pr = repo.get_pull(pr_info.number) for ic in pr.get_issue_comments(): - if ic.body.startswith(comment_header): + if ic.body.startswith(comment_service_header): comment = ic break - statuses_limit = 25 # the arbitrary limit to use
html tag - statuses = sorted(get_commit_filtered_statuses(commit), key=lambda x: x.context) - details = "" - details_ending = "" - if statuses_limit < len(statuses): - details = "
Show the statuses\n\n" - details_ending = "\n\n
" + if comment is None: + pr.create_issue_comment(comment_body) + return + + if comment.body == comment_body: + logging.info("The status comment is already updated, no needs to change it") + return + comment.edit(comment_body) + + +def generate_status_comment(pr_info: PRInfo, statuses: CommitStatuses) -> str: + """The method generates the comment body, as well it updates the CI report""" + + def beauty_state(state: str) -> str: + if state == "success": + return f"🟢 {state}" + if state == "pending": + return f"🟡 {state}" + if state in ["error", "failure"]: + return f"🔴 {state}" + return state + + report_url = create_ci_report(pr_info, statuses) + worst_state = get_worst_state(statuses) + if not worst_state: + # Theoretically possible, although + # the function should not be used on empty statuses + worst_state = "The commit doesn't have the statuses yet" + else: + worst_state = f"The overall status of the commit is {beauty_state(worst_state)}" + comment_body = ( - f"This is an automated comment for commit `{commit.sha}` with " - f"description of existing statuses\n\n{details}" - "" - "\n" + f"\n" + f"This is an automated comment for commit `{pr_info.sha}` with " + f"description of existing statuses\n" + f"The full report is available [here]({report_url})\n" + f"{worst_state}\n\n
Check nameDescriptionStatusStatus messageOptional report URL
" + "\n" + "" ) - table_rows = [] # type: List[str] + # group checks by the name to get the worst one per each + grouped_statuses = {} # type: Dict[CheckDescription, CommitStatuses] for status in statuses: - description = "" # it's impossible to have nothing, but safer to define - for cd in CHECK_DESCRIPTIONS: - if cd.match_func(status.context): - description = cd.description + cd = None + for c in CHECK_DESCRIPTIONS: + if c.match_func(status.context): + cd = c break - if status.state == "success": - state = "🟢 " - elif status.state == "pending": - state = "🟡 " - elif status.state in ["error", "failure"]: - state = "🔴 " - else: - state = "" + if cd is None or cd == CHECK_DESCRIPTIONS[-1]: + # This is the case for either non-found description or a fallback + cd = CheckDescription( + status.context, + CHECK_DESCRIPTIONS[-1].description, + CHECK_DESCRIPTIONS[-1].match_func, + ) - if status.target_url: - link = f'Report' + if cd in grouped_statuses: + grouped_statuses[cd].append(status) else: - link = "" + grouped_statuses[cd] = [status] + table_rows = [] # type: List[str] + for desc, gs in grouped_statuses.items(): table_rows.append( - f"" - f"" - f"\n" + f"" + f"\n" ) - comment_footer = f"
Check nameDescriptionStatus
{status.context}{description}{state}{status.state}{status.description}{link}
{desc.name}{desc.description}{beauty_state(get_worst_state(gs))}
{details_ending}" - comment_body = "".join([comment_header, comment_body, *table_rows, comment_footer]) - if comment is not None: - comment.edit(comment_body) - return - pr.create_issue_comment(comment_body) + table_rows.sort() + + comment_footer = "" + return "".join([comment_body, *table_rows, comment_footer]) + + +def get_worst_state(statuses: CommitStatuses) -> str: + worst_status = None + states = {"error": 0, "failure": 1, "pending": 2, "success": 3} + for status in statuses: + if worst_status is None: + worst_status = status + continue + if states[status.state] < states[worst_status.state]: + worst_status = status + if worst_status.state == "error": + break + + if worst_status is None: + return "" + return worst_status.state + + +def create_ci_report(pr_info: PRInfo, statuses: CommitStatuses) -> str: + """The function converst the statuses to TestResults and uploads the report + to S3 tests bucket. Then it returns the URL""" + test_results = [] # type: TestResults + for status in statuses: + log_urls = None + if status.target_url is not None: + log_urls = [status.target_url] + test_results.append(TestResult(status.context, status.state, log_urls=log_urls)) + return upload_results( + S3Helper(), pr_info.number, pr_info.sha, test_results, [], CI_STATUS_NAME + ) def post_commit_status_to_file( diff --git a/tests/ci/run_check.py b/tests/ci/run_check.py index 344ca208883d..351e740bd3cc 100644 --- a/tests/ci/run_check.py +++ b/tests/ci/run_check.py @@ -9,6 +9,7 @@ from commit_status_helper import ( CI_STATUS_NAME, NotSet, + create_ci_report, format_description, get_commit, post_commit_status, @@ -288,16 +289,17 @@ def main(): ) sys.exit(1) + ci_report_url = create_ci_report(pr_info, []) if not can_run: print("::notice ::Cannot run") post_commit_status( - commit, labels_state, NotSet, description, CI_STATUS_NAME, pr_info + commit, labels_state, ci_report_url, description, CI_STATUS_NAME, pr_info ) sys.exit(1) else: print("::notice ::Can run") post_commit_status( - commit, "pending", NotSet, description, CI_STATUS_NAME, pr_info + commit, "pending", ci_report_url, description, CI_STATUS_NAME, pr_info ) From 20c8f0f1b464c6e3e8bcdc7c374ec74f5be609be Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Wed, 26 Apr 2023 17:26:34 +0200 Subject: [PATCH 283/406] Use only one comment for all statuses in the PR --- tests/ci/commit_status_helper.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/ci/commit_status_helper.py b/tests/ci/commit_status_helper.py index a8e89ac14659..666eb27919ea 100644 --- a/tests/ci/commit_status_helper.py +++ b/tests/ci/commit_status_helper.py @@ -163,9 +163,10 @@ def beauty_state(state: str) -> str: worst_state = f"The overall status of the commit is {beauty_state(worst_state)}" comment_body = ( - f"\n" - f"This is an automated comment for commit `{pr_info.sha}` with " - f"description of existing statuses\n" + f"\n" + f"This is an automated comment for commit {pr_info.sha} with " + f"description of existing statuses. It's updated for the latest CI running\n" f"The full report is available [here]({report_url})\n" f"{worst_state}\n\n" "\n" From 886ba0b01dcfd5bf6d196474e0a42428383234e3 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Wed, 26 Apr 2023 20:08:09 +0200 Subject: [PATCH 284/406] Replace markdown with pure HTML for check descriptions --- tests/ci/ci_config.py | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/tests/ci/ci_config.py b/tests/ci/ci_config.py index ee7f08c319a8..9dca9f2d8bdd 100644 --- a/tests/ci/ci_config.py +++ b/tests/ci/ci_config.py @@ -428,8 +428,8 @@ def __hash__(self) -> int: ), CheckDescription( "CI running", - "A meta-check that indicates the running CI. Normally, it's in `success` or " - "`pending` state. The failed status indicates some problems with the PR", + "A meta-check that indicates the running CI. Normally, it's in success or " + "pending state. The failed status indicates some problems with the PR", lambda x: x == "CI running", ), CheckDescription( @@ -437,14 +437,14 @@ def __hash__(self) -> int: "Builds ClickHouse in various configurations for use in further steps. " "You have to fix the builds that fail. Build logs often has enough " "information to fix the error, but you might have to reproduce the failure " - "locally. The `cmake` options can be found in the build log, grepping for " - "`cmake`. Use these options and follow the [general build process]" - "(https://clickhouse.com/docs/en/development/build).", + "locally. The cmake options can be found in the build log, grepping for " + 'cmake. Use these options and follow the general build process.', lambda x: x.startswith("ClickHouse") and x.endswith("build check"), ), CheckDescription( "Compatibility check", - "Checks that `clickhouse` binary runs on distributions with old libc " + "Checks that clickhouse binary runs on distributions with old libc " "versions. If it fails, ask a maintainer for help.", lambda x: x.startswith("Compatibility check"), ), @@ -460,12 +460,12 @@ def __hash__(self) -> int: CheckDescription( "Fast test", "Normally this is the first check that is ran for a PR. It builds ClickHouse " - "and runs most of [stateless functional tests]" - "(https://clickhouse.com/docs/en/development/tests#functional-tests), " + 'and runs most of stateless functional tests, ' "omitting some. If it fails, further checks are not started until it is fixed. " "Look at the report to see which tests fail, then reproduce the failure " - "locally as described [here]" - "(https://clickhouse.com/docs/en/development/tests#functional-test-locally).", + 'locally as described here.', lambda x: x == "Fast test", ), CheckDescription( @@ -492,8 +492,8 @@ def __hash__(self) -> int: CheckDescription( "Performance Comparison", "Measure changes in query performance. The performance test report is " - "described in detail [here](https://github.com/ClickHouse/ClickHouse/tree/" - "master/docker/test/performance-comparison#how-to-read-the-report). " + 'described in detail here. ' "In square brackets are the optional part/total tests", lambda x: x.startswith("Performance Comparison"), ), @@ -504,14 +504,15 @@ def __hash__(self) -> int: ), CheckDescription( "Sqllogic", - "Run clickhouse on the [sqllogic](https://www.sqlite.org/sqllogictest) " + "Run clickhouse on the " + 'sqllogic ' "test set against sqlite and checks that all statements are passed.", lambda x: x.startswith("Sqllogic test"), ), CheckDescription( "SQLancer", "Fuzzing tests that detect logical bugs with " - "[SQLancer](https://github.com/sqlancer/sqlancer) tool.", + 'SQLancer tool.', lambda x: x.startswith("SQLancer"), ), CheckDescription( From d336bee056697b0d1028b27d2f6871b729fd2925 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Fri, 28 Apr 2023 23:51:53 +0200 Subject: [PATCH 285/406] Fix the `Logs` header for case when only log_urls is set --- tests/ci/report.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/ci/report.py b/tests/ci/report.py index a40eb559792d..cdef8409e7ea 100644 --- a/tests/ci/report.py +++ b/tests/ci/report.py @@ -370,6 +370,7 @@ def create_test_html_report( colspan += 1 if test_result.log_urls is not None: + has_log_urls = True test_logs_html = "
".join( [_get_html_url(url) for url in test_result.log_urls] ) From f1ff2c0135a573d3af0c06c8a94be3bef883dd23 Mon Sep 17 00:00:00 2001 From: Nikita Taranov Date: Sat, 29 Apr 2023 18:44:50 +0200 Subject: [PATCH 286/406] Fix oss-fuzz build errors (#49236) --- .../AggregateFunctionQuantile.h | 4 --- ..._function_state_deserialization_fuzzer.cpp | 1 + .../MergeTree/MergeTreeBackgroundExecutor.cpp | 22 ++++--------- .../MergeTree/MergeTreeBackgroundExecutor.h | 33 ++++++++++++------- 4 files changed, 28 insertions(+), 32 deletions(-) diff --git a/src/AggregateFunctions/AggregateFunctionQuantile.h b/src/AggregateFunctions/AggregateFunctionQuantile.h index c03b7f16a2a8..13320ad90b6e 100644 --- a/src/AggregateFunctions/AggregateFunctionQuantile.h +++ b/src/AggregateFunctions/AggregateFunctionQuantile.h @@ -163,15 +163,11 @@ class AggregateFunctionQuantile final if constexpr (std::is_same_v>) { /// QuantileTiming only supports unsigned integers. Too large values are also meaningless. -#ifdef OS_DARWIN # pragma clang diagnostic push # pragma clang diagnostic ignored "-Wimplicit-const-int-float-conversion" -#endif if (isNaN(value) || value > std::numeric_limits::max() || value < 0) return; -#ifdef OS_DARWIN # pragma clang diagnostic pop -#endif } if constexpr (has_second_arg) diff --git a/src/AggregateFunctions/fuzzers/aggregate_function_state_deserialization_fuzzer.cpp b/src/AggregateFunctions/fuzzers/aggregate_function_state_deserialization_fuzzer.cpp index 39f57e00c483..2ea01e1d5bcc 100644 --- a/src/AggregateFunctions/fuzzers/aggregate_function_state_deserialization_fuzzer.cpp +++ b/src/AggregateFunctions/fuzzers/aggregate_function_state_deserialization_fuzzer.cpp @@ -13,6 +13,7 @@ #include +#include extern "C" int LLVMFuzzerTestOneInput(const uint8_t * data, size_t size) try diff --git a/src/Storages/MergeTree/MergeTreeBackgroundExecutor.cpp b/src/Storages/MergeTree/MergeTreeBackgroundExecutor.cpp index 6f1e41d27913..6512aad9260b 100644 --- a/src/Storages/MergeTree/MergeTreeBackgroundExecutor.cpp +++ b/src/Storages/MergeTree/MergeTreeBackgroundExecutor.cpp @@ -32,13 +32,15 @@ MergeTreeBackgroundExecutor::MergeTreeBackgroundExecutor( size_t threads_count_, size_t max_tasks_count_, CurrentMetrics::Metric metric_, - CurrentMetrics::Metric max_tasks_metric_) + CurrentMetrics::Metric max_tasks_metric_, + std::string_view policy) : name(name_) , threads_count(threads_count_) , max_tasks_count(max_tasks_count_) , metric(metric_) , max_tasks_metric(max_tasks_metric_, 2 * max_tasks_count) // active + pending - , pool(std::make_unique(CurrentMetrics::MergeTreeBackgroundExecutorThreads, CurrentMetrics::MergeTreeBackgroundExecutorThreadsActive)) + , pool(std::make_unique( + CurrentMetrics::MergeTreeBackgroundExecutorThreads, CurrentMetrics::MergeTreeBackgroundExecutorThreadsActive)) { if (max_tasks_count == 0) throw Exception(ErrorCodes::INVALID_CONFIG_PARAMETER, "Task count for MergeTreeBackgroundExecutor must not be zero"); @@ -52,20 +54,9 @@ MergeTreeBackgroundExecutor::MergeTreeBackgroundExecutor( for (size_t number = 0; number < threads_count; ++number) pool->scheduleOrThrowOnError([this] { threadFunction(); }); -} -template -MergeTreeBackgroundExecutor::MergeTreeBackgroundExecutor( - String name_, - size_t threads_count_, - size_t max_tasks_count_, - CurrentMetrics::Metric metric_, - CurrentMetrics::Metric max_tasks_metric_, - std::string_view policy) - requires requires(Queue queue) { queue.updatePolicy(policy); } // Because we use explicit template instantiation - : MergeTreeBackgroundExecutor(name_, threads_count_, max_tasks_count_, metric_, max_tasks_metric_) -{ - pending.updatePolicy(policy); + if (!policy.empty()) + pending.updatePolicy(policy); } template @@ -326,5 +317,4 @@ void MergeTreeBackgroundExecutor::threadFunction() template class MergeTreeBackgroundExecutor; template class MergeTreeBackgroundExecutor; template class MergeTreeBackgroundExecutor; - } diff --git a/src/Storages/MergeTree/MergeTreeBackgroundExecutor.h b/src/Storages/MergeTree/MergeTreeBackgroundExecutor.h index 9bfea32c7f93..8142e383d0c8 100644 --- a/src/Storages/MergeTree/MergeTreeBackgroundExecutor.h +++ b/src/Storages/MergeTree/MergeTreeBackgroundExecutor.h @@ -13,16 +13,22 @@ #include #include +#include +#include #include -#include +#include #include -#include -#include +#include namespace DB { +namespace ErrorCodes +{ + extern const int NOT_IMPLEMENTED; +} + struct TaskRuntimeData; using TaskRuntimeDataPtr = std::shared_ptr; @@ -92,6 +98,11 @@ class RoundRobinRuntimeQueue void setCapacity(size_t count) { queue.set_capacity(count); } bool empty() { return queue.empty(); } + [[noreturn]] void updatePolicy(std::string_view) + { + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method updatePolicy() is not implemented"); + } + static constexpr std::string_view name = "round_robin"; private: @@ -126,6 +137,11 @@ class PriorityRuntimeQueue void setCapacity(size_t count) { buffer.reserve(count); } bool empty() { return buffer.empty(); } + [[noreturn]] void updatePolicy(std::string_view) + { + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method updatePolicy() is not implemented"); + } + static constexpr std::string_view name = "shortest_task_first"; private: @@ -239,20 +255,14 @@ template class MergeTreeBackgroundExecutor final : boost::noncopyable { public: - MergeTreeBackgroundExecutor( - String name_, - size_t threads_count_, - size_t max_tasks_count_, - CurrentMetrics::Metric metric_, - CurrentMetrics::Metric max_tasks_metric_); MergeTreeBackgroundExecutor( String name_, size_t threads_count_, size_t max_tasks_count_, CurrentMetrics::Metric metric_, CurrentMetrics::Metric max_tasks_metric_, - std::string_view policy) - requires requires(Queue queue) { queue.updatePolicy(policy); }; // Because we use explicit template instantiation + std::string_view policy = {}); + ~MergeTreeBackgroundExecutor(); /// Handler for hot-reloading @@ -271,7 +281,6 @@ class MergeTreeBackgroundExecutor final : boost::noncopyable /// Update scheduling policy for pending tasks. It does nothing if `new_policy` is the same or unknown. void updateSchedulingPolicy(std::string_view new_policy) - requires requires(Queue queue) { queue.updatePolicy(new_policy); } // Because we use explicit template instantiation { std::lock_guard lock(mutex); pending.updatePolicy(new_policy); From d945089df830e24251e49e68b750d47cce98c757 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 29 Apr 2023 23:48:45 +0200 Subject: [PATCH 287/406] Fix tests visualizer --- utils/tests-visualizer/index.html | 45 +++++++------------------------ 1 file changed, 10 insertions(+), 35 deletions(-) diff --git a/utils/tests-visualizer/index.html b/utils/tests-visualizer/index.html index 11b2d6504e44..b2db5dbed338 100644 --- a/utils/tests-visualizer/index.html +++ b/utils/tests-visualizer/index.html @@ -20,9 +20,7 @@ width: 130px; display: block; margin: 30px auto; - -webkit-animation: spin 2s ease-in-out infinite; - -moz-animation: spin 2s ease-in-out infinite; - animation: spin 2s ease-in-out infinite; + animation: spin 10s ease-in-out infinite; } h1 { @@ -45,16 +43,9 @@ cursor: pointer; } - @-moz-keyframes spin { - 100% { -moz-transform: rotate(360deg); } - } - - @-webkit-keyframes spin { - 100% { -webkit-transform: rotate(360deg); } - } - @keyframes spin { - 100% { transform:rotate(360deg); } + 50% { transform:scale(150%); } + 100% { transform:scale(100%); } } @@ -67,33 +58,26 @@

Loading (~10 seconds, ~20 MB)

Check nameDescriptionStatus