diff --git a/ci/test/03_test_script.sh b/ci/test/03_test_script.sh index ff9f2fd60a5e..1b6831ae562b 100755 --- a/ci/test/03_test_script.sh +++ b/ci/test/03_test_script.sh @@ -201,7 +201,7 @@ fi if [[ "${RUN_IWYU}" == true ]]; then # TODO: Consider enforcing IWYU across the entire codebase. - FILES_WITH_ENFORCED_IWYU="/src/((crypto|index|kernel|primitives|univalue/(lib|test)|zmq)/.*\\.cpp|node/blockstorage\\.cpp|node/utxo_snapshot\\.cpp|core_io\\.cpp|signet\\.cpp)" + FILES_WITH_ENFORCED_IWYU="/src/((crypto|index|kernel|primitives|univalue/(lib|test)|zmq)/.*\\.cpp|node/blockstorage\\.cpp|node/utxo_snapshot\\.cpp|core_io\\.cpp|signet\\.cpp|common/system_ram\\.cpp|node/dbcache\\.cpp)" jq --arg patterns "$FILES_WITH_ENFORCED_IWYU" 'map(select(.file | test($patterns)))' "${BASE_BUILD_DIR}/compile_commands.json" > "${BASE_BUILD_DIR}/compile_commands_iwyu_errors.json" jq --arg patterns "$FILES_WITH_ENFORCED_IWYU" 'map(select(.file | test($patterns) | not))' "${BASE_BUILD_DIR}/compile_commands.json" > "${BASE_BUILD_DIR}/compile_commands_iwyu_warnings.json" diff --git a/doc/reduce-memory.md b/doc/reduce-memory.md index 44d90bcf5005..f8fbd88dfa4f 100644 --- a/doc/reduce-memory.md +++ b/doc/reduce-memory.md @@ -15,7 +15,8 @@ Bitcoin Core may warn at startup when `-dbcache` looks too large for the detecte The size of some in-memory caches can be reduced. As caches trade off memory usage for performance, reducing these will usually have a negative effect on performance. -- `-dbcache=` - the UTXO database cache size, this defaults to `1024` (or `450` if less than `4096` MiB system RAM is detected). The unit is MiB (1024). +- `-dbcache=` - the UTXO database cache size. The unit is MiB (1024). + - Automatic default scales with system RAM, up to 2 GiB. - The minimum value for `-dbcache` is 4. - A lower `-dbcache` makes initial sync time much longer. After the initial sync, the effect is less pronounced for most use-cases, unless fast validation of blocks is important, such as for mining. diff --git a/doc/release-notes-34641.md b/doc/release-notes-34641.md new file mode 100644 index 000000000000..863556df62a1 --- /dev/null +++ b/doc/release-notes-34641.md @@ -0,0 +1,7 @@ +Updated `dbcache` default settings +---------------------------------- + +- When `-dbcache` is not set explicitly, Bitcoin Core now chooses a RAM-aware default between 100 MiB and 2 GiB. + The selected `-dbcache` value is reduced after IBD for steady-state operation and unused mempool allocation may still be shared with this cache while syncing. + In environments with external memory limits (e.g. containers), automatic sizing may not match effective limits. + Set `-dbcache` explicitly if needed, to maintain the previous behavior, set `-dbcache=450`. (#34641) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index ad18115bbc5f..cf597cd6c702 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -104,6 +104,7 @@ add_library(bitcoin_common STATIC EXCLUDE_FROM_ALL common/settings.cpp common/signmessage.cpp common/system.cpp + common/system_ram.cpp common/url.cpp compressor.cpp core_io.cpp @@ -210,6 +211,7 @@ add_library(bitcoin_node STATIC EXCLUDE_FROM_ALL node/blockmanager_args.cpp node/blockstorage.cpp node/caches.cpp + node/dbcache.cpp node/chainstate.cpp node/chainstatemanager_args.cpp node/coin.cpp diff --git a/src/common/system.cpp b/src/common/system.cpp index 08c0c692711b..3abb60cc458e 100644 --- a/src/common/system.cpp +++ b/src/common/system.cpp @@ -111,22 +111,6 @@ int GetNumCores() return std::thread::hardware_concurrency(); } -std::optional GetTotalRAM() -{ - [[maybe_unused]] auto clamp{[](uint64_t v) { return size_t(std::min(v, uint64_t{std::numeric_limits::max()})); }}; -#ifdef WIN32 - if (MEMORYSTATUSEX m{}; (m.dwLength = sizeof(m), GlobalMemoryStatusEx(&m))) return clamp(m.ullTotalPhys); -#elif defined(__APPLE__) || \ - defined(__FreeBSD__) || \ - defined(__NetBSD__) || \ - defined(__OpenBSD__) || \ - defined(__illumos__) || \ - defined(__linux__) - if (long p{sysconf(_SC_PHYS_PAGES)}, s{sysconf(_SC_PAGESIZE)}; p > 0 && s > 0) return clamp(1ULL * p * s); -#endif - return std::nullopt; -} - namespace { const auto g_startup_time{SteadyClock::now()}; } // namespace diff --git a/src/common/system.h b/src/common/system.h index a3100fecbc17..c6c0305cf3fa 100644 --- a/src/common/system.h +++ b/src/common/system.h @@ -8,10 +8,6 @@ #include // IWYU pragma: keep #include - -#include -#include -#include #include /// Monotonic uptime (not affected by system time changes). @@ -32,9 +28,4 @@ void runCommand(const std::string& strCommand); */ int GetNumCores(); -/** - * Return the total RAM available on the current system, if detectable. - */ -std::optional GetTotalRAM(); - #endif // BITCOIN_COMMON_SYSTEM_H diff --git a/src/common/system_ram.cpp b/src/common/system_ram.cpp new file mode 100644 index 000000000000..5904e924adec --- /dev/null +++ b/src/common/system_ram.cpp @@ -0,0 +1,30 @@ +// Copyright (c) 2009-2010 Satoshi Nakamoto +// Copyright (c) 2009-present The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include + +#ifdef WIN32 +#include +#else +#include +#endif + +#include +#include +#include + +std::optional TryGetTotalRam() +{ + static const auto total_ram{[]() -> std::optional { + [[maybe_unused]] auto clamp{[](uint64_t v) { return size_t(std::min(v, uint64_t{std::numeric_limits::max()})); }}; +#ifdef WIN32 + if (MEMORYSTATUSEX m{}; (m.dwLength = sizeof(m), GlobalMemoryStatusEx(&m))) return clamp(m.ullTotalPhys); +#elif defined(__APPLE__) || defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) || defined(__illumos__) || defined(__linux__) + if (long p{sysconf(_SC_PHYS_PAGES)}, s{sysconf(_SC_PAGESIZE)}; p > 0 && s > 0) return clamp(1ULL * p * s); +#endif + return std::nullopt; + }()}; + return total_ram; +} diff --git a/src/common/system_ram.h b/src/common/system_ram.h new file mode 100644 index 000000000000..031f9c2768c6 --- /dev/null +++ b/src/common/system_ram.h @@ -0,0 +1,17 @@ +// Copyright (c) 2009-2010 Satoshi Nakamoto +// Copyright (c) 2009-present The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_COMMON_SYSTEM_RAM_H +#define BITCOIN_COMMON_SYSTEM_RAM_H + +#include +#include + +/** + * Return the total RAM available on the current system, if detectable. + */ +std::optional TryGetTotalRam(); + +#endif // BITCOIN_COMMON_SYSTEM_RAM_H diff --git a/src/init.cpp b/src/init.cpp index fb4b8fa9e27a..107b6fcd4784 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -51,6 +51,7 @@ #include #include #include +#include #include #include #include @@ -503,7 +504,7 @@ void SetupServerArgs(ArgsManager& argsman, bool can_listen_ipc) argsman.AddArg("-conf=", strprintf("Specify path to read-only configuration file. Relative paths will be prefixed by datadir location (only useable from command line, not configuration file) (default: %s)", BITCOIN_CONF_FILENAME), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-datadir=", "Specify data directory", ArgsManager::ALLOW_ANY | ArgsManager::DISALLOW_NEGATION, OptionsCategory::OPTIONS); argsman.AddArg("-dbbatchsize", strprintf("Maximum database write batch size in bytes (default: %u)", DEFAULT_DB_CACHE_BATCH), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::OPTIONS); - argsman.AddArg("-dbcache=", strprintf("Maximum database cache size MiB (minimum %d, default: %d). Make sure you have enough RAM. In addition, unused memory allocated to the mempool is shared with this cache (see -maxmempool).", MIN_DB_CACHE >> 20, node::GetDefaultDBCache() >> 20), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); + argsman.AddArg("-dbcache=", strprintf("Maximum database cache size MiB (minimum %s, default: %s MiB). In addition, unused memory allocated to the mempool is shared with this cache (see -maxmempool).", MIN_DB_CACHE >> 20, node::GetDefaultDBCache() >> 20), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-includeconf=", "Specify additional configuration file, relative to the -datadir path (only useable from configuration file, not command line)", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-allowignoredconf", strprintf("For backwards compatibility, treat an unused %s file in the datadir as a warning, not an error.", BITCOIN_CONF_FILENAME), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-loadblock=", "Imports blocks from external file on startup", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); @@ -659,6 +660,7 @@ void SetupServerArgs(ArgsManager& argsman, bool can_listen_ipc) argsman.AddArg("-limitclustersize=", strprintf("Do not accept transactions whose virtual size with all in-mempool connected transactions exceeds kilobytes (default: %u)", DEFAULT_CLUSTER_SIZE_LIMIT_KVB), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); argsman.AddArg("-capturemessages", "Capture all P2P messages to disk", ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); argsman.AddArg("-mocktime=", "Replace actual time with " + UNIX_EPOCH_TIME + " (default: 0)", ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); + argsman.AddArg("-mocktotalram=", "Replace detected system RAM with MiB for automatic dbcache sizing tests (default: 0, disabled)", ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); argsman.AddArg("-maxsigcachesize=", strprintf("Limit sum of signature cache and script execution cache sizes to MiB (default: %u)", DEFAULT_VALIDATION_CACHE_BYTES >> 20), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); argsman.AddArg("-maxtipage=", strprintf("Maximum tip age in seconds to consider node in initial block download (default: %u)", @@ -1302,6 +1304,7 @@ static ChainstateLoadResult InitAndLoadChainstate( NodeContext& node, bool do_reindex, const bool do_reindex_chainstate, + const size_t total_dbcache, const kernel::CacheSizes& cache_sizes, const ArgsManager& args) { @@ -1383,6 +1386,9 @@ static ChainstateLoadResult InitAndLoadChainstate( }; node::ChainstateLoadOptions options; options.mempool = Assert(node.mempool.get()); + options.auto_dbcache = !args.GetIntArg("-dbcache"); + options.total_ram_bytes = node::GetTotalRam(); + options.fixed_index_cache_bytes = total_dbcache - (cache_sizes.block_tree_db + cache_sizes.coins_db + cache_sizes.coins); options.wipe_chainstate_db = do_reindex || do_reindex_chainstate; options.prune = chainman.m_blockman.IsPruneMode(); options.check_blocks = args.GetIntArg("-checkblocks", DEFAULT_CHECKBLOCKS); @@ -1827,8 +1833,19 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) // ********************************************************* Step 7: load block chain // cache size calculations - node::LogOversizedDbCache(args); + if (args.GetIntArg("-dbcache")) { + node::LogOversizedDbCache(args); + } else { + node::LogAutoDbCacheSettings(); + } const auto [index_cache_sizes, kernel_cache_sizes] = CalculateCacheSizes(args, g_enabled_filter_types.size()); + const size_t total_dbcache{ + index_cache_sizes.tx_index + + index_cache_sizes.txospender_index + + index_cache_sizes.filter_index * g_enabled_filter_types.size() + + kernel_cache_sizes.block_tree_db + + kernel_cache_sizes.coins_db + + kernel_cache_sizes.coins}; LogInfo("Cache configuration:"); LogInfo("* Using %.1f MiB for block index database", kernel_cache_sizes.block_tree_db * (1.0 / 1024 / 1024)); @@ -1855,6 +1872,7 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) node, do_reindex, do_reindex_chainstate, + total_dbcache, kernel_cache_sizes, args); if (status == ChainstateLoadStatus::FAILURE && !do_reindex && !ShutdownRequested(node)) { @@ -1875,6 +1893,7 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) node, do_reindex, do_reindex_chainstate, + total_dbcache, kernel_cache_sizes, args); } diff --git a/src/kernel/CMakeLists.txt b/src/kernel/CMakeLists.txt index 244f0098fdf9..9e240f66085b 100644 --- a/src/kernel/CMakeLists.txt +++ b/src/kernel/CMakeLists.txt @@ -18,6 +18,7 @@ add_library(bitcoinkernel cs_main.cpp disconnected_transactions.cpp mempool_removal_reason.cpp + ../common/system_ram.cpp ../arith_uint256.cpp ../chain.cpp ../coins.cpp @@ -32,6 +33,7 @@ add_library(bitcoinkernel ../hash.cpp ../logging.cpp ../node/blockstorage.cpp + ../node/dbcache.cpp ../node/chainstate.cpp ../node/utxo_snapshot.cpp ../policy/ephemeral_policy.cpp diff --git a/src/kernel/bitcoinkernel.cpp b/src/kernel/bitcoinkernel.cpp index 09b4853fa91e..5da4ea24795d 100644 --- a/src/kernel/bitcoinkernel.cpp +++ b/src/kernel/bitcoinkernel.cpp @@ -19,6 +19,7 @@ #include #include #include +#include #include #include #include