node: shrink automatic dbcache after IBD

ci/test/03_test_script.sh

@@ -201,7 +201,7 @@ fi
 
 if [[ "${RUN_IWYU}" == true ]]; then
   # TODO: Consider enforcing IWYU across the entire codebase.
-  FILES_WITH_ENFORCED_IWYU="/src/((crypto|index|kernel|primitives|univalue/(lib|test)|zmq)/.*\\.cpp|node/blockstorage\\.cpp|node/utxo_snapshot\\.cpp|core_io\\.cpp|signet\\.cpp)"
+  FILES_WITH_ENFORCED_IWYU="/src/((crypto|index|kernel|primitives|univalue/(lib|test)|zmq)/.*\\.cpp|node/blockstorage\\.cpp|node/utxo_snapshot\\.cpp|core_io\\.cpp|signet\\.cpp|common/system_ram\\.cpp|node/dbcache\\.cpp)"
   jq --arg patterns "$FILES_WITH_ENFORCED_IWYU" 'map(select(.file | test($patterns)))' "${BASE_BUILD_DIR}/compile_commands.json" > "${BASE_BUILD_DIR}/compile_commands_iwyu_errors.json"
   jq --arg patterns "$FILES_WITH_ENFORCED_IWYU" 'map(select(.file | test($patterns) | not))' "${BASE_BUILD_DIR}/compile_commands.json" > "${BASE_BUILD_DIR}/compile_commands_iwyu_warnings.json"
 

doc/reduce-memory.md

@@ -15,7 +15,8 @@ Bitcoin Core may warn at startup when `-dbcache` looks too large for the detecte
 
 The size of some in-memory caches can be reduced. As caches trade off memory usage for performance, reducing these will usually have a negative effect on performance.
 
-- `-dbcache=<n>` - the UTXO database cache size, this defaults to `1024` (or `450` if less than `4096` MiB system RAM is detected). The unit is MiB (1024).
+- `-dbcache=<n>` - the UTXO database cache size. The unit is MiB (1024).
+  - Automatic default scales with system RAM, up to 2 GiB.
   - The minimum value for `-dbcache` is 4.
   - A lower `-dbcache` makes initial sync time much longer. After the initial sync, the effect is less pronounced for most use-cases, unless fast validation of blocks is important, such as for mining.
 

doc/release-notes-34641.md

@@ -0,0 +1,7 @@
+Updated `dbcache` default settings
+----------------------------------
+
+- When `-dbcache` is not set explicitly, Bitcoin Core now chooses a RAM-aware default between 100 MiB and 2 GiB.
+  The selected `-dbcache` value is reduced after IBD for steady-state operation and unused mempool allocation may still be shared with this cache while syncing.
+  In environments with external memory limits (e.g. containers), automatic sizing may not match effective limits.
+  Set `-dbcache` explicitly if needed, to maintain the previous behavior, set `-dbcache=450`. (#34641)

src/CMakeLists.txt

@@ -104,6 +104,7 @@ add_library(bitcoin_common STATIC EXCLUDE_FROM_ALL
   common/settings.cpp
   common/signmessage.cpp
   common/system.cpp
+  common/system_ram.cpp
   common/url.cpp
   compressor.cpp
   core_io.cpp
@@ -210,6 +211,7 @@ add_library(bitcoin_node STATIC EXCLUDE_FROM_ALL
   node/blockmanager_args.cpp
   node/blockstorage.cpp
   node/caches.cpp
+  node/dbcache.cpp
   node/chainstate.cpp
   node/chainstatemanager_args.cpp
   node/coin.cpp

src/common/system.cpp

@@ -111,22 +111,6 @@ int GetNumCores()
     return std::thread::hardware_concurrency();
 }
 
-std::optional<size_t> GetTotalRAM()
-{
-    [[maybe_unused]] auto clamp{[](uint64_t v) { return size_t(std::min(v, uint64_t{std::numeric_limits<size_t>::max()})); }};
-#ifdef WIN32
-    if (MEMORYSTATUSEX m{}; (m.dwLength = sizeof(m), GlobalMemoryStatusEx(&m))) return clamp(m.ullTotalPhys);
-#elif defined(__APPLE__) || \
-      defined(__FreeBSD__) || \
-      defined(__NetBSD__) || \
-      defined(__OpenBSD__) || \
-      defined(__illumos__) || \
-      defined(__linux__)
-    if (long p{sysconf(_SC_PHYS_PAGES)}, s{sysconf(_SC_PAGESIZE)}; p > 0 && s > 0) return clamp(1ULL * p * s);
-#endif
-    return std::nullopt;
-}
-
 namespace {
     const auto g_startup_time{SteadyClock::now()};
 } // namespace

src/common/system.h

@@ -8,10 +8,6 @@
 
 #include <bitcoin-build-config.h> // IWYU pragma: keep
 #include <util/time.h>
-
-#include <chrono>
-#include <cstdint>
-#include <optional>
 #include <string>
 
 /// Monotonic uptime (not affected by system time changes).
@@ -32,9 +28,4 @@ void runCommand(const std::string& strCommand);
  */
 int GetNumCores();
 
-/**
- * Return the total RAM available on the current system, if detectable.
- */
-std::optional<size_t> GetTotalRAM();
-
 #endif // BITCOIN_COMMON_SYSTEM_H

src/common/system_ram.cpp

@@ -0,0 +1,30 @@
+// Copyright (c) 2009-2010 Satoshi Nakamoto
+// Copyright (c) 2009-present The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#include <common/system_ram.h>
+
+#ifdef WIN32
+#include <windows.h>
+#else
+#include <unistd.h>
+#endif
+
+#include <algorithm>
+#include <cstdint>
+#include <limits>
+
+std::optional<size_t> TryGetTotalRam()
+{
+    static const auto total_ram{[]() -> std::optional<size_t> {
+        [[maybe_unused]] auto clamp{[](uint64_t v) { return size_t(std::min(v, uint64_t{std::numeric_limits<size_t>::max()})); }};
+#ifdef WIN32
+        if (MEMORYSTATUSEX m{}; (m.dwLength = sizeof(m), GlobalMemoryStatusEx(&m))) return clamp(m.ullTotalPhys);
+#elif defined(__APPLE__) || defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) || defined(__illumos__) || defined(__linux__)
+        if (long p{sysconf(_SC_PHYS_PAGES)}, s{sysconf(_SC_PAGESIZE)}; p > 0 && s > 0) return clamp(1ULL * p * s);
+#endif
+        return std::nullopt;
+    }()};
+    return total_ram;
+}

src/common/system_ram.h

@@ -0,0 +1,17 @@
+// Copyright (c) 2009-2010 Satoshi Nakamoto
+// Copyright (c) 2009-present The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#ifndef BITCOIN_COMMON_SYSTEM_RAM_H
+#define BITCOIN_COMMON_SYSTEM_RAM_H
+
+#include <cstddef>
+#include <optional>
+
+/**
+ * Return the total RAM available on the current system, if detectable.
+ */
+std::optional<size_t> TryGetTotalRam();
+
+#endif // BITCOIN_COMMON_SYSTEM_RAM_H

src/init.cpp

@@ -51,6 +51,7 @@
 #include <node/chainstate.h>
 #include <node/chainstatemanager_args.h>
 #include <node/context.h>
+#include <node/dbcache.h>
 #include <node/interface_ui.h>
 #include <node/kernel_notifications.h>
 #include <node/mempool_args.h>
@@ -503,7 +504,7 @@ void SetupServerArgs(ArgsManager& argsman, bool can_listen_ipc)
     argsman.AddArg("-conf=<file>", strprintf("Specify path to read-only configuration file. Relative paths will be prefixed by datadir location (only useable from command line, not configuration file) (default: %s)", BITCOIN_CONF_FILENAME), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
     argsman.AddArg("-datadir=<dir>", "Specify data directory", ArgsManager::ALLOW_ANY | ArgsManager::DISALLOW_NEGATION, OptionsCategory::OPTIONS);
     argsman.AddArg("-dbbatchsize", strprintf("Maximum database write batch size in bytes (default: %u)", DEFAULT_DB_CACHE_BATCH), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::OPTIONS);
-    argsman.AddArg("-dbcache=<n>", strprintf("Maximum database cache size <n> MiB (minimum %d, default: %d). Make sure you have enough RAM. In addition, unused memory allocated to the mempool is shared with this cache (see -maxmempool).", MIN_DB_CACHE >> 20, node::GetDefaultDBCache() >> 20), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
+    argsman.AddArg("-dbcache=<n>", strprintf("Maximum database cache size <n> MiB (minimum %s, default: %s MiB). In addition, unused memory allocated to the mempool is shared with this cache (see -maxmempool).", MIN_DB_CACHE >> 20, node::GetDefaultDBCache() >> 20), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
     argsman.AddArg("-includeconf=<file>", "Specify additional configuration file, relative to the -datadir path (only useable from configuration file, not command line)", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
     argsman.AddArg("-allowignoredconf", strprintf("For backwards compatibility, treat an unused %s file in the datadir as a warning, not an error.", BITCOIN_CONF_FILENAME), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
     argsman.AddArg("-loadblock=<file>", "Imports blocks from external file on startup", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
@@ -659,6 +660,7 @@ void SetupServerArgs(ArgsManager& argsman, bool can_listen_ipc)
     argsman.AddArg("-limitclustersize=<n>", strprintf("Do not accept transactions whose virtual size with all in-mempool connected transactions exceeds <n> kilobytes (default: %u)", DEFAULT_CLUSTER_SIZE_LIMIT_KVB), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST);
     argsman.AddArg("-capturemessages", "Capture all P2P messages to disk", ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST);
     argsman.AddArg("-mocktime=<n>", "Replace actual time with " + UNIX_EPOCH_TIME + " (default: 0)", ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST);
+    argsman.AddArg("-mocktotalram=<n>", "Replace detected system RAM with <n> MiB for automatic dbcache sizing tests (default: 0, disabled)", ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST);
     argsman.AddArg("-maxsigcachesize=<n>", strprintf("Limit sum of signature cache and script execution cache sizes to <n> MiB (default: %u)", DEFAULT_VALIDATION_CACHE_BYTES >> 20), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST);
     argsman.AddArg("-maxtipage=<n>",
                    strprintf("Maximum tip age in seconds to consider node in initial block download (default: %u)",
@@ -1302,6 +1304,7 @@ static ChainstateLoadResult InitAndLoadChainstate(
     NodeContext& node,
     bool do_reindex,
     const bool do_reindex_chainstate,
+    const size_t total_dbcache,
     const kernel::CacheSizes& cache_sizes,
     const ArgsManager& args)
 {
@@ -1383,6 +1386,9 @@ static ChainstateLoadResult InitAndLoadChainstate(
     };
     node::ChainstateLoadOptions options;
     options.mempool = Assert(node.mempool.get());
+    options.auto_dbcache = !args.GetIntArg("-dbcache");
+    options.total_ram_bytes = node::GetTotalRam();
+    options.fixed_index_cache_bytes = total_dbcache - (cache_sizes.block_tree_db + cache_sizes.coins_db + cache_sizes.coins);
     options.wipe_chainstate_db = do_reindex || do_reindex_chainstate;
     options.prune = chainman.m_blockman.IsPruneMode();
     options.check_blocks = args.GetIntArg("-checkblocks", DEFAULT_CHECKBLOCKS);
@@ -1827,8 +1833,19 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info)
     // ********************************************************* Step 7: load block chain
 
     // cache size calculations
-    node::LogOversizedDbCache(args);
+    if (args.GetIntArg("-dbcache")) {
+        node::LogOversizedDbCache(args);
+    } else {
+        node::LogAutoDbCacheSettings();
+    }
     const auto [index_cache_sizes, kernel_cache_sizes] = CalculateCacheSizes(args, g_enabled_filter_types.size());
+    const size_t total_dbcache{
+        index_cache_sizes.tx_index +
+        index_cache_sizes.txospender_index +
+        index_cache_sizes.filter_index * g_enabled_filter_types.size() +
+        kernel_cache_sizes.block_tree_db +
+        kernel_cache_sizes.coins_db +
+        kernel_cache_sizes.coins};
 
     LogInfo("Cache configuration:");
     LogInfo("* Using %.1f MiB for block index database", kernel_cache_sizes.block_tree_db * (1.0 / 1024 / 1024));
@@ -1855,6 +1872,7 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info)
         node,
         do_reindex,
         do_reindex_chainstate,
+        total_dbcache,
         kernel_cache_sizes,
         args);
     if (status == ChainstateLoadStatus::FAILURE && !do_reindex && !ShutdownRequested(node)) {
@@ -1875,6 +1893,7 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info)
             node,
             do_reindex,
             do_reindex_chainstate,
+            total_dbcache,
             kernel_cache_sizes,
             args);
     }

src/kernel/CMakeLists.txt

@@ -18,6 +18,7 @@ add_library(bitcoinkernel
   cs_main.cpp
   disconnected_transactions.cpp
   mempool_removal_reason.cpp
+  ../common/system_ram.cpp
   ../arith_uint256.cpp
   ../chain.cpp
   ../coins.cpp
@@ -32,6 +33,7 @@ add_library(bitcoinkernel
   ../hash.cpp
   ../logging.cpp
   ../node/blockstorage.cpp
+  ../node/dbcache.cpp
   ../node/chainstate.cpp
   ../node/utxo_snapshot.cpp
   ../policy/ephemeral_policy.cpp

src/kernel/bitcoinkernel.cpp

@@ -19,6 +19,7 @@
 #include <logging.h>
 #include <node/blockstorage.h>
 #include <node/chainstate.h>
+#include <node/dbcache.h>
 #include <primitives/block.h>
 #include <primitives/transaction.h>
 #include <script/interpreter.h>
@@ -460,7 +461,7 @@ struct ChainstateManagerOptions {
               .notifications = *context->m_notifications,
               .block_tree_db_params = DBParams{
                   .path = data_dir / "blocks" / "index",
-                  .cache_bytes = kernel::CacheSizes{DEFAULT_KERNEL_CACHE}.block_tree_db,
+                  .cache_bytes = kernel::CacheSizes{node::GetDefaultDBCache()}.block_tree_db,
               }}},
           m_context{context}, m_chainstate_load_options{node::ChainstateLoadOptions{}}
     {
@@ -1010,7 +1011,7 @@ btck_ChainstateManager* btck_chainstate_manager_create(
     try {
         const auto chainstate_load_opts{WITH_LOCK(opts.m_mutex, return opts.m_chainstate_load_options)};
 
-        kernel::CacheSizes cache_sizes{DEFAULT_KERNEL_CACHE};
+        kernel::CacheSizes cache_sizes{node::GetDefaultDBCache()};
         auto [status, chainstate_err]{node::LoadChainstate(*chainman, cache_sizes, chainstate_load_opts)};
         if (status != node::ChainstateLoadStatus::SUCCESS) {
             LogError("Failed to load chain state from your data directory: %s", chainstate_err.original);

src/kernel/caches.h

@@ -9,8 +9,6 @@
 
 #include <algorithm>
 
-//! Suggested default amount of cache reserved for the kernel (bytes)
-static constexpr size_t DEFAULT_KERNEL_CACHE{450_MiB};
 //! Default LevelDB write batch size
 static constexpr size_t DEFAULT_DB_CACHE_BATCH{32_MiB};
 

src/node/caches.cpp

@@ -3,18 +3,22 @@
 // file COPYING or http://www.opensource.org/licenses/mit-license.php.
 
 #include <node/caches.h>
-
 #include <common/args.h>
-#include <common/system.h>
+#include <common/system_ram.h>
 #include <index/txindex.h>
 #include <index/txospenderindex.h>
 #include <kernel/caches.h>
-#include <logging.h>
+#include <node/dbcache.h>
 #include <node/interface_ui.h>
 #include <tinyformat.h>
 #include <util/byte_units.h>
+#include <util/log.h>
+#include <util/overflow.h>
+#include <util/translation.h>
 
 #include <algorithm>
+#include <cstdint>
+#include <optional>
 #include <string>
 
 // Unlike for the UTXO database, for the txindex scenario the leveldb cache make
@@ -25,31 +29,14 @@ static constexpr size_t MAX_TX_INDEX_CACHE{1024_MiB};
 static constexpr size_t MAX_FILTER_INDEX_CACHE{1024_MiB};
 //! Max memory allocated to tx spenderindex DB specific cache in bytes.
 static constexpr size_t MAX_TXOSPENDER_INDEX_CACHE{1024_MiB};
-//! Maximum dbcache size on 32-bit systems.
-static constexpr size_t MAX_32BIT_DBCACHE{1024_MiB};
-//! Larger default dbcache on 64-bit systems with enough RAM.
-static constexpr size_t HIGH_DEFAULT_DBCACHE{1024_MiB};
-//! Minimum detected RAM required for HIGH_DEFAULT_DBCACHE.
-static constexpr uint64_t HIGH_DEFAULT_DBCACHE_MIN_TOTAL_RAM{4096ULL << 20};
 
 namespace node {
-size_t GetDefaultDBCache()
-{
-    if constexpr (sizeof(void*) >= 8) {
-        if (GetTotalRAM().value_or(0) >= HIGH_DEFAULT_DBCACHE_MIN_TOTAL_RAM) {
-            return HIGH_DEFAULT_DBCACHE;
-        }
-    }
-    return DEFAULT_DB_CACHE;
-}
-
 size_t CalculateDbCacheBytes(const ArgsManager& args)
 {
     if (auto db_cache{args.GetIntArg("-dbcache")}) {
         if (*db_cache < 0) db_cache = 0;
         const uint64_t db_cache_bytes{SaturatingLeftShift<uint64_t>(*db_cache, 20)};
-        constexpr auto max_db_cache{sizeof(void*) == 4 ? MAX_32BIT_DBCACHE : std::numeric_limits<size_t>::max()};
-        return std::max<size_t>(MIN_DB_CACHE, std::min<uint64_t>(db_cache_bytes, max_db_cache));
+        return std::max<size_t>(MIN_DB_CACHE, std::min<uint64_t>(db_cache_bytes, MAX_DBCACHE_BYTES));
     }
     return GetDefaultDBCache();
 }
@@ -73,12 +60,20 @@ CacheSizes CalculateCacheSizes(const ArgsManager& args, size_t n_indexes)
 
 void LogOversizedDbCache(const ArgsManager& args) noexcept
 {
-    if (const auto total_ram{GetTotalRAM()}) {
+    if (const auto total_ram{TryGetTotalRam()}) {
         const size_t db_cache{CalculateDbCacheBytes(args)};
         if (ShouldWarnOversizedDbCache(db_cache, *total_ram)) {
-            InitWarning(bilingual_str{tfm::format(_("A %zu MiB dbcache may be too large for a system memory of only %zu MiB."),
+            InitWarning(bilingual_str{tfm::format(_("A %s MiB dbcache may be too large for a system with only %s MiB of memory."),
                         db_cache >> 20, *total_ram >> 20)});
         }
     }
 }
+
+void LogAutoDbCacheSettings() noexcept
+{
+    LogInfo("Automatically selected -dbcache=%s MiB based on %s system memory of %s MiB.",
+            GetDefaultDBCache(GetTotalRam()) >> 20,
+            TryGetTotalRam() ? "detected" : "assumed",
+            GetTotalRam() >> 20);
+}
 } // namespace node

src/node/caches.h

@@ -6,19 +6,11 @@
 #define BITCOIN_NODE_CACHES_H
 
 #include <kernel/caches.h>
-#include <util/byte_units.h>
-
 #include <cstddef>
 
 class ArgsManager;
 
-//! min. -dbcache (bytes)
-static constexpr size_t MIN_DB_CACHE{4_MiB};
-//! -dbcache default (bytes)
-static constexpr size_t DEFAULT_DB_CACHE{DEFAULT_KERNEL_CACHE};
-
 namespace node {
-size_t GetDefaultDBCache();
 struct IndexCacheSizes {
     size_t tx_index{0};
     size_t filter_index{0};
@@ -29,13 +21,9 @@ struct CacheSizes {
     kernel::CacheSizes kernel;
 };
 CacheSizes CalculateCacheSizes(const ArgsManager& args, size_t n_indexes = 0);
-constexpr bool ShouldWarnOversizedDbCache(size_t dbcache, size_t total_ram) noexcept
-{
-    const size_t cap{(total_ram < 2048_MiB) ? DEFAULT_DB_CACHE : (total_ram / 100) * 75};
-    return dbcache > cap;
-}
 
 void LogOversizedDbCache(const ArgsManager& args) noexcept;
+void LogAutoDbCacheSettings() noexcept;
 } // namespace node
 
 #endif // BITCOIN_NODE_CACHES_H