Merge pull request #96 from ajtowns/202509-inq29-sendtemplate1

net: Provide block templates to peers on request

Author: ajtowns

src/blockencodings.cpp

@@ -18,14 +18,22 @@
 #include <unordered_map>
 
 CBlockHeaderAndShortTxIDs::CBlockHeaderAndShortTxIDs(const CBlock& block, const uint64_t nonce) :
-        nonce(nonce),
-        shorttxids(block.vtx.size() - 1), prefilledtxn(1), header(block) {
+        nonce(nonce), header(block)
+{
+    const bool prefill_cb = block.vtx.size() != 0 && block.vtx[0]->IsCoinBase();
+
     FillShortTxIDSelector();
+    shorttxids.resize(block.vtx.size() - (size_t)prefill_cb);
+
     //TODO: Use our mempool prior to block acceptance to predictively fill more than just the coinbase
-    prefilledtxn[0] = {0, block.vtx[0]};
-    for (size_t i = 1; i < block.vtx.size(); i++) {
+    prefilledtxn.resize((size_t)prefill_cb);
+    if (prefill_cb) {
+        prefilledtxn[0] = {0, block.vtx[0]};
+    }
+
+    for (size_t i = prefill_cb; i < block.vtx.size(); i++) {
         const CTransaction& tx = *block.vtx[i];
-        shorttxids[i - 1] = GetShortID(tx.GetWitnessHash());
+        shorttxids[i - (size_t)prefill_cb] = GetShortID(tx.GetWitnessHash());
     }
 }
 

src/init.cpp

@@ -576,6 +576,7 @@ void SetupServerArgs(ArgsManager& argsman, bool can_listen_ipc)
         "-whitebind. "
         "Additional flags \"in\" and \"out\" control whether permissions apply to incoming connections and/or manual (default: incoming only). "
         "Can be specified multiple times.", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION);
+    argsman.AddArg("-sharetemplatecount", strprintf("Number of block templates to store and share to peers (default: %d). Set to 0 to disable sharing block templates.", DEFAULT_SHARETMPL_COUNT), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::NODE_RELAY);
 
     g_wallet_init_interface.AddWalletOptions(argsman);
 
@@ -1081,6 +1082,13 @@ bool AppInitParameterInteraction(const ArgsManager& args)
         }
     }
 
+    if (args.GetBoolArg("-printpriority", DEFAULT_PRINT_MODIFIED_FEE)) {
+        if (!LogInstance().WillLogCategory(BCLog::MINER)) {
+            LogInstance().EnableCategory(BCLog::MINER);
+            LogInfo("parameter interaction: -printpriority -> setting -debug=miner\n");
+        }
+    }
+
     // Also report errors from parsing before daemonization
     {
         kernel::Notifications notifications{};

src/logging.cpp

@@ -201,6 +201,8 @@ static const std::map<std::string, BCLog::LogFlags, std::less<>> LOG_CATEGORIES_
     {"txreconciliation", BCLog::TXRECONCILIATION},
     {"scan", BCLog::SCAN},
     {"txpackages", BCLog::TXPACKAGES},
+    {"miner", BCLog::MINER},
+    {"sharetmpl", BCLog::SHARETMPL},
 };
 
 static const std::unordered_map<BCLog::LogFlags, std::string> LOG_CATEGORIES_BY_FLAG{

src/logging.h

@@ -94,6 +94,8 @@ namespace BCLog {
         TXRECONCILIATION = (CategoryMask{1} << 26),
         SCAN        = (CategoryMask{1} << 27),
         TXPACKAGES  = (CategoryMask{1} << 28),
+        MINER       = (CategoryMask{1} << 29),
+        SHARETMPL   = (CategoryMask{1} << 30),
         ALL         = ~NONE,
     };
     enum class Level {

src/net_processing.cpp

@@ -23,6 +23,7 @@
 #include <netbase.h>
 #include <netmessagemaker.h>
 #include <node/blockstorage.h>
+#include <node/miner.h>
 #include <node/timeoffsets.h>
 #include <node/txdownloadman.h>
 #include <node/txreconciliation.h>
@@ -165,6 +166,11 @@ static constexpr double MAX_ADDR_RATE_PER_SECOND{0.1};
 static constexpr size_t MAX_ADDR_PROCESSING_TOKEN_BUCKET{MAX_ADDR_TO_SEND};
 /** The compactblocks version we support. See BIP 152. */
 static constexpr uint64_t CMPCTBLOCKS_VERSION{2};
+/** How frequently to update templates for sharing */
+static constexpr std::chrono::microseconds TEMPLATE_UPDATE_INTERVAL{30s};
+/** Template weight limit */
+static constexpr unsigned int MAX_TEMPLATE_WEIGHT{8000000};
+static_assert(MAX_TEMPLATE_WEIGHT == 2 * MAX_BLOCK_WEIGHT);
 
 // Internal stuff
 namespace {
@@ -176,6 +182,57 @@ struct QueuedBlock {
     std::unique_ptr<PartiallyDownloadedBlock> partialBlock;
 };
 
+struct TemplateTx
+{
+    CTransactionRef tx;
+    uint32_t num_templates{0};
+
+    explicit TemplateTx(CTransactionRef tx) : tx{std::move(tx)} { }
+};
+using TemplateTxSet = std::map<Wtxid, TemplateTx>;
+using TemplateTxRefVec = std::vector<TemplateTxSet::iterator>;
+
+struct MyTemplate {
+    uint256 hash;
+    TemplateTxRefVec txs;
+    CBlockHeaderAndShortTxIDs compact;
+
+    // don't relay this template to peers whose m_last_sequence isn't at least this value
+    uint64_t inv_sequence;
+    uint32_t weight;
+};
+
+class TemplateManager
+{
+public:
+    TemplateTxSet template_txs;
+
+    std::deque<MyTemplate> my_templates;
+
+    void DiscardTxs(TemplateTxRefVec& txrv)
+    {
+        for (auto& it : txrv) {
+            if (--it->second.num_templates == 0) {
+                template_txs.erase(it);
+            }
+        }
+        txrv.clear();
+    }
+
+    TemplateTxRefVec AddTxs(const std::vector<CTransactionRef>& txs)
+    {
+        TemplateTxRefVec result;
+        result.reserve(txs.size());
+        for (auto& tx : txs) {
+            const auto& wtxid = tx->GetWitnessHash();
+            auto [it, inserted] = template_txs.try_emplace(wtxid, tx);
+            ++it->second.num_templates;
+            result.emplace_back(it);
+        }
+        return result;
+    }
+};
+
 /**
  * Data structure for an individual peer. This struct is not protected by
  * cs_main since it does not contain validation-critical data.
@@ -286,6 +343,9 @@ struct Peer {
 
         /** Minimum fee rate with which to filter transaction announcements to this node. See BIP133. */
         std::atomic<CAmount> m_fee_filter_received{0};
+
+        /** Whether this peer negotiated SENDTEMPLATE */
+        std::atomic<bool> m_support_sendtemplate{false};
     };
 
     /* Initializes a TxRelay struct for this peer. Can be called at most once for a peer. */
@@ -493,7 +553,7 @@ class PeerManagerImpl final : public PeerManager
     void FinalizeNode(const CNode& node) override EXCLUSIVE_LOCKS_REQUIRED(!m_peer_mutex, !m_headers_presync_mutex, !m_tx_download_mutex);
     bool HasAllDesirableServiceFlags(ServiceFlags services) const override;
     bool ProcessMessages(CNode* pfrom, std::atomic<bool>& interrupt) override
-        EXCLUSIVE_LOCKS_REQUIRED(!m_peer_mutex, !m_most_recent_block_mutex, !m_headers_presync_mutex, g_msgproc_mutex, !m_tx_download_mutex);
+        EXCLUSIVE_LOCKS_REQUIRED(!m_peer_mutex, !m_most_recent_block_mutex, !m_headers_presync_mutex, g_msgproc_mutex, !m_tx_download_mutex, !m_templatestats_mutex);
     bool SendMessages(CNode* pto) override
         EXCLUSIVE_LOCKS_REQUIRED(!m_peer_mutex, !m_most_recent_block_mutex, g_msgproc_mutex, !m_tx_download_mutex);
 
@@ -504,6 +564,7 @@ class PeerManagerImpl final : public PeerManager
         EXCLUSIVE_LOCKS_REQUIRED(!m_peer_mutex);
     bool GetNodeStateStats(NodeId nodeid, CNodeStateStats& stats) const override EXCLUSIVE_LOCKS_REQUIRED(!m_peer_mutex);
     std::vector<TxOrphanage::OrphanTxBase> GetOrphanTransactions() override EXCLUSIVE_LOCKS_REQUIRED(!m_tx_download_mutex);
+    TemplateStats GetTemplateStats() const override EXCLUSIVE_LOCKS_REQUIRED(!m_templatestats_mutex);
     PeerManagerInfo GetInfo() const override EXCLUSIVE_LOCKS_REQUIRED(!m_peer_mutex);
     void SendPings() override EXCLUSIVE_LOCKS_REQUIRED(!m_peer_mutex);
     void RelayTransaction(const uint256& txid, const uint256& wtxid) override EXCLUSIVE_LOCKS_REQUIRED(!m_peer_mutex);
@@ -1041,6 +1102,15 @@ class PeerManagerImpl final : public PeerManager
 
     void AddAddressKnown(Peer& peer, const CAddress& addr) EXCLUSIVE_LOCKS_REQUIRED(g_msgproc_mutex);
     void PushAddress(Peer& peer, const CAddress& addr) EXCLUSIVE_LOCKS_REQUIRED(g_msgproc_mutex);
+
+    TemplateManager m_templateman GUARDED_BY(g_msgproc_mutex);
+    NodeClock::time_point m_next_template_update GUARDED_BY(g_msgproc_mutex){NodeClock::time_point::min()};
+
+    mutable Mutex m_templatestats_mutex;
+    TemplateStats m_templatestats GUARDED_BY(m_templatestats_mutex);
+
+    void MaybeGenerateNewTemplate() EXCLUSIVE_LOCKS_REQUIRED(g_msgproc_mutex, !m_templatestats_mutex);
+    void SendTemplateTransactions(CNode& pfrom, Peer& peer, const MyTemplate& mytmp, const BlockTransactionsRequest& req) EXCLUSIVE_LOCKS_REQUIRED(g_msgproc_mutex);
 };
 
 const CNodeState* PeerManagerImpl::State(NodeId pnode) const
@@ -3473,6 +3543,12 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type,
             MakeAndPushMessage(pfrom, NetMsgType::WTXIDRELAY);
         }
 
+        if (greatest_common_version >= SENDTEMPLATE_VERSION) {
+            if (m_opts.share_template_count != 0) {
+                MakeAndPushMessage(pfrom, NetMsgType::SENDTEMPLATE);
+            }
+        }
+
         // Signal ADDRv2 support (BIP155).
         if (greatest_common_version >= 70016) {
             // BIP155 defines addrv2 and sendaddrv2 for all protocol versions, but some
@@ -3694,6 +3770,20 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type,
         return;
     }
 
+    if (msg_type == NetMsgType::SENDTEMPLATE) {
+        if (pfrom.fSuccessfullyConnected) {
+            LogDebug(BCLog::NET, "sendtemplate received after verack, %s\n", pfrom.DisconnectMsg(fLogIPs));
+            pfrom.fDisconnect = true;
+            return;
+        }
+        if (m_opts.share_template_count == 0) {
+            return;
+        }
+        auto* tx_relay = peer->GetTxRelay();
+        if (tx_relay) tx_relay->m_support_sendtemplate = true;
+        return;
+    }
+
     // BIP339 defines feature negotiation of wtxidrelay, which must happen between
     // VERSION and VERACK to avoid relay problems from switching after a connection is up.
     if (msg_type == NetMsgType::WTXIDRELAY) {
@@ -4084,6 +4174,16 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type,
             return;
         }
 
+        if (auto* tx_relay = peer->GetTxRelay(); tx_relay) {
+            for (const auto& mytmp : m_templateman.my_templates) {
+                if (mytmp.hash == req.blockhash && tx_relay->m_last_inv_sequence >= mytmp.inv_sequence) {
+                    LogDebug(BCLog::SHARETMPL, "Sending requested txns for template %s peer=%d\n", mytmp.hash.ToString(), peer->m_id);
+                    SendTemplateTransactions(pfrom, *peer, mytmp, req);
+                    return;
+                }
+            }
+        }
+
         FlatFilePos block_pos{};
         {
             LOCK(cs_main);
@@ -4279,6 +4379,23 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type,
         return;
     }
 
+    if (msg_type == NetMsgType::GETTEMPLATE) {
+        auto tx_relay = peer->GetTxRelay();
+        if (tx_relay == nullptr || !tx_relay->m_support_sendtemplate) return;
+
+        for (const auto& mytmp : m_templateman.my_templates) {
+            if (mytmp.inv_sequence <= tx_relay->m_last_inv_sequence) {
+                MakeAndPushMessage(pfrom, NetMsgType::TEMPLATE, mytmp.compact);
+                break;
+            }
+        }
+        return;
+    }
+
+    if (msg_type == NetMsgType::TEMPLATE) {
+        return; // ignore these for now
+    }
+
     if (msg_type == NetMsgType::CMPCTBLOCK)
     {
         // Ignore cmpctblock received while importing
@@ -4928,6 +5045,94 @@ bool PeerManagerImpl::MaybeDiscourageAndDisconnect(CNode& pnode, Peer& peer)
     return true;
 }
 
+void PeerManagerImpl::SendTemplateTransactions(CNode& pfrom, Peer& peer, const MyTemplate& mytmp, const BlockTransactionsRequest& req)
+{
+    BlockTransactions resp(req);
+    unsigned int tx_requested_size = 0;
+    for (size_t i = 0; i < req.indexes.size(); i++) {
+        if (req.indexes[i] >= mytmp.txs.size()) {
+            Misbehaving(peer, "getblocktxn with out-of-bounds tx indices");
+            return;
+        }
+        resp.txn[i] = mytmp.txs[req.indexes[i]]->second.tx;
+        tx_requested_size += resp.txn[i]->GetTotalSize();
+    }
+
+    LogDebug(BCLog::SHARETMPL, "Peer %d sent us a GETBLOCKTXN for template %s, sending a BLOCKTXN with %u txns. (%u bytes)\n", pfrom.GetId(), mytmp.hash.ToString(), resp.txn.size(), tx_requested_size);
+    MakeAndPushMessage(pfrom, NetMsgType::BLOCKTXN, resp);
+}
+
+TemplateStats PeerManagerImpl::GetTemplateStats() const
+{
+    LOCK(m_templatestats_mutex);
+    return m_templatestats;
+}
+
+void PeerManagerImpl::MaybeGenerateNewTemplate()
+{
+    if (m_opts.share_template_count == 0) return;
+    if (m_next_template_update == NodeClock::time_point::min()) {
+        if (m_chainman.IsInitialBlockDownload() || !m_mempool.GetLoadTried()) {
+             return;
+        } else if (WITH_LOCK(cs_main, return !CanDirectFetch())) {
+             return;
+        }
+    }
+
+    auto now = NodeClock::now();
+    if (now < m_next_template_update) return;
+    m_next_template_update = now + TEMPLATE_UPDATE_INTERVAL;
+
+    auto& my_templates = m_templateman.my_templates;
+    while (my_templates.size() >= m_opts.share_template_count) {
+        m_templateman.DiscardTxs(my_templates.back().txs);
+        my_templates.pop_back();
+    }
+
+    const auto assemble_options = []() {
+        node::BlockAssembler::Options opt;
+        opt.nBlockMaxWeight=MAX_TEMPLATE_WEIGHT;
+        opt.blockMinFeeRate=CFeeRate(0);
+        opt.test_block_validity=false;
+        opt.print_modified_fee=false;
+        return opt;
+    }();
+    node::BlockAssembler assembler{m_chainman.ActiveChainstate(), &m_mempool, assemble_options, node::BlockAssembler::ALLOW_OVERSIZED_BLOCKS};
+    auto& new_template = my_templates.emplace_front();
+
+    auto block_template = assembler.CreateNewBlock();
+    auto& block = block_template->block;
+    assert(block.vtx[0]->IsCoinBase());
+    block.vtx.erase(block.vtx.begin());
+    block.nNonce = 0;
+    block.nTime = std::numeric_limits<uint32_t>::max();
+
+    new_template.hash = block.GetHash();
+    new_template.compact = CBlockHeaderAndShortTxIDs(block, FastRandomContext().rand64());
+    new_template.weight = 0;
+    for (auto& tx : block.vtx) {
+        new_template.weight += GetTransactionWeight(*tx);
+    }
+    new_template.txs = m_templateman.AddTxs(block.vtx);
+    new_template.inv_sequence = WITH_LOCK(m_mempool.cs, return m_mempool.GetSequence());
+
+    LogDebug(BCLog::SHARETMPL, "Generated template for sharing hash=%s (%d txs, %d weight)\n", new_template.hash.ToString(), new_template.txs.size(), new_template.weight);
+
+    LOCK(m_templatestats_mutex);
+    m_templatestats.num_templates = my_templates.size();
+    m_templatestats.max_templates = m_opts.share_template_count;
+    m_templatestats.num_transactions = m_templateman.template_txs.size();
+    if (!m_templateman.my_templates.empty()) {
+        const auto& tmp = m_templateman.my_templates.front();
+        m_templatestats.latest_template_weight = tmp.weight;
+        m_templatestats.latest_template_tx = tmp.txs.size();
+    } else {
+        m_templatestats.latest_template_weight = 0;
+        m_templatestats.latest_template_tx = 0;
+    }
+    m_templatestats.next_update = m_next_template_update;
+}
+
 bool PeerManagerImpl::ProcessMessages(CNode* pfrom, std::atomic<bool>& interruptMsgProc)
 {
     AssertLockNotHeld(m_tx_download_mutex);
@@ -4936,6 +5141,8 @@ bool PeerManagerImpl::ProcessMessages(CNode* pfrom, std::atomic<bool>& interrupt
     PeerRef peer = GetPeerRef(pfrom->GetId());
     if (peer == nullptr) return false;
 
+    MaybeGenerateNewTemplate();
+
     // For outbound connections, ensure that the initial VERSION message
     // has been sent first before processing any incoming messages
     if (!pfrom->IsInboundConn() && !peer->m_outbound_version_message_sent) return false;