From 971a4d4bbe6d3c7fedb21ab7dc6a574cf08e4967 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C5=91rinc?= Date: Tue, 27 Jan 2026 11:21:46 +0100 Subject: [PATCH 1/2] test: exercise large RPC batch replies Add a functional test that creates a large JSON-RPC batch reply (using repeated getblock verbosity=0 calls) and asserts the HTTP server logs that the large response body was copied into the libevent output buffer. This establishes baseline behavior related to bitcoin/bitcoin#31041 (memory pressure from large batch replies) before switching the reply path to avoid the extra copy. --- src/httpserver.cpp | 4 ++ test/functional/rpc_batch_memory.py | 92 +++++++++++++++++++++++++++++ test/functional/test_runner.py | 1 + 3 files changed, 97 insertions(+) create mode 100644 test/functional/rpc_batch_memory.py diff --git a/src/httpserver.cpp b/src/httpserver.cpp index 71c6f5b1ee24..4294f44a5057 100644 --- a/src/httpserver.cpp +++ b/src/httpserver.cpp @@ -658,6 +658,10 @@ void HTTPRequest::WriteReply(int nStatus, std::span reply) // Send event to main http thread to send reply message struct evbuffer* evb = evhttp_request_get_output_buffer(req); assert(evb); + static constexpr size_t LARGE_HTTP_REPLY_BYTES{16 * 1024 * 1024}; + if (reply.size() >= LARGE_HTTP_REPLY_BYTES) { + LogDebug(BCLog::HTTP, "Large HTTP reply body copied: status=%d bytes=%u\n", nStatus, reply.size()); + } evbuffer_add(evb, reply.data(), reply.size()); auto req_copy = req; HTTPEvent* ev = new HTTPEvent(eventBase, true, [req_copy, nStatus]{ diff --git a/test/functional/rpc_batch_memory.py b/test/functional/rpc_batch_memory.py new file mode 100644 index 000000000000..2a19a455728e --- /dev/null +++ b/test/functional/rpc_batch_memory.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python3 +# Copyright (c) 2026-present The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Exercise large JSON-RPC batch replies. + +bitcoin/bitcoin#31041 reported out-of-memory termination triggered by large +JSON-RPC batch replies. This test creates a large RPC reply and asserts it is +served through the expected HTTP reply path. +""" + +import base64 +import http.client +import json +from urllib.parse import urlparse + +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import ( + assert_equal, + create_lots_of_big_transactions, + gen_return_txouts, +) +from test_framework.wallet import MiniWallet + + +def raw_jsonrpc_request(url, payload, *, timeout): + parsed = urlparse(url) + assert parsed.hostname + assert parsed.port + assert parsed.username is not None + assert parsed.password is not None + + auth_b64 = base64.b64encode(f"{parsed.username}:{parsed.password}".encode()).decode() + headers = { + "Content-Type": "application/json", + "Authorization": f"Basic {auth_b64}", + } + + conn = http.client.HTTPConnection(parsed.hostname, parsed.port, timeout=timeout) + conn.request("POST", "/", body=payload, headers=headers) + response = conn.getresponse() + while response.read(64 * 1024): + pass + conn.close() + return response.status + + +class RPCBatchMemoryTest(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 1 + self.setup_clean_chain = True + + def run_test(self): + self.skip_if_running_under_valgrind() + + node = self.nodes[0] + mini_wallet = MiniWallet(node) + # Create enough mature coinbase UTXOs for multiple spends. + self.generate(mini_wallet, 120) + + txouts = gen_return_txouts() + fee = 100 * node.getnetworkinfo()["relayfee"] + create_lots_of_big_transactions(mini_wallet, node, fee, tx_batch_size=8, txouts=txouts) + self.generate(mini_wallet, 1) + + block_hash = node.getbestblockhash() + # Avoid calling getblock(verbosity=0) here because it would load the + # block hex into this process, defeating the purpose of exercising the + # node's reply path with a large response. + block_size = node.getblock(block_hash, 1)["size"] + block_hex_len = block_size * 2 + + # Ensure the response body is large enough to trigger the large-reply + # log line in the HTTP server. + target_response_bytes = 32 * 1024 * 1024 + batch_size = (target_response_bytes + block_hex_len - 1) // block_hex_len + + batch = [ + {"jsonrpc": "2.0", "id": i, "method": "getblock", "params": [block_hash, 0]} + for i in range(batch_size) + ] + payload = json.dumps(batch).encode() + + assert node.process.poll() is None + with node.assert_debug_log(["Large HTTP reply body copied:"]): + status = raw_jsonrpc_request(node.url, payload, timeout=120) + assert_equal(status, 200) + assert node.process.poll() is None + + +if __name__ == "__main__": + RPCBatchMemoryTest(__file__).main() diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index 6877aa474c57..04058742d1d5 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -140,6 +140,7 @@ 'wallet_taproot.py', 'feature_bip68_sequence.py', 'rpc_packages.py', + 'rpc_batch_memory.py', 'rpc_bind.py --ipv4', 'rpc_bind.py --ipv6', 'rpc_bind.py --nonloopback', From 766dc0cbb9fd09616d544670765396a437432a87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C5=91rinc?= Date: Tue, 27 Jan 2026 11:25:16 +0100 Subject: [PATCH 2/2] http: avoid copying large reply bodies Add `HTTPRequest::WriteReply(int, std::string&&)` that stores the reply body and uses `evbuffer_add_reference()` so libevent can send large responses without an extra full copy. This reduces peak memory when serving large RPC/REST replies (eg JSON-RPC batch responses implicated in bitcoin/bitcoin#31041). Update the functional test to assert the referenced-reply path is used for large batch replies. --- src/httpserver.cpp | 55 +++++++++++++++++++++++++++++ src/httpserver.h | 5 +++ test/functional/rpc_batch_memory.py | 2 +- 3 files changed, 61 insertions(+), 1 deletion(-) diff --git a/src/httpserver.cpp b/src/httpserver.cpp index 4294f44a5057..23891907a9a4 100644 --- a/src/httpserver.cpp +++ b/src/httpserver.cpp @@ -683,6 +683,61 @@ void HTTPRequest::WriteReply(int nStatus, std::span reply) req = nullptr; // transferred back to main thread } +void HTTPRequest::WriteReply(int nStatus, std::string&& reply) +{ + assert(!replySent && req); + if (m_interrupt) { + WriteHeader("Connection", "close"); + } + + // Move the reply body into heap storage so libevent can reference it + // without copying, and free it once the buffer is done with it. + auto* reply_ref = new std::string(std::move(reply)); + + struct evbuffer* evb = evhttp_request_get_output_buffer(req); + assert(evb); + static constexpr size_t LARGE_HTTP_REPLY_BYTES{16 * 1024 * 1024}; + + const bool is_large{reply_ref->size() >= LARGE_HTTP_REPLY_BYTES}; + if (evbuffer_add_reference( + evb, + reply_ref->data(), + reply_ref->size(), + [](const void*, size_t, void* arg) { delete static_cast(arg); }, + reply_ref) != 0) { + // If reference insertion fails, fall back to copying the reply body and + // free the heap allocation immediately. + if (is_large) { + LogDebug(BCLog::HTTP, "Large HTTP reply body copied: status=%d bytes=%u\n", nStatus, reply_ref->size()); + } + evbuffer_add(evb, reply_ref->data(), reply_ref->size()); + delete reply_ref; + } else { + if (is_large) { + LogDebug(BCLog::HTTP, "Large HTTP reply body referenced: status=%d bytes=%u\n", nStatus, reply_ref->size()); + } + } + + auto req_copy = req; + HTTPEvent* ev = new HTTPEvent(eventBase, true, [req_copy, nStatus] { + evhttp_send_reply(req_copy, nStatus, nullptr, nullptr); + // Re-enable reading from the socket. This is the second part of the libevent + // workaround above. + if (event_get_version_number() >= 0x02010600 && event_get_version_number() < 0x02010900) { + evhttp_connection* conn = evhttp_request_get_connection(req_copy); + if (conn) { + bufferevent* bev = evhttp_connection_get_bufferevent(conn); + if (bev) { + bufferevent_enable(bev, EV_READ | EV_WRITE); + } + } + } + }); + ev->trigger(nullptr); + replySent = true; + req = nullptr; // transferred back to main thread +} + CService HTTPRequest::GetPeer() const { evhttp_connection* con = evhttp_request_get_connection(req); diff --git a/src/httpserver.h b/src/httpserver.h index 1ef3aaeb0ab1..31f6000fd109 100644 --- a/src/httpserver.h +++ b/src/httpserver.h @@ -138,10 +138,15 @@ class HTTPRequest * @note Can be called only once. As this will give the request back to the * main thread, do not call any other HTTPRequest methods after calling this. */ + void WriteReply(int nStatus, const char* reply) + { + WriteReply(nStatus, std::string_view{reply}); + } void WriteReply(int nStatus, std::string_view reply = "") { WriteReply(nStatus, std::as_bytes(std::span{reply})); } + void WriteReply(int nStatus, std::string&& reply); void WriteReply(int nStatus, std::span reply); }; diff --git a/test/functional/rpc_batch_memory.py b/test/functional/rpc_batch_memory.py index 2a19a455728e..5a4128eeeec9 100644 --- a/test/functional/rpc_batch_memory.py +++ b/test/functional/rpc_batch_memory.py @@ -82,7 +82,7 @@ def run_test(self): payload = json.dumps(batch).encode() assert node.process.poll() is None - with node.assert_debug_log(["Large HTTP reply body copied:"]): + with node.assert_debug_log(["Large HTTP reply body referenced:"], unexpected_msgs=["Large HTTP reply body copied:"]): status = raw_jsonrpc_request(node.url, payload, timeout=120) assert_equal(status, 200) assert node.process.poll() is None