diff --git a/src/httpserver.cpp b/src/httpserver.cpp index 71c6f5b1ee24..23891907a9a4 100644 --- a/src/httpserver.cpp +++ b/src/httpserver.cpp @@ -658,6 +658,10 @@ void HTTPRequest::WriteReply(int nStatus, std::span reply) // Send event to main http thread to send reply message struct evbuffer* evb = evhttp_request_get_output_buffer(req); assert(evb); + static constexpr size_t LARGE_HTTP_REPLY_BYTES{16 * 1024 * 1024}; + if (reply.size() >= LARGE_HTTP_REPLY_BYTES) { + LogDebug(BCLog::HTTP, "Large HTTP reply body copied: status=%d bytes=%u\n", nStatus, reply.size()); + } evbuffer_add(evb, reply.data(), reply.size()); auto req_copy = req; HTTPEvent* ev = new HTTPEvent(eventBase, true, [req_copy, nStatus]{ @@ -679,6 +683,61 @@ void HTTPRequest::WriteReply(int nStatus, std::span reply) req = nullptr; // transferred back to main thread } +void HTTPRequest::WriteReply(int nStatus, std::string&& reply) +{ + assert(!replySent && req); + if (m_interrupt) { + WriteHeader("Connection", "close"); + } + + // Move the reply body into heap storage so libevent can reference it + // without copying, and free it once the buffer is done with it. + auto* reply_ref = new std::string(std::move(reply)); + + struct evbuffer* evb = evhttp_request_get_output_buffer(req); + assert(evb); + static constexpr size_t LARGE_HTTP_REPLY_BYTES{16 * 1024 * 1024}; + + const bool is_large{reply_ref->size() >= LARGE_HTTP_REPLY_BYTES}; + if (evbuffer_add_reference( + evb, + reply_ref->data(), + reply_ref->size(), + [](const void*, size_t, void* arg) { delete static_cast(arg); }, + reply_ref) != 0) { + // If reference insertion fails, fall back to copying the reply body and + // free the heap allocation immediately. + if (is_large) { + LogDebug(BCLog::HTTP, "Large HTTP reply body copied: status=%d bytes=%u\n", nStatus, reply_ref->size()); + } + evbuffer_add(evb, reply_ref->data(), reply_ref->size()); + delete reply_ref; + } else { + if (is_large) { + LogDebug(BCLog::HTTP, "Large HTTP reply body referenced: status=%d bytes=%u\n", nStatus, reply_ref->size()); + } + } + + auto req_copy = req; + HTTPEvent* ev = new HTTPEvent(eventBase, true, [req_copy, nStatus] { + evhttp_send_reply(req_copy, nStatus, nullptr, nullptr); + // Re-enable reading from the socket. This is the second part of the libevent + // workaround above. + if (event_get_version_number() >= 0x02010600 && event_get_version_number() < 0x02010900) { + evhttp_connection* conn = evhttp_request_get_connection(req_copy); + if (conn) { + bufferevent* bev = evhttp_connection_get_bufferevent(conn); + if (bev) { + bufferevent_enable(bev, EV_READ | EV_WRITE); + } + } + } + }); + ev->trigger(nullptr); + replySent = true; + req = nullptr; // transferred back to main thread +} + CService HTTPRequest::GetPeer() const { evhttp_connection* con = evhttp_request_get_connection(req); diff --git a/src/httpserver.h b/src/httpserver.h index 1ef3aaeb0ab1..31f6000fd109 100644 --- a/src/httpserver.h +++ b/src/httpserver.h @@ -138,10 +138,15 @@ class HTTPRequest * @note Can be called only once. As this will give the request back to the * main thread, do not call any other HTTPRequest methods after calling this. */ + void WriteReply(int nStatus, const char* reply) + { + WriteReply(nStatus, std::string_view{reply}); + } void WriteReply(int nStatus, std::string_view reply = "") { WriteReply(nStatus, std::as_bytes(std::span{reply})); } + void WriteReply(int nStatus, std::string&& reply); void WriteReply(int nStatus, std::span reply); }; diff --git a/test/functional/rpc_batch_memory.py b/test/functional/rpc_batch_memory.py new file mode 100644 index 000000000000..5a4128eeeec9 --- /dev/null +++ b/test/functional/rpc_batch_memory.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python3 +# Copyright (c) 2026-present The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Exercise large JSON-RPC batch replies. + +bitcoin/bitcoin#31041 reported out-of-memory termination triggered by large +JSON-RPC batch replies. This test creates a large RPC reply and asserts it is +served through the expected HTTP reply path. +""" + +import base64 +import http.client +import json +from urllib.parse import urlparse + +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import ( + assert_equal, + create_lots_of_big_transactions, + gen_return_txouts, +) +from test_framework.wallet import MiniWallet + + +def raw_jsonrpc_request(url, payload, *, timeout): + parsed = urlparse(url) + assert parsed.hostname + assert parsed.port + assert parsed.username is not None + assert parsed.password is not None + + auth_b64 = base64.b64encode(f"{parsed.username}:{parsed.password}".encode()).decode() + headers = { + "Content-Type": "application/json", + "Authorization": f"Basic {auth_b64}", + } + + conn = http.client.HTTPConnection(parsed.hostname, parsed.port, timeout=timeout) + conn.request("POST", "/", body=payload, headers=headers) + response = conn.getresponse() + while response.read(64 * 1024): + pass + conn.close() + return response.status + + +class RPCBatchMemoryTest(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 1 + self.setup_clean_chain = True + + def run_test(self): + self.skip_if_running_under_valgrind() + + node = self.nodes[0] + mini_wallet = MiniWallet(node) + # Create enough mature coinbase UTXOs for multiple spends. + self.generate(mini_wallet, 120) + + txouts = gen_return_txouts() + fee = 100 * node.getnetworkinfo()["relayfee"] + create_lots_of_big_transactions(mini_wallet, node, fee, tx_batch_size=8, txouts=txouts) + self.generate(mini_wallet, 1) + + block_hash = node.getbestblockhash() + # Avoid calling getblock(verbosity=0) here because it would load the + # block hex into this process, defeating the purpose of exercising the + # node's reply path with a large response. + block_size = node.getblock(block_hash, 1)["size"] + block_hex_len = block_size * 2 + + # Ensure the response body is large enough to trigger the large-reply + # log line in the HTTP server. + target_response_bytes = 32 * 1024 * 1024 + batch_size = (target_response_bytes + block_hex_len - 1) // block_hex_len + + batch = [ + {"jsonrpc": "2.0", "id": i, "method": "getblock", "params": [block_hash, 0]} + for i in range(batch_size) + ] + payload = json.dumps(batch).encode() + + assert node.process.poll() is None + with node.assert_debug_log(["Large HTTP reply body referenced:"], unexpected_msgs=["Large HTTP reply body copied:"]): + status = raw_jsonrpc_request(node.url, payload, timeout=120) + assert_equal(status, 200) + assert node.process.poll() is None + + +if __name__ == "__main__": + RPCBatchMemoryTest(__file__).main() diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index 6877aa474c57..04058742d1d5 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -140,6 +140,7 @@ 'wallet_taproot.py', 'feature_bip68_sequence.py', 'rpc_packages.py', + 'rpc_batch_memory.py', 'rpc_bind.py --ipv4', 'rpc_bind.py --ipv6', 'rpc_bind.py --nonloopback',