Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
59 changes: 59 additions & 0 deletions src/httpserver.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -658,6 +658,10 @@ void HTTPRequest::WriteReply(int nStatus, std::span<const std::byte> reply)
// Send event to main http thread to send reply message
struct evbuffer* evb = evhttp_request_get_output_buffer(req);
assert(evb);
static constexpr size_t LARGE_HTTP_REPLY_BYTES{16 * 1024 * 1024};
if (reply.size() >= LARGE_HTTP_REPLY_BYTES) {
LogDebug(BCLog::HTTP, "Large HTTP reply body copied: status=%d bytes=%u\n", nStatus, reply.size());
}
evbuffer_add(evb, reply.data(), reply.size());
auto req_copy = req;
HTTPEvent* ev = new HTTPEvent(eventBase, true, [req_copy, nStatus]{
Expand All @@ -679,6 +683,61 @@ void HTTPRequest::WriteReply(int nStatus, std::span<const std::byte> reply)
req = nullptr; // transferred back to main thread
}

void HTTPRequest::WriteReply(int nStatus, std::string&& reply)
{
assert(!replySent && req);
if (m_interrupt) {
WriteHeader("Connection", "close");
}

// Move the reply body into heap storage so libevent can reference it
// without copying, and free it once the buffer is done with it.
auto* reply_ref = new std::string(std::move(reply));

struct evbuffer* evb = evhttp_request_get_output_buffer(req);
assert(evb);
static constexpr size_t LARGE_HTTP_REPLY_BYTES{16 * 1024 * 1024};

const bool is_large{reply_ref->size() >= LARGE_HTTP_REPLY_BYTES};
if (evbuffer_add_reference(
evb,
reply_ref->data(),
reply_ref->size(),
[](const void*, size_t, void* arg) { delete static_cast<std::string*>(arg); },
reply_ref) != 0) {
// If reference insertion fails, fall back to copying the reply body and
// free the heap allocation immediately.
if (is_large) {
LogDebug(BCLog::HTTP, "Large HTTP reply body copied: status=%d bytes=%u\n", nStatus, reply_ref->size());
}
evbuffer_add(evb, reply_ref->data(), reply_ref->size());
delete reply_ref;
} else {
if (is_large) {
LogDebug(BCLog::HTTP, "Large HTTP reply body referenced: status=%d bytes=%u\n", nStatus, reply_ref->size());
}
}

auto req_copy = req;
HTTPEvent* ev = new HTTPEvent(eventBase, true, [req_copy, nStatus] {
evhttp_send_reply(req_copy, nStatus, nullptr, nullptr);
// Re-enable reading from the socket. This is the second part of the libevent
// workaround above.
if (event_get_version_number() >= 0x02010600 && event_get_version_number() < 0x02010900) {
evhttp_connection* conn = evhttp_request_get_connection(req_copy);
if (conn) {
bufferevent* bev = evhttp_connection_get_bufferevent(conn);
if (bev) {
bufferevent_enable(bev, EV_READ | EV_WRITE);
}
}
}
});
ev->trigger(nullptr);
replySent = true;
req = nullptr; // transferred back to main thread
}

CService HTTPRequest::GetPeer() const
{
evhttp_connection* con = evhttp_request_get_connection(req);
Expand Down
5 changes: 5 additions & 0 deletions src/httpserver.h
Original file line number Diff line number Diff line change
Expand Up @@ -138,10 +138,15 @@ class HTTPRequest
* @note Can be called only once. As this will give the request back to the
* main thread, do not call any other HTTPRequest methods after calling this.
*/
void WriteReply(int nStatus, const char* reply)
{
WriteReply(nStatus, std::string_view{reply});
}
void WriteReply(int nStatus, std::string_view reply = "")
{
WriteReply(nStatus, std::as_bytes(std::span{reply}));
}
void WriteReply(int nStatus, std::string&& reply);
void WriteReply(int nStatus, std::span<const std::byte> reply);
};

Expand Down
92 changes: 92 additions & 0 deletions test/functional/rpc_batch_memory.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,92 @@
#!/usr/bin/env python3
# Copyright (c) 2026-present The Bitcoin Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
"""Exercise large JSON-RPC batch replies.

bitcoin/bitcoin#31041 reported out-of-memory termination triggered by large
JSON-RPC batch replies. This test creates a large RPC reply and asserts it is
served through the expected HTTP reply path.
"""

import base64
import http.client
import json
from urllib.parse import urlparse

from test_framework.test_framework import BitcoinTestFramework
from test_framework.util import (
assert_equal,
create_lots_of_big_transactions,
gen_return_txouts,
)
from test_framework.wallet import MiniWallet


def raw_jsonrpc_request(url, payload, *, timeout):
parsed = urlparse(url)
assert parsed.hostname
assert parsed.port
assert parsed.username is not None
assert parsed.password is not None

auth_b64 = base64.b64encode(f"{parsed.username}:{parsed.password}".encode()).decode()
headers = {
"Content-Type": "application/json",
"Authorization": f"Basic {auth_b64}",
}

conn = http.client.HTTPConnection(parsed.hostname, parsed.port, timeout=timeout)
conn.request("POST", "/", body=payload, headers=headers)
response = conn.getresponse()
while response.read(64 * 1024):
pass
conn.close()
return response.status


class RPCBatchMemoryTest(BitcoinTestFramework):
def set_test_params(self):
self.num_nodes = 1
self.setup_clean_chain = True

def run_test(self):
self.skip_if_running_under_valgrind()

node = self.nodes[0]
mini_wallet = MiniWallet(node)
# Create enough mature coinbase UTXOs for multiple spends.
self.generate(mini_wallet, 120)

txouts = gen_return_txouts()
fee = 100 * node.getnetworkinfo()["relayfee"]
create_lots_of_big_transactions(mini_wallet, node, fee, tx_batch_size=8, txouts=txouts)
self.generate(mini_wallet, 1)

block_hash = node.getbestblockhash()
# Avoid calling getblock(verbosity=0) here because it would load the
# block hex into this process, defeating the purpose of exercising the
# node's reply path with a large response.
block_size = node.getblock(block_hash, 1)["size"]
block_hex_len = block_size * 2

# Ensure the response body is large enough to trigger the large-reply
# log line in the HTTP server.
target_response_bytes = 32 * 1024 * 1024
batch_size = (target_response_bytes + block_hex_len - 1) // block_hex_len

batch = [
{"jsonrpc": "2.0", "id": i, "method": "getblock", "params": [block_hash, 0]}
for i in range(batch_size)
]
payload = json.dumps(batch).encode()

assert node.process.poll() is None
with node.assert_debug_log(["Large HTTP reply body referenced:"], unexpected_msgs=["Large HTTP reply body copied:"]):
status = raw_jsonrpc_request(node.url, payload, timeout=120)
assert_equal(status, 200)
assert node.process.poll() is None


if __name__ == "__main__":
RPCBatchMemoryTest(__file__).main()
1 change: 1 addition & 0 deletions test/functional/test_runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -140,6 +140,7 @@
'wallet_taproot.py',
'feature_bip68_sequence.py',
'rpc_packages.py',
'rpc_batch_memory.py',
'rpc_bind.py --ipv4',
'rpc_bind.py --ipv6',
'rpc_bind.py --nonloopback',
Expand Down
Loading