rpc: Generate UTXO set hints

Generate hints for the location of UTXOs in each block. This RPC allows
for a rollback of the chainstate to an arbitary height. Outputs not
included in the index of the hintsfile are:
1. provably unspendable outputs
2. BIP30 unspendable outputs
3. outputs spent within the same block

Author: rustaceanrob

src/rpc/blockchain.cpp

@@ -39,6 +39,7 @@
 #include <script/descriptor.h>
 #include <serialize.h>
 #include <streams.h>
+#include <swiftsync.h>
 #include <sync.h>
 #include <tinyformat.h>
 #include <txdb.h>
@@ -3421,6 +3422,147 @@ static RPCHelpMan loadtxoutset()
     };
 }
 
+static RPCHelpMan generatetxohints()
+{
+    return RPCHelpMan{
+        "generatetxohints",
+        "Build a file of hints for the state of the UTXO set at a particular height.\n"
+        "The purpose of said hints is to allow clients performing initial block download\n"
+        "to omit unnecessary disk I/O and CPU usage.\n"
+        "The hint file is constructed by reading in blocks sequentially and determining what outputs\n"
+        "will remain in the UTXO set. Network activity will be suspended during this process, and the\n"
+        "hint file may take a few hours to build."
+        "Make sure to use no RPC timeout (bitcoin-cli -rpcclienttimeout=0)",
+        {
+            {"path",
+                RPCArg::Type::STR,
+                RPCArg::Optional::NO,
+                "Path to the hint file. If relative, will be prefixed by datadir."},
+            {"rollback",
+                RPCArg::Type::NUM,
+                RPCArg::Optional::OMITTED,
+                "The block hash or height to build the hint file up to. If none is provided, the file will be built from the current block tip.",
+                RPCArgOptions{
+                    .skip_type_check = true,
+                    .type_str = {"", "string or numeric"},}},
+        },
+        RPCResult{
+            RPCResult::Type::OBJ, "", "",
+                {
+                    {RPCResult::Type::NUM, "height", "The stopping height encoded by the hint file."},
+                    {RPCResult::Type::STR, "path", "Absolute path where the file was written."},
+                    {RPCResult::Type::STR, "duration", "Time taken to build the file."},
+                }
+        },
+        RPCExamples{
+            HelpExampleCli("--rpcclienttimeout=0 generatetxohints", "signet.hints 270000"),
+        },
+        [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue
+{
+    const auto start{SteadyClock::now()};
+    NodeContext& node{EnsureAnyNodeContext(request.context)};
+    if (node.chainman->m_blockman.IsPruneMode()) {
+        throw JSONRPCError(RPC_MISC_ERROR, "Creating a hint file in pruned mode is not possible.");
+    }
+    const ArgsManager& args{EnsureAnyArgsman(request.context)};
+    const fs::path path = fsbridge::AbsPathJoin(args.GetDataDirNet(), fs::u8path(self.Arg<std::string_view>("path")));
+    const fs::path temppath = path + ".incomplete";
+    if (fs::exists(path)) {
+        throw JSONRPCError(
+            RPC_INVALID_PARAMETER,
+            path.utf8string() + " already exists. If you are sure this is what you want, "
+            "move it out of the way first");
+    }
+    FILE* file{fsbridge::fopen(temppath, "wb")};
+    AutoFile afile{file};
+    if (afile.IsNull()) {
+        throw JSONRPCError(
+            RPC_INVALID_PARAMETER,
+            "Couldn't open file " + temppath.utf8string() + " for writing.");
+    }
+    CConnman& connman{EnsureConnman(node)};
+    NetworkDisable disable_net{NetworkDisable(connman)};
+    std::optional<TemporaryRollback> rollback;
+    if (!request.params[1].isNull()) {
+        const CBlockIndex* invalidate_index = ParseHashOrHeight(request.params[1], *node.chainman);
+        invalidate_index = WITH_LOCK(::cs_main, return node.chainman->ActiveChain().Next(invalidate_index));
+        rollback.emplace(*node.chainman, *invalidate_index);
+    }
+    node.rpc_interruption_point();
+    LOCK(node.chainman->GetMutex());
+    CChain& active_chain{node.chainman->ActiveChain()};
+    Chainstate& active_state{node.chainman->ActiveChainstate()};
+    active_state.ForceFlushStateToDisk();
+    const CBlockIndex* end_index{active_chain.Tip()};
+    const auto tip_height{end_index->nHeight};
+    LogDebug(BCLog::RPC, "Active chain best tip %d", tip_height);
+    swiftsync::HintsfileWriter writer{swiftsync::HintsfileWriter(afile)};
+    if (!writer.WriteStopHeight(tip_height)) {
+        throw JSONRPCError(RPC_DATABASE_ERROR, "Failed to commit changes to hint file.");
+    }
+    CBlockIndex* curr{active_chain.Next(active_chain.Genesis())};
+    uint64_t total_outputs_written{0};
+    while (curr) {
+        auto height{curr->nHeight};
+        if (height % 10000 == 0) {
+            LogDebug(BCLog::RPC, "Wrote hints up to height %s, total outputs written %d", height, total_outputs_written);
+        }
+        FlatFilePos file_pos = curr->GetBlockPos();
+        std::unique_ptr<CBlock> pblock = std::make_unique<CBlock>();
+        bool read = node.chainman->m_blockman.ReadBlock(*pblock, file_pos, curr->GetBlockHash());
+        if (!read) {
+            throw JSONRPCError(RPC_DATABASE_ERROR, "Block could not be read from disk.");
+        }
+        std::set<COutPoint> spent_outputs{};
+        for (const auto& transaction : pblock->vtx) {
+            if (transaction->IsCoinBase()) {
+                continue;
+            }
+            for (const auto& txin : transaction->vin) {
+                spent_outputs.insert(txin.prevout);
+            }
+        }
+        uint32_t output_index{};
+        std::vector<uint32_t> unspent;
+        for (const auto& tx: pblock->vtx) {
+            if (tx->IsCoinBase() && IsBIP30Unspendable(curr->GetBlockHash(), curr->nHeight)) {
+                continue;
+            };
+            const Txid& txid = tx->GetHash();
+            for (size_t vout{}; vout < tx->vout.size(); ++vout) {
+                if (tx->vout[vout].scriptPubKey.IsUnspendable()) {
+                    continue;
+                }
+                const COutPoint outpoint = COutPoint(txid, vout);
+                if (spent_outputs.contains(outpoint)) {
+                    continue;
+                }
+                if (active_state.CoinsDB().HaveCoin(outpoint)) {
+                    unspent.push_back(output_index);
+                }
+                ++output_index;
+                ++total_outputs_written;
+            }
+        }
+        auto ef{swiftsync::EliasFano::Compress(unspent)};
+        if (!writer.WriteHints(ef)) {
+            throw JSONRPCError(RPC_DATABASE_ERROR, "Failed to commit changes to hint file.");
+        }
+        node.rpc_interruption_point();
+        curr = active_chain.Next(curr);
+    }
+    fs::rename(temppath, path);
+    const auto end{SteadyClock::now()};
+    const auto duration = std::chrono::duration_cast<std::chrono::minutes>(end - start).count();
+    UniValue result(UniValue::VOBJ);
+    result.pushKV("height", tip_height);
+    result.pushKV("path", path.utf8string());
+    result.pushKV("duration", tfm::format("%d minutes", duration));
+    return result;
+},
+    };
+}
+
 const std::vector<RPCResult> RPCHelpForChainstate{
     {RPCResult::Type::NUM, "blocks", "number of blocks in this chainstate"},
     {RPCResult::Type::STR_HEX, "bestblockhash", "blockhash of the tip"},
@@ -3520,6 +3662,7 @@ void RegisterBlockchainRPCCommands(CRPCTable& t)
         {"blockchain", &getblockfilter},
         {"blockchain", &dumptxoutset},
         {"blockchain", &loadtxoutset},
+        {"blockchain", &generatetxohints},
         {"blockchain", &getchainstates},
         {"hidden", &invalidateblock},
         {"hidden", &reconsiderblock},

src/rpc/client.cpp

@@ -388,6 +388,7 @@ static const CRPCConvertParam vRPCConvertParams[] =
     { "signmessagewithprivkey", 1, "message", ParamFormat::STRING },
     { "walletpassphrasechange", 0, "oldpassphrase", ParamFormat::STRING },
     { "walletpassphrasechange", 1, "newpassphrase", ParamFormat::STRING },
+    { "generatetxohints", 1, "rollback", ParamFormat::JSON_OR_STRING }
 };
 // clang-format on
 

src/test/fuzz/rpc.cpp

@@ -79,6 +79,7 @@ const std::vector<std::string> RPC_COMMANDS_NOT_SAFE_FOR_FUZZING{
     "echoipc",              // avoid assertion failure (Assertion `"EnsureAnyNodeContext(request.context).init" && check' failed.)
     "generatetoaddress",    // avoid prohibitively slow execution (when `num_blocks` is large)
     "generatetodescriptor", // avoid prohibitively slow execution (when `nblocks` is large)
+    "generatetxohints",     // avoid writing to disk
     "gettxoutproof",        // avoid prohibitively slow execution
     "importmempool", // avoid reading from disk
     "loadtxoutset",   // avoid reading from disk