[Bug]: crashed at `IndexPageManager::FreeIndexPage` for the page is Pinned

[lzxddz]

What happened?

Steps to Reproduce

Run jepsen test on "jepsen-eloqkv-eloqstorecloud-standby"

Environment

• OS:
• Version:
• Any other relevant details:
• Any related issues:

[lzxddz]

crash info

(gdb) bt
#0  __pthread_kill_implementation (no_tid=0, signo=6, threadid=139783372179008) at ./nptl/pthread_kill.c:44
#1  __pthread_kill_internal (signo=6, threadid=139783372179008) at ./nptl/pthread_kill.c:78
#2  __GI___pthread_kill (threadid=139783372179008, signo=signo@entry=6) at ./nptl/pthread_kill.c:89
#3  0x00007f21ea042476 in __GI_raise (sig=sig@entry=6) at ../sysdeps/posix/raise.c:26
#4  0x00007f21ea0287f3 in __GI_abort () at ./stdlib/abort.c:79
#5  0x00007f21ea02871b in __assert_fail_base (fmt=0x7f21ea1dd130 "%s%s%s:%u: %s%sAssertion `%s' failed.\n%n",
    assertion=0x5a52239e6360 "!page->IsPinned()",
    file=0x5a52239e60a0 "/home/eloq/eloqkv/data_substrate/store_handler/eloq_data_store_service/eloqstore/src/storage/index_page_manager.cpp", line=127, function=<optimized out>) at ./assert/assert.c:94
#6  0x00007f21ea039e96 in __GI___assert_fail (assertion=0x5a52239e6360 "!page->IsPinned()",
    file=0x5a52239e60a0 "/home/eloq/eloqkv/data_substrate/store_handler/eloq_data_store_service/eloqstore/src/storage/index_page_manager.cpp", line=127, function=0x5a52239e62a0 "void eloqstore::IndexPageManager::FreeIndexPage(eloqstore::MemIndexPage*)")
    at ./assert/assert.c:103
#7  0x00005a5223038d71 in eloqstore::IndexPageManager::FreeIndexPage (this=0x51d000304d40, page=0x5070005402c0)
    at /home/eloq/eloqkv/data_substrate/store_handler/eloq_data_store_service/eloqstore/src/storage/index_page_manager.cpp:127
#8  0x00005a522303dc53 in eloqstore::IndexPageManager::FindPage (this=0x51d000304d40, mapping=0x50f0002c41f0, page_id=0)
    at /home/eloq/eloqkv/data_substrate/store_handler/eloq_data_store_service/eloqstore/src/storage/index_page_manager.cpp:455
#9  0x00005a5222f168b8 in eloqstore::ScanIterator::BuildIndexStack (this=0x7f215e6a9f10, key=...)
    at /home/eloq/eloqkv/data_substrate/store_handler/eloq_data_store_service/eloqstore/src/tasks/scan_task.cpp:93
#10 0x00005a5222f16331 in eloqstore::ScanIterator::Seek (this=0x7f215e6a9f10, key=..., ttl=false)
    at /home/eloq/eloqkv/data_substrate/store_handler/eloq_data_store_service/eloqstore/src/tasks/scan_task.cpp:57
#11 0x00005a5222f182c9 in eloqstore::ScanTask::Scan (this=0x7f21248e1288)
    at /home/eloq/eloqkv/data_substrate/store_handler/eloq_data_store_service/eloqstore/src/tasks/scan_task.cpp:299
#12 0x00005a5222e871b6 in operator() (__closure=0x7f215e6aaf38)
    at /home/eloq/eloqkv/data_substrate/store_handler/eloq_data_store_service/eloqstore/src/storage/shard.cpp:455
#13 0x00005a5222e8c935 in operator() (__closure=0x7f215e6aaf38, sink=...)
    at /home/eloq/eloqkv/data_substrate/store_handler/eloq_data_store_service/eloqstore/include/storage/shard.h:130
#14 0x00005a5222e9f09a in std::__invoke_impl<boost::context::continuation, eloqstore::Shard::StartTask<eloqstore::Shard::ProcessReq(eloqstore::KvRequest*)::<lambda()> >(eloqstore::KvTask*, eloqstore::KvRequest*, eloqstore::Shard::ProcessReq(eloqstore::KvRequest*)::<lambda()>)::<lambda(boost::context::continuation&&)>&, boost::context::continuation>(std::__invoke_other, struct {...} &) (__f=...)
    at /usr/include/c++/11/bits/invoke.h:61
#15 0x00005a5222e9e9d9 in std::__invoke<eloqstore::Shard::StartTask<eloqstore::Shard::ProcessReq(eloqstore::KvRequest*)::<lambda()> >(eloqstore::KvTask*, eloqstore::KvRequest*, eloqstore::Shard::ProcessReq(eloqstore::KvRequest*)::<lambda()>)::<lambda(boost::context::continuation&&)>&, boost::context::continuation>(struct {...} &) (__fn=...) at /usr/include/c++/11/bits/invoke.h:97
#16 0x00005a5222e9e35e in std::invoke<eloqstore::Shard::StartTask<eloqstore::Shard::ProcessReq(eloqstore::KvRequest*)::<lambda()> >(eloqstore::KvTask*, eloqstore::KvRequest*, eloqstore::Shard::ProcessReq(eloqstore::KvRequest*)::<lambda()>)::<lambda(boost::context::continuation&&)>&, boost::context::continuation>(struct {...} &) (__fn=...) at /usr/include/c++/11/functional:98
#17 0x00005a5222e9baa5 in boost::context::detail::capture_record<boost::context::continuation, boost::context::basic_protected_fixedsize_stack<boost::context::stack_traits>&, eloqstore::Shard::StartTask<eloqstore::Shard::ProcessReq(eloqstore::KvRequest*)::<lambda()> >(eloqstore::KvTask*, eloqstore::KvRequest*, eloqstore::Shard::ProcessReq(eloqstore::KvRequest*)::<lambda()>)::<lambda(boost::context::continuation&&)> >::run(void) (this=0x7f215e6aab00) at /usr/include/boost/context/continuation_ucontext.hpp:267
#18 0x00005a5222e9974c in boost::context::detail::entry_func<boost::context::detail::capture_record<boost::context::continuation, boost::context::basic_protected_fixedsize_stack<boost::context::stack_traits>&, eloqstore::Shard::StartTask<eloqstore::Shard::ProcessReq(eloqstore::KvRequest*)::<lambda()> >(eloqstore::KvTask*, eloqstore::KvRequest*, eloqstore::Shard::ProcessReq(eloqstore::KvRequest*)::<lambda()>)::<lambda(boost::context::continuation&&)> > >(void *) (data=0x7f215e6aab00)
    at /usr/include/boost/context/continuation_ucontext.hpp:68
#19 0x00007f21ea05a130 in ?? () at ../sysdeps/unix/sysv/linux/x86_64/__start_context.S:90 from /lib/x86_64-linux-gnu/libc.so.6
#20 0x0000000000000000 in ?? ()


(gdb) f 8
#8  0x00005a522303dc53 in eloqstore::IndexPageManager::FindPage (this=0x51d000304d40, mapping=0x50f0002c41f0, page_id=0)
    at /home/eloq/eloqkv/data_substrate/store_handler/eloq_data_store_service/eloqstore/src/storage/index_page_manager.cpp:455
455     in /home/eloq/eloqkv/data_substrate/store_handler/eloq_data_store_service/eloqstore/src/storage/index_page_manager.cpp
(gdb) p new_page
$9 = (eloqstore::MemIndexPage *) 0x5070005402c0
(gdb) p *new_page
$10 = {static max_page_size = 65536, static page_size_offset = 9, static leftmost_ptr_offset = 11, page_id_ = 0, ref_cnt_ = 3,
  file_page_id_ = 3, page_ = {static kRegisteredMask = 9223372036854775808, ptr_ = 9223511815428648960}, waiting_ = {head_ = 0x0,
    tail_ = 0x0}, next_ = 0x0, prev_ = 0x0, tbl_ident_ = 0x0, in_free_list_ = false}
(gdb)

[lzxddz]

log:

[2026-03-13T15:20:23.305181 E 1323941] [async_io_manager.cpp:1141] open failed eloqkv_data_table_0.46 file id 0 : Resource not found

[lzxddz]

crashed codes:

std::pair<MemIndexPage::Handle, KvError> IndexPageManager::FindPage(
    MappingSnapshot *mapping, PageId page_id)
{
    while (true)
    {
        // First checks swizzling pointers.
        MemIndexPage::Handle handle = mapping->GetSwizzlingHandle(page_id);
        if (!handle)
        {
            // This is the first request to load the page.
            MemIndexPage *new_page = AllocIndexPage();  
            if (new_page == nullptr)
            {
                return {MemIndexPage::Handle(), KvError::OutOfMem};
            }
            FilePageId file_page_id = mapping->ToFilePage(page_id);
            new_page->SetPageId(page_id);
            new_page->SetFilePageId(file_page_id);
            mapping->AddSwizzling(page_id, new_page);

            // Read the page async.
            auto [page, err] = IoMgr()->ReadPage(
                *mapping->tbl_ident_, file_page_id, std::move(new_page->page_));
            new_page->page_ = std::move(page);
            if (err != KvError::NoError)   // this err is "NotFound"
            {
                new_page->waiting_.WakeAll();
                mapping->Unswizzling(new_page);
                FreeIndexPage(new_page);  // crashed at this line.
                return {MemIndexPage::Handle(), err};
            }
            FinishIo(mapping, new_page);
            new_page->waiting_.WakeAll();
            return {MemIndexPage::Handle(new_page), KvError::NoError};
        }
...

Through the codes, we can see the page is new allocated at line MemIndexPage *new_page = AllocIndexPage(); (in AllocIndexPage() functions, checked the page is not Pinned),
then, read the page but page not found, then do FreeIndexPage(new_page) crashed for Page::Pinned() is true( ref_cnt=3).

[lzxddz]

This is an unexpected state, indicating that another part of the system has incorrectly pinned "pages that have not been fully initialized" or that previously pinned pages were mistakenly returned to the free list.

[lzxddz]

The reason for this bug and another bug (#410) may be similar