From 04c28d18e3bb525fd083c89be8c89f7b19f0d7c3 Mon Sep 17 00:00:00 2001 From: Xinyang Ge Date: Wed, 3 Dec 2025 00:58:08 +0000 Subject: [PATCH] Extend layer limit from 255 to 65535 by using 16-bit tag field MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The SegmentMapping struct's tag field was 8-bit, limiting overlaybd to 255 stacked layers. This change extends the tag to 16-bit by borrowing 8 bits from moffset (55->47 bits), which still allows 64 PB per layer file - far exceeding practical needs. Changes: - index.h: Change tag from uint8_t to uint16_t, moffset from 55 to 47 bits - file.h: Update MAX_STACK_LAYERS from 255 to 65535 - file.cpp: Update m_rw_tag, min_tag, warp_file_tag types to uint16_t - index.cpp: Update ro_index_count parameter and merge limit to 65535 - config.h: Update MAX_LAYER_CNT from 256 to 65536 Note: This is a breaking change - layers created with this version are not compatible with older overlaybd versions. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- src/config.h | 2 +- src/overlaybd/lsmt/file.cpp | 20 ++++++++++---------- src/overlaybd/lsmt/file.h | 2 +- src/overlaybd/lsmt/index.cpp | 8 ++++---- src/overlaybd/lsmt/index.h | 12 ++++++------ 5 files changed, 22 insertions(+), 22 deletions(-) diff --git a/src/config.h b/src/config.h index 33ce1d80..1f373b4e 100644 --- a/src/config.h +++ b/src/config.h @@ -21,7 +21,7 @@ #include "overlaybd/config_util.h" namespace ImageConfigNS { -const int MAX_LAYER_CNT = 256; +const int MAX_LAYER_CNT = 65536; struct LayerConfig : public ConfigUtils::Config { APPCFG_CLASS; diff --git a/src/overlaybd/lsmt/file.cpp b/src/overlaybd/lsmt/file.cpp index 1a2af5b0..87ccbc19 100644 --- a/src/overlaybd/lsmt/file.cpp +++ b/src/overlaybd/lsmt/file.cpp @@ -692,7 +692,7 @@ class LSMTFile : public LSMTReadOnlyFile { bool m_init_concurrency = false; uint64_t m_data_offset = HeaderTrailer::SPACE / ALIGNMENT; - uint8_t m_rw_tag = 0; + uint16_t m_rw_tag = 0; Mutex m_rw_mtx; IFile *m_findex = nullptr; @@ -1120,7 +1120,7 @@ class LSMTWarpFile : public LSMTFile { virtual ssize_t pwrite(const void *buf, size_t count, off_t offset) override { LOG_DEBUG("write fs meta {offset: `, len: `}", offset, count); - auto tag = m_rw_tag + (uint8_t)SegmentType::fsMeta; + auto tag = m_rw_tag + (uint16_t)SegmentType::fsMeta; SegmentMapping m{ (uint64_t)offset / (uint64_t)ALIGNMENT, (uint32_t)count / (uint32_t)ALIGNMENT, @@ -1159,7 +1159,7 @@ class LSMTWarpFile : public LSMTFile { m.length = (Segment::MAX_LENGTH < lba.count / ALIGNMENT ? Segment::MAX_LENGTH : lba.count / ALIGNMENT); m.moffset = lba.roffset / ALIGNMENT; - m.tag = m_rw_tag + (uint8_t)SegmentType::remoteData; + m.tag = m_rw_tag + (uint16_t)SegmentType::remoteData; LOG_DEBUG("insert segment: ` into findex: `", m, m_findex); static_cast(m_index)->insert(m); append_index(m); @@ -1179,7 +1179,7 @@ class LSMTWarpFile : public LSMTFile { size_t compacted_idx_size = 0; moffset /= ALIGNMENT; for (auto &m : marray) { - if (m.tag == (uint8_t)SegmentType::remoteData) { + if (m.tag == (uint16_t)SegmentType::remoteData) { compact_index.push_back(m); compacted_idx_size++; continue; @@ -1271,7 +1271,7 @@ static HeaderTrailer *verify_ht(IFile *file, char *buf, bool is_trailer, ssize_t } static SegmentMapping *do_load_index(IFile *file, HeaderTrailer *pheader_trailer, bool trailer, - uint8_t warp_file_tag = 0) { + uint16_t warp_file_tag = 0) { ALIGNED_MEM(buf, HeaderTrailer::SPACE, ALIGNMENT4K); auto pht = verify_ht(file, buf); @@ -1315,7 +1315,7 @@ static SegmentMapping *do_load_index(IFile *file, HeaderTrailer *pheader_trailer } size_t index_size = 0; - uint8_t min_tag = 255; + uint16_t min_tag = 65535; for (size_t i = 0; i < pht->index_size; ++i) { if (ibuf[i].offset != SegmentMapping::INVALID_OFFSET) { ibuf[index_size] = ibuf[i]; @@ -1329,9 +1329,9 @@ static SegmentMapping *do_load_index(IFile *file, HeaderTrailer *pheader_trailer LOG_INFO("rebuild index tag for LSMTWarpFile."); for (size_t i = 0; i < index_size; i++) { if (warp_file_tag == 1) /* only fsmeta */ - ibuf[i].tag = (uint8_t)SegmentType::fsMeta; + ibuf[i].tag = (uint16_t)SegmentType::fsMeta; if (warp_file_tag == 2) /* only remote data */ - ibuf[i].tag = (uint8_t)SegmentType::remoteData; + ibuf[i].tag = (uint16_t)SegmentType::remoteData; if (warp_file_tag == 3) /* only remote data */ ibuf[i].tag -= min_tag; LOG_DEBUG("`", ibuf[i]); @@ -1485,8 +1485,8 @@ IFileRW *create_warpfile(WarpFileArgs &args, bool ownership) { write_header_trailer(rst->m_findex, true, false, false, HeaderTrailer::SPACE, 0, info); rst->m_index = create_memory_index0((const SegmentMapping *)nullptr, 0, 0, 0); rst->m_files.resize(2); - rst->m_files[(uint8_t)SegmentType::fsMeta] = args.fsmeta; - rst->m_files[(uint8_t)SegmentType::remoteData] = args.target_file; + rst->m_files[(uint16_t)SegmentType::fsMeta] = args.fsmeta; + rst->m_files[(uint16_t)SegmentType::remoteData] = args.target_file; rst->m_vsize = args.virtual_size; rst->m_file_ownership = ownership; UUID raw; diff --git a/src/overlaybd/lsmt/file.h b/src/overlaybd/lsmt/file.h index 1fc7b5bc..0dc4b099 100644 --- a/src/overlaybd/lsmt/file.h +++ b/src/overlaybd/lsmt/file.h @@ -33,7 +33,7 @@ IMemoryIndex -> IMemoryIndex0 -> IComboIndex -> Index0 ( set ) -> Co namespace LSMT { -static const int MAX_STACK_LAYERS = 255; +static const int MAX_STACK_LAYERS = 65535; static const uint32_t ALIGNMENT = 512; // same as trim block size. static const uint32_t ALIGNMENT4K = 4096; diff --git a/src/overlaybd/lsmt/index.cpp b/src/overlaybd/lsmt/index.cpp index d4dadf9f..737fcff9 100644 --- a/src/overlaybd/lsmt/index.cpp +++ b/src/overlaybd/lsmt/index.cpp @@ -629,7 +629,7 @@ class ComboIndex : public Index0 { Index *m_backing_index{nullptr}; bool m_ownership; - ComboIndex(Index0 *index0, const Index *index, uint8_t ro_layers_count, bool ownership) { + ComboIndex(Index0 *index0, const Index *index, uint16_t ro_layers_count, bool ownership) { m_index0 = index0; m_backing_index = const_cast(index); mapping = index0->mapping; @@ -853,7 +853,7 @@ static void merge_indexes(uint8_t level, vector &mapping, const } IComboIndex *create_combo_index(IMemoryIndex0 *index0, const IMemoryIndex *index, - uint8_t ro_index_count, bool ownership) { + uint16_t ro_index_count, bool ownership) { if (!index0 || !index) LOG_ERROR_RETURN(EINVAL, nullptr, "invalid argument(s)"); @@ -903,8 +903,8 @@ size_t compress_raw_index_predict(const SegmentMapping *mapping, size_t n) { } IMemoryIndex *merge_memory_indexes(const IMemoryIndex **pindexes, size_t n) { - if (n > 255) { - LOG_ERROR("too many indexes to merge, 255 at most!"); + if (n > 65535) { + LOG_ERROR("too many indexes to merge, 65535 at most!"); return nullptr; } if (n == 0 || pindexes == nullptr) diff --git a/src/overlaybd/lsmt/index.h b/src/overlaybd/lsmt/index.h index 2de95695..fe0d23ee 100644 --- a/src/overlaybd/lsmt/index.h +++ b/src/overlaybd/lsmt/index.h @@ -51,15 +51,15 @@ struct Segment { } } __attribute__((packed)); -struct SegmentMapping : public Segment { // 64 + 55 + 9 == 128 - uint64_t moffset : 55; // mapped offset (2^64 B if in sector) +struct SegmentMapping : public Segment { // 64 + 47 + 1 + 16 == 128 + uint64_t moffset : 47; // mapped offset (64 PB if in sector) uint32_t zeroed : 1; // indicating a zero-filled segment - uint8_t tag; - const static uint64_t MAX_MOFFSET = (1UL << 55) - 1; + uint16_t tag; // layer index (up to 65535 layers) + const static uint64_t MAX_MOFFSET = (1UL << 47) - 1; SegmentMapping() { } - SegmentMapping(uint64_t loffset, uint32_t length, uint64_t moffset, uint8_t tag = 0) + SegmentMapping(uint64_t loffset, uint32_t length, uint64_t moffset, uint16_t tag = 0) : Segment{loffset, length}, moffset(moffset), zeroed(0), tag(tag) { assert(length <= Segment::MAX_LENGTH); } @@ -186,7 +186,7 @@ extern "C" IMemoryIndex *merge_memory_indexes(const IMemoryIndex **pindexes, std // were one single index; inserting into a combo effectively inserting into the index0 part; // the mapped offset must be within [moffset_begin, moffset_end) extern "C" IComboIndex *create_combo_index(IMemoryIndex0 *index0, const IMemoryIndex *index, - uint8_t ro_index_count, bool ownership); + uint16_t ro_index_count, bool ownership); // compress raw index array by mergeing adjacent continuous mappings // returning compressed size of the array