DataTableOptimizer.lua

--[[
How to use:

Put all lua files into DatabaseRoot then call ExportDatabaseLocalText( tofile = true, newStringBank = false )  at the end of file. ( beware: all your original files will be replaced with optimized files )
If you want to exlucde some input files in DatabaseRoot, just add theirs names into ExcludedFiles

Optimization Features:
1. Table deduplication: Identical tables are extracted as shared references
2. Default value extraction: Common field values across records are extracted as defaults
3. Nil-safety: Only fields that exist in ALL records are extracted as defaults to prevent
   the metatable from returning default values for originally nil fields

--]] -- source code: https://github.com/lujian101/LuaTableOptimizer
local Root = "."
local SubDir = ""
if arg and arg[1] then
    SubDir = arg[1]
    print(arg[1])
end
Root = string.gsub(Root, '\\', '/')
local DatabaseRoot = Root .. "/../Config/lua_client/XLS/" .. SubDir

package.path = package.path .. ';' .. DatabaseRoot .. '/?.lua'

-- 获取绝对路径的函数
local function getAbsolutePath(path)
    if path:match("^[A-Za-z]:") or path:match("^/") then
        return path -- 已经是绝对路径
    end

    -- 使用io.popen获取绝对路径
    local cmd
    if package.config:sub(1, 1) == '\\' then -- Windows
        cmd = 'cd /d "' .. path .. '" && cd'
    else -- Unix/Linux
        cmd = 'cd "' .. path .. '" && pwd'
    end

    local handle = io.popen(cmd)
    if handle then
        local result = handle:read("*a")
        handle:close()
        if result then
            return result:gsub("[\r\n]+$", ""):gsub('\\', '/') -- 移除尾部换行符并统一路径分隔符
        end
    end
    return path -- 如果失败，返回原路径
end

local EnableDatasetOptimize = true
local EnableDefaultValueOptimize = true
local EnableReadOnlyOptimize = true

local Database = {}
local DefaultNumberSerializedFormat = "%.14g"
local NumberSerializedFormat = DefaultNumberSerializedFormat
local DatabaseLocaleTextName = "_LocaleText"
local StringBankOutput = DatabaseRoot .. "/" .. DatabaseLocaleTextName .. ".lua"
local StringBankCSVOutput = DatabaseRoot .. "/" .. DatabaseLocaleTextName .. ".csv"
local MaxStringBankRedundancy = 100
local MaxStringBankBinSize = 524288
local LocaleTextLeadingTag = '@'
local MaxLocalVariableNum = 160 -- lparser.c #define MAXVARS 200
local RefTableName = "__rt"
local PrintTableRefCount = false
local UnknownName = "___noname___"

local fmod = math.fmod

local ExcludedFiles = {
    -- Add file name to exclude from build
    _LocaleText = true,
	-- ["sound/lipsync_data"] = true,
    [".*_attr.*"] = true,
}

-- 支持正则匹配的文件排除检查函数
local function IsFileExcluded(path)
    for pattern, enabled in pairs(ExcludedFiles) do
        if enabled and string.match(path, pattern) then
            return true
        end
    end
    return false
end

local UniquifyTables = {} -- hash -> table
local UniquifyTablesIds = {} -- id -> hash
local UniquifyTablesInvIds = {} -- table -> id
local UniquifyTablesRefCounter = {} -- table -> refcount

local function HashString(v)
    local val = 0
    local fmod = fmod
    local gmatch = string.gmatch
    local byte = string.byte
    local MaxStringBankBinSize = MaxStringBankBinSize
    local c
    for _c in gmatch(v, ".") do
        c = byte(_c)
        val = val + c * 193951
        val = fmod(val, MaxStringBankBinSize)
        val = val * 399283
        val = fmod(val, MaxStringBankBinSize)
    end
    return val
end

local function AddStringToBank(stringBank, str)
    local meta = getmetatable(stringBank)
    local reversed = nil
    local counter = nil
    if not meta then
        meta = {
            __counter = {
                used = {}
            }, -- mark used hash value
            __reversed = {} -- string -> hash reverse lookup
        }
        reversed = meta.__reversed
        counter = meta.__counter
        setmetatable(stringBank, meta)
        local remove = {}
        -- lazy initialize reverse lut
        for h, s in pairs(stringBank) do
            local _h = reversed[s]
            assert(_h == nil)
            reversed[s] = h
        end
    end
    reversed = reversed or meta.__reversed
    counter = counter or meta.__counter
    local hash = reversed[str]
    if hash then
        counter.used[hash] = true
        return hash
    end
    hash = HashString(str)
    local _v = stringBank[hash]
    while _v do
        hash = hash + 1
        hash = fmod(hash, MaxStringBankBinSize)
        _v = stringBank[hash]
    end
    assert(not reversed[str])
    stringBank[hash] = str
    reversed[str] = hash
    counter.used[hash] = true
    return hash
end

local function OrderedForeach(_table, _func, _filter)
    local _type = type
    if _type(_table) == "table" then
        local kv = {}
        for k, v in pairs(_table) do
            if not _filter or _filter(k, v) then
                kv[#kv + 1] = {k, v}
            end
        end
        local _tostring = tostring
        table.sort(kv, function(_l, _r)
            local l = _l[1]
            local r = _r[1]
            local lt = _type(l)
            local rt = _type(r)
            if lt == rt and lt ~= "table" then
                return l < r
            else
                return _tostring(l) < _tostring(r)
            end
        end)
        for _, _v in ipairs(kv) do
            local k = _v[1]
            local v = _v[2]
            _func(k, v)
        end
    end
end

local function OrderedForeachByValue(_table, _func, _filter)
    local _type = type
    if _type(_table) == "table" then
        local kv = {}
        for k, v in pairs(_table) do
            if not _filter or _filter(k, v) then
                kv[#kv + 1] = {k, v}
            end
        end
        local _tostring = tostring
        table.sort(kv, function(_l, _r)
            local l = _l[2]
            local r = _r[2]
            local lt = _type(l)
            local rt = _type(r)
            if lt == rt and lt ~= "table" then
                return l < r
            else
                return _tostring(l) < _tostring(r)
            end
        end)
        for _, _v in ipairs(kv) do
            local k = _v[1]
            local v = _v[2]
            if not pcall(_func, k, v) then
                return false
            end
        end
        return true
    end
end

local function EncodeEscapeString(s)
    local buf = {}
    buf[#buf + 1] = "\""
    string.gsub(s, ".", function(c)
        if c == '\n' then
            buf[#buf + 1] = "\\n"
        elseif c == '\t' then
            buf[#buf + 1] = "\\t"
        elseif c == '\r' then
            buf[#buf + 1] = "\\r"
        elseif c == '\a' then
            buf[#buf + 1] = "\\a"
        elseif c == '\b' then
            buf[#buf + 1] = "\\b"
        elseif c == '\\' then
            buf[#buf + 1] = "\\\\"
        elseif c == '\"' then
            buf[#buf + 1] = "\\\""
        elseif c == '\'' then
            buf[#buf + 1] = "\\\'"
        elseif c == '\v' then
            buf[#buf + 1] = "\\\v"
        elseif c == '\f' then
            buf[#buf + 1] = "\\\f"
        else
            buf[#buf + 1] = c
        end
    end)
    buf[#buf + 1] = "\""
    return table.concat(buf, "")
end

local function StringBuilder()
    local sb = {}
    local f = function(str)
        if str then
            sb[#sb + 1] = str
        end
        return f, sb
    end
    return f
end

local function CreateFileWriter(fileName, mode)
    local file = nil
    local indent = 0
    if mode and fileName then
        local _file, err = io.open(fileName)
        if _file ~= nil then
            -- print( "remove file "..fileName )
            os.remove(fileName)
        end
        file = io.open(fileName, mode)
    end
    local ret = nil
    if file then
        ret = {
            write = function(...)
                if indent > 0 then
                    for i = 0, indent - 1 do
                        file:write("\t")
                    end
                end
                return file:write(...)
            end,
            close = function(...)
                return file:close()
            end
        }
    else
        ret = {
            write = function(...)
                for i = 0, indent - 1 do
                    io.write("\t")
                end
                return io.write(...)
            end,
            close = function(...)
            end
        }
    end
    ret.indent = function(count)
        count = count or 1
        indent = indent + count or 1
    end
    ret.outdent = function(count)
        count = count or 1
        if indent >= count then
            indent = indent - count
        end
    end
    return ret
end

local function SetNumberSerializedFormat(f)
    NumberSerializedFormat = f or DefaultNumberSerializedFormat
    if NumberSerializedFormat == "" then
        NumberSerializedFormat = DefaultNumberSerializedFormat
    end
    print("set NumberSerializedFormat: " .. NumberSerializedFormat)
end

local DefaultVisitor = {
    recursive = true,
    iVisit = function(i, v, curPath)
        print(string.format("%s[%d] = %s", curPath, i, tostring(v)))
        return true
    end,
    nVisit = function(n, v, curPath)
        print(string.format("%s[%g] = %s", curPath, n, tostring(v)))
        return true
    end,
    sVisit = function(s, v, curPath)
        local _v = tostring(v)
        print(#curPath > 0 and curPath .. "." .. s .. " = " .. _v or s .. " = " .. _v)
        return true
    end,
    xVisit = function(k, v, curPath)
        local sk = tostring(k)
        local sv = tostring(v)
        print(#curPath > 0 and curPath .. "." .. sk .. " = " .. sv or sk .. " = " .. sv)
        return true
    end
}

local function WalkDataset(t, visitor, parent)
    if not parent then
        parent = ""
    end
    -- all integer key
    local continue = true
    if visitor.iVisit then
        for i, v in ipairs(t) do
            local _t = type(v)
            if _t == "table" and visitor.recursive then
                continue = WalkDataset(v, visitor, string.format("%s[%g]", parent, i))
            elseif _t == "string" or _t == "number" then
                continue = visitor.iVisit(i, v, parent)
            else
                -- not support value type
                if visitor.xVisit then
                    continue = visitor.xVisit(i, v, parent)
                end
            end
            if not continue then
                return continue
            end
        end
    end

    local len = #t
    local keys = {}
    local idict = {}
    for k, v in pairs(t) do
        local _t = type(k)
        if _t == "number" then
            local intKey = k == math.floor(k);
            if k > len or k <= 0 or not intKey then
                idict[k] = v
            end
        elseif _t == "string" then
            keys[#keys + 1] = k
        else
            -- table, function, ...
            -- not support data type for key
            if visitor.xVisit then
                continue = visitor.xVisit(k, v, parent)
            end
        end
        if not continue then
            return continue
        end
    end
    -- for all number keys those are not in array part
    -- key must be number
    for k, v in pairs(idict) do
        local intKey = k == math.floor(k);
        local _t = type(v)
        if _t ~= "table" then
            if _t == "number" or _t == "string" then
                if intKey then
                    if visitor.iVisit then
                        continue = visitor.iVisit(k, v, parent)
                    end
                else
                    if visitor.nVisit then
                        continue = visitor.nVisit(k, v, parent)
                    end
                end
            else
                -- not support value data type
                if visitor.xVisit then
                    continue = visitor.xVisit(k, v, parent)
                end
            end
        elseif visitor.recursive then
            if intKey then
                continue = WalkDataset(v, visitor, string.format("%s[%d]", parent, k))
            else
                continue = WalkDataset(v, visitor, string.format("%s[%g]", parent, k))
            end
        end
        if not continue then
            return continue
        end
    end
    -- sort all string keys
    table.sort(keys)
    -- for all none-table value
    local tableValue
    for k, v in pairs(keys) do
        local value = t[v]
        local _t = type(value)
        if _t == "number" or _t == "string" then
            -- print all number or string value here
            if visitor.sVisit then
                continue = visitor.sVisit(v, value, parent)
            end
        elseif _t == "table" then
            -- for table value
            if not tableValue then
                tableValue = {}
            end
            tableValue[k] = v
        else
            if visitor.xVisit then
                continue = visitor.xVisit(v, value, parent)
            end
        end
        if not continue then
            return continue
        end
    end
    if visitor.recursive then
        -- for all table value
        if tableValue then
            for k, v in pairs(tableValue) do
                local value = t[v]
                continue = WalkDataset(value, visitor, #parent > 0 and parent .. "." .. v or v)
                if not continue then
                    return continue
                end
            end
        end
    end
    return continue
end

local function PrintDataset(t, parent)
    if not parent then
        parent = ""
    end
    local string_format = string.format
    -- all integer key
    for i, v in ipairs(t) do
        local _t = type(v)
        if _t == "table" then
            PrintDataset(v, string_format("%s[%g]", parent, i))
        elseif _t == "string" or _t == "number" then
            print(string.format("%s[%d] = %s", parent, i, tostring(v)))
        else
            -- not support value type
        end
    end
    local len = #t
    local keys = {}
    local idict = {}
    for k, v in pairs(t) do
        local _t = type(k)
        if _t == "number" then
            if k > len or k <= 0 then
                idict[k] = v
            end
        elseif _t == "string" then
            keys[#keys + 1] = k
        else
            -- table, function, ...
            -- not support data type for key
        end
    end
    -- for all number keys those are not in array part
    -- key must be number
    for k, v in pairs(idict) do
        local intKey = k == math.floor(k)
        local _t = type(v)
        if _t ~= "table" then
            if _t == "number" or _t == "string" then
                if intKey then
                    print(string_format("%s[%d] = %s", parent, k, tostring(v)))
                else
                    print(string_format("%s[%g] = %s", parent, k, tostring(v)))
                end
            else
                -- not support value data type
            end
        else
            if intKey then
                PrintDataset(v, string_format("%s[%d]", parent, k))
            else
                PrintDataset(v, string_format("%s[%g]", parent, k))
            end
        end
    end
    -- sort all string keys
    table.sort(keys)
    -- for all none-table value
    local tableValue
    for k, v in pairs(keys) do
        local value = t[v]
        local _t = type(value)
        if _t ~= "table" then
            -- print all number or string value here
            local _value = tostring(value)
            print(#parent > 0 and parent .. "." .. v .. " = " .. _value or v .. " = " .. _value)
        else
            -- for table value
            if not tableValue then
                tableValue = {}
            end
            tableValue[k] = v
        end
    end
    -- for all table value
    if tableValue then
        for k, v in pairs(tableValue) do
            local value = t[v]
            PrintDataset(value, #parent > 0 and parent .. "." .. v or v)
        end
    end
end

local function DeserializeTable(val)
    local loader = loadstring or load -- lua5.2 compat
    local chunk = loader("return " .. val)
    local ok, ret = pcall(chunk)
    if not ok then
        ret = nil
        print("DeserializeTable failed!" .. val)
    end
    return ret
end

local function _SerializeTable(val, name, skipnewlines, campact, depth, tableRef)
    local valt = type(val)
    depth = depth or 0
    campact = campact or false
    local append = StringBuilder()
    local eqSign = " = "
    local tmp = ""
    local string_format = string.format
    if not campact then
        append(string.rep("\t", depth))
        skipnewlines = skipnewlines or false
    else
        skipnewlines = true
        eqSign = "="
    end
    if name then
        local nt = type(name)
        if nt == "string" then
            if name ~= "" then
                -- 检查是否需要使用方括号表示法
                -- 如果包含特殊字符或以数字开头，使用方括号
                if string.match(name, '^%d') or string.match(name, '[^%w_]') then
                    append("[\"")
                    append(name)
                    append("\"]")
                else
                    append(name)
                end
            else
                append("[\"\"]")
            end
            append(eqSign)
        elseif nt == "number" then
            append(string_format("[%s]", tostring(name)))
            append(eqSign)
        else
            tmp = tmp .. "\"[inserializeable datatype for key:" .. nt .. "]\""
        end
    end
    local ending = not skipnewlines and "\n" or ""
    if tableRef then
        local refName = tableRef[val]
        if refName then
            valt = "ref"
            val = refName
        end
    end
    if valt == "table" then
        append("{")
        append(ending)
        local array_part = {}
        local count = 0
        for k, v in ipairs(val) do
            if type(v) ~= "function" then
                array_part[k] = true
                if count > 0 then
                    append(",")
                    append(ending)
                end
                append(_SerializeTable(v, nil, skipnewlines, campact, depth + 1, tableRef))
                count = count + 1
            end
        end
        local sortedK = {}
        for k, v in pairs(val) do
            if type(v) ~= "function" then
                if not array_part[k] then
                    sortedK[#sortedK + 1] = k
                end
            end
        end
        table.sort(sortedK)
        for i, k in ipairs(sortedK) do
            local v = val[k]
            if count > 0 then
                append(",")
                append(ending)
            end
            append(_SerializeTable(v, k, skipnewlines, campact, depth + 1, tableRef))
            count = count + 1
        end
        if count >= 1 then
            append(ending)
        end
        if not campact then
            append(string.rep("\t", depth))
        end
        append("}")
    elseif valt == "number" then
        if DefaultNumberSerializedFormat == NumberSerializedFormat or math.floor(val) == val then
            append(tostring(val))
        else
            append(string_format(NumberSerializedFormat, val))
        end
    elseif valt == "string" then
        append(EncodeEscapeString(val))
    elseif valt == "boolean" then
        append(val and "true" or "false")
    elseif valt == "ref" then
        append(val or "nil")
    else
        tmp = tmp .. "\"[inserializeable datatype:" .. valt .. "]\""
    end
    local _, slist = append()
    return table.concat(slist, "")
end

local function SerializeTable(val, skipnewlines, campact, tableRef, name)
    getmetatable("").__lt = function(a, b)
        return tostring(a):lower() < tostring(b):lower()
    end
    local ret = _SerializeTable(val, name, skipnewlines, campact, 0, tableRef)
    getmetatable("").__lt = nil
    return ret
end

local function DumpStringBank(stringBank)
    print('dump database local string bank begin...')
    for k, v in pairs(stringBank) do
        print(string.format("\t[%g] = %s", k, v))
    end
    print('dump database local string bank end.')
end

local function SaveStringBankToLua(stringBank, tofile)
    if tofile then
        local fileName = StringBankOutput
        local _file, err = io.open(fileName)
        if _file ~= nil then
            _file:close()
            os.remove(fileName)
        end
        file = io.open(fileName, "w")
        local fmt = string.format
        file:write(fmt("local %s = {\n", DatabaseLocaleTextName))
        for k, v in pairs(stringBank) do
            file:write(fmt("\t[%g] = %s,\n", k, EncodeEscapeString(v)))
        end
        file:write("}\n")
        file:write(fmt("return %s\n--EOF", DatabaseLocaleTextName))
        file:close()
    else
        DumpStringBank(stringBank)
    end
end

local function SaveStringBankToCSV(stringBank, tofile)
    local _exists = {}
    for k, v in pairs(stringBank) do
        assert(not _exists[v])
        _exists[v] = k
    end
    local csv = CSV
    if tofile and csv then
        local fileName = StringBankCSVOutput
        local _file, err = io.open(fileName)
        if _file ~= nil then
            _file:close()
            os.remove(fileName)
        end
        local t = {}
        local count = 1
        for k, v in pairs(stringBank) do
            t[count] = {k, v}
            count = count + 1
        end
        table.sort(t, function(a, b)
            return a[1] < b[1]
        end)
        csv.save(fileName, t, true)
    else
        SaveStringBankToLua(stringBank, tofile)
    end
end

local function LoadStringBankFromLua(info)
    local stringBank = {}
    local chunk = loadfile(StringBankOutput)
    if chunk then
        print('load string bank: ' .. StringBankOutput)
        local last = chunk()
        if last and type(last) == "table" then
            for k, v in pairs(last) do
                stringBank[k] = v
            end
        end
    end
    return stringBank
end

local function LoadStringBankFromCSV()
    local stringBank = {}
    local csv = CSV
    if csv then
        local fileName = StringBankCSVOutput
        local file, err = io.open(fileName)
        if file then
            print("Load StringBank: " .. fileName)
            file:close()
            local b = csv.load(fileName, true)
            local allstr = {}
            for i = 1, #b do
                local key = b[i][1]
                local value = b[i][2]
                local oldHash = allstr[value]
                if not oldHash then
                    stringBank[key] = value
                    allstr[value] = key
                else
                    print(string.format("\"%s\" already exists in StringBank with hash %d", value, oldHash))
                end
            end
        end
    else
        return LoadStringBankFromLua()
    end
    return stringBank
end

local function TrimStringBank(stringBank)
    -- remove useless values
    local meta = getmetatable(stringBank)
    if meta then
        local counter = meta.__counter
        if counter then
            local used = counter.used
            if used then
                local count = 0
                for hash, str in pairs(stringBank) do
                    count = count + 1
                end
                local unused = {}
                for hash, str in pairs(stringBank) do
                    if not used[hash] then
                        unused[#unused + 1] = hash
                    end
                end
                if #unused > MaxStringBankRedundancy then
                    for _, h in ipairs(unused) do
                        stringBank[h] = nil
                    end
                end
            end
        end
    end
end

-- 获取文件大小（字节）
local function GetFileSize(filePath)
    local file = io.open(filePath, "rb")
    if not file then
        return 0
    end
    local size = file:seek("end")
    file:close()
    return size or 0
end

-- 格式化文件大小显示
local function FormatFileSize(bytes)
    if bytes < 1024 then
        return string.format("%.0f B", bytes)
    elseif bytes < 1024 * 1024 then
        return string.format("%.1f KB", bytes / 1024)
    else
        return string.format("%.1f MB", bytes / (1024 * 1024))
    end
end

local function GetAllFileRelativePathsAtPath(path)
    local absolutePath = getAbsolutePath(path)
    absolutePath = absolutePath:gsub("/", "\\")

    local ret = {}
    local filteredFiles = {}

    -- 确保路径以反斜杠结尾，方便后续处理
    if not absolutePath:match("\\$") then
        absolutePath = absolutePath .. "\\"
    end

    print(string.format("扫描目录: %s", absolutePath))
    print("----------------------------------------")

    for dir in io.popen(string.format("dir \"%s\" /S/b", absolutePath)):lines() do
        if dir:match("%.lua$") then
            -- 计算相对于DatabaseRoot的相对路径
            local relativePath = dir:gsub("^" .. absolutePath:gsub("([%(%)%.%+%-%*%?%[%]%^%$%%])", "%%%1"), "")
            -- 将反斜杠转换为正斜杠，并去掉.lua扩展名
            relativePath = relativePath:gsub("\\", "/"):gsub("%.lua$", "")
            table.insert(ret, relativePath)
        end
    end

    print("----------------------------------------")
    print(string.format("总计找到 %d 个lua文件", #ret + #filteredFiles))
    print(string.format("符合条件: %d 个文件", #ret))
    print(string.format("跳过小文件: %d 个文件", #filteredFiles))
    print("----------------------------------------")

    table.sort(ret)
    return ret
end

local function LoadDataset(relativePathWithoutLuaExtension)
    if not Database then
        _G["Database"] = {}
        Database.loaded = {}
    end
    local loader = function(name)
        Database.loaded = Database.loaded or {}
        local r = Database.loaded[name]
        if r then
            return r
        end
        local pname = string.gsub(name, "%.", "/")
        local split = function(s, p)
            local rt = {}
            string.gsub(s, '[^' .. p .. ']+', function(w)
                table.insert(rt, w)
            end)
            return rt
        end

        local reletivePathWithLuaExtension = pname .. ".lua"
        local filePath = DatabaseRoot .. "/" .. reletivePathWithLuaExtension
        local checkFileName = function(root, relaPath)
            local absolutePath = getAbsolutePath(root)
            absolutePath = absolutePath:gsub("/", "\\")
            if not absolutePath:match("\\$") then
                absolutePath = absolutePath .. "\\"
            end
            local _name = string.lower(relaPath)
            for dir in io.popen(string.format("dir \"%s\" /s/b", absolutePath)):lines() do
                if dir:match("%.lua$") then
                    local relativePath = dir:gsub("^" .. absolutePath:gsub("([%(%)%.%+%-%*%?%[%]%^%$%%])", "%%%1"), "")
                    relativePath = relativePath:gsub("\\", "/")
                    local _f = string.lower(relativePath)
                    if _name == _f then
                        return relaPath == relativePath, relativePath -- not match, real name
                    end
                end
            end
        end

        local m, real = checkFileName(DatabaseRoot, reletivePathWithLuaExtension)
        if not m and real then
            local msg = string.format("filename must be matched by case! realname: \"%s\", you pass: \"%s\"",
                reletivePathWithLuaExtension, real)
            print(msg)
            os.execute("pause")
        end

        local chunk = loadfile(filePath)
        print(filePath)
        if not chunk or not chunk() or type(chunk()) ~= "table" then
            print("loadfile failed: " .. filePath)
            return nil
        end
        local rval = chunk()
        if rval.__name ~= nil then
            os.execute("pause table's key must not be '__name' which is the reserved keyword.")
        end
        if rval.__sourcefile ~= nil then
            os.execute("pause table's key must not be '__sourcefile' which is the reserved keyword.")
        end
        local namespace = Database
        local ns = split(pname, '/')
        local xname = ns[#ns] -- last one

        -- 使用文件名作为表格名，而不是完整路径
        rval.__name = "t"
        rval.__sourcefile = filePath
        if #ns > 1 then
            for i = 1, #ns - 1 do
                local n = ns[i];
                if namespace[n] == nil then
                    namespace[n] = {}
                end
                namespace = namespace[n]
            end
        end
        namespace[xname] = rval
        print("dataset: " .. name .. " has been loaded")
        Database.loaded[name] = rval
        return rval
    end
    return loader(relativePathWithoutLuaExtension)
end

local function CheckNotAscii(v)
    if v ~= nil and type(v) == "string" then
        local byte = string.byte
        for _c in string.gmatch(v, ".") do
            local c = byte(_c)
            if c < 0 or c > 127 then
                return true
            end
        end
    end
    return false
end

local function LocalizeRecord(id, record, genCode, StringBank)
    local localized_fields = nil
    local subTable = nil
    OrderedForeach(record, function(k, v)
        local vt = type(v)
        if vt == "string" then
            if CheckNotAscii(v) then
                if #v > 0 and string.sub(v, 1, 1) == LocaleTextLeadingTag then
                    print(string.format("invalid leading character for localized text! key, value: %s, %s", k, v))
                    os.execute("pause")
                end
                if not localized_fields then
                    localized_fields = {}
                end
                -- build localized id string with tag
                local sid = AddStringToBank(StringBank, v)
                localized_fields[k] = string.format("%s%g", LocaleTextLeadingTag, sid)
                if genCode then
                    genCode[#genCode + 1] = {id, sid, v}
                end
            end
        elseif vt == "table" then
            if not subTable then
                subTable = {}
            end
            subTable[#subTable + 1] = v
        end
    end)
    local localized = false
    if localized_fields then
        -- override localized string with tag
        localized = true
        for k, v in pairs(localized_fields) do
            record[k] = localized_fields[k]
        end
    end
    if subTable then
        for _, sub in ipairs(subTable) do
            localized = LocalizeRecord(0, sub, genCode, StringBank) or localized
        end
    end
    return localized
end

local function GetValueTypeNameCS(value)
    local t = type(value)
    if t == "string" then
        return "string"
    elseif t == "number" then
        if value == math.floor(value) then
            return "int"
        else
            return "float"
        end
    elseif t == "boolean" then
        return "bool"
    elseif t == "table" then
        return "table"
    else
        return "void"
    end
end

local function UniquifyTable(t)
    if t == nil or type(t) ~= "table" then
        return nil
    end
    
    local hash = SerializeTable(t, true, true)
    local ref = UniquifyTables[hash]
    if ref then
        local refcount = UniquifyTablesRefCounter[ref] or 1
        UniquifyTablesRefCounter[ref] = refcount + 1
        return ref
    end

    local overwrites = nil
    local _type = type
    OrderedForeach(t, function(k, v)
        overwrites = overwrites or {}
        overwrites[k] = UniquifyTable(v)
    end, function(k, v)
        return _type(v) == "table"
    end)
    if overwrites then
        for k, v in pairs(overwrites) do
            t[k] = overwrites[k]
        end
    end
    local id = #UniquifyTablesIds + 1
    UniquifyTablesIds[id] = hash
    UniquifyTables[hash] = t
    UniquifyTablesInvIds[t] = id
    UniquifyTablesRefCounter[t] = 1
    return t
end

local function IsPlanarKeyValue(table)
	-- 检测是否为平铺的键值对结构（如多语言文件）
	local isPlanarKeyValue = true
	local hasNonTableValues = false

	local sampleCount = 0

    for k, v in pairs(table) do
        local _sk = tostring(k)
        if _sk ~= "__name" and _sk ~= "__sourcefile" then
            sampleCount = sampleCount + 1
            if type(v) == "table" then
                isPlanarKeyValue = false
            else
                hasNonTableValues = true
            end
            -- 只检查前20个样本就足够判断结构
            if sampleCount >= 20 then
                break
            end
        end
    end

    if isPlanarKeyValue and hasNonTableValues then
        return true
    end
	return false
end

-- 收集所有表格的公共字段（在所有表格中都存在且不为nil的字段）
local function IsArray(tab)
    if type(tab) ~= "table" then return false end
    local count = 0
    for k,_ in pairs(tab) do
        if type(k) ~= "number" then return false end
        if k <=0 or k ~= math.floor(k) then return false end
        count = math.max(count,k)
    end
    -- treat as array if numeric keys only and at least 1 element
    return count>0
end

local function GetCommonFields(tables)
    if #tables == 0 then
        return {}
    end

    local candidateFields = {}
    for field, value in pairs(tables[1]) do
        if value ~= nil and type(field)~="number" then -- 跳过数字键(数组)
            candidateFields[field] = true
        end
    end

    for i=2,#tables do
        local t = tables[i]
        for field,_ in pairs(candidateFields) do
            if t[field]==nil then candidateFields[field]=nil end
        end
    end

    local common = {}
    for field,_ in pairs(candidateFields) do
        common[#common+1]=field
    end
    return common
end

-- 为特定字段统计值的出现频率并找到默认值
-- 只有当字段在所有表格中都不为nil时才提取默认值
local function FindDefaultValue(tables, field)
    local valueCounts = {}
    local totalCount = 0
    local valueToOriginal = {} -- 保存原始值的映射

    -- 如果所有表格都有该字段，则统计值的出现频率
    for _, table in ipairs(tables) do
		if table[field] == nil then
            -- 如果任何一个表格中该字段为nil，则不提取默认值
            return nil, 0, 0
        end
        local value = table[field]
        local serialized = SerializeTable(value, true, true)
        valueCounts[serialized] = (valueCounts[serialized] or 0) + 1
        if not valueToOriginal[serialized] then
            valueToOriginal[serialized] = value -- 保存第一个遇到的原始值
        end
        totalCount = totalCount + 1
    end
    
    -- 找到最大出现次数
    local maxCount = 0
    for _, count in pairs(valueCounts) do
        if count > maxCount then
            maxCount = count
        end
    end
    
    -- 收集所有具有最大计数的序列化值
    local candidates = {}
    for serialized, count in pairs(valueCounts) do
        if count == maxCount then
            table.insert(candidates, serialized)
        end
    end
    
    -- 如果没有候选，返回 nil
    if #candidates == 0 then
        return nil, 0, totalCount
    end
    
    -- 对候选进行排序：首先按长度升序，然后按字典序
    table.sort(candidates, function(a, b)
        if #a ~= #b then
            return #a < #b
        else
            return a < b
        end
    end)
    
    -- 选择第一个（最短，且字典序最小）
    local defaultSerialized = candidates[1]
    local defaultValue = valueToOriginal[defaultSerialized]
    
    return defaultValue, maxCount, totalCount
end

-- 递归提取默认值表
local function ExtractDefaultValuesRecursive(tables, path)
    path = path or ""
    local defaultValues = {}
    local allDefaultValues = {}
    
    -- 过滤出有效的表格
    local validTables = {}
    for _, table in ipairs(tables) do
        if type(table) == "table" then
            validTables[#validTables + 1] = table
        end
    end
    
    if #validTables == 0 then
        return {}, {}
    end
    
    -- 获取所有表格的公共字段
    local commonFields = GetCommonFields(validTables)
    
    -- 检查值是否包含嵌套数组
    local function containsNestedArray(value)
        if type(value) ~= "table" then
            return false
        end
        -- 检查是否为数组
        local isArray = true
        local maxIndex = 0
        for k, v in pairs(value) do
            if type(k) ~= "number" or k <= 0 or k ~= math.floor(k) then
                isArray = false
                break
            end
            maxIndex = math.max(maxIndex, k)
        end
        if isArray and maxIndex > 0 then
            -- 检查数组元素是否也是数组
            for i = 1, maxIndex do
                if type(value[i]) == "table" then
                    local elementIsArray = true
                    for k, _ in pairs(value[i]) do
                        if type(k) ~= "number" or k <= 0 or k ~= math.floor(k) then
                            elementIsArray = false
                            break
                        end
                    end
                    if elementIsArray then
                        return true -- 发现嵌套数组
                    end
                end
            end
        end
        return false
    end
    
    -- 为每个公共字段提取默认值
    for _, field in ipairs(commonFields) do
        local defaultValue, maxCount, totalCount = FindDefaultValue(validTables, field)
        if defaultValue ~= nil then
            defaultValues[field] = defaultValue
        end
    end
    
    -- 添加当前层级的默认值到总集合
    if next(defaultValues) then
        local currentPath = path == "" and "root" or path
        allDefaultValues[currentPath] = defaultValues
    end
    
    -- 递归处理嵌套表格
    local nestedTablesByField = {}
    for _, table in ipairs(validTables) do
        for field, value in pairs(table) do
            if type(value) == "table" and (not containsNestedArray(value)) and (not IsArray(value)) then -- 跳过纯数组表
                if not nestedTablesByField[field] then
                    nestedTablesByField[field] = {}
                end
                nestedTablesByField[field][#nestedTablesByField[field] + 1] = value
            end
        end
    end
    
    -- 对每个字段的嵌套表格递归处理
    for field, nestedTables in pairs(nestedTablesByField) do
        -- 严格检查：只有当该字段在所有表格中都存在且都是table时才递归处理
        local fieldExistsInAllTablesAsTable = true
        for _, table in ipairs(validTables) do
            local fieldValue = table[field]
            -- 如果字段不存在、为nil或不是table，则不进行递归处理
            if fieldValue == nil or type(fieldValue) ~= "table" then
                fieldExistsInAllTablesAsTable = false
                break
            end
        end
        
        if fieldExistsInAllTablesAsTable and #nestedTables > 1 then
            local fieldStr = tostring(field) -- 确保字段名是字符串
            local nestedPath = path == "" and fieldStr or (path .. "." .. fieldStr)
            local _, nestedAllDefaults = ExtractDefaultValuesRecursive(nestedTables, nestedPath)
            
            -- 合并嵌套默认值
            for nestedPath, nestedDefaults in pairs(nestedAllDefaults) do
                allDefaultValues[nestedPath] = nestedDefaults
            end
        end
    end
    
    return defaultValues, allDefaultValues
end

-- 应用默认值优化（移除与默认值相同的字段）
local function ApplyDefaultValues(tables, defaultValues, path)
    path = path or ""
    
    for _, table in ipairs(tables) do
        if type(table) == "table" then
            -- 移除与默认值相同的字段
            for field, defaultVal in pairs(defaultValues) do
                local value = table[field]
                if value ~= nil then
                    local valueStr = SerializeTable(value, true, true)
                    local defaultStr = SerializeTable(defaultVal, true, true)
                    if valueStr == defaultStr then
                        -- 与默认值相同时：
                        -- 1. 非 table 值可安全移除，减少体积
                        -- 2. table 值保持不变，避免丢失数组元素或依赖元表取值导致行为改变
                        if type(value) ~= "table" then
                            table[field] = nil
                        end
                    end
                end
            end
        end
    end
end

-- 递归应用所有层级的默认值
local function ApplyAllDefaultValues(tables, allDefaultValues, path)
    path = path or ""
    
    -- 应用当前层级的默认值
    local currentKey = path == "" and "root" or path
    if allDefaultValues[currentKey] then
        ApplyDefaultValues(tables, allDefaultValues[currentKey], path)
    end
    
    -- 递归处理嵌套表格
    local nestedTablesByField = {}
    local validTables = {}
    
    -- 过滤出有效的表格
    for _, table in ipairs(tables) do
        if type(table) == "table" then
            validTables[#validTables + 1] = table
            for field, value in pairs(table) do
                if type(value) == "table" then
                    if not nestedTablesByField[field] then
                        nestedTablesByField[field] = {}
                    end
                    nestedTablesByField[field][#nestedTablesByField[field] + 1] = value
                end
            end
        end
    end
    
    -- 对每个字段的嵌套表格递归处理
    for field, nestedTables in pairs(nestedTablesByField) do
        -- 严格检查：只有当该字段在所有表格中都存在且都是table时才递归处理
        local fieldExistsInAllTablesAsTable = true
        for _, table in ipairs(validTables) do
            local fieldValue = table[field]
            -- 如果字段不存在、为nil或不是table，则不进行递归处理
            if fieldValue == nil or type(fieldValue) ~= "table" then
                fieldExistsInAllTablesAsTable = false
                break
            end
        end
        
        if fieldExistsInAllTablesAsTable then
            local fieldStr = tostring(field) -- 确保字段名是字符串
            local nestedPath = path == "" and fieldStr or (path .. "." .. fieldStr)
            ApplyAllDefaultValues(nestedTables, allDefaultValues, nestedPath)
        end
    end
end

local function OptimizeDataset(dataset)
    -- 如果是平铺的键值对结构，跳过优化
    if IsPlanarKeyValue(dataset) then
        print("检测到平铺键值对结构, 跳过字段优化: " .. dataset.__sourcefile)
        return nil, {}
    end

    local keysArray = {}
    local keyType = nil
    
    -- 收集所有有效的记录
    local allRecords = {}
    for k, v in pairs(dataset) do
        local _sk = tostring(k)
        if _sk ~= "__name" and _sk ~= "__sourcefile" and type(v) == "table" then
            if not keyType then
                keyType = type(k)
            end
            if keyType == type(k) then
                keysArray[#keysArray + 1] = k
                allRecords[#allRecords + 1] = v
            end
        end
    end

    -- **先进行表格去重，基于原始完整内容**
    local overwrites = nil
    OrderedForeach(dataset, function(k, v)
        local _sk = tostring(k)
        if _sk ~= "__name" and _sk ~= "__sourcefile" then
            if type(v) == "table" then
                overwrites = overwrites or {}
                overwrites[k] = UniquifyTable(v)
            end
        end
    end)
    if overwrites then
        for k, v in pairs(overwrites) do
            dataset[k] = overwrites[k]
        end
    end

    -- **然后进行默认值提取和优化**
    local allDefaultValues = {}
    local topLevelDefaults = nil
    
    if EnableDefaultValueOptimize and #allRecords > 1 then
        -- 重新收集记录（可能因为去重发生了变化）
        allRecords = {}
        for k, v in pairs(dataset) do
            local _sk = tostring(k)
            if _sk ~= "__name" and _sk ~= "__sourcefile" and type(v) == "table" then
                allRecords[#allRecords + 1] = v
            end
        end
        
        -- 提取默认值
        topLevelDefaults, allDefaultValues = ExtractDefaultValuesRecursive(allRecords, "")
        
        -- 应用所有层级的默认值优化
        if allDefaultValues and next(allDefaultValues) then
            ApplyAllDefaultValues(allRecords, allDefaultValues, "")
        end
    end
    
    return allDefaultValues
end

local function ToUniqueTableRefName(id, maxLocalVarNum)
    maxLocalVarNum = maxLocalVarNum or MaxLocalVariableNum
    if id <= maxLocalVarNum then
        return string.format(RefTableName .. "_%d", id)
    else
        return string.format(RefTableName .. "[%d]", id - maxLocalVarNum)
    end
end

local function SaveDatasetToFile(dataset, tofile, tableRef, name)
    -- 如果没有tableRef（平铺结构文件），无需优化
    if not tableRef then
        return
    end

    if tofile then
        outFile = CreateFileWriter(dataset.__sourcefile, "w")
    else
        outFile = CreateFileWriter()
    end

    local ptr2ref = nil
    if tableRef and tableRef.ptr2ref then
        ptr2ref = tableRef.ptr2ref
    end
    if tableRef and tableRef.name2value then
        local name2table = tableRef.name2value
        local tables = tableRef.tables
        local tableIds = tableRef.tableIds
        local ptr2ref = tableRef.ptr2ref
        local refcounter = tableRef.refcounter
        local maxLocalVariableNum = tableRef.maxLocalVariableNum or MaxLocalVariableNum
        local refTableName = tableRef.refTableName or "__rt"
        local tableNum = #tableIds
        
        -- 统计所有将要生成的局部变量数量
        local totalLocalVars = 0
        
        -- 1. __rt_xx 变量数量
        local rtLocalVars = math.min(tableNum, maxLocalVariableNum)
        totalLocalVars = totalLocalVars + rtLocalVars
        
        -- 2. __rt 索引表（如果需要的话）
        if tableNum > maxLocalVariableNum then
            totalLocalVars = totalLocalVars + 1
        end
        
        -- 3. 数据集名称
        totalLocalVars = totalLocalVars + 1
        
        -- 4. 默认值表和元表数量
        local defaultTableCount = 0
        if tableRef.allDefaultValues then
            for _ in pairs(tableRef.allDefaultValues) do
                defaultTableCount = defaultTableCount + 1
            end
        end
        -- 默认值表和元表都使用索引表方式，各占用1个局部变量
        if defaultTableCount > 0 then
            totalLocalVars = totalLocalVars + 2 -- __defaults 和 __metas 两个索引表
        end
        
        -- 5. 预留一些空间给其他变量（原始元表等）
        local reservedVars = 5
        totalLocalVars = totalLocalVars + reservedVars
        
        -- 如果总数超过限制，调整策略
        local actualMaxLocalVarNum = maxLocalVariableNum
        if totalLocalVars > 190 then -- 保留10个变量的缓冲区
            local excess = totalLocalVars - 190
            actualMaxLocalVarNum = math.max(10, maxLocalVariableNum - excess)
            print(string.format("警告: 局部变量总数将达到 %d，调整__rt变量上限为 %d", totalLocalVars, actualMaxLocalVarNum))
            
            -- 重新生成ptr2ref映射，使用正确的actualMaxLocalVarNum
            ptr2ref = {}
            for id, hash in ipairs(tableIds) do
                local table = tables[hash]
                if table then
                    local refName = ToUniqueTableRefName(id, actualMaxLocalVarNum)
                    ptr2ref[table] = refName
                end
            end
        end
        
        for id, hash in ipairs(tableIds) do
            local table = tables[hash]
            if table and id <= actualMaxLocalVarNum then
                local refname = ptr2ref[table]
                -- temp comment out top level ref
                ptr2ref[table] = nil
                local refcount = refcounter[table]
                outFile.write(string.format("%slocal %s = %s\n",
                    PrintTableRefCount and string.format("--ref:%d\n", refcount) or "", refname,
                    SerializeTable(table, false, false, ptr2ref)))
                ptr2ref[table] = refname
            else
                break
            end
        end
        if tableNum > actualMaxLocalVarNum then
            local maxCount = tableNum - actualMaxLocalVarNum
            outFile.write(string.format("local %s = createtable and createtable( %d, 0 ) or {}\n", refTableName,
                maxCount))
            for id = actualMaxLocalVarNum + 1, tableNum do
                local offset = id - actualMaxLocalVarNum
                local hash = tableIds[id]
                local table = tables[hash]
                local refname = ptr2ref[table]
                -- temp comment out top level ref
                ptr2ref[table] = nil
                local refcount = refcounter[table]
                outFile.write(string.format("%s%s[%d] = %s\n", PrintTableRefCount and
                    string.format("-- %s, ref:%d\n", refname, refcount) or "", refTableName, offset,
                    SerializeTable(table, false, false, ptr2ref)))
                ptr2ref[table] = refname
            end
        end
    end
    local datasetName = dataset.__name or name
    if not datasetName then
        datasetName = UnknownName
        dataset.__name = datasetName
    end
    outFile.write(string.format("local %s = \n", datasetName))

    -- 序列化前移除内部维护字段，避免出现在最终配置
    local tmpName, tmpSrc = dataset.__name, dataset.__sourcefile
    dataset.__name, dataset.__sourcefile = nil, nil

    local serialize = SerializeTable(dataset, false, false, ptr2ref)
    dataset.__name, dataset.__sourcefile = tmpName, tmpSrc

    outFile.write(serialize)
    outFile.write("\n")
    if tableRef and tableRef.postOutput then
        tableRef.postOutput(outFile)
    end
    outFile.write(string.format("\nreturn %s\n", datasetName))
    outFile.close()
end

local function ExportOptimizedDataset(t, StringBank)
    UniquifyTables = {}
    UniquifyTablesIds = {}
    UniquifyTablesInvIds = {}
    UniquifyTablesRefCounter = {}

    -- 如果是平铺的键值对结构，直接跳过所有优化
    if IsPlanarKeyValue(t) then
        print("检测到平铺键值对结构, 跳过字段优化: " .. t.__sourcefile)
        return t, nil, false
    end

    local datasetName = t.__name
    if not datasetName then
        datasetName = UnknownName
        t.__name = datasetName
    end
    local localized = false
    local genCode = false
    local tableRef = nil
    local allDefaultValues = nil
    if EnableDatasetOptimize then
        allDefaultValues = OptimizeDataset(t)
        -- 移除表格引用计数为1的表并重新映射ID
        local newid = 1
        local newIds = {}
        local newInvIds = {}

        OrderedForeach(UniquifyTablesIds, function(id, hash)
            local table = UniquifyTables[hash]
            local refcount = UniquifyTablesRefCounter[table]
            if refcount == 1 then
                UniquifyTables[hash] = nil
            else
                newIds[newid] = hash
                newInvIds[table] = newid
                newid = newid + 1
            end
        end)

        UniquifyTablesIds = newIds
        UniquifyTablesInvIds = newInvIds
        
        local reftables = nil
        local ptr2ref = nil
        -- create ref table: table -> refname
        for _, hash in pairs(UniquifyTablesIds) do
            local t = UniquifyTables[hash]
            if t then
                local refName = ToUniqueTableRefName(UniquifyTablesInvIds[t])
                reftables = reftables or {}
                ptr2ref = ptr2ref or {}
                reftables[refName] = t
                ptr2ref[t] = refName
            end
        end
        tableRef = {
            name2value = reftables,
            tables = UniquifyTables, -- hash -> table
            tableIds = UniquifyTablesIds, -- id -> hash
            ptr2ref = ptr2ref, -- table -> refname
            refcounter = UniquifyTablesRefCounter, -- table -> refcount
            maxLocalVariableNum = MaxLocalVariableNum,
            refTableName = RefTableName,
            allDefaultValues = allDefaultValues, -- 所有层级的默认值表
            postOutput = function(outFile)
                if allDefaultValues and next(allDefaultValues) then
                    ---------------------------------------------------------
                    -- 1. 生成 __defaults 与 __metas
                    ---------------------------------------------------------
                    outFile.write("\n-- 默认值表定义\n")
                    outFile.write("local __defaults = {}\n")
                    outFile.write("local __metas = {}\n")

                    -- 收集并排序路径
                    local sortedPaths = {}
                    for path, _ in pairs(allDefaultValues) do
                        table.insert(sortedPaths, path)
                    end
                    table.sort(sortedPaths)

                    -- 使用排序后的路径分配索引
                    local defaultTableNames  = {}
                    local defaultTableIndexes = {}
                    local idx = 1
                    for _, path in ipairs(sortedPaths) do
                        local defaults = allDefaultValues[path]
                        local tblName   = string.format("__defaults[%d]", idx)
                        defaultTableNames[path]   = tblName
                        defaultTableIndexes[path] = idx
                        outFile.write(string.format("%s = %s\n", tblName, SerializeTable(defaults, false, false, ptr2ref)))
                        idx = idx + 1
                    end

                    ---------------------------------------------------------
                    -- 2. 创建元表
                    ---------------------------------------------------------
                    outFile.write("\n-- 设置默认值元表\n")
                    outFile.write("do\n")
                    outFile.write("\t-- 创建元表\n")
                    local sortedDefaultTableNamesKeys = {}
                    for pStr, tblName in pairs(defaultTableNames) do
                        table.insert(sortedDefaultTableNamesKeys, pStr)
                    end
                    table.sort(sortedDefaultTableNamesKeys)
                    for _, pStr in ipairs(sortedDefaultTableNamesKeys) do
                        local tblName = defaultTableNames[pStr]
                        local _idx = defaultTableIndexes[pStr]
                        outFile.write(string.format("\t__metas[%d] = { __index = %s }\n", _idx, tblName))
                    end

                    ---------------------------------------------------------
                    -- 3. 收集 指针 -> 路径(字符串) + segments 的映射
                    ---------------------------------------------------------
                    local ptrPathMap  = {}
                    local visited_ptr = {}

                    local function copySegments(segs)
                        local t = {}
                        for i,s in ipairs(segs) do t[i]=s end
                        return t
                    end

                    local function traverse(obj, segments)
                        if type(obj) ~= "table" or visited_ptr[obj] then return end
                        visited_ptr[obj] = true
                        local pathStr
                        if #segments==0 then pathStr = "root" else
                            -- 构建路径字符串时使用键的原始值
                            local pathParts = {}
                            for i, segInfo in ipairs(segments) do
                                pathParts[i] = tostring(segInfo.key)
                            end
                            pathStr = table.concat(pathParts, ".")
                        end
                        ptrPathMap[obj] = {path=pathStr, seg=segments}

                        for k, v in pairs(obj) do
                            if type(v) == "table" then
                                local newSegs = copySegments(segments)
                                -- 保存键的原始类型和值
                                table.insert(newSegs, {key=k, keyType=type(k)})
                                traverse(v, newSegs)
                            end
                        end
                    end

                    traverse(t, {})

                    ---------------------------------------------------------
                    -- 4. 根据 segments 生成 Lua 访问表达式
                    ---------------------------------------------------------
                    local function segmentsToExpr(segs)
                        if #segs==0 then return datasetName end
                        local expr = datasetName
                        for _,segInfo in ipairs(segs) do
                            local key = segInfo.key
                            local keyType = segInfo.keyType
                            
                            if keyType == "number" then
                                -- 数值键：使用数值访问
                                expr = string.format("%s[%s]", expr, tostring(key))
                            elseif keyType == "string" then
                                -- 字符串键：根据是否为标识符决定访问方式
                                if string.match(key, "^[A-Za-z_][A-Za-z0-9_]*$") then
                                    expr = string.format("%s.%s", expr, key)
                                else
                                    expr = string.format("%s[%s]", expr, EncodeEscapeString(key))
                                end
                            else
                                -- 其他类型键：使用字符串形式
                                expr = string.format("%s[%s]", expr, EncodeEscapeString(tostring(key)))
                            end
                        end
                        return expr
                    end

                    ---------------------------------------------------------
                    -- 5. 对遍历到的每个表决定绑定元表
                    ---------------------------------------------------------
                    -- 收集并排序路径
                    local sortedPtrPaths = {}
                    for tblPtr, info in pairs(ptrPathMap) do
                        local pathStr = info.path
                        table.insert(sortedPtrPaths, {ptr = tblPtr, path = pathStr, info = info})
                    end
                    table.sort(sortedPtrPaths, function(a, b)
                        return a.path < b.path
                    end)

                    local sortedRefTables = {}
                    -- 使用排序后的列表生成 setmetatable 语句
                    for _, item in ipairs(sortedPtrPaths) do
                        local tblPtr = item.ptr
                        local path = item.path
                        local info = item.info
                        local metaIdx = defaultTableIndexes[path]
                        if not metaIdx then
                            -- 尝试最后段匹配
                            local lastSeg = path and path:match("([^.]+)$")
                            metaIdx = lastSeg and defaultTableIndexes[lastSeg]
                        end
                        if not metaIdx then
                            -- 对顶层记录使用 root 默认元表
                            metaIdx = defaultTableIndexes["root"] or defaultTableIndexes[""]
                        end
                        if metaIdx then
                            local expr
                            -- 如果有唯一引用名（如 __rt_xx），直接用它
                            local refName = ptr2ref and ptr2ref[tblPtr]
                            if not refName then
                                expr = segmentsToExpr(info.seg)
                                outFile.write(string.format("\tsetmetatable(%s, __metas[%d])\n", expr, metaIdx))
                            else
                                sortedRefTables[refName] = metaIdx
                            end
                        end
                    end

                    -- 收集键并排序
                    local refKeys = {}
                    for refName in pairs(sortedRefTables) do
                        table.insert(refKeys, refName)
                    end
                    table.sort(refKeys)

                    -- 按排序后的键输出
                    for _, refName in ipairs(refKeys) do
                        local metaIdx = sortedRefTables[refName]
                        outFile.write(string.format("\tsetmetatable(%s, __metas[%d])\n", refName, metaIdx))
                    end

                    ---------------------------------------------------------
                    -- 6. 保护顶层数据表只读
                    ---------------------------------------------------------
                    if EnableReadOnlyOptimize then
                        outFile.write("\n\t-- 设置顶层数据表只读\n")
                        outFile.write(string.format("\tlocal _data = %s\n", datasetName))
                        outFile.write("\tlocal _mt = getmetatable(_data) or {}\n")
                        outFile.write("\t_mt.__newindex = function() error(\"Attempt to modify read-only table\") end\n")
                        outFile.write("\tsetmetatable(_data, _mt)\n")
                    end
                    outFile.write("end\n")
                end
            end
        }
    end
    return t, tableRef, localized
end

-- 辅助函数：根据路径获取嵌套的数据库对象
local function GetDatabaseObject(path)
    local split = function(s, p)
        local rt = {}
        string.gsub(s, '[^' .. p .. ']+', function(w)
            table.insert(rt, w)
        end)
        return rt
    end

    local namespace = Database
    local ns = split(path, '/')

    for i = 1, #ns do
        local n = ns[i]
        if namespace[n] == nil then
            return nil -- 路径不存在
        end
        namespace = namespace[n]
    end

    return namespace
end

local M = {}

---优化DatabaseRoot目录下全部lua配置
---@param is_write_to_file 是否将优化后的结果写入到源文件
---@param newStringBank if false, exporter will use existing string hash for increamental building
function M.ExportDatabaseRoot(is_write_to_file, newStringBank)
    print("Root (absolute): " .. getAbsolutePath(Root))
    print("DatabaseRoot (absolute): " .. getAbsolutePath(DatabaseRoot))

    local StringBank = nil
    if newStringBank then
        StringBank = {}
    else
        StringBank = LoadStringBankFromCSV()
    end
    StringBank = StringBank or {}
    local localized_dirty = false
    local relativePathsWithoutLuaExtension = GetAllFileRelativePathsAtPath(DatabaseRoot)
    for _, relativePathWithoutLuaExtension in ipairs(relativePathsWithoutLuaExtension) do
        if not IsFileExcluded(relativePathWithoutLuaExtension) then
            print("LoadDataset :" .. relativePathWithoutLuaExtension)
            LoadDataset(relativePathWithoutLuaExtension)

            local t = GetDatabaseObject(relativePathWithoutLuaExtension)
            local localized = false
            if t then
                local _t, tableRef, localized = ExportOptimizedDataset(t, StringBank)
                assert(_t == t)
                localized_dirty = localized_dirty or localized
                
                -- 输出优化信息
                if tableRef and tableRef.allDefaultValues then
                    local defaultCount = 0
                    for path, defaults in pairs(tableRef.allDefaultValues) do
                        local fieldCount = 0
                        for _ in pairs(defaults) do
                            fieldCount = fieldCount + 1
                        end
                        if fieldCount > 0 then
                            print(string.format("  - 路径 '%s': 提取了 %d 个默认值字段", tostring(path), fieldCount))
                            defaultCount = defaultCount + fieldCount
                        end
                    end
                    if defaultCount > 0 then
                        print(string.format("  总计提取了 %d 个默认值字段", defaultCount))
                    end
                end
                
                SaveDatasetToFile(t, is_write_to_file, tableRef)
            else
                print("Warning: Could not find database object for " .. relativePathWithoutLuaExtension)
            end
		else
			print("Exclude file: " .. relativePathWithoutLuaExtension)
        end
    end
    TrimStringBank(StringBank)
    if localized_dirty then
        SaveStringBankToCSV(StringBank, is_write_to_file)
    else
        print("\nDatabase LocaleText is up to date.\n")
    end
    print("Database Exporting LocaleText done.")
end

--- 优化单个lua配置
---@param input_lua_table 输入的lua table
---@param path 输入的lua table的文件路径
---@param enable_default_val_opt 是否启用默认值优化
---@param enable_read_only_opt 是否启用只读优化
function M.Export(input_lua_table, path, enable_default_val_opt, enable_read_only_opt)
    if not input_lua_table or type(input_lua_table) ~= "table" then
        print("input_lua_table is not a table. path: " .. path)
        return false
    end

    if not path or type(path) ~= "string" then
        print("path is not a string. path: " .. tostring(path))
        return false
    end

    local _src = SerializeTable( input_lua_table )
    local _clone = DeserializeTable( _src )

    EnableDefaultValueOptimize = enable_default_val_opt == nil and true or enable_default_val_opt
    EnableReadOnlyOptimize = enable_read_only_opt == nil and true or enable_read_only_opt

    _clone.__name = "t"
    _clone.__sourcefile = path
    local t, tableRef = ExportOptimizedDataset( _clone)

    SaveDatasetToFile( _clone, true, tableRef )
    return true
end

-- M.ExportDatabaseRoot(true)

return M