From 11cb99f4265a92d91b1306762a2844f2f51624ed Mon Sep 17 00:00:00 2001 From: fx Date: Tue, 21 Jul 2020 12:49:26 +0800 Subject: [PATCH] stat: add mem stat for jemalloc/os 1, add opts arg for mallctl() 2, collect jemalloc.mem info 3, collect os.mem info by /proc 4, add new debug_console cmd: jmem, osmem, i, q 5, make mem/cmem/stat cmd more human-readable --- lualib-src/lua-memory.c | 34 +++++++++++++++++++++++- lualib/skynet/debug.lua | 1 + service/debug_console.lua | 55 +++++++++++++++++++++++++++++++++++---- service/launcher.lua | 11 ++++++-- skynet-src/malloc_hook.c | 6 ++--- skynet-src/malloc_hook.h | 2 +- 6 files changed, 97 insertions(+), 12 deletions(-) diff --git a/lualib-src/lua-memory.c b/lualib-src/lua-memory.c index b9b07a611..6dff13ac3 100644 --- a/lualib-src/lua-memory.c +++ b/lualib-src/lua-memory.c @@ -23,11 +23,41 @@ lblock(lua_State *L) { static int ldumpinfo(lua_State *L) { - memory_info_dump(); + const char *opts = NULL; + if (lua_isstring(L, 1)) { + opts = luaL_checkstring(L,1); + } + memory_info_dump(opts); return 0; } +static int +ljestat(lua_State *L) { + static const char* names[] = { + "stats.allocated", + "stats.resident", + "stats.retained", + "stats.mapped", + "stats.active" }; + mallctl_int64("epoch", 1); //refresh je.stats.cache + lua_newtable(L); + int i; + for (i = 0; i < (sizeof(names)/sizeof(names[0])); i++) { + lua_pushstring(L, names[i]); + lua_pushinteger(L, (lua_Integer) mallctl_int64(names[i], NULL)); + lua_settable(L, -3); + } + return 1; +} + +static int +lmallctl(lua_State *L) { + const char *name = luaL_checkstring(L,1); + lua_pushinteger(L, (lua_Integer) mallctl_int64(name, NULL)); + return 1; +} + static int ldump(lua_State *L) { dump_c_mem(); @@ -69,6 +99,8 @@ luaopen_skynet_memory(lua_State *L) { { "total", ltotal }, { "block", lblock }, { "dumpinfo", ldumpinfo }, + { "jestat", ljestat }, + { "mallctl", lmallctl }, { "dump", ldump }, { "info", dump_mem_lua }, { "current", lcurrent }, diff --git a/lualib/skynet/debug.lua b/lualib/skynet/debug.lua index 78fd1a166..e206a760f 100644 --- a/lualib/skynet/debug.lua +++ b/lualib/skynet/debug.lua @@ -29,6 +29,7 @@ local function init(skynet, export) stat.mqlen = skynet.stat "mqlen" stat.cpu = skynet.stat "cpu" stat.message = skynet.stat "message" + stat.mem = collectgarbage "count" skynet.ret(skynet.pack(stat)) end diff --git a/service/debug_console.lua b/service/debug_console.lua index 07572bbbf..07b22c9de 100644 --- a/service/debug_console.lua +++ b/service/debug_console.lua @@ -55,11 +55,17 @@ local function split_cmdline(cmdline) return split end +local function htime() + return skynet.hpc() / 1000000000 +end + local function docmd(cmdline, print, fd) local split = split_cmdline(cmdline) local command = split[1] local cmd = COMMAND[command] local ok, list + local t_start = htime() + print(string.format("\n---------------------------------", command)) if cmd then ok, list = pcall(cmd, table.unpack(split,2)) else @@ -73,6 +79,7 @@ local function docmd(cmdline, print, fd) end end + local t_cost = htime() - t_start if ok then if list then if type(list) == "string" then @@ -81,10 +88,10 @@ local function docmd(cmdline, print, fd) dump_list(print, list) end end - print("") + print(string.format(" cost=%.4f sec", t_cost)) else print(list) - print("") + print(string.format(" cost=%.4f sec", t_cost)) end end @@ -94,7 +101,7 @@ local function console_main_loop(stdin, print) local ok, err = pcall(function() while true do local cmdline = socket.readline(stdin, "\n") - if not cmdline then + if not cmdline or cmdline == "q" or cmdline == "q\r" then break end if cmdline:sub(1,4) == "GET " then @@ -156,12 +163,16 @@ function COMMAND.help() debug = "debug address : debug a lua service", signal = "signal address sig", cmem = "Show C memory info", + jmem = "Show jemalloc stats", + osmem = "Show OS memory stats like ps: vsz/rss", ping = "ping address", call = "call address ...", trace = "trace address [proto] [on|off]", netstat = "netstat : show netstat", profactive = "profactive [on|off] : active/deactive jemalloc heap profilling", dumpheap = "dumpheap : dump heap profilling", + i = "info of this server", + q = "close console connection, bye", } end @@ -334,14 +345,48 @@ function COMMAND.cmem() local info = memory.info() local tmp = {} for k,v in pairs(info) do - tmp[skynet.address(k)] = v + tmp[skynet.address(k)] = string.format("%11d %8.2f Mb", v, v/1048576) end - tmp.total = memory.total() + local total = memory.total() + tmp.total = string.format("%d %.2f Mb", total, total/1048576) tmp.block = memory.block() return tmp end +function COMMAND.jmem() + local info = memory.jestat() + local tmp = {} + for k,v in pairs(info) do + tmp[k] = string.format("%11d %8.2f Mb", v, v/1048576) + end + return tmp +end + +function COMMAND.osmem() + local info = {} + local fproc = io.open("/proc/self/statm") + if fproc then + info['vsz'],info['rss'],info['shr'],info['txt'],info['dat'] = string.match( + fproc:read("a"), + '(%d+)%s+(%d+)%s+(%d+)%s+(%d+)%s+%d+%s+(%d+)%s+%d+') + if info['vsz'] then + for k,v in pairs(info) do + info[k] = string.format("%8d Kb %5d Mb", v*4, math.ceil(v*4/1024)) + end + end + end + return info +end + +function COMMAND.i() + local info = {} + info.harbor = skynet.getenv("harbor") + info.address = skynet.getenv("address") + info.path = os.getenv("PWD") + return info +end + function COMMAND.ping(address) address = adjust_address(address) local ti = skynet.now() diff --git a/service/launcher.lua b/service/launcher.lua index 77300a23f..023a1c1f7 100644 --- a/service/launcher.lua +++ b/service/launcher.lua @@ -25,10 +25,14 @@ end function command.STAT() local list = {} for k,v in pairs(services) do + local t0 = skynet.time() local ok, stat = pcall(skynet.call,k,"debug","STAT") if not ok then - stat = string.format("ERROR (%s)",v) + stat = {err=stat or "ERROR"} end + stat.name = v + stat.lag = skynet.time() - t0 + stat.mem = string.format("%.3f", stat.mem) list[skynet.address(k)] = stat end return list @@ -44,14 +48,17 @@ end function command.MEM() local list = {} + local sum = 0 for k,v in pairs(services) do local ok, kb = pcall(skynet.call,k,"debug","MEM") if not ok then list[skynet.address(k)] = string.format("ERROR (%s)",v) else - list[skynet.address(k)] = string.format("%.2f Kb (%s)",kb,v) + list[skynet.address(k)] = string.format("%11.2f %8.2f Mb (%s)", kb, kb/1024, v) + sum = sum + kb end end + list["sum"] = string.format("%11.2f Kb %.2f Mb (sum)", sum, sum/1024) return list end diff --git a/skynet-src/malloc_hook.c b/skynet-src/malloc_hook.c index 150c2cc61..a3fed30ec 100644 --- a/skynet-src/malloc_hook.c +++ b/skynet-src/malloc_hook.c @@ -127,8 +127,8 @@ static void malloc_oom(size_t size) { } void -memory_info_dump(void) { - je_malloc_stats_print(0,0,0); +memory_info_dump(const char* opts) { + je_malloc_stats_print(0,0, opts); } bool @@ -241,7 +241,7 @@ skynet_posix_memalign(void **memptr, size_t alignment, size_t size) { #define raw_free free void -memory_info_dump(void) { +memory_info_dump(const char* opts) { skynet_error(NULL, "No jemalloc"); } diff --git a/skynet-src/malloc_hook.h b/skynet-src/malloc_hook.h index 04f522ac7..ae95b9a5e 100644 --- a/skynet-src/malloc_hook.h +++ b/skynet-src/malloc_hook.h @@ -7,7 +7,7 @@ extern size_t malloc_used_memory(void); extern size_t malloc_memory_block(void); -extern void memory_info_dump(void); +extern void memory_info_dump(const char *opts); extern size_t mallctl_int64(const char* name, size_t* newval); extern int mallctl_opt(const char* name, int* newval); extern bool mallctl_bool(const char* name, bool* newval);