From 4093b15b2356256f2a86ceaf1a0c0a0837290f32 Mon Sep 17 00:00:00 2001 From: BuckarooBanzay Date: Tue, 20 May 2025 13:13:27 +0200 Subject: [PATCH 1/3] async env --- api.async.lua | 2 ++ api.lua | 6 +++++- deserialize_chunk.lua | 4 ++-- init.lua | 19 +++++++++++++++++-- mapgen.async.lua | 16 ++++++++++++++++ mapgen.lua | 2 +- parse_chunk.lua | 4 ++-- 7 files changed, 45 insertions(+), 8 deletions(-) create mode 100644 api.async.lua create mode 100644 mapgen.async.lua diff --git a/api.async.lua b/api.async.lua new file mode 100644 index 0000000..767fa3b --- /dev/null +++ b/api.async.lua @@ -0,0 +1,2 @@ + +mapsync = {} diff --git a/api.lua b/api.lua index cf09253..2322bd0 100644 --- a/api.lua +++ b/api.lua @@ -17,6 +17,10 @@ function mapsync.register_backend(name, backend_def) -- register backends[name] = backend_def + if minetest.ipc_set then + -- register async + minetest.ipc_set("mapsync:backend", backends) + end end -- unregisters a backend @@ -64,4 +68,4 @@ end function mapsync.get_data_backend() return data_backend_def -end \ No newline at end of file +end diff --git a/deserialize_chunk.lua b/deserialize_chunk.lua index 263b162..b80c5d2 100644 --- a/deserialize_chunk.lua +++ b/deserialize_chunk.lua @@ -1,4 +1,4 @@ -local global_env = ... +local global_env = ... or _G local param1, param2, node_data @@ -70,4 +70,4 @@ function mapsync.get_key_manifest(filename) return false, m_err_msg end return minetest.parse_json(key_str) -end \ No newline at end of file +end diff --git a/init.lua b/init.lua index f6b3a11..1d69f68 100644 --- a/init.lua +++ b/init.lua @@ -40,7 +40,22 @@ dofile(MP.."/auto_update.lua") dofile(MP.."/save.lua") loadfile(MP.."/data.lua")(global_env) dofile(MP.."/load.lua") -dofile(MP.."/mapgen.lua") + +if not minetest.register_mapgen_script then + -- sync mapgen + dofile(MP.."/mapgen.lua") +else + -- async mapgen + minetest.register_mapgen_script(MP.."/api.async.lua") + minetest.register_mapgen_script(MP.."/pos_iterator.lua") + minetest.register_mapgen_script(MP.."/encoding.lua") + minetest.register_mapgen_script(MP.."/functions.lua") + minetest.register_mapgen_script(MP.."/mapgen.async.lua") + + minetest.register_mapgen_script(MP.."/load.lua") + minetest.register_mapgen_script(MP.."/parse_chunk.lua") + minetest.register_mapgen_script(MP.."/deserialize_chunk.lua") +end -- hud stuff dofile(MP.."/hud.lua") @@ -74,4 +89,4 @@ if minetest.get_modpath("mtt") and mtt.enabled then dofile(MP.."/data.spec.lua") dofile(MP.."/api.spec.lua") dofile(MP.."/serialize_chunk.spec.lua") -end \ No newline at end of file +end diff --git a/mapgen.async.lua b/mapgen.async.lua new file mode 100644 index 0000000..f5b9f00 --- /dev/null +++ b/mapgen.async.lua @@ -0,0 +1,16 @@ +minetest.register_on_generated(function(vmanip, minp) + local chunk_pos = mapsync.get_chunkpos(minp) + + local t1 = minetest.get_us_time() + mapsync.load(chunk_pos, vmanip) + local t2 = minetest.get_us_time() + + local micros = t2 - t1 + if micros > 10000 then + -- log slow chunks + minetest.log( + "action", + "[mapsync] emerge of " .. minetest.pos_to_string(chunk_pos) .. " took " .. micros .. " us" + ) + end +end) diff --git a/mapgen.lua b/mapgen.lua index 04b2043..7bcfbf5 100644 --- a/mapgen.lua +++ b/mapgen.lua @@ -14,4 +14,4 @@ minetest.register_on_generated(function(minp) "[mapsync] emerge of " .. minetest.pos_to_string(chunk_pos) .. " took " .. micros .. " us" ) end -end) \ No newline at end of file +end) diff --git a/parse_chunk.lua b/parse_chunk.lua index 0824d20..cb6b0f2 100644 --- a/parse_chunk.lua +++ b/parse_chunk.lua @@ -1,4 +1,4 @@ -local global_env = ... +local global_env = ... or _G -- local vars for faster access local insert, byte, decode_uint16 = table.insert, string.byte, mapsync.decode_uint16 @@ -65,4 +65,4 @@ function mapsync.parse_chunk(filename) end return chunk -end \ No newline at end of file +end From e847f6e6dd3cd5fd9e3f11e5696ecc7ed9941940 Mon Sep 17 00:00:00 2001 From: BuckarooBanzay Date: Tue, 20 May 2025 13:18:46 +0200 Subject: [PATCH 2/3] async api --- api.async.lua | 8 +++++++- api.lua | 6 +++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/api.async.lua b/api.async.lua index 767fa3b..2f0e77a 100644 --- a/api.async.lua +++ b/api.async.lua @@ -1,2 +1,8 @@ -mapsync = {} +mapsync = { + async_env = true +} + +function mapsync.select_backend(chunk_pos) + return minetest.ipc_get("mapsync:backend") +end diff --git a/api.lua b/api.lua index 2322bd0..f4df56a 100644 --- a/api.lua +++ b/api.lua @@ -19,7 +19,11 @@ function mapsync.register_backend(name, backend_def) backends[name] = backend_def if minetest.ipc_set then -- register async - minetest.ipc_set("mapsync:backend", backends) + minetest.ipc_set("mapsync:backend", { + type = backend_def.type, + name = backend_def.name, + path = backend_def.path + }) end end From 5feeb3ebd1b6cbe6129ce9929633e50a073cc536 Mon Sep 17 00:00:00 2001 From: BuckarooBanzay Date: Thu, 22 May 2025 07:34:48 +0200 Subject: [PATCH 3/3] async tests --- data.spec.lua | 1 - init.lua | 4 +++- init.spec.lua | 2 -- mapgen.async.lua | 4 ++-- mapgen.spec.lua | 18 ++++++++++++++++++ test/map/chunk_(0,0,0).zip | Bin 0 -> 3895 bytes 6 files changed, 23 insertions(+), 6 deletions(-) delete mode 100644 init.spec.lua create mode 100644 mapgen.spec.lua create mode 100644 test/map/chunk_(0,0,0).zip diff --git a/data.spec.lua b/data.spec.lua index f45420a..1cdc8ee 100644 --- a/data.spec.lua +++ b/data.spec.lua @@ -21,7 +21,6 @@ mtt.register("data", function(callback) assert(mapsync.get_data_file("myfile2.txt", "w")) -- write to data file - assert(not mapsync.get_data_file("myfile3.txt")) local f = mapsync.get_data_file("myfile3.txt", "w") assert(f) f:write("stuff") diff --git a/init.lua b/init.lua index 1d69f68..b22a078 100644 --- a/init.lua +++ b/init.lua @@ -55,6 +55,8 @@ else minetest.register_mapgen_script(MP.."/load.lua") minetest.register_mapgen_script(MP.."/parse_chunk.lua") minetest.register_mapgen_script(MP.."/deserialize_chunk.lua") + minetest.register_mapgen_script(MP.."/deserialize_mapblock.lua") + minetest.register_mapgen_script(MP.."/localize_nodeids.lua") end -- hud stuff @@ -84,9 +86,9 @@ end -- testing if minetest.get_modpath("mtt") and mtt.enabled then - dofile(MP.."/init.spec.lua") dofile(MP.."/functions.spec.lua") dofile(MP.."/data.spec.lua") dofile(MP.."/api.spec.lua") dofile(MP.."/serialize_chunk.spec.lua") + dofile(MP.."/mapgen.spec.lua") end diff --git a/init.spec.lua b/init.spec.lua deleted file mode 100644 index 6ea8cc6..0000000 --- a/init.spec.lua +++ /dev/null @@ -1,2 +0,0 @@ -local pos = { x=0, y=0, z=0 } -mtt.emerge_area(pos, pos) \ No newline at end of file diff --git a/mapgen.async.lua b/mapgen.async.lua index f5b9f00..bef4380 100644 --- a/mapgen.async.lua +++ b/mapgen.async.lua @@ -6,11 +6,11 @@ minetest.register_on_generated(function(vmanip, minp) local t2 = minetest.get_us_time() local micros = t2 - t1 - if micros > 10000 then + if micros > 0 then -- log slow chunks minetest.log( "action", - "[mapsync] emerge of " .. minetest.pos_to_string(chunk_pos) .. " took " .. micros .. " us" + "[mapsync] async emerge of " .. minetest.pos_to_string(chunk_pos) .. " took " .. micros .. " us" ) end end) diff --git a/mapgen.spec.lua b/mapgen.spec.lua new file mode 100644 index 0000000..86b3bea --- /dev/null +++ b/mapgen.spec.lua @@ -0,0 +1,18 @@ +local BACKEND_NAME = "test-mapgen-backend" + +mtt.register("mapgen-test (register)", function(callback) + -- backend for mapgen test + mapsync.register_backend(BACKEND_NAME, { + type = "fs", + path = minetest.get_modpath("mapsync") .. "/test/map" + }) + callback() +end) + +local pos = { x=0, y=0, z=0 } +mtt.emerge_area(pos, pos) + +mtt.register("mapgen-test (unregister)", function(callback) + mapsync.unregister_backend(BACKEND_NAME) + callback() +end) \ No newline at end of file diff --git a/test/map/chunk_(0,0,0).zip b/test/map/chunk_(0,0,0).zip new file mode 100644 index 0000000000000000000000000000000000000000..d518acab553a7f327843b8afdccefd61651a4f1a GIT binary patch literal 3895 zcmbtXdpuO@`lp0$bauKrnXW2|E~4C~vQ;=4LPG9aB8(kk#7sLCyVxA%lG`YiT$%}6 z?z1YJNiGvc;}R2NObiBNW?FMu?-~2I&#!i8pTEvp@8^BqcfHT|v!3U9pY>huV`sBi zX_TINQ1S3*SIg?D@E-D%Tj+H|lLq3T+M@#yew%LGaIMacAX$U#Y0ED3VCyrD} zh$gX!vA_9q6q76~smT&oPn3lK5rHu$W03({hbJ&`Og*wU2IZ&0|2WBwMivRGjhjba zKVk;-1QHxA7J8mIP1{KRdDkxK2W}g7QG$@!uqX5e3;;Aql8Lc6Y%ZxWB0Z;$B z*ziNq8M7#FiM5hG6-yBp9M z{x#Fg;0Pfo=gvcu#xhz#-h}u%2i;vwVIf5EgXnJIG0;6kfTN3P8J|EeSHuM!WF}Uw z&5I_8K)ggED#?+6>Ee5cC^E($&>ld+0y=GH_N5$PL+SMdB@|1$4eqZ6^3> zzg|-tQ*U4%<(s&_dHt7;wYz3&FOOab)=GP0=<4!3UeEgQL74rLUg%E!O79EekI|2a ztr@r$-7gk+xo(j9tQpCuTmJrYYwi+Fsq4C++0zY2XG8-_Zu4F&*8nx{&pmlIxKZ~# zSWmoQLNI`ZB#cNVB~}(#=JF2najluT*78Du>!9AcJG819qU*`o1EcIIHS|^C2nSM) zYVeA8NW7tWjXNmiR(en)VZT>fN1Hn9SGeFWTbQ}yOPnMbKZx!Y>O&74dkkQ@l$KLm z7Z4hR5DzD%)pY>h!m>~%Oc=eVtDXzuZEHd&r07uVrW!VA3)p3w0Sa26&(#%pf*b$J zD~Fvxre;!7*h&bVTRwpF3J4?g*Ur@U$?YH;+=R^}(!5wa%^dP4GAn4jK8|RTe^rPx z^*pkHV*0|oQkr4AH>UF{a=!(A5cbywr9|mE+gqGIHm?Y*g!x?Xe1HHXP&;~ZmUnS_ zFE~nt&S|+Jq$46}Y*+{!KSqZg`0wB_7KP90aOa>jsbeS;ViqCL^vbgp1n;#m0c(53 zz2SjFWTE+_XLy6PRga;TKGuTbZmSD94MZiz@X;=h*_NtXZmc@~rF+x1lbMaB7@=aZ z6!B1uumPSfBz)%&$v#Tq3S8NJR6|7!zwU1~jdzYkt(jx-S+c7BS#-7$w4A>+2LQ)*Nk=lc~znT%}%@Zz8Tgup#)RJ>rE*=sj zb(c_6JkIAh-)dwzIrUJTsi`N9^n!G!9XQ#vhf^xUxx|_g>F`WPgui=As8TD0=X4uW zon7X1xU){@=5%a4{93p(mJvH~?v=&(h6;*)F-9GQG`O#7MMTF2IIZLrHh8}l@*X&C z2UI@loGyKv%CygAmbn0aA21J<#K39Ze_r-Um%HZ1tHVDUoG|LtsqBg}fh!c5PI zn!_?VQ9}N!Smc6Sb=K}bTBxIF!J>1vc?7L7BF&^7jV1Y)%RFxkfD5K-SfE}Q1g_#x zCdgw`io5$QYE^rb615}(?=md@%vMv7hejMiX{e{>0A(|#`%R9?rev`+(!aw~@1;Gb zuA#oq9hfd96T(d=xWu{yE>W&}9=J{mqCW>jnMvoZqRdKRTJ6watE5+!JQ_Ua!AmYC zD2YxIop^DQf?~0GHQE?Knq}3^(R>-1?ZJ!ZO0P59hEzn2S)f!h=n`gjmxGGH(|GNC zbt1&&pbBtE1cSyWcbbB5xDJmT$>K2e2n-rz1Ej3lgGqcod3w%mrJ>f&vU!4Jr=kbc zPdIWkc>-%{JA6~^r7;C}vPA4g)p3Ea zFTcM~Si|8N^E4|qUopLPHb91T2#`Q)V2jgcXbItIml*5higdjokO;@q-UGw57r^bG zXbspx8x2QV;^!h;YR2ia3G5A7*Y)s$QHDlA+=KizxHqKkcR8+)IVZm`$A2F^n38_3 z8O~i2f=EAvHoWBfy~!A6Fm{zzvp0R1(IXvs*=L!MyZa44CS=a63MOh&mhzc5VC-wD zkvEdP51h^f7kZ_q6ThaZ|NgT7r!#*pIHPDWVfUv~VePq76W5=k=|sNUdIhHP4OT{|H$@p99}tWy==S!##X zhWnGc@!LxR!jl&1tD;cVJsY>b&P(36UUloRYw6k4;+YJ7fy?)&%$a!k&Y9h>IN|M%213<$_)m-Xves=JlK*dvlGYg{ofuncbo+w)>bp9w z%SfMGZ@RQatMv47btw*NgT0w>}>~v~knbb%|)pXC0Oy+51nSm5chV zd{ZWV3BGf_+4JM>hfn|EGQ4Ga&t<>+-9CD(pcfkXxDVG^*H+LTBYPfeZCRxF_q43p zw0G2>WLnmVEj}~+5}nQ4b{9@eoNc}yBK z7K(8`8iiUmHiKu<*eJ?f zn)Q%Xm$Ga&XP+5?QaU^<39u;3)!mGV$k*|#+EP;A|7_w)WygAbZyScunY@iZtqS}8 zQg+~jLmMFRwPMkeyq2D@H=W~Kyrn8LDxs7))X3XpB)B>WJn6cd>@t2!^U7gmwvNY$ zL4DbsJ4rnRJS#5`@1#f3?}(} zZlqbFx+{%whI=c?t*3gpAWv!BtE#_NR^!riy!=K}4CPeQT;Eb@0lK99*y?VBjvc2z zY!HUBwppugnb~)AfQc!G;6Eq7qlTfqfue$f$KpSd-#aAWf?rs;-a_(6bWMf4-mIbg z@#=7uOHhFBW!=`}w#$vTE?M+xEKy|c)%;k4aIB*-z|%bBqT>bQ{g+H1@{Hz_cHsv? z-mU30OhV)8vBlS&uX_qVI$=a}=*RWb<7dll?X1Q9RONOrnGW`Siw^BYlTDfiysp<^sZw*^g^z;Jae?Wq6`f6P;-bl4nO0-h6wJq$uR@Ewg^zrTC z>eVL3TYRpQP&&F=)y%a~b}0`>C@$}l)#@>Dt#xjteO4Y_)xV$dwAFzA6l0*_cK29Y zo7?BE@$Vn<(n%i|wPjai&#B#hWPSdIPr_?}9oosFWLOTN^c z-?EFGVE(w)%9Z}aHQz$NEm!{y{6G0s@%k