Use string.buffer for inter-VM communication

[DaddelZeit]

This moves from JSON for inter-VM communication to the LuaJIT String Buffer library. This is important for modernization and performance reasons, as this library performs much better than the JSON or Lpack serializer.

This also fixes the redundant string concatenation between string.format and base strings introduced in #174 .

jsonEncode in lines 150-152 of kiss_electrics.lua needs different handling (or can be left as-is), as it calls network.send_data directly. And since communications to the bridge are JSON only, re-encoding the data is obviously less performant than right now.

    obj:queueGameEngineLua(string.format(
      "network.send_data(%q, true)",
      jsonEncode(data)))

[Vlad118]

Looks good to me.

I played around with this as well to also see the differences for engine VM compilation and garbage collection (as binary data is inflated by %q as opposed to JSON). Regular lpack does indeed take a long time to encode (omitted from test below).

local string_buffer = require("string.buffer")
local lpack = require("lpack")

local data = {
  v0 = {12.345, -12.123, 1.111},
  v1 = {1.1, -2.2, 3.3},
  v2 = {0.1, 0.2, 0.3, 0.9},
  v3 = 123.45
}

local ITERATIONS = 150000

local function test_encode(name, encode_fn)
  collectgarbage("collect")
  
  local queue, bytes, t0 = {}, 0, os.clock()
  
  for i = 1, ITERATIONS do
    local s = string.format("%q", encode_fn(data))
    bytes = bytes + #s
    queue[i % 33] = s
  end
  
  local t_loop = os.clock() - t0
  
  local gc0 = os.clock()
  collectgarbage("collect")
  local t_gc = os.clock() - gc0
  local t_total = t_loop + t_gc
  
  print(string.format("[Encode]  %-10s | Size: %5.2fMB | Loop: %.4fs | GC: %.4fs | Total: %.4fs", 
    name, bytes / 1048576, t_loop, t_gc, t_total))
    
  return t_total
end

local function test_compile(name, encode_fn)
  collectgarbage("collect")
  
  local cross_vm_string = string.format("%q", encode_fn(data))
  local source_code = "local dummy = " .. cross_vm_string
  
  local t0 = os.clock()
  
  for i = 1, ITERATIONS do
    local _ = load(source_code) 
  end
  
  local t_compile = os.clock() - t0
  
  print(string.format("[Compile] %-10s | Size: %7d B | Compile Time: %.4fs", 
    name, #source_code, t_compile))
    
  return t_compile
end

local function test_decode(name, encode_fn, decode_fn)
  local payload = encode_fn(data)
  collectgarbage("collect")
  
  local queue, t0 = {}, os.clock()
  
  for i = 1, ITERATIONS do
    queue[i % 33] = decode_fn(payload)
  end
  
  local t_loop = os.clock() - t0
  
  local gc0 = os.clock()
  collectgarbage("collect")
  local t_gc = os.clock() - gc0
  local t_total = t_loop + t_gc
  
  print(string.format("[Decode]  %-10s | Loop: %.4fs   | GC: %.4fs | Total: %.4fs", 
    name, t_loop, t_gc, t_total))
    
  return t_total
end

print("\n=== VM Boundary Pipeline Load Test (" .. ITERATIONS .. " updates) ===")
local json_time = test_encode("JSON", jsonEncode)
local buf_time = test_encode("Buffer", string_buffer.encode)
local lpack_time = test_encode("Lpack Bin", lpack.encodeBin)
print("--------------------------------------------------------------------------------")
json_time = json_time + test_compile("JSON", jsonEncode)
buf_time = buf_time + test_compile("Buffer", string_buffer.encode)
lpack_time = lpack_time + test_compile("Lpack Bin", lpack.encodeBin)
print("--------------------------------------------------------------------------------")
json_time = json_time + test_decode("JSON", jsonEncode, jsonDecode)
buf_time = buf_time + test_decode("Buffer", string_buffer.encode, string_buffer.decode)
lpack_time = lpack_time + test_decode("Lpack Bin", lpack.encodeBin, lpack.decode)
print("--------------------------------------------------------------------------------")
print(string.format("[TOTAL]   JSON: %.4fs | Buffer: %.4fs | Lpack Bin: %.4fs", json_time, buf_time, lpack_time))
print("================================================================================\n")