From a1e3a6977ae7d07f46f2ff6b27e63139537e3792 Mon Sep 17 00:00:00 2001 From: ruslandoga Date: Tue, 15 Jul 2025 19:28:28 +0300 Subject: [PATCH 1/3] support native json --- lib/ch/row_binary.ex | 50 +++++++++++++++++++++++++++++++++++++++++++ lib/ch/types.ex | 2 ++ test/ch/json_test.exs | 45 ++++++++++++++++++++++++++++++++++++++ test/test_helper.exs | 4 ++-- 4 files changed, 99 insertions(+), 2 deletions(-) create mode 100644 test/ch/json_test.exs diff --git a/lib/ch/row_binary.ex b/lib/ch/row_binary.ex index 8cefaed..95e8959 100644 --- a/lib/ch/row_binary.ex +++ b/lib/ch/row_binary.ex @@ -582,6 +582,7 @@ defmodule Ch.RowBinary do :ipv4, :ipv6, :point, + :json, :nothing ], do: t @@ -754,6 +755,52 @@ defmodule Ch.RowBinary do end end + for {pattern, count} <- varints do + defp decode_json_decode_rows( + <>, + types_rest, + row, + rows, + types + ) do + decode_json_continue(bin, unquote(count), [], types_rest, row, rows, types) + end + end + + for {pattern, size} <- varints do + defp decode_json_continue( + <>, + count, + acc, + types_rest, + row, + rows, + types + ) + when count > 0 do + decode_json_continue_value(bin, name, count, acc, types_rest, row, rows, types) + end + end + + defp decode_json_continue(bin, _count = 0, acc, types_rest, row, rows, types) do + decode_rows(types_rest, bin, [Map.new(acc) | row], rows, types) + end + + for {pattern, size} <- varints do + defp decode_json_continue_value( + <<0x15, unquote(pattern), string::size(unquote(size))-bytes, bin::bytes>>, + name, + count, + acc, + types_rest, + row, + rows, + types + ) do + decode_json_continue(bin, count - 1, [{name, string} | acc], types_rest, row, rows, types) + end + end + @compile inline: [decode_array_decode_rows: 6] defp decode_array_decode_rows(<<0, bin::bytes>>, _type, types_rest, row, rows, types) do decode_rows(types_rest, bin, [[] | row], rows, types) @@ -884,6 +931,9 @@ defmodule Ch.RowBinary do :binary -> decode_binary_decode_rows(bin, types_rest, row, rows, types) + :json -> + decode_json_decode_rows(bin, types_rest, row, rows, types) + # TODO utf8? {:fixed_string, size} -> <> = bin diff --git a/lib/ch/types.ex b/lib/ch/types.ex index 0c96da8..63a833c 100644 --- a/lib/ch/types.ex +++ b/lib/ch/types.ex @@ -28,6 +28,7 @@ defmodule Ch.Types do {"Time", :time, []}, {"Date32", :date32, []}, {"Date", :date, []}, + {"JSON", :json, []}, {"LowCardinality", :low_cardinality, [:type]}, for size <- [32, 64, 128, 256] do {"Decimal#{size}", :"decimal#{size}", [:int]} @@ -324,6 +325,7 @@ defmodule Ch.Types do end def decode("DateTime"), do: :datetime + def decode("JSON" <> _), do: :json def decode(type) do try do diff --git a/test/ch/json_test.exs b/test/ch/json_test.exs new file mode 100644 index 0000000..3ec266d --- /dev/null +++ b/test/ch/json_test.exs @@ -0,0 +1,45 @@ +defmodule Ch.JSONTest do + use ExUnit.Case, async: true + + @moduletag :json + + @table "json_test" + + setup do + conn = start_supervised!({Ch, database: Ch.Test.database(), settings: [enable_json_type: 1]}) + + on_exit(fn -> + Ch.Test.query("DROP TABLE IF EXISTS #{@table}", [], database: Ch.Test.database()) + end) + + {:ok, conn: conn} + end + + test "simple json", %{conn: conn} do + assert Ch.query!(conn, ~s|select '{"a":"b","c":"d"}'::json|).rows == [ + [%{"a" => "b", "c" => "d"}] + ] + end + + # TODO + @tag :skip + test "creating json", %{conn: conn} do + Ch.query!(conn, "CREATE TABLE #{@table} (json JSON) ENGINE = Memory") + + Ch.query!(conn, """ + INSERT INTO #{@table} VALUES + ('{"a" : {"b" : 42}, "c" : [1, 2, 3]}'), + ('{"f" : "Hello, World!"}'), + ('{"a" : {"b" : 43, "e" : 10}, "c" : [4, 5, 6]}') + """) + + assert Ch.query!( + conn, + "SELECT json FROM #{@table}" + ).rows == [ + [%{"a" => %{"b" => "42"}, "c" => ["1", "2", "3"]}], + [%{"f" => "Hello, World!"}], + [%{"a" => %{"b" => "43", "e" => "10"}, "c" => ["4", "5", "6"]}] + ] + end +end diff --git a/test/test_helper.exs b/test/test_helper.exs index dd11412..189b4d4 100644 --- a/test/test_helper.exs +++ b/test/test_helper.exs @@ -30,8 +30,8 @@ extra_exclude = if ch_version >= "25" do [] else - # Time type is not supported in ClickHouse < 25 - [:time] + # Time and JSON types are not supported in ClickHouse < 25 + [:time, :json] end ExUnit.start(exclude: [:slow | extra_exclude]) From 611af151bd295444a70ba536700b8e34e62b54be Mon Sep 17 00:00:00 2001 From: ruslandoga Date: Tue, 15 Jul 2025 19:33:31 +0300 Subject: [PATCH 2/3] more types --- lib/ch/row_binary.ex | 26 ++++++++++++++++++++++++++ test/ch/json_test.exs | 6 ++++++ 2 files changed, 32 insertions(+) diff --git a/lib/ch/row_binary.ex b/lib/ch/row_binary.ex index 95e8959..fe88958 100644 --- a/lib/ch/row_binary.ex +++ b/lib/ch/row_binary.ex @@ -786,6 +786,32 @@ defmodule Ch.RowBinary do decode_rows(types_rest, bin, [Map.new(acc) | row], rows, types) end + defp decode_json_continue_value( + <<0x0A, i64::64-little-signed, bin::bytes>>, + name, + count, + acc, + types_rest, + row, + rows, + types + ) do + decode_json_continue(bin, count - 1, [{name, i64} | acc], types_rest, row, rows, types) + end + + defp decode_json_continue_value( + <<0x0E, f64::64-little-float, bin::bytes>>, + name, + count, + acc, + types_rest, + row, + rows, + types + ) do + decode_json_continue(bin, count - 1, [{name, f64} | acc], types_rest, row, rows, types) + end + for {pattern, size} <- varints do defp decode_json_continue_value( <<0x15, unquote(pattern), string::size(unquote(size))-bytes, bin::bytes>>, diff --git a/test/ch/json_test.exs b/test/ch/json_test.exs index 3ec266d..d5834f4 100644 --- a/test/ch/json_test.exs +++ b/test/ch/json_test.exs @@ -19,6 +19,12 @@ defmodule Ch.JSONTest do assert Ch.query!(conn, ~s|select '{"a":"b","c":"d"}'::json|).rows == [ [%{"a" => "b", "c" => "d"}] ] + + assert Ch.query!(conn, ~s|select '{"a":42}'::json|).rows == [[%{"a" => 42}]] + + assert Ch.query!(conn, ~s|select '{}'::json|).rows == [[%{}]] + + assert Ch.query!(conn, ~s|select '{"a":3.14}'::json|).rows == [[%{"a" => 3.14}]] end # TODO From eaf5c8205935d18bffa797d5dd8f0394678c4758 Mon Sep 17 00:00:00 2001 From: ruslandoga Date: Tue, 15 Jul 2025 21:05:42 +0300 Subject: [PATCH 3/3] begin arrays --- lib/ch/row_binary.ex | 161 ++++++++++++++++++++++++++++++++++++------ test/ch/json_test.exs | 26 ++++++- 2 files changed, 162 insertions(+), 25 deletions(-) diff --git a/lib/ch/row_binary.ex b/lib/ch/row_binary.ex index fe88958..64582f2 100644 --- a/lib/ch/row_binary.ex +++ b/lib/ch/row_binary.ex @@ -778,7 +778,47 @@ defmodule Ch.RowBinary do types ) when count > 0 do - decode_json_continue_value(bin, name, count, acc, types_rest, row, rows, types) + case bin do + # Int64 + <<0x0A, i64::64-little-signed, rest::bytes>> -> + decode_json_continue(rest, count - 1, [{name, i64} | acc], types_rest, row, rows, types) + + # Float64 + <<0x0E, f64::64-little-float, rest::bytes>> -> + decode_json_continue(rest, count - 1, [{name, f64} | acc], types_rest, row, rows, types) + + # Boolean + <<0x2D, b, rest::bytes>> -> + b = + case b do + 0 -> false + 1 -> true + end + + decode_json_continue(rest, count - 1, [{name, b} | acc], types_rest, row, rows, types) + + # String + <<0x15, rest::bytes>> -> + decode_json_string(rest, name, count, acc, types_rest, row, rows, types) + + # Array of nullable strings + <<0x1E, 0x23, 0x15, rest::bytes>> -> + decode_json_nullable_array( + rest, + :string, + name, + count, + acc, + types_rest, + row, + rows, + types + ) + + _ -> + raise "oops, what is " <> + Enum.join(for(<>, do: "0x" <> Integer.to_string(b, 16)), " ") + end end end @@ -786,47 +826,122 @@ defmodule Ch.RowBinary do decode_rows(types_rest, bin, [Map.new(acc) | row], rows, types) end - defp decode_json_continue_value( - <<0x0A, i64::64-little-signed, bin::bytes>>, - name, - count, - acc, - types_rest, - row, - rows, - types - ) do - decode_json_continue(bin, count - 1, [{name, i64} | acc], types_rest, row, rows, types) + for {pattern, size} <- varints do + defp decode_json_string( + <>, + name, + count, + acc, + types_rest, + row, + rows, + types + ) do + decode_json_continue(rest, count - 1, [{name, s} | acc], types_rest, row, rows, types) + end end - defp decode_json_continue_value( - <<0x0E, f64::64-little-float, bin::bytes>>, + for {pattern, array_count} <- varints do + defp decode_json_nullable_array( + <>, + type, + name, + json_count, + json_acc, + types_rest, + row, + rows, + types + ) do + case type do + :string -> + decode_json_nullable_string_array( + rest, + unquote(array_count), + [], + name, + json_count, + json_acc, + types_rest, + row, + rows, + types + ) + end + end + end + + defp decode_json_nullable_string_array( + <<1, rest::bytes>>, + array_count, + array_acc, name, - count, - acc, + json_count, + json_acc, types_rest, row, rows, types ) do - decode_json_continue(bin, count - 1, [{name, f64} | acc], types_rest, row, rows, types) + decode_json_nullable_string_array( + rest, + array_count - 1, + [nil | array_acc], + name, + json_count, + json_acc, + types_rest, + row, + rows, + types + ) end for {pattern, size} <- varints do - defp decode_json_continue_value( - <<0x15, unquote(pattern), string::size(unquote(size))-bytes, bin::bytes>>, + defp decode_json_nullable_string_array( + <<0, unquote(pattern), s::size(unquote(size))-bytes, rest::bytes>>, + array_count, + array_acc, name, - count, - acc, + json_count, + json_acc, types_rest, row, rows, types - ) do - decode_json_continue(bin, count - 1, [{name, string} | acc], types_rest, row, rows, types) + ) + when array_count > 0 do + decode_json_nullable_string_array( + rest, + array_count - 1, + [s | array_acc], + name, + json_count, + json_acc, + types_rest, + row, + rows, + types + ) end end + defp decode_json_nullable_string_array( + <>, + 0, + array_acc, + name, + json_count, + json_acc, + types_rest, + row, + rows, + types + ) do + acc = [{name, :lists.reverse(array_acc)} | json_acc] + decode_json_continue(rest, json_count - 1, acc, types_rest, row, rows, types) + end + @compile inline: [decode_array_decode_rows: 6] defp decode_array_decode_rows(<<0, bin::bytes>>, _type, types_rest, row, rows, types) do decode_rows(types_rest, bin, [[] | row], rows, types) diff --git a/test/ch/json_test.exs b/test/ch/json_test.exs index d5834f4..0720871 100644 --- a/test/ch/json_test.exs +++ b/test/ch/json_test.exs @@ -21,10 +21,32 @@ defmodule Ch.JSONTest do ] assert Ch.query!(conn, ~s|select '{"a":42}'::json|).rows == [[%{"a" => 42}]] - assert Ch.query!(conn, ~s|select '{}'::json|).rows == [[%{}]] - + assert Ch.query!(conn, ~s|select '{"a":null}'::json|).rows == [[%{}]] assert Ch.query!(conn, ~s|select '{"a":3.14}'::json|).rows == [[%{"a" => 3.14}]] + assert Ch.query!(conn, ~s|select '{"a":true}'::json|).rows == [[%{"a" => true}]] + assert Ch.query!(conn, ~s|select '{"a":false}'::json|).rows == [[%{"a" => false}]] + + assert Ch.query!(conn, ~s|select '{"a":{"b":"c"}}'::json|).rows == [ + [%{"a.b" => "c"}] + ] + + assert Ch.query!(conn, ~s|select '{"a":[]}'::json|).rows == [ + [%{"a" => []}] + ] + + assert Ch.query!(conn, ~s|select '{"a":[null]}'::json|).rows == [ + [%{"a" => [nil]}] + ] + + assert Ch.query!(conn, ~s|select '{"a":[1,3.14,"hello",null]}'::json|).rows == [ + [%{"a" => ["1", "3.14", "hello", nil]}] + ] + + # # <<31, 4, 35, 10, 35, 14, 35, 21, 48, 0, 128, 2, 16, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 10, 215, 163, 112, 61, 10, 1, 64, 0, 1, 115, 1, 1, 97, 21, 1, 98>> + # assert Ch.query!(conn, ~s|select '{"a":[1,2.13,"s",{"a":"b"}]}'::json|).rows == [ + # [%{"a.b" => 42}] + # ] end # TODO