From 2ab86081667d879574d7be11c6aa2750f908cd6d Mon Sep 17 00:00:00 2001 From: hawkyre Date: Fri, 11 Jul 2025 12:51:10 +0200 Subject: [PATCH 01/21] multipart requests --- lib/ch/connection.ex | 35 +++++++++++++++++++++++++++++++++-- mix.exs | 1 + mix.lock | 2 ++ 3 files changed, 36 insertions(+), 2 deletions(-) diff --git a/lib/ch/connection.ex b/lib/ch/connection.ex index 40970ae..9e247c4 100644 --- a/lib/ch/connection.ex +++ b/lib/ch/connection.ex @@ -2,6 +2,7 @@ defmodule Ch.Connection do @moduledoc false use DBConnection require Logger + alias Multipart.Part alias Ch.{Error, Query, Result} alias Mint.HTTP1, as: HTTP @@ -214,7 +215,9 @@ defmodule Ch.Connection do def handle_execute(%Query{command: :insert} = query, params, opts, conn) do conn = maybe_reconnect(conn) - {query_params, extra_headers, body} = params + + multipart_request = Keyword.get(opts, :multipart_request, true) + {query_params, extra_headers, body} = parse_params(params, multipart_request) path = path(conn, query_params, opts) headers = headers(conn, extra_headers, opts) @@ -233,7 +236,9 @@ defmodule Ch.Connection do def handle_execute(query, params, opts, conn) do conn = maybe_reconnect(conn) - {query_params, extra_headers, body} = params + + multipart_request = Keyword.get(opts, :multipart_request, true) + {query_params, extra_headers, body} = parse_params(params, multipart_request) path = path(conn, query_params, opts) headers = headers(conn, extra_headers, opts) @@ -379,6 +384,32 @@ defmodule Ch.Connection do end end + @spec parse_params(tuple, boolean) :: tuple + defp parse_params({query_params, headers, body}, true = _multipart?) when is_binary(body) do + body = to_string(body) + + multipart = + query_params + |> Enum.reduce(Multipart.new(), fn {k, v}, acc -> + Multipart.add_part(acc, Part.text_field(v, k)) + end) + |> Multipart.add_part(Part.text_field(body, "query")) + + content_length = Multipart.content_length(multipart) + content_type = Multipart.content_type(multipart, "multipart/form-data") + + multipart_headers = [ + {"Content-Type", content_type}, + {"Content-Length", to_string(content_length)} + ] + + {[], headers ++ multipart_headers, Multipart.body_binary(multipart)} + end + + defp parse_params(params, _) do + params + end + defp get_header(headers, key) do case List.keyfind(headers, key, 0) do {_, value} -> value diff --git a/mix.exs b/mix.exs index dd11049..c8f704e 100644 --- a/mix.exs +++ b/mix.exs @@ -41,6 +41,7 @@ defmodule Ch.MixProject do {:db_connection, "~> 2.0"}, {:jason, "~> 1.0"}, {:decimal, "~> 2.0"}, + {:multipart, "~> 0.4.0"}, {:ecto, "~> 3.13.0", optional: true}, {:benchee, "~> 1.0", only: [:bench]}, {:dialyxir, "~> 1.0", only: [:dev], runtime: false}, diff --git a/mix.lock b/mix.lock index 0e57a37..94448ac 100644 --- a/mix.lock +++ b/mix.lock @@ -13,7 +13,9 @@ "makeup": {:hex, :makeup, "1.2.1", "e90ac1c65589ef354378def3ba19d401e739ee7ee06fb47f94c687016e3713d1", [:mix], [{:nimble_parsec, "~> 1.4", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "d36484867b0bae0fea568d10131197a4c2e47056a6fbe84922bf6ba71c8d17ce"}, "makeup_elixir": {:hex, :makeup_elixir, "1.0.1", "e928a4f984e795e41e3abd27bfc09f51db16ab8ba1aebdba2b3a575437efafc2", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.2.3 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "7284900d412a3e5cfd97fdaed4f5ed389b8f2b4cb49efc0eb3bd10e2febf9507"}, "makeup_erlang": {:hex, :makeup_erlang, "1.0.2", "03e1804074b3aa64d5fad7aa64601ed0fb395337b982d9bcf04029d68d51b6a7", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "af33ff7ef368d5893e4a267933e7744e46ce3cf1f61e2dccf53a111ed3aa3727"}, + "mime": {:hex, :mime, "2.0.7", "b8d739037be7cd402aee1ba0306edfdef982687ee7e9859bee6198c1e7e2f128", [:mix], [], "hexpm", "6171188e399ee16023ffc5b76ce445eb6d9672e2e241d2df6050f3c771e80ccd"}, "mint": {:hex, :mint, "1.7.1", "113fdb2b2f3b59e47c7955971854641c61f378549d73e829e1768de90fc1abf1", [:mix], [{:castore, "~> 0.1.0 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:hpax, "~> 0.1.1 or ~> 0.2.0 or ~> 1.0", [hex: :hpax, repo: "hexpm", optional: false]}], "hexpm", "fceba0a4d0f24301ddee3024ae116df1c3f4bb7a563a731f45fdfeb9d39a231b"}, + "multipart": {:hex, :multipart, "0.4.0", "634880a2148d4555d050963373d0e3bbb44a55b2badd87fa8623166172e9cda0", [:mix], [{:mime, "~> 1.2 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}], "hexpm", "3c5604bc2fb17b3137e5d2abdf5dacc2647e60c5cc6634b102cf1aef75a06f0a"}, "nimble_parsec": {:hex, :nimble_parsec, "1.4.2", "8efba0122db06df95bfaa78f791344a89352ba04baedd3849593bfce4d0dc1c6", [:mix], [], "hexpm", "4b21398942dda052b403bbe1da991ccd03a053668d147d53fb8c4e0efe09c973"}, "statistex": {:hex, :statistex, "1.1.0", "7fec1eb2f580a0d2c1a05ed27396a084ab064a40cfc84246dbfb0c72a5c761e5", [:mix], [], "hexpm", "f5950ea26ad43246ba2cce54324ac394a4e7408fdcf98b8e230f503a0cba9cf5"}, "telemetry": {:hex, :telemetry, "1.3.0", "fedebbae410d715cf8e7062c96a1ef32ec22e764197f70cda73d82778d61e7a2", [:rebar3], [], "hexpm", "7015fc8919dbe63764f4b4b87a95b7c0996bd539e0d499be6ec9d7f3875b79e6"}, From 76274b77363f1414033d4b5eb9fef57d3480ad3b Mon Sep 17 00:00:00 2001 From: hawkyre Date: Fri, 11 Jul 2025 12:52:54 +0200 Subject: [PATCH 02/21] to false --- lib/ch/connection.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/ch/connection.ex b/lib/ch/connection.ex index 9e247c4..46c3600 100644 --- a/lib/ch/connection.ex +++ b/lib/ch/connection.ex @@ -216,7 +216,7 @@ defmodule Ch.Connection do def handle_execute(%Query{command: :insert} = query, params, opts, conn) do conn = maybe_reconnect(conn) - multipart_request = Keyword.get(opts, :multipart_request, true) + multipart_request = Keyword.get(opts, :multipart_request, false) {query_params, extra_headers, body} = parse_params(params, multipart_request) path = path(conn, query_params, opts) @@ -237,7 +237,7 @@ defmodule Ch.Connection do def handle_execute(query, params, opts, conn) do conn = maybe_reconnect(conn) - multipart_request = Keyword.get(opts, :multipart_request, true) + multipart_request = Keyword.get(opts, :multipart_request, false) {query_params, extra_headers, body} = parse_params(params, multipart_request) path = path(conn, query_params, opts) From 828133c5fabfad10e13ccec9dde49b4830c10519 Mon Sep 17 00:00:00 2001 From: hawkyre Date: Fri, 11 Jul 2025 13:11:05 +0200 Subject: [PATCH 03/21] update docs and interface --- README.md | 20 ++++++++++++++++++++ lib/ch.ex | 1 + lib/ch/connection.ex | 8 ++++---- 3 files changed, 25 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 22de38c..6222ed5 100644 --- a/README.md +++ b/README.md @@ -161,6 +161,26 @@ settings = [async_insert: 1] Ch.query!(pid, "SHOW SETTINGS LIKE 'async_insert'", [], settings: settings) ``` +#### Sending request as multipart + +```elixir +{:ok, pid} = Ch.start_link() + +Ch.query!(pid, "CREATE TABLE IF NOT EXISTS ch_demo(id UInt64) ENGINE Null") + +{:ok, %Ch.Result{rows: [[0], [1], [2]]}} = + Ch.query(pid, "SELECT * FROM system.numbers LIMIT {$0:UInt8}", [3], multipart: true) + +{:ok, %Ch.Result{rows: [[0], [1], [2]]}} = + Ch.query(pid, "SELECT * FROM system.numbers LIMIT {limit:UInt8}", %{"limit" => 3}, multipart: true) + +%Ch.Result{num_rows: 2} = + Ch.query!(pid, "INSERT INTO ch_demo(id) VALUES ({$0:UInt8}), ({$1:UInt32})", [0, 1], multipart: true) +``` + +Adding the `multipart: true` option to a query sends its Clickhouse request as a multipart request. +This encodes all the request data inside the body, and leaves the URL empty. + ## Caveats #### NULL in RowBinary diff --git a/lib/ch.ex b/lib/ch.ex index 52a5f6b..758627f 100644 --- a/lib/ch.ex +++ b/lib/ch.ex @@ -76,6 +76,7 @@ defmodule Ch do * `:headers` - Custom HTTP headers for the request * `:format` - Custom response format for the request * `:decode` - Whether to automatically decode the response + * `:multipart` - Whether to use multipart/form-data encoding for the request. Not supported for RowBinary inserts or streaming. * [`DBConnection.connection_option()`](https://hexdocs.pm/db_connection/DBConnection.html#t:connection_option/0) """ diff --git a/lib/ch/connection.ex b/lib/ch/connection.ex index 46c3600..351a5d7 100644 --- a/lib/ch/connection.ex +++ b/lib/ch/connection.ex @@ -216,8 +216,8 @@ defmodule Ch.Connection do def handle_execute(%Query{command: :insert} = query, params, opts, conn) do conn = maybe_reconnect(conn) - multipart_request = Keyword.get(opts, :multipart_request, false) - {query_params, extra_headers, body} = parse_params(params, multipart_request) + multipart = Keyword.get(opts, :multipart, false) + {query_params, extra_headers, body} = parse_params(params, multipart) path = path(conn, query_params, opts) headers = headers(conn, extra_headers, opts) @@ -237,8 +237,8 @@ defmodule Ch.Connection do def handle_execute(query, params, opts, conn) do conn = maybe_reconnect(conn) - multipart_request = Keyword.get(opts, :multipart_request, false) - {query_params, extra_headers, body} = parse_params(params, multipart_request) + multipart = Keyword.get(opts, :multipart, false) + {query_params, extra_headers, body} = parse_params(params, multipart) path = path(conn, query_params, opts) headers = headers(conn, extra_headers, opts) From 23db2b9056f7c4a590c92697fa8cbf3473f23286 Mon Sep 17 00:00:00 2001 From: hawkyre Date: Fri, 11 Jul 2025 13:13:16 +0200 Subject: [PATCH 04/21] remove to string --- lib/ch/connection.ex | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/ch/connection.ex b/lib/ch/connection.ex index 351a5d7..76679f5 100644 --- a/lib/ch/connection.ex +++ b/lib/ch/connection.ex @@ -386,8 +386,6 @@ defmodule Ch.Connection do @spec parse_params(tuple, boolean) :: tuple defp parse_params({query_params, headers, body}, true = _multipart?) when is_binary(body) do - body = to_string(body) - multipart = query_params |> Enum.reduce(Multipart.new(), fn {k, v}, acc -> From 66020ff9cf1df99bce5a45b4437f41ef0bcb13ac Mon Sep 17 00:00:00 2001 From: hawkyre Date: Tue, 29 Jul 2025 09:49:24 +0200 Subject: [PATCH 05/21] custom multipart and slight refactor --- lib/ch/connection.ex | 33 +---------- lib/ch/encode/multipart.ex | 86 ++++++++++++++++++++++++++++ lib/ch/encode/parameters.ex | 109 ++++++++++++++++++++++++++++++++++++ lib/ch/query.ex | 104 +--------------------------------- mix.exs | 1 - 5 files changed, 200 insertions(+), 133 deletions(-) create mode 100644 lib/ch/encode/multipart.ex create mode 100644 lib/ch/encode/parameters.ex diff --git a/lib/ch/connection.ex b/lib/ch/connection.ex index 76679f5..40970ae 100644 --- a/lib/ch/connection.ex +++ b/lib/ch/connection.ex @@ -2,7 +2,6 @@ defmodule Ch.Connection do @moduledoc false use DBConnection require Logger - alias Multipart.Part alias Ch.{Error, Query, Result} alias Mint.HTTP1, as: HTTP @@ -215,9 +214,7 @@ defmodule Ch.Connection do def handle_execute(%Query{command: :insert} = query, params, opts, conn) do conn = maybe_reconnect(conn) - - multipart = Keyword.get(opts, :multipart, false) - {query_params, extra_headers, body} = parse_params(params, multipart) + {query_params, extra_headers, body} = params path = path(conn, query_params, opts) headers = headers(conn, extra_headers, opts) @@ -236,9 +233,7 @@ defmodule Ch.Connection do def handle_execute(query, params, opts, conn) do conn = maybe_reconnect(conn) - - multipart = Keyword.get(opts, :multipart, false) - {query_params, extra_headers, body} = parse_params(params, multipart) + {query_params, extra_headers, body} = params path = path(conn, query_params, opts) headers = headers(conn, extra_headers, opts) @@ -384,30 +379,6 @@ defmodule Ch.Connection do end end - @spec parse_params(tuple, boolean) :: tuple - defp parse_params({query_params, headers, body}, true = _multipart?) when is_binary(body) do - multipart = - query_params - |> Enum.reduce(Multipart.new(), fn {k, v}, acc -> - Multipart.add_part(acc, Part.text_field(v, k)) - end) - |> Multipart.add_part(Part.text_field(body, "query")) - - content_length = Multipart.content_length(multipart) - content_type = Multipart.content_type(multipart, "multipart/form-data") - - multipart_headers = [ - {"Content-Type", content_type}, - {"Content-Length", to_string(content_length)} - ] - - {[], headers ++ multipart_headers, Multipart.body_binary(multipart)} - end - - defp parse_params(params, _) do - params - end - defp get_header(headers, key) do case List.keyfind(headers, key, 0) do {_, value} -> value diff --git a/lib/ch/encode/multipart.ex b/lib/ch/encode/multipart.ex new file mode 100644 index 0000000..25fa42e --- /dev/null +++ b/lib/ch/encode/multipart.ex @@ -0,0 +1,86 @@ +defmodule Ch.Encode.Multipart do + @moduledoc false + + alias Ch.Encode.Parameters + + @doc """ + Encodes a query statement and params into a multipart request. + """ + @spec encode(iodata, map, [Ch.query_option()]) :: + {list, Mint.Types.headers(), iodata} + def encode(statement, params, opts) do + types = Keyword.get(opts, :types) + default_format = if types, do: "RowBinary", else: "RowBinaryWithNamesAndTypes" + format = Keyword.get(opts, :format) || default_format + + boundary = "ChFormBoundary" <> Base.url_encode64(:crypto.strong_rand_bytes(24)) + content_type = "multipart/form-data; boundary=\"#{boundary}\"" + enc_boundary = "--#{boundary}\r\n" + + multipart = + params + |> multipart_params(enc_boundary) + |> add_multipart_part("query", statement, enc_boundary) + |> then(&[&1 | "--#{boundary}--\r\n"]) + + headers = [{"x-clickhouse-format", format}, {"content-type", content_type} | headers(opts)] + + {_no_query_params = [], headers, multipart} + end + + defp multipart_params(params, boundary) when is_map(params) do + multipart_named_params(Map.to_list(params), boundary, []) + end + + defp multipart_params(params, boundary) when is_list(params) do + multipart_positional_params(params, 0, boundary, []) + end + + defp multipart_named_params([{name, value} | params], boundary, acc) do + acc = + add_multipart_part( + acc, + "param_" <> URI.encode_www_form(name), + Parameters.encode(value), + boundary + ) + + multipart_named_params(params, boundary, acc) + end + + defp multipart_named_params([], _boundary, acc), do: acc + + defp multipart_positional_params([value | params], idx, boundary, acc) do + acc = + add_multipart_part( + acc, + "param_$" <> Integer.to_string(idx), + Parameters.encode(value), + boundary + ) + + multipart_positional_params(params, idx + 1, boundary, acc) + end + + defp multipart_positional_params([], _idx, _boundary, acc), do: acc + + @compile inline: [add_multipart_part: 4] + defp add_multipart_part(multipart, name, value, boundary) do + part = [ + boundary, + "content-disposition: form-data; name=\"", + name, + "\"\r\n\r\n", + value, + "\r\n" + ] + + case multipart do + [] -> part + _ -> [multipart | part] + end + end + + @spec headers(Keyword.t()) :: Mint.Types.headers() + defp headers(opts), do: Keyword.get(opts, :headers, []) +end diff --git a/lib/ch/encode/parameters.ex b/lib/ch/encode/parameters.ex new file mode 100644 index 0000000..818f4fb --- /dev/null +++ b/lib/ch/encode/parameters.ex @@ -0,0 +1,109 @@ +defmodule Ch.Encode.Parameters do + @moduledoc false + + @doc """ + Encodes a map/list of parameters into a list of clickhouse parameter tuples. + + The format is `[{"param_", ""}, ...]`. + """ + @spec encode_many(map | [term]) :: [{String.t(), String.t()}] + def encode_many(params) when is_map(params) do + Enum.map(params, fn {k, v} -> {"param_#{k}", encode(v)} end) + end + + def encode_many(params) when is_list(params) do + params + |> Enum.with_index() + |> Enum.map(fn {v, idx} -> {"param_$#{idx}", encode(v)} end) + end + + @doc """ + Encodes a clickhouse parameter to a string. + """ + @spec encode(term) :: binary + def encode(n) when is_integer(n), do: Integer.to_string(n) + def encode(f) when is_float(f), do: Float.to_string(f) + + # TODO possibly speed up + # For more info see + # https://clickhouse.com/docs/en/interfaces/http#tabs-in-url-parameters + # "escaped" format is the same as https://clickhouse.com/docs/en/interfaces/formats#tabseparated-data-formatting + def encode(b) when is_binary(b) do + escape_param([{"\\", "\\\\"}, {"\t", "\\\t"}, {"\n", "\\\n"}], b) + end + + def encode(b) when is_boolean(b), do: Atom.to_string(b) + def encode(%Decimal{} = d), do: Decimal.to_string(d, :normal) + def encode(%Date{} = date), do: Date.to_iso8601(date) + def encode(%NaiveDateTime{} = naive), do: NaiveDateTime.to_iso8601(naive) + + def encode(%DateTime{microsecond: microsecond} = dt) do + dt = DateTime.shift_zone!(dt, "Etc/UTC") + + case microsecond do + {val, precision} when val > 0 and precision > 0 -> + size = round(:math.pow(10, precision)) + unix = DateTime.to_unix(dt, size) + seconds = div(unix, size) + fractional = rem(unix, size) + + IO.iodata_to_binary([ + Integer.to_string(seconds), + ?., + String.pad_leading(Integer.to_string(fractional), precision, "0") + ]) + + _ -> + dt |> DateTime.to_unix(:second) |> Integer.to_string() + end + end + + def encode(tuple) when is_tuple(tuple) do + IO.iodata_to_binary([?(, encode_array_params(Tuple.to_list(tuple)), ?)]) + end + + def encode(a) when is_list(a) do + IO.iodata_to_binary([?[, encode_array_params(a), ?]]) + end + + def encode(m) when is_map(m) do + IO.iodata_to_binary([?{, encode_map_params(Map.to_list(m)), ?}]) + end + + defp encode_array_params([last]), do: encode_array_param(last) + + defp encode_array_params([s | rest]) do + [encode_array_param(s), ?, | encode_array_params(rest)] + end + + defp encode_array_params([] = empty), do: empty + + defp encode_map_params([last]), do: encode_map_param(last) + + defp encode_map_params([kv | rest]) do + [encode_map_param(kv), ?, | encode_map_params(rest)] + end + + defp encode_map_params([] = empty), do: empty + + defp encode_array_param(s) when is_binary(s) do + [?', escape_param([{"'", "''"}, {"\\", "\\\\"}], s), ?'] + end + + defp encode_array_param(%s{} = param) when s in [Date, NaiveDateTime] do + [?', encode(param), ?'] + end + + defp encode_array_param(v), do: encode(v) + + defp encode_map_param({k, v}) do + [encode_array_param(k), ?:, encode_array_param(v)] + end + + defp escape_param([{pattern, replacement} | escapes], param) do + param = String.replace(param, pattern, replacement) + escape_param(escapes, param) + end + + defp escape_param([], param), do: param +end diff --git a/lib/ch/query.ex b/lib/ch/query.ex index f5a9805..c1d9159 100644 --- a/lib/ch/query.ex +++ b/lib/ch/query.ex @@ -72,6 +72,7 @@ end defimpl DBConnection.Query, for: Ch.Query do alias Ch.{Query, Result, RowBinary} + alias Ch.Encode.{Multipart, Parameters} @spec parse(Query.t(), [Ch.query_option()]) :: Query.t() def parse(query, _opts), do: query @@ -123,15 +124,12 @@ defimpl DBConnection.Query, for: Ch.Query do {_query_params = [], headers(opts), [statement, ?\n | data]} true -> - {query_params(params), headers(opts), statement} + {Parameters.encode_many(params), headers(opts), statement} end end def encode(%Query{statement: statement}, params, opts) do - types = Keyword.get(opts, :types) - default_format = if types, do: "RowBinary", else: "RowBinaryWithNamesAndTypes" - format = Keyword.get(opts, :format) || default_format - {query_params(params), [{"x-clickhouse-format", format} | headers(opts)], statement} + Multipart.encode(statement, params, opts) end defp format_row_binary?(statement) when is_binary(statement) do @@ -207,102 +205,6 @@ defimpl DBConnection.Query, for: Ch.Query do end end - defp query_params(params) when is_map(params) do - Enum.map(params, fn {k, v} -> {"param_#{k}", encode_param(v)} end) - end - - defp query_params(params) when is_list(params) do - params - |> Enum.with_index() - |> Enum.map(fn {v, idx} -> {"param_$#{idx}", encode_param(v)} end) - end - - defp encode_param(n) when is_integer(n), do: Integer.to_string(n) - defp encode_param(f) when is_float(f), do: Float.to_string(f) - - # TODO possibly speed up - # For more info see - # https://clickhouse.com/docs/en/interfaces/http#tabs-in-url-parameters - # "escaped" format is the same as https://clickhouse.com/docs/en/interfaces/formats#tabseparated-data-formatting - defp encode_param(b) when is_binary(b) do - escape_param([{"\\", "\\\\"}, {"\t", "\\\t"}, {"\n", "\\\n"}], b) - end - - defp encode_param(b) when is_boolean(b), do: Atom.to_string(b) - defp encode_param(%Decimal{} = d), do: Decimal.to_string(d, :normal) - defp encode_param(%Date{} = date), do: Date.to_iso8601(date) - defp encode_param(%NaiveDateTime{} = naive), do: NaiveDateTime.to_iso8601(naive) - - defp encode_param(%DateTime{microsecond: microsecond} = dt) do - dt = DateTime.shift_zone!(dt, "Etc/UTC") - - case microsecond do - {val, precision} when val > 0 and precision > 0 -> - size = round(:math.pow(10, precision)) - unix = DateTime.to_unix(dt, size) - seconds = div(unix, size) - fractional = rem(unix, size) - - IO.iodata_to_binary([ - Integer.to_string(seconds), - ?., - String.pad_leading(Integer.to_string(fractional), precision, "0") - ]) - - _ -> - dt |> DateTime.to_unix(:second) |> Integer.to_string() - end - end - - defp encode_param(tuple) when is_tuple(tuple) do - IO.iodata_to_binary([?(, encode_array_params(Tuple.to_list(tuple)), ?)]) - end - - defp encode_param(a) when is_list(a) do - IO.iodata_to_binary([?[, encode_array_params(a), ?]]) - end - - defp encode_param(m) when is_map(m) do - IO.iodata_to_binary([?{, encode_map_params(Map.to_list(m)), ?}]) - end - - defp encode_array_params([last]), do: encode_array_param(last) - - defp encode_array_params([s | rest]) do - [encode_array_param(s), ?, | encode_array_params(rest)] - end - - defp encode_array_params([] = empty), do: empty - - defp encode_map_params([last]), do: encode_map_param(last) - - defp encode_map_params([kv | rest]) do - [encode_map_param(kv), ?, | encode_map_params(rest)] - end - - defp encode_map_params([] = empty), do: empty - - defp encode_array_param(s) when is_binary(s) do - [?', escape_param([{"'", "''"}, {"\\", "\\\\"}], s), ?'] - end - - defp encode_array_param(%s{} = param) when s in [Date, NaiveDateTime] do - [?', encode_param(param), ?'] - end - - defp encode_array_param(v), do: encode_param(v) - - defp encode_map_param({k, v}) do - [encode_array_param(k), ?:, encode_array_param(v)] - end - - defp escape_param([{pattern, replacement} | escapes], param) do - param = String.replace(param, pattern, replacement) - escape_param(escapes, param) - end - - defp escape_param([], param), do: param - @spec headers(Keyword.t()) :: Mint.Types.headers() defp headers(opts), do: Keyword.get(opts, :headers, []) end diff --git a/mix.exs b/mix.exs index c8f704e..dd11049 100644 --- a/mix.exs +++ b/mix.exs @@ -41,7 +41,6 @@ defmodule Ch.MixProject do {:db_connection, "~> 2.0"}, {:jason, "~> 1.0"}, {:decimal, "~> 2.0"}, - {:multipart, "~> 0.4.0"}, {:ecto, "~> 3.13.0", optional: true}, {:benchee, "~> 1.0", only: [:bench]}, {:dialyxir, "~> 1.0", only: [:dev], runtime: false}, From 482a42fd9ef538462c4e0ee0323fea9a4fa54068 Mon Sep 17 00:00:00 2001 From: hawkyre Date: Tue, 29 Jul 2025 09:53:24 +0200 Subject: [PATCH 06/21] merge --- lib/ch/encode/parameters.ex | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/ch/encode/parameters.ex b/lib/ch/encode/parameters.ex index 818f4fb..0ac0828 100644 --- a/lib/ch/encode/parameters.ex +++ b/lib/ch/encode/parameters.ex @@ -33,9 +33,11 @@ defmodule Ch.Encode.Parameters do end def encode(b) when is_boolean(b), do: Atom.to_string(b) + def encode(nil), do: "\\N" def encode(%Decimal{} = d), do: Decimal.to_string(d, :normal) def encode(%Date{} = date), do: Date.to_iso8601(date) def encode(%NaiveDateTime{} = naive), do: NaiveDateTime.to_iso8601(naive) + def encode(%Time{} = time), do: Time.to_iso8601(time) def encode(%DateTime{microsecond: microsecond} = dt) do dt = DateTime.shift_zone!(dt, "Etc/UTC") @@ -90,6 +92,8 @@ defmodule Ch.Encode.Parameters do [?', escape_param([{"'", "''"}, {"\\", "\\\\"}], s), ?'] end + defp encode_array_param(nil), do: "null" + defp encode_array_param(%s{} = param) when s in [Date, NaiveDateTime] do [?', encode(param), ?'] end From 5191d07e060c2b0bd6f0a8925f6b413fa9467953 Mon Sep 17 00:00:00 2001 From: hawkyre Date: Tue, 29 Jul 2025 10:01:05 +0200 Subject: [PATCH 07/21] docs --- README.md | 19 ++----------------- 1 file changed, 2 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index f044dfe..0f0b1f5 100644 --- a/README.md +++ b/README.md @@ -163,23 +163,8 @@ settings = [async_insert: 1] #### Sending request as multipart -```elixir -{:ok, pid} = Ch.start_link() - -Ch.query!(pid, "CREATE TABLE IF NOT EXISTS ch_demo(id UInt64) ENGINE Null") - -{:ok, %Ch.Result{rows: [[0], [1], [2]]}} = - Ch.query(pid, "SELECT * FROM system.numbers LIMIT {$0:UInt8}", [3], multipart: true) - -{:ok, %Ch.Result{rows: [[0], [1], [2]]}} = - Ch.query(pid, "SELECT * FROM system.numbers LIMIT {limit:UInt8}", %{"limit" => 3}, multipart: true) - -%Ch.Result{num_rows: 2} = - Ch.query!(pid, "INSERT INTO ch_demo(id) VALUES ({$0:UInt8}), ({$1:UInt32})", [0, 1], multipart: true) -``` - -Adding the `multipart: true` option to a query sends its Clickhouse request as a multipart request. -This encodes all the request data inside the body, and leaves the URL empty. +SELECT queries will be automatically sent as multipart requests. +INSERT queries and streams are treated normally. ## Caveats From 22207b1a9f0a4b3f45ed20407615a87c524a4154 Mon Sep 17 00:00:00 2001 From: hawkyre Date: Tue, 29 Jul 2025 10:03:14 +0200 Subject: [PATCH 08/21] doc title --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 0f0b1f5..0bece3e 100644 --- a/README.md +++ b/README.md @@ -161,7 +161,7 @@ settings = [async_insert: 1] Ch.query!(pid, "SHOW SETTINGS LIKE 'async_insert'", [], settings: settings) ``` -#### Sending request as multipart +#### Multipart requests SELECT queries will be automatically sent as multipart requests. INSERT queries and streams are treated normally. From d21d7abf2a7280cc2526d56c1d0df92ba5c738e0 Mon Sep 17 00:00:00 2001 From: hawkyre Date: Tue, 29 Jul 2025 10:04:55 +0200 Subject: [PATCH 09/21] more cleanup --- lib/ch.ex | 1 - mix.lock | 2 -- 2 files changed, 3 deletions(-) diff --git a/lib/ch.ex b/lib/ch.ex index 7346345..8b12e2c 100644 --- a/lib/ch.ex +++ b/lib/ch.ex @@ -76,7 +76,6 @@ defmodule Ch do * `:headers` - Custom HTTP headers for the request * `:format` - Custom response format for the request * `:decode` - Whether to automatically decode the response - * `:multipart` - Whether to use multipart/form-data encoding for the request. Not supported for RowBinary inserts or streaming. * [`DBConnection.connection_option()`](https://hexdocs.pm/db_connection/DBConnection.html#t:connection_option/0) """ diff --git a/mix.lock b/mix.lock index 94448ac..0e57a37 100644 --- a/mix.lock +++ b/mix.lock @@ -13,9 +13,7 @@ "makeup": {:hex, :makeup, "1.2.1", "e90ac1c65589ef354378def3ba19d401e739ee7ee06fb47f94c687016e3713d1", [:mix], [{:nimble_parsec, "~> 1.4", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "d36484867b0bae0fea568d10131197a4c2e47056a6fbe84922bf6ba71c8d17ce"}, "makeup_elixir": {:hex, :makeup_elixir, "1.0.1", "e928a4f984e795e41e3abd27bfc09f51db16ab8ba1aebdba2b3a575437efafc2", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.2.3 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "7284900d412a3e5cfd97fdaed4f5ed389b8f2b4cb49efc0eb3bd10e2febf9507"}, "makeup_erlang": {:hex, :makeup_erlang, "1.0.2", "03e1804074b3aa64d5fad7aa64601ed0fb395337b982d9bcf04029d68d51b6a7", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "af33ff7ef368d5893e4a267933e7744e46ce3cf1f61e2dccf53a111ed3aa3727"}, - "mime": {:hex, :mime, "2.0.7", "b8d739037be7cd402aee1ba0306edfdef982687ee7e9859bee6198c1e7e2f128", [:mix], [], "hexpm", "6171188e399ee16023ffc5b76ce445eb6d9672e2e241d2df6050f3c771e80ccd"}, "mint": {:hex, :mint, "1.7.1", "113fdb2b2f3b59e47c7955971854641c61f378549d73e829e1768de90fc1abf1", [:mix], [{:castore, "~> 0.1.0 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:hpax, "~> 0.1.1 or ~> 0.2.0 or ~> 1.0", [hex: :hpax, repo: "hexpm", optional: false]}], "hexpm", "fceba0a4d0f24301ddee3024ae116df1c3f4bb7a563a731f45fdfeb9d39a231b"}, - "multipart": {:hex, :multipart, "0.4.0", "634880a2148d4555d050963373d0e3bbb44a55b2badd87fa8623166172e9cda0", [:mix], [{:mime, "~> 1.2 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}], "hexpm", "3c5604bc2fb17b3137e5d2abdf5dacc2647e60c5cc6634b102cf1aef75a06f0a"}, "nimble_parsec": {:hex, :nimble_parsec, "1.4.2", "8efba0122db06df95bfaa78f791344a89352ba04baedd3849593bfce4d0dc1c6", [:mix], [], "hexpm", "4b21398942dda052b403bbe1da991ccd03a053668d147d53fb8c4e0efe09c973"}, "statistex": {:hex, :statistex, "1.1.0", "7fec1eb2f580a0d2c1a05ed27396a084ab064a40cfc84246dbfb0c72a5c761e5", [:mix], [], "hexpm", "f5950ea26ad43246ba2cce54324ac394a4e7408fdcf98b8e230f503a0cba9cf5"}, "telemetry": {:hex, :telemetry, "1.3.0", "fedebbae410d715cf8e7062c96a1ef32ec22e764197f70cda73d82778d61e7a2", [:rebar3], [], "hexpm", "7015fc8919dbe63764f4b4b87a95b7c0996bd539e0d499be6ec9d7f3875b79e6"}, From 3bee9258c615827b3d64d670ee5fed0e8047b2c5 Mon Sep 17 00:00:00 2001 From: hawkyre Date: Mon, 18 Aug 2025 15:03:07 +0200 Subject: [PATCH 10/21] send settings as params --- lib/ch/encode/multipart.ex | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/ch/encode/multipart.ex b/lib/ch/encode/multipart.ex index 25fa42e..c058fde 100644 --- a/lib/ch/encode/multipart.ex +++ b/lib/ch/encode/multipart.ex @@ -10,6 +10,7 @@ defmodule Ch.Encode.Multipart do {list, Mint.Types.headers(), iodata} def encode(statement, params, opts) do types = Keyword.get(opts, :types) + settings = Keyword.get(opts, :settings, []) default_format = if types, do: "RowBinary", else: "RowBinaryWithNamesAndTypes" format = Keyword.get(opts, :format) || default_format @@ -25,7 +26,7 @@ defmodule Ch.Encode.Multipart do headers = [{"x-clickhouse-format", format}, {"content-type", content_type} | headers(opts)] - {_no_query_params = [], headers, multipart} + {settings, headers, multipart} end defp multipart_params(params, boundary) when is_map(params) do From 33ecfaf1fee0a55698c335257c9ce5d01a9d0414 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 19 Aug 2025 08:39:22 +0900 Subject: [PATCH 11/21] Bump actions/checkout from 4 to 5 (#270) Bumps [actions/checkout](https://github.com/actions/checkout) from 4 to 5. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v4...v5) --- updated-dependencies: - dependency-name: actions/checkout dependency-version: '5' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/bench.yml | 2 +- .github/workflows/spellcheck.yml | 4 ++-- .github/workflows/test.yml | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index aa08d9e..a9d40fa 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -27,7 +27,7 @@ jobs: --health-retries 5 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - id: beam uses: erlef/setup-beam@v1 diff --git a/.github/workflows/spellcheck.yml b/.github/workflows/spellcheck.yml index ff0ab78..3d39fe8 100644 --- a/.github/workflows/spellcheck.yml +++ b/.github/workflows/spellcheck.yml @@ -9,7 +9,7 @@ jobs: codespell: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - uses: codespell-project/actions-codespell@v2 with: check_filenames: true @@ -18,5 +18,5 @@ jobs: typos: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - uses: crate-ci/typos@master diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 06b25d0..05b2a6a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -62,7 +62,7 @@ jobs: --health-retries 5 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - id: beam uses: erlef/setup-beam@v1 @@ -87,7 +87,7 @@ jobs: format: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - uses: erlef/setup-beam@v1 with: elixir-version: 1 From 574c14bfcc540a398183b5e43cac7a3ffdbf88ab Mon Sep 17 00:00:00 2001 From: ruslandoga Date: Tue, 26 Aug 2025 09:22:38 +0900 Subject: [PATCH 12/21] fix version check (#274) --- lib/ch/connection.ex | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/lib/ch/connection.ex b/lib/ch/connection.ex index f94b50a..b84846a 100644 --- a/lib/ch/connection.ex +++ b/lib/ch/connection.ex @@ -21,7 +21,7 @@ defmodule Ch.Connection do case DBConnection.Query.decode(handshake, responses, _opts = []) do %Result{rows: [[1, version]]} -> conn = - if version >= "24.10" do + if parse_version(version) >= parse_version("24.10") do settings = HTTP.get_private(conn, :settings, []) |> Keyword.put_new(:input_format_binary_read_json_as_string, 1) @@ -51,6 +51,17 @@ defmodule Ch.Connection do end end + defp parse_version(version) do + version + |> String.split(".") + |> Enum.flat_map(fn segment -> + case Integer.parse(segment) do + {int, _rest} -> [int] + :error -> [] + end + end) + end + @impl true @spec ping(conn) :: {:ok, conn} | {:disconnect, Mint.Types.error() | Error.t(), conn} def ping(conn) do From a018df865801eecb288773ee9ee222e76efab0db Mon Sep 17 00:00:00 2001 From: ruslandoga Date: Tue, 26 Aug 2025 09:23:35 +0900 Subject: [PATCH 13/21] changelog --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8f48d1a..6845417 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## Unreleased + +- fix version check for adding JSON settings https://github.com/plausible/ch/pull/274 + ## 0.5.4 (2025-07-22) - allow `nil` in params https://github.com/plausible/ch/pull/268 From ecc8a19f7002125548cee5942796d051e2e14551 Mon Sep 17 00:00:00 2001 From: ruslandoga Date: Tue, 26 Aug 2025 09:24:25 +0900 Subject: [PATCH 14/21] add older ClickHouse to CI --- .github/workflows/test.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 05b2a6a..66f32da 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -45,6 +45,9 @@ jobs: otp: 27.3.1 clickhouse: 24.12.2.29 timezone: UTC + - elixir: 1.18 + otp: 28 + clickhouse: 24.5.4.49 services: clickhouse: From cac0abd2c0ff245fb20079b70c84453c15a36399 Mon Sep 17 00:00:00 2001 From: ruslandoga Date: Tue, 26 Aug 2025 09:27:06 +0900 Subject: [PATCH 15/21] tag 'json as string' test as json --- test/ch/connection_test.exs | 1 + 1 file changed, 1 insertion(+) diff --git a/test/ch/connection_test.exs b/test/ch/connection_test.exs index b9003ed..394dcb6 100644 --- a/test/ch/connection_test.exs +++ b/test/ch/connection_test.exs @@ -593,6 +593,7 @@ defmodule Ch.ConnectionTest do assert_raise ArgumentError, fn -> Ch.query!(conn, "SELECT o FROM json") end end + @tag :json test "json as string", %{conn: conn} do # after v25 ClickHouse started rendering numbers in JSON as strings [[version]] = Ch.query!(conn, "select version()").rows From a59d141a1ff1ff468188e2335e1ad84b929d2e8a Mon Sep 17 00:00:00 2001 From: ruslandoga Date: Tue, 26 Aug 2025 09:29:49 +0900 Subject: [PATCH 16/21] release v0.5.5 --- CHANGELOG.md | 2 +- mix.exs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6845417..88b6a72 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -## Unreleased +## 0.5.5 (2025-08-26) - fix version check for adding JSON settings https://github.com/plausible/ch/pull/274 diff --git a/mix.exs b/mix.exs index 48ff301..e0dc05c 100644 --- a/mix.exs +++ b/mix.exs @@ -2,7 +2,7 @@ defmodule Ch.MixProject do use Mix.Project @source_url "https://github.com/plausible/ch" - @version "0.5.4" + @version "0.5.5" def project do [ From e0ccd54e435028314f17d25aea1b2bc5eb48c111 Mon Sep 17 00:00:00 2001 From: ruslandoga Date: Tue, 26 Aug 2025 09:30:08 +0900 Subject: [PATCH 17/21] update deps --- mix.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mix.lock b/mix.lock index 0e57a37..4abb55a 100644 --- a/mix.lock +++ b/mix.lock @@ -3,11 +3,11 @@ "db_connection": {:hex, :db_connection, "2.8.0", "64fd82cfa6d8e25ec6660cea73e92a4cbc6a18b31343910427b702838c4b33b2", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "008399dae5eee1bf5caa6e86d204dcb44242c82b1ed5e22c881f2c34da201b15"}, "decimal": {:hex, :decimal, "2.3.0", "3ad6255aa77b4a3c4f818171b12d237500e63525c2fd056699967a3e7ea20f62", [:mix], [], "hexpm", "a4d66355cb29cb47c3cf30e71329e58361cfcb37c34235ef3bf1d7bf3773aeac"}, "deep_merge": {:hex, :deep_merge, "1.0.0", "b4aa1a0d1acac393bdf38b2291af38cb1d4a52806cf7a4906f718e1feb5ee961", [:mix], [], "hexpm", "ce708e5f094b9cd4e8f2be4f00d2f4250c4095be93f8cd6d018c753894885430"}, - "dialyxir": {:hex, :dialyxir, "1.4.5", "ca1571ac18e0f88d4ab245f0b60fa31ff1b12cbae2b11bd25d207f865e8ae78a", [:mix], [{:erlex, ">= 0.2.7", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "b0fb08bb8107c750db5c0b324fa2df5ceaa0f9307690ee3c1f6ba5b9eb5d35c3"}, + "dialyxir": {:hex, :dialyxir, "1.4.6", "7cca478334bf8307e968664343cbdb432ee95b4b68a9cba95bdabb0ad5bdfd9a", [:mix], [{:erlex, ">= 0.2.7", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "8cf5615c5cd4c2da6c501faae642839c8405b49f8aa057ad4ae401cb808ef64d"}, "earmark_parser": {:hex, :earmark_parser, "1.4.44", "f20830dd6b5c77afe2b063777ddbbff09f9759396500cdbe7523efd58d7a339c", [:mix], [], "hexpm", "4778ac752b4701a5599215f7030989c989ffdc4f6df457c5f36938cc2d2a2750"}, "ecto": {:hex, :ecto, "3.13.2", "7d0c0863f3fc8d71d17fc3ad3b9424beae13f02712ad84191a826c7169484f01", [:mix], [{:decimal, "~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "669d9291370513ff56e7b7e7081b7af3283d02e046cf3d403053c557894a0b3e"}, "erlex": {:hex, :erlex, "0.2.7", "810e8725f96ab74d17aac676e748627a07bc87eb950d2b83acd29dc047a30595", [:mix], [], "hexpm", "3ed95f79d1a844c3f6bf0cea61e0d5612a42ce56da9c03f01df538685365efb0"}, - "ex_doc": {:hex, :ex_doc, "0.38.2", "504d25eef296b4dec3b8e33e810bc8b5344d565998cd83914ffe1b8503737c02", [:mix], [{:earmark_parser, "~> 1.4.44", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_c, ">= 0.1.0", [hex: :makeup_c, repo: "hexpm", optional: true]}, {:makeup_elixir, "~> 0.14 or ~> 1.0", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1 or ~> 1.0", [hex: :makeup_erlang, repo: "hexpm", optional: false]}, {:makeup_html, ">= 0.1.0", [hex: :makeup_html, repo: "hexpm", optional: true]}], "hexpm", "732f2d972e42c116a70802f9898c51b54916e542cc50968ac6980512ec90f42b"}, + "ex_doc": {:hex, :ex_doc, "0.38.3", "ddafe36b8e9fe101c093620879f6604f6254861a95133022101c08e75e6c759a", [:mix], [{:earmark_parser, "~> 1.4.44", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_c, ">= 0.1.0", [hex: :makeup_c, repo: "hexpm", optional: true]}, {:makeup_elixir, "~> 0.14 or ~> 1.0", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1 or ~> 1.0", [hex: :makeup_erlang, repo: "hexpm", optional: false]}, {:makeup_html, ">= 0.1.0", [hex: :makeup_html, repo: "hexpm", optional: true]}], "hexpm", "ecaa785456a67f63b4e7d7f200e8832fa108279e7eb73fd9928e7e66215a01f9"}, "hpax": {:hex, :hpax, "1.0.3", "ed67ef51ad4df91e75cc6a1494f851850c0bd98ebc0be6e81b026e765ee535aa", [:mix], [], "hexpm", "8eab6e1cfa8d5918c2ce4ba43588e894af35dbd8e91e6e55c817bca5847df34a"}, "jason": {:hex, :jason, "1.4.4", "b9226785a9aa77b6857ca22832cffa5d5011a667207eb2a0ad56adb5db443b8a", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "c5eb0cab91f094599f94d55bc63409236a8ec69a21a67814529e8d5f6cc90b3b"}, "makeup": {:hex, :makeup, "1.2.1", "e90ac1c65589ef354378def3ba19d401e739ee7ee06fb47f94c687016e3713d1", [:mix], [{:nimble_parsec, "~> 1.4", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "d36484867b0bae0fea568d10131197a4c2e47056a6fbe84922bf6ba71c8d17ce"}, From b3edc2fd4c4022b0aed9fe5a6f8069575b506c89 Mon Sep 17 00:00:00 2001 From: ruslandoga Date: Tue, 26 Aug 2025 09:33:09 +0900 Subject: [PATCH 18/21] comment on why older version in ci --- .github/workflows/test.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 66f32da..8dbc3fa 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -45,6 +45,8 @@ jobs: otp: 27.3.1 clickhouse: 24.12.2.29 timezone: UTC + # some older pre-JSON ClickHouse version + # https://github.com/plausible/ch/issues/273 - elixir: 1.18 otp: 28 clickhouse: 24.5.4.49 From 27e11d8b0a9246b1c53106c2d0749ff6c3f6d5a2 Mon Sep 17 00:00:00 2001 From: ruslandoga Date: Tue, 26 Aug 2025 09:34:30 +0900 Subject: [PATCH 19/21] shorter cache key --- .github/workflows/test.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8dbc3fa..f2e1dd7 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -80,10 +80,10 @@ jobs: path: | deps _build - key: test-${{ steps.beam.outputs.elixir-version }}-${{ steps.beam.outputs.otp-version }}-${{ github.head_ref || github.ref }}-${{ hashFiles('**/mix.lock') }} + key: test-${{ steps.beam.outputs.elixir-version }}-${{ github.head_ref || github.ref }}-${{ hashFiles('**/mix.lock') }} restore-keys: | - test-${{ steps.beam.outputs.elixir-version }}-${{ steps.beam.outputs.otp-version }}-${{ github.head_ref || github.ref }}- - test-${{ steps.beam.outputs.elixir-version }}-${{ steps.beam.outputs.otp-version }}-refs/heads/master- + test-${{ steps.beam.outputs.elixir-version }}-${{ github.head_ref || github.ref }}- + test-${{ steps.beam.outputs.elixir-version }}-refs/heads/master- - run: mix deps.get --only $MIX_ENV - run: mix compile --warnings-as-errors From 116322b9fd077cba2c21c107c6af133b3dd13c03 Mon Sep 17 00:00:00 2001 From: ruslandoga Date: Tue, 26 Aug 2025 14:47:22 +0900 Subject: [PATCH 20/21] fix internal type ordering in Variant (#275) * fix internal type ordering in Variant * cleanup * link pr --- CHANGELOG.md | 4 ++++ lib/ch/types.ex | 2 +- test/ch/variant_test.exs | 18 ++++++++++++++++++ 3 files changed, 23 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 88b6a72..ba42b84 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## Unreleased + +- fix internal type ordering in Variant https://github.com/plausible/ch/pull/275 + ## 0.5.5 (2025-08-26) - fix version check for adding JSON settings https://github.com/plausible/ch/pull/274 diff --git a/lib/ch/types.ex b/lib/ch/types.ex index de29ba2..e86b16c 100644 --- a/lib/ch/types.ex +++ b/lib/ch/types.ex @@ -502,7 +502,7 @@ defmodule Ch.Types do defp named_columns_to_types([], acc), do: acc defp build_variant(types) do - Enum.sort_by(types, &__MODULE__.encode/1) + Enum.sort_by(types, fn t -> IO.iodata_to_binary(encode(t)) end) end # TODO '', \' diff --git a/test/ch/variant_test.exs b/test/ch/variant_test.exs index 04b5d8e..23a2c57 100644 --- a/test/ch/variant_test.exs +++ b/test/ch/variant_test.exs @@ -19,6 +19,24 @@ defmodule Ch.VariantTest do [["Hello, World!"]] end + # https://github.com/plausible/ch/issues/272 + test "ordering internal types", %{conn: conn} do + test = %{ + "'hello'" => "hello", + "-10" => -10, + "true" => true, + "map('hello', null::Nullable(String))" => %{"hello" => nil}, + "map('hello', 'world'::Nullable(String))" => %{"hello" => "world"} + } + + for {value, expected} <- test do + assert Ch.query!( + conn, + "select #{value}::Variant(String, Int32, Bool, Map(String, Nullable(String)))" + ).rows == [[expected]] + end + end + test "with a table", %{conn: conn} do # https://clickhouse.com/docs/sql-reference/data-types/variant#creating-variant Ch.query!(conn, """ From d8cb624409678529dd9ca8839cd96d5baefd0ccc Mon Sep 17 00:00:00 2001 From: ruslandoga Date: Tue, 26 Aug 2025 14:48:11 +0900 Subject: [PATCH 21/21] release v0.5.6 --- CHANGELOG.md | 2 +- mix.exs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ba42b84..75810b0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -## Unreleased +## 0.5.6 (2025-08-26) - fix internal type ordering in Variant https://github.com/plausible/ch/pull/275 diff --git a/mix.exs b/mix.exs index e0dc05c..4f86353 100644 --- a/mix.exs +++ b/mix.exs @@ -2,7 +2,7 @@ defmodule Ch.MixProject do use Mix.Project @source_url "https://github.com/plausible/ch" - @version "0.5.5" + @version "0.5.6" def project do [