From 05d3e5c93838b854bec74b4d91b49ec6dd3cc3db Mon Sep 17 00:00:00 2001 From: Paul Engel Date: Wed, 3 May 2017 11:58:43 +0200 Subject: [PATCH 1/3] Extracted `to_sql()` to SqlDust.QueryUtils --- lib/ecto/sql_dust.ex | 2 +- lib/sql_dust/query.ex | 2 +- lib/sql_dust/utils/compose_utils.ex | 18 ------------------ lib/sql_dust/utils/query_utils.ex | 26 ++++++++++++++++++++++++++ 4 files changed, 28 insertions(+), 20 deletions(-) create mode 100644 lib/sql_dust/utils/query_utils.ex diff --git a/lib/ecto/sql_dust.ex b/lib/ecto/sql_dust.ex index 37c91a3..e4aa401 100644 --- a/lib/ecto/sql_dust.ex +++ b/lib/ecto/sql_dust.ex @@ -1,5 +1,5 @@ defmodule Ecto.SqlDust do - use SqlDust.ComposeUtils + use SqlDust.QueryUtils def from(options, model) do resource = model.__schema__(:source) diff --git a/lib/sql_dust/query.ex b/lib/sql_dust/query.ex index 28f12c5..43f64cc 100644 --- a/lib/sql_dust/query.ex +++ b/lib/sql_dust/query.ex @@ -1,3 +1,3 @@ defmodule SqlDust.Query do - use SqlDust.ComposeUtils + use SqlDust.QueryUtils end diff --git a/lib/sql_dust/utils/compose_utils.ex b/lib/sql_dust/utils/compose_utils.ex index 92d5967..7507a36 100644 --- a/lib/sql_dust/utils/compose_utils.ex +++ b/lib/sql_dust/utils/compose_utils.ex @@ -56,24 +56,6 @@ defmodule SqlDust.ComposeUtils do end end - def to_sql(options) do - options = parse(options) - - if (is_nil(options.from)) do; - raise "missing :from option in query dust" - end - - from = options.from - schema = options.schema - options = Map.take(options, [:select, :join_on, :where, :group_by, :order_by, :limit, :offset, :unique, :variables, :adapter]) - |> Enum.reduce(%{from: options.from}, fn - ({_key, nil}, map) -> map - ({key, value}, map) -> Map.put(map, key, value) - end) - - SqlDust.from(from, options, schema || %{}) - end - defp options do %SqlDust{} end diff --git a/lib/sql_dust/utils/query_utils.ex b/lib/sql_dust/utils/query_utils.ex new file mode 100644 index 0000000..44a0e90 --- /dev/null +++ b/lib/sql_dust/utils/query_utils.ex @@ -0,0 +1,26 @@ +defmodule SqlDust.QueryUtils do + defmacro __using__(_) do + quote do + use SqlDust.ComposeUtils + + def to_sql(options) do + options = parse(options) + + if (is_nil(options.from)) do; + raise "missing :from option in query dust" + end + + from = options.from + schema = options.schema + options = Map.take(options, [:select, :join_on, :where, :group_by, :order_by, :limit, :offset, :unique, :variables, :adapter]) + |> Enum.reduce(%{from: options.from}, fn + ({_key, nil}, map) -> map + ({key, value}, map) -> Map.put(map, key, value) + end) + + SqlDust.from(from, options, schema || %{}) + end + + end + end +end From 8f6c2d8ef236f5d06129c2062ffd985adf13dea7 Mon Sep 17 00:00:00 2001 From: Paul Engel Date: Wed, 3 May 2017 12:34:18 +0200 Subject: [PATCH 2/3] Being able to query data (fetching both lists and maps) --- config/config.exs | 6 ++- config/test.exs | 13 +++++++ lib/sql_dust/utils/query_utils.ex | 38 +++++++++++++++++++ mix.exs | 16 +++++++- mix.lock | 3 ++ .../20170502224942_create_cities.exs | 9 +++++ test/ecto/sql_dust_test.exs | 38 ++++++++++++++++++- test/sql_dust/query_test.exs | 37 +++++++++++++++++- test/support/test_repo.ex | 3 ++ test/test_helper.exs | 20 ++++++++++ 10 files changed, 177 insertions(+), 6 deletions(-) create mode 100644 config/test.exs create mode 100644 priv/test_repo/migrations/20170502224942_create_cities.exs create mode 100644 test/support/test_repo.ex diff --git a/config/config.exs b/config/config.exs index 85e8d52..b7bba19 100644 --- a/config/config.exs +++ b/config/config.exs @@ -26,5 +26,7 @@ use Mix.Config # by uncommenting the line below and defining dev.exs, test.exs and such. # Configuration from the imported file will override the ones defined # here (which is why it is important to import them last). -# -# import_config "#{Mix.env}.exs" + +if Mix.env == :test do + import_config "test.exs" +end diff --git a/config/test.exs b/config/test.exs new file mode 100644 index 0000000..2706464 --- /dev/null +++ b/config/test.exs @@ -0,0 +1,13 @@ +use Mix.Config + +config :sql_dust, TestRepo,[ + adapter: Ecto.Adapters.MySQL, + database: "sql_dust_test", + username: "root", + password: "", + hostname: "localhost", + pool: Ecto.Adapters.SQL.Sandbox +] + +config :sql_dust, ecto_repos: [TestRepo] +config :logger, level: :info diff --git a/lib/sql_dust/utils/query_utils.ex b/lib/sql_dust/utils/query_utils.ex index 44a0e90..cf34969 100644 --- a/lib/sql_dust/utils/query_utils.ex +++ b/lib/sql_dust/utils/query_utils.ex @@ -21,6 +21,44 @@ defmodule SqlDust.QueryUtils do SqlDust.from(from, options, schema || %{}) end + def to_lists(options, repo) do + %{columns: columns, rows: rows} = query(options, repo) + + case length(columns) do + 1 -> List.flatten(rows) + _ -> rows + end + end + + def to_maps(options, repo) do + %{columns: columns, rows: rows} = query(options, repo) + + rows + |> Enum.map(fn(row) -> + columns + |> Enum.zip(row) + |> Enum.into(%{}) + end) + end + + defp query(options, repo) do + repo_adapter = + repo.config[:adapter] + |> Module.split() + |> Enum.at(-1) + |> String.downcase() + |> String.to_atom() + + [sql, vars] = + options + |> adapter(repo_adapter) + |> to_sql() + |> Tuple.to_list + |> Enum.take(2) + + Ecto.Adapters.SQL.query!(repo, sql, vars) + end + end end end diff --git a/mix.exs b/mix.exs index 1e1b408..152c16d 100644 --- a/mix.exs +++ b/mix.exs @@ -9,7 +9,9 @@ defmodule SqlDust.Mixfile do start_permanent: Mix.env == :prod, description: description(), package: package(), - deps: deps()] + deps: deps(), + elixirc_paths: elixirc_paths(Mix.env), + aliases: aliases()] end def application do @@ -25,7 +27,9 @@ defmodule SqlDust.Mixfile do {:earmark, "~> 0.1", only: :dev}, {:ex_doc, "~> 0.11", only: :dev}, {:credo, "~> 0.2", only: [:dev, :test]}, - {:inch_ex, ">= 0.0.0", only: :docs} + {:inch_ex, ">= 0.0.0", only: :docs}, + {:db_connection, ">= 0.0.0", only: :test}, + {:mariaex, ">= 0.0.0", only: :test} ] end @@ -42,4 +46,12 @@ defmodule SqlDust.Mixfile do links: %{github: "https://github.com/bettyblocks/sql_dust"} ] end + + defp elixirc_paths(:test), do: ~w(lib test/support) + defp elixirc_paths(_), do: ~w(lib) + + defp aliases do + ["test": ["ecto.create --quiet", "ecto.migrate --quiet", "test"]] + end + end diff --git a/mix.lock b/mix.lock index 310b1d3..81e6f61 100644 --- a/mix.lock +++ b/mix.lock @@ -1,6 +1,8 @@ %{"benchfella": {:hex, :benchfella, "0.3.4", "41d2c017b361ece5225b5ba2e3b30ae53578c57c6ebc434417b4f1c2c94cf4f3", [:mix], []}, "bunt": {:hex, :bunt, "0.1.5", "c378ea1698232597d3778e4b83234dcea4a60e7c28114b0fe53657a2c0d8885e", [:mix], []}, + "connection": {:hex, :connection, "1.0.4", "a1cae72211f0eef17705aaededacac3eb30e6625b04a6117c1b2db6ace7d5976", [:mix], []}, "credo": {:hex, :credo, "0.3.13", "90d2d2deb9d376bb2a63f81126a320c3920ce65acb1294982ab49a8aacc7d89f", [:mix], [{:bunt, "~> 0.1.4", [hex: :bunt, optional: false]}]}, + "db_connection": {:hex, :db_connection, "1.0.0", "63c03e520d54886a66104d34e32397ba960db6e74b596ce221592c07d6a40d8d", [:mix], [{:connection, "~> 1.0.2", [hex: :connection, optional: false]}, {:poolboy, "~> 1.5", [hex: :poolboy, optional: true]}, {:sbroker, "~> 1.0", [hex: :sbroker, optional: true]}]}, "decimal": {:hex, :decimal, "1.3.1", "157b3cedb2bfcb5359372a7766dd7a41091ad34578296e951f58a946fcab49c6", [:mix], []}, "earmark": {:hex, :earmark, "0.2.1", "ba6d26ceb16106d069b289df66751734802777a3cbb6787026dd800ffeb850f3", [:mix], []}, "ecto": {:hex, :ecto, "2.0.5", "7f4c79ac41ffba1a4c032b69d7045489f0069c256de606523c65d9f8188e502d", [:mix], [{:db_connection, "~> 1.0-rc.4", [hex: :db_connection, optional: true]}, {:decimal, "~> 1.1.2 or ~> 1.2", [hex: :decimal, optional: false]}, {:mariaex, "~> 0.7.7", [hex: :mariaex, optional: true]}, {:poison, "~> 1.5 or ~> 2.0", [hex: :poison, optional: true]}, {:poolboy, "~> 1.5", [hex: :poolboy, optional: false]}, {:postgrex, "~> 0.12.0", [hex: :postgrex, optional: true]}, {:sbroker, "~> 1.0-beta", [hex: :sbroker, optional: true]}]}, @@ -9,5 +11,6 @@ "exprof": {:hex, :exprof, "0.2.0", "b03f50d0d33e2f18c8e047d9188ba765dc32daba0b553ed717a98a78561d5eaf", [:mix], [{:exprintf, "~> 0.1", [hex: :exprintf, optional: false]}]}, "inch_ex": {:hex, :inch_ex, "0.5.1", "c1c18966c935944cbb2d273796b36e44fab3c54fd59f906ff026a686205b4e14", [:mix], [{:poison, "~> 1.5 or ~> 2.0", [hex: :poison, optional: false]}]}, "inflex": {:hex, :inflex, "1.6.0", "2c0c9928cd865e3b47cfaaa15c3520dfdb8c6ba5829d036296deb766f2aba770", [:mix], []}, + "mariaex": {:hex, :mariaex, "0.7.9", "52f837cf1b0717f95a0e62624bb99707329cba599885cf3bd2fdecc0172a98ad", [:mix], [{:db_connection, "~> 1.0.0-rc", [hex: :db_connection, optional: false]}, {:decimal, "~> 1.0", [hex: :decimal, optional: false]}]}, "poison": {:hex, :poison, "2.2.0", "4763b69a8a77bd77d26f477d196428b741261a761257ff1cf92753a0d4d24a63", [:mix], []}, "poolboy": {:hex, :poolboy, "1.5.1", "6b46163901cfd0a1b43d692657ed9d7e599853b3b21b95ae5ae0a777cf9b6ca8", [:rebar], []}} diff --git a/priv/test_repo/migrations/20170502224942_create_cities.exs b/priv/test_repo/migrations/20170502224942_create_cities.exs new file mode 100644 index 0000000..7eb088f --- /dev/null +++ b/priv/test_repo/migrations/20170502224942_create_cities.exs @@ -0,0 +1,9 @@ +defmodule TestRepo.Migrations.CreateCities do + use Ecto.Migration + + def change do + create table(:cities) do + add :name, :string + end + end +end diff --git a/test/ecto/sql_dust_test.exs b/test/ecto/sql_dust_test.exs index 0818a5a..c7fad35 100644 --- a/test/ecto/sql_dust_test.exs +++ b/test/ecto/sql_dust_test.exs @@ -9,6 +9,7 @@ defmodule Test do defmodule City do use Ecto.Schema schema "cities" do + field :name, :string has_many :local_weather, Test.Weather belongs_to :country, Test.Country end @@ -24,7 +25,7 @@ defmodule Test do end defmodule Ecto.SqlDustTest do - use ExUnit.Case + use SqlDust.ExUnit.Case doctest Ecto.SqlDust import Ecto.SqlDust @@ -88,4 +89,39 @@ defmodule Ecto.SqlDustTest do WHERE (`local_weather`.`wdate` = ?) """, ["2015-09-12"]} end + + describe "querying data" do + test ".to_lists()" do + assert [ + [1, "Amsterdam"], + [2, "New York"], + [3, "Barcelona"], + [4, "London"] + ] == Test.City |> to_lists(TestRepo) + + assert [ + "New York", + "London", + "Barcelona", + "Amsterdam" + ] == Test.City |> select(:name) |> order_by("name DESC") |> to_lists(TestRepo) + end + + test ".to_maps()" do + assert [ + %{"id" => 1, "name" => "Amsterdam"}, + %{"id" => 2, "name" => "New York"}, + %{"id" => 3, "name" => "Barcelona"}, + %{"id" => 4, "name" => "London"} + ] == Test.City |> to_maps(TestRepo) + + assert [ + %{"name" => "New York"}, + %{"name" => "London"}, + %{"name" => "Barcelona"}, + %{"name" => "Amsterdam"} + ] == Test.City |> select(:name) |> order_by("name DESC") |> to_maps(TestRepo) + end + end + end diff --git a/test/sql_dust/query_test.exs b/test/sql_dust/query_test.exs index ba0b4f4..b08174c 100644 --- a/test/sql_dust/query_test.exs +++ b/test/sql_dust/query_test.exs @@ -1,5 +1,6 @@ defmodule SqlDust.QueryTest do - use ExUnit.Case + use SqlDust.ExUnit.Case + doctest SqlDust.Query import SqlDust.Query @@ -454,4 +455,38 @@ defmodule SqlDust.QueryTest do """, [1, 100, "%Engel%", 20]} end + + describe "querying data" do + test ".to_lists()" do + assert [ + [1, "Amsterdam"], + [2, "New York"], + [3, "Barcelona"], + [4, "London"] + ] == "cities" |> to_lists(TestRepo) + + assert [ + "New York", + "London", + "Barcelona", + "Amsterdam" + ] == "cities" |> select(:name) |> order_by("name DESC") |> to_lists(TestRepo) + end + + test ".to_maps()" do + assert [ + %{"id" => 1, "name" => "Amsterdam"}, + %{"id" => 2, "name" => "New York"}, + %{"id" => 3, "name" => "Barcelona"}, + %{"id" => 4, "name" => "London"} + ] == "cities" |> to_maps(TestRepo) + + assert [ + %{"name" => "New York"}, + %{"name" => "London"}, + %{"name" => "Barcelona"}, + %{"name" => "Amsterdam"} + ] == "cities" |> select(:name) |> order_by("name DESC") |> to_maps(TestRepo) + end + end end diff --git a/test/support/test_repo.ex b/test/support/test_repo.ex new file mode 100644 index 0000000..847c61d --- /dev/null +++ b/test/support/test_repo.ex @@ -0,0 +1,3 @@ +defmodule TestRepo do + use Ecto.Repo, otp_app: :sql_dust +end diff --git a/test/test_helper.exs b/test/test_helper.exs index 869559e..52c5bf1 100644 --- a/test/test_helper.exs +++ b/test/test_helper.exs @@ -1 +1,21 @@ ExUnit.start() + +defmodule SqlDust.ExUnit.Case do + use ExUnit.CaseTemplate + + setup do + :ok = Ecto.Adapters.SQL.Sandbox.checkout(TestRepo) + :ok = Ecto.Adapters.SQL.Sandbox.mode(TestRepo, {:shared, self()}) + + Ecto.Adapters.SQL.query!(TestRepo, "TRUNCATE `cities`", []) + Ecto.Adapters.SQL.query!(TestRepo, "INSERT INTO `cities` (`id`, `name`) VALUES (1, 'Amsterdam')", []) + Ecto.Adapters.SQL.query!(TestRepo, "INSERT INTO `cities` (`id`, `name`) VALUES (2, 'New York')", []) + Ecto.Adapters.SQL.query!(TestRepo, "INSERT INTO `cities` (`id`, `name`) VALUES (3, 'Barcelona')", []) + Ecto.Adapters.SQL.query!(TestRepo, "INSERT INTO `cities` (`id`, `name`) VALUES (4, 'London')", []) + + :ok + end +end + +{:ok, _pid} = TestRepo.start_link() +Ecto.Adapters.SQL.Sandbox.mode(TestRepo, {:shared, self()}) From d4957b6a6b9fb506b1cf24c75626a8fc780840da Mon Sep 17 00:00:00 2001 From: Paul Engel Date: Wed, 3 May 2017 13:19:44 +0200 Subject: [PATCH 3/3] Being able to query and load Ecto schema structs (w00t!) --- lib/ecto/sql_dust.ex | 27 ++++++++++++++++++++++----- test/ecto/sql_dust_test.exs | 16 ++++++++++++++++ 2 files changed, 38 insertions(+), 5 deletions(-) diff --git a/lib/ecto/sql_dust.ex b/lib/ecto/sql_dust.ex index e4aa401..74d7a5d 100644 --- a/lib/ecto/sql_dust.ex +++ b/lib/ecto/sql_dust.ex @@ -1,14 +1,31 @@ defmodule Ecto.SqlDust do use SqlDust.QueryUtils - def from(options, model) do - resource = model.__schema__(:source) - derived_schema = derive_schema(model) + def from(options, struct) do + source = struct.__schema__(:source) + derived_schema = derive_schema(struct) options - |> __from__(resource) - |> adapter(:postgres) + |> __from__(source) |> schema(derived_schema) + |> ecto_schema(struct) + |> adapter(:postgres) + end + + def ecto_schema(options, arg) do + put(options, :ecto_schema, arg) + end + + def to_structs(options, repo) do + %{columns: columns, rows: rows} = query(options, repo) + + columns = columns |> Enum.map(&String.to_atom/1) + ecto_schema = parse(options).ecto_schema + + Enum.map(rows, fn(row) -> + Ecto.Schema.__load__(ecto_schema, nil, nil, nil, {columns, row}, + &Ecto.Type.adapter_load(repo.config[:adapter], &1, &2)) + end) end defp parse(options) do diff --git a/test/ecto/sql_dust_test.exs b/test/ecto/sql_dust_test.exs index c7fad35..1aa0684 100644 --- a/test/ecto/sql_dust_test.exs +++ b/test/ecto/sql_dust_test.exs @@ -122,6 +122,22 @@ defmodule Ecto.SqlDustTest do %{"name" => "Amsterdam"} ] == Test.City |> select(:name) |> order_by("name DESC") |> to_maps(TestRepo) end + + test ".to_structs()" do + assert [ + %Test.City{id: 1, name: "Amsterdam"}, + %Test.City{id: 2, name: "New York"}, + %Test.City{id: 3, name: "Barcelona"}, + %Test.City{id: 4, name: "London"} + ] = Test.City |> to_structs(TestRepo) + + assert [ + %Test.City{id: 2, name: "New York"}, + %Test.City{id: 4, name: "London"}, + %Test.City{id: 3, name: "Barcelona"}, + %Test.City{id: 1, name: "Amsterdam"} + ] = Test.City |> order_by("name DESC") |> to_structs(TestRepo) + end end end