From bd64089c83cd0f847ef11238e80b8e32eb0c0f67 Mon Sep 17 00:00:00 2001 From: Jesse Kelly Date: Mon, 20 May 2024 17:26:32 -0500 Subject: [PATCH 1/5] Upgrade all dependencies --- .tool-versions | 4 ++-- mix.lock | 34 +++++++++++++++++----------------- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/.tool-versions b/.tool-versions index bb91489..8b0d80d 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1,2 +1,2 @@ -erlang 24.1.6 -elixir 1.13.4-otp-24 +erlang 25.3.2.8 +elixir 1.16.1-otp-25 diff --git a/mix.lock b/mix.lock index 1a1d7b4..48ee5c2 100644 --- a/mix.lock +++ b/mix.lock @@ -1,30 +1,30 @@ %{ - "certifi": {:hex, :certifi, "2.6.1", "dbab8e5e155a0763eea978c913ca280a6b544bfa115633fa20249c3d396d9493", [:rebar3], [], "hexpm", "524c97b4991b3849dd5c17a631223896272c6b0af446778ba4675a1dff53bb7e"}, - "decimal": {:hex, :decimal, "2.0.0", "a78296e617b0f5dd4c6caf57c714431347912ffb1d0842e998e9792b5642d697", [:mix], [], "hexpm", "34666e9c55dea81013e77d9d87370fe6cb6291d1ef32f46a1600230b1d44f577"}, + "certifi": {:hex, :certifi, "2.12.0", "2d1cca2ec95f59643862af91f001478c9863c2ac9cb6e2f89780bfd8de987329", [:rebar3], [], "hexpm", "ee68d85df22e554040cdb4be100f33873ac6051387baf6a8f6ce82272340ff1c"}, + "decimal": {:hex, :decimal, "2.1.1", "5611dca5d4b2c3dd497dec8f68751f1f1a54755e8ed2a966c2633cf885973ad6", [:mix], [], "hexpm", "53cfe5f497ed0e7771ae1a475575603d77425099ba5faef9394932b35020ffcc"}, "decorator": {:hex, :decorator, "1.4.0", "a57ac32c823ea7e4e67f5af56412d12b33274661bb7640ec7fc882f8d23ac419", [:mix], [], "hexpm", "0a07cedd9083da875c7418dea95b78361197cf2bf3211d743f6f7ce39656597f"}, - "earmark": {:hex, :earmark, "1.2.6", "b6da42b3831458d3ecc57314dff3051b080b9b2be88c2e5aa41cd642a5b044ed", [:mix], [], "hexpm", "b42a23e9bd92d65d16db2f75553982e58519054095356a418bb8320bbacb58b1"}, + "earmark": {:hex, :earmark, "1.4.46", "8c7287bd3137e99d26ae4643e5b7ef2129a260e3dcf41f251750cb4563c8fb81", [:mix], [], "hexpm", "798d86db3d79964e759ddc0c077d5eb254968ed426399fbf5a62de2b5ff8910a"}, "earmark_parser": {:hex, :earmark_parser, "1.4.13", "0c98163e7d04a15feb62000e1a891489feb29f3d10cb57d4f845c405852bbef8", [:mix], [], "hexpm", "d602c26af3a0af43d2f2645613f65841657ad6efc9f0e361c3b6c06b578214ba"}, - "ecto": {:hex, :ecto, "3.6.2", "efdf52acfc4ce29249bab5417415bd50abd62db7b0603b8bab0d7b996548c2bc", [:mix], [{:decimal, "~> 1.6 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "efad6dfb04e6f986b8a3047822b0f826d9affe8e4ebdd2aeedbfcb14fd48884e"}, + "ecto": {:hex, :ecto, "3.11.2", "e1d26be989db350a633667c5cda9c3d115ae779b66da567c68c80cfb26a8c9ee", [:mix], [{:decimal, "~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "3c38bca2c6f8d8023f2145326cc8a80100c3ffe4dcbd9842ff867f7fc6156c65"}, "elastix": {:hex, :elastix, "0.10.0", "7567da885677ba9deffc20063db5f3ca8cd10f23cff1ab3ed9c52b7063b7e340", [:mix], [{:httpoison, "~> 1.4", [hex: :httpoison, repo: "hexpm", optional: false]}, {:poison, "~> 3.0 or ~> 4.0", [hex: :poison, repo: "hexpm", optional: true]}, {:retry, "~> 0.8", [hex: :retry, repo: "hexpm", optional: false]}], "hexpm", "5fb342ce068b20f7845f5dd198c2dc80d967deafaa940a6e51b846db82696d1d"}, "ex_doc": {:hex, :ex_doc, "0.19.3", "3c7b0f02851f5fc13b040e8e925051452e41248f685e40250d7e40b07b9f8c10", [:mix], [{:earmark, "~> 1.2", [hex: :earmark, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.10", [hex: :makeup_elixir, repo: "hexpm", optional: false]}], "hexpm", "0e11d67e662142fc3945b0ee410c73c8c956717fbeae4ad954b418747c734973"}, - "flow": {:hex, :flow, "1.1.0", "b569c1042cb2da97103f6d70a0267a5657dce1402f41b4020bef98bbef9c7c1e", [:mix], [{:gen_stage, "~> 1.0", [hex: :gen_stage, repo: "hexpm", optional: false]}], "hexpm", "066f42f7a1ea6a86cb4ef763310338981a5cfb93bcebce10863a23a4859fd785"}, - "gen_stage": {:hex, :gen_stage, "1.1.0", "dd0c0f8d2f3b993fdbd3d58e94abbe65380f4e78bdee3fa93d5618d7d14abe60", [:mix], [], "hexpm", "7f2b36a6d02f7ef2ba410733b540ec423af65ec9c99f3d1083da508aca3b9305"}, - "hackney": {:hex, :hackney, "1.17.4", "99da4674592504d3fb0cfef0db84c3ba02b4508bae2dff8c0108baa0d6e0977c", [:rebar3], [{:certifi, "~>2.6.1", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "~>6.1.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "~>1.0.0", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~>1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:parse_trans, "3.3.1", [hex: :parse_trans, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "~>1.1.0", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}, {:unicode_util_compat, "~>0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "de16ff4996556c8548d512f4dbe22dd58a587bf3332e7fd362430a7ef3986b16"}, - "httpoison": {:hex, :httpoison, "1.8.0", "6b85dea15820b7804ef607ff78406ab449dd78bed923a49c7160e1886e987a3d", [:mix], [{:hackney, "~> 1.17", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "28089eaa98cf90c66265b6b5ad87c59a3729bea2e74e9d08f9b51eb9729b3c3a"}, - "idna": {:hex, :idna, "6.1.1", "8a63070e9f7d0c62eb9d9fcb360a7de382448200fbbd1b106cc96d3d8099df8d", [:rebar3], [{:unicode_util_compat, "~>0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "92376eb7894412ed19ac475e4a86f7b413c1b9fbb5bd16dccd57934157944cea"}, - "makeup": {:hex, :makeup, "1.0.5", "d5a830bc42c9800ce07dd97fa94669dfb93d3bf5fcf6ea7a0c67b2e0e4a7f26c", [:mix], [{:nimble_parsec, "~> 0.5 or ~> 1.0", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "cfa158c02d3f5c0c665d0af11512fed3fba0144cf1aadee0f2ce17747fba2ca9"}, - "makeup_elixir": {:hex, :makeup_elixir, "0.15.1", "b5888c880d17d1cc3e598f05cdb5b5a91b7b17ac4eaf5f297cb697663a1094dd", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.1", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "db68c173234b07ab2a07f645a5acdc117b9f99d69ebf521821d89690ae6c6ec8"}, + "flow": {:hex, :flow, "1.2.4", "1dd58918287eb286656008777cb32714b5123d3855956f29aa141ebae456922d", [:mix], [{:gen_stage, "~> 1.0", [hex: :gen_stage, repo: "hexpm", optional: false]}], "hexpm", "874adde96368e71870f3510b91e35bc31652291858c86c0e75359cbdd35eb211"}, + "gen_stage": {:hex, :gen_stage, "1.2.1", "19d8b5e9a5996d813b8245338a28246307fd8b9c99d1237de199d21efc4c76a1", [:mix], [], "hexpm", "83e8be657fa05b992ffa6ac1e3af6d57aa50aace8f691fcf696ff02f8335b001"}, + "hackney": {:hex, :hackney, "1.20.1", "8d97aec62ddddd757d128bfd1df6c5861093419f8f7a4223823537bad5d064e2", [:rebar3], [{:certifi, "~> 2.12.0", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "~> 6.1.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "~> 1.0.0", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~> 1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:parse_trans, "3.4.1", [hex: :parse_trans, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "~> 1.1.0", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}, {:unicode_util_compat, "~> 0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "fe9094e5f1a2a2c0a7d10918fee36bfec0ec2a979994cff8cfe8058cd9af38e3"}, + "httpoison": {:hex, :httpoison, "1.8.2", "9eb9c63ae289296a544842ef816a85d881d4a31f518a0fec089aaa744beae290", [:mix], [{:hackney, "~> 1.17", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "2bb350d26972e30c96e2ca74a1aaf8293d61d0742ff17f01e0279fef11599921"}, + "idna": {:hex, :idna, "6.1.1", "8a63070e9f7d0c62eb9d9fcb360a7de382448200fbbd1b106cc96d3d8099df8d", [:rebar3], [{:unicode_util_compat, "~> 0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "92376eb7894412ed19ac475e4a86f7b413c1b9fbb5bd16dccd57934157944cea"}, + "makeup": {:hex, :makeup, "1.1.2", "9ba8837913bdf757787e71c1581c21f9d2455f4dd04cfca785c70bbfff1a76a3", [:mix], [{:nimble_parsec, "~> 1.2.2 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "cce1566b81fbcbd21eca8ffe808f33b221f9eee2cbc7a1706fc3da9ff18e6cac"}, + "makeup_elixir": {:hex, :makeup_elixir, "0.16.2", "627e84b8e8bf22e60a2579dad15067c755531fea049ae26ef1020cad58fe9578", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.2.3 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "41193978704763f6bbe6cc2758b84909e62984c7752b3784bd3c218bb341706b"}, "makeup_erlang": {:hex, :makeup_erlang, "0.1.1", "3fcb7f09eb9d98dc4d208f49cc955a34218fc41ff6b84df7c75b3e6e533cc65f", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "174d0809e98a4ef0b3309256cbf97101c6ec01c4ab0b23e926a9e17df2077cbb"}, "metrics": {:hex, :metrics, "1.0.1", "25f094dea2cda98213cecc3aeff09e940299d950904393b2a29d191c346a8486", [:rebar3], [], "hexpm", "69b09adddc4f74a40716ae54d140f93beb0fb8978d8636eaded0c31b6f099f16"}, - "mimerl": {:hex, :mimerl, "1.2.0", "67e2d3f571088d5cfd3e550c383094b47159f3eee8ffa08e64106cdf5e981be3", [:rebar3], [], "hexpm", "f278585650aa581986264638ebf698f8bb19df297f66ad91b18910dfc6e19323"}, - "nimble_parsec": {:hex, :nimble_parsec, "1.1.0", "3a6fca1550363552e54c216debb6a9e95bd8d32348938e13de5eda962c0d7f89", [:mix], [], "hexpm", "08eb32d66b706e913ff748f11694b17981c0b04a33ef470e33e11b3d3ac8f54b"}, - "parse_trans": {:hex, :parse_trans, "3.3.1", "16328ab840cc09919bd10dab29e431da3af9e9e7e7e6f0089dd5a2d2820011d8", [:rebar3], [], "hexpm", "07cd9577885f56362d414e8c4c4e6bdf10d43a8767abb92d24cbe8b24c54888b"}, + "mimerl": {:hex, :mimerl, "1.3.0", "d0cd9fc04b9061f82490f6581e0128379830e78535e017f7780f37fea7545726", [:rebar3], [], "hexpm", "a1e15a50d1887217de95f0b9b0793e32853f7c258a5cd227650889b38839fe9d"}, + "nimble_parsec": {:hex, :nimble_parsec, "1.4.0", "51f9b613ea62cfa97b25ccc2c1b4216e81df970acd8e16e8d1bdc58fef21370d", [:mix], [], "hexpm", "9c565862810fb383e9838c1dd2d7d2c437b3d13b267414ba6af33e50d2d1cf28"}, + "parse_trans": {:hex, :parse_trans, "3.4.1", "6e6aa8167cb44cc8f39441d05193be6e6f4e7c2946cb2759f015f8c56b76e5ff", [:rebar3], [], "hexpm", "620a406ce75dada827b82e453c19cf06776be266f5a67cff34e1ef2cbb60e49a"}, "poison": {:hex, :poison, "3.1.0", "d9eb636610e096f86f25d9a46f35a9facac35609a7591b3be3326e99a0484665", [:mix], [], "hexpm", "fec8660eb7733ee4117b85f55799fd3833eb769a6df71ccf8903e8dc5447cfce"}, "poolboy": {:hex, :poolboy, "1.5.1", "6b46163901cfd0a1b43d692657ed9d7e599853b3b21b95ae5ae0a777cf9b6ca8", [:rebar], [], "hexpm"}, - "retry": {:hex, :retry, "0.14.1", "722d1b0cf87096b71213f5801d99fface7ca76adc83fc9dbf3e1daee952aef10", [:mix], [], "hexpm", "b3a609f286f6fe4f6b2c15f32cd4a8a60427d78d05d7b68c2dd9110981111ae0"}, + "retry": {:hex, :retry, "0.18.0", "dc58ebe22c95aa00bc2459f9e0c5400e6005541cf8539925af0aa027dc860543", [:mix], [], "hexpm", "9483959cc7bf69c9e576d9dfb2b678b71c045d3e6f39ab7c9aa1489df4492d73"}, "scrivener": {:hex, :scrivener, "2.7.2", "1d913c965ec352650a7f864ad7fd8d80462f76a32f33d57d1e48bc5e9d40aba2", [:mix], [], "hexpm", "7866a0ec4d40274efbee1db8bead13a995ea4926ecd8203345af8f90d2b620d9"}, "scrivener_ecto": {:hex, :scrivener_ecto, "2.7.0", "cf64b8cb8a96cd131cdbcecf64e7fd395e21aaa1cb0236c42a7c2e34b0dca580", [:mix], [{:ecto, "~> 3.3", [hex: :ecto, repo: "hexpm", optional: false]}, {:scrivener, "~> 2.4", [hex: :scrivener, repo: "hexpm", optional: false]}], "hexpm", "e809f171687806b0031129034352f5ae44849720c48dd839200adeaf0ac3e260"}, - "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.6", "cf344f5692c82d2cd7554f5ec8fd961548d4fd09e7d22f5b62482e5aeaebd4b0", [:make, :mix, :rebar3], [], "hexpm", "bdb0d2471f453c88ff3908e7686f86f9be327d065cc1ec16fa4540197ea04680"}, - "telemetry": {:hex, :telemetry, "0.4.3", "a06428a514bdbc63293cd9a6263aad00ddeb66f608163bdec7c8995784080818", [:rebar3], [], "hexpm", "eb72b8365ffda5bed68a620d1da88525e326cb82a75ee61354fc24b844768041"}, + "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.7", "354c321cf377240c7b8716899e182ce4890c5938111a1296add3ec74cf1715df", [:make, :mix, :rebar3], [], "hexpm", "fe4c190e8f37401d30167c8c405eda19469f34577987c76dde613e838bbc67f8"}, + "telemetry": {:hex, :telemetry, "1.2.1", "68fdfe8d8f05a8428483a97d7aab2f268aaff24b49e0f599faa091f1d4e7f61c", [:rebar3], [], "hexpm", "dad9ce9d8effc621708f99eac538ef1cbe05d6a874dd741de2e689c47feafed5"}, "unicode_util_compat": {:hex, :unicode_util_compat, "0.7.0", "bc84380c9ab48177092f43ac89e4dfa2c6d62b40b8bd132b1059ecc7232f9a78", [:rebar3], [], "hexpm", "25eee6d67df61960cf6a794239566599b09e17e668d3700247bc498638152521"}, } From 74ab2d277cacb93fb958e5cf9e176bba0855bceb Mon Sep 17 00:00:00 2001 From: Jesse Kelly Date: Mon, 20 May 2024 17:43:36 -0500 Subject: [PATCH 2/5] Fix failing tests --- test/exlasticsearch/repo_test.exs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/exlasticsearch/repo_test.exs b/test/exlasticsearch/repo_test.exs index 90c289c..470ae4e 100644 --- a/test/exlasticsearch/repo_test.exs +++ b/test/exlasticsearch/repo_test.exs @@ -177,7 +177,7 @@ defmodule ExlasticSearch.RepoTest do aggregation = Aggregation.new() - |> Aggregation.terms(:group, field: :group) + |> Aggregation.terms(:group, field: String.to_atom("group.keyword")) |> Aggregation.nest(:group, nested) {:ok, @@ -214,7 +214,7 @@ defmodule ExlasticSearch.RepoTest do Repo.refresh(TestModel) sources = [ - Aggregation.composite_source(:group, :terms, field: :group, order: :desc), + Aggregation.composite_source(:group, :terms, field: String.to_atom("group.keyword"), order: :desc), Aggregation.composite_source(:age, :terms, field: :age, order: :asc) ] From 980fc7fe299ae2fb44b62f063a7b5dc2588fbac3 Mon Sep 17 00:00:00 2001 From: Jesse Kelly Date: Mon, 20 May 2024 18:10:38 -0500 Subject: [PATCH 3/5] I think it is ES8 compliant now, need to add unit tests --- lib/exlasticsearch/bulk.ex | 34 ++++++++++++++++++++----------- lib/exlasticsearch/model.ex | 6 ++++-- lib/exlasticsearch/repo.ex | 20 ++++++++++-------- test/exlasticsearch/repo_test.exs | 6 ++++++ test/support/test_model.ex | 18 +++++++++++++++- 5 files changed, 61 insertions(+), 23 deletions(-) diff --git a/lib/exlasticsearch/bulk.ex b/lib/exlasticsearch/bulk.ex index b4ea9e4..2f2f43f 100644 --- a/lib/exlasticsearch/bulk.ex +++ b/lib/exlasticsearch/bulk.ex @@ -28,14 +28,18 @@ defmodule ExlasticSearch.BulkOperation do do: bulk_operation_default({op_type, struct, :index}) defp bulk_operation_default({op_type, %{__struct__: model} = struct, index}) do + op = %{ + _id: Indexable.id(struct), + _index: model.__es_index__(index) + } + + op = + if doc_type = model.__doc_type__(), + do: Map.put(op, :_type, doc_type), + else: op + [ - %{ - op_type => %{ - _id: Indexable.id(struct), - _index: model.__es_index__(index), - _type: model.__doc_type__() - } - }, + %{op_type => op}, build_document(struct, index) ] end @@ -48,13 +52,19 @@ defmodule ExlasticSearch.BulkOperation do end defp bulk_operation_delete({:delete, %{__struct__: model} = struct, index}) do + op = %{ + _id: Indexable.id(struct), + _index: model.__es_index__(index) + } + + op = + if doc_type = model.__doc_type__(), + do: Map.put(op, :_type, doc_type), + else: op + [ %{ - delete: %{ - _id: Indexable.id(struct), - _index: model.__es_index__(index), - _type: model.__doc_type__() - } + delete: op } ] end diff --git a/lib/exlasticsearch/model.ex b/lib/exlasticsearch/model.ex index dc7a444..89d09d4 100644 --- a/lib/exlasticsearch/model.ex +++ b/lib/exlasticsearch/model.ex @@ -67,13 +67,15 @@ defmodule ExlasticSearch.Model do * `type` - the indexes type (and index name will be `type <> "s"`) * `block` - the definition of the index """ - defmacro indexes(type, block) do + defmacro indexes(type, opts \\ [], block) do + doc_type = Keyword.get(opts, :doc_type, type) + quote do Module.register_attribute(__MODULE__, :es_mappings, accumulate: true) @read_version :ignore @index_version :ignore - def __doc_type__(), do: unquote(type) + def __doc_type__(), do: unquote(doc_type) unquote(block) diff --git a/lib/exlasticsearch/repo.ex b/lib/exlasticsearch/repo.ex index f8b668a..f3c6091 100644 --- a/lib/exlasticsearch/repo.ex +++ b/lib/exlasticsearch/repo.ex @@ -60,7 +60,7 @@ defmodule ExlasticSearch.Repo do @spec create_mapping(atom) :: response def create_mapping(model, index \\ :index, opts \\ []) do es_url(index) - |> Mapping.put(model.__es_index__(index), model.__doc_type__(), model.__es_mappings__(), opts) + |> Mapping.put(model.__es_index__(index), model.__doc_type__() || "", model.__es_mappings__(), opts) end @doc """ @@ -156,7 +156,7 @@ defmodule ExlasticSearch.Repo do document = build_document(struct, index) es_url(index) - |> Document.index(model.__es_index__(index), model.__doc_type__(), id, document) + |> Document.index(model.__es_index__(index), model.__doc_type__() || "_doc", id, document) |> log_response() |> mark_failure() end @@ -167,7 +167,7 @@ defmodule ExlasticSearch.Repo do @decorate retry() def update(model, id, data, index \\ :index) do es_url(index) - |> Document.update(model.__es_index__(index), model.__doc_type__(), id, data) + |> Document.update(model.__es_index__(index), model.__doc_type__() || "_update", id, data) |> log_response() |> mark_failure() end @@ -215,7 +215,7 @@ defmodule ExlasticSearch.Repo do @spec get(struct) ::{:ok, %Response.Record{}} | {:error, any} def get(%{__struct__: model} = struct, index_type \\ :read) do es_url(index_type) - |> Document.get(model.__es_index__(index_type), model.__doc_type__(), Indexable.id(struct)) + |> Document.get(model.__es_index__(index_type), model.__doc_type__() || "_doc", Indexable.id(struct)) |> log_response() |> decode(Response.Record, model) end @@ -250,9 +250,11 @@ defmodule ExlasticSearch.Repo do defp model_to_doc_types(models) when is_list(models) do models - |> Enum.map(& &1.__doc_type__()) + |> Enum.flat_map(&model_to_doc_types/1) + end + defp model_to_doc_types(model) do + if doc_type = model.__doc_type__(), do: [doc_type], else: [] end - defp model_to_doc_types(model), do: [model.__doc_type__()] @doc """ Performs an aggregation against a query, and returns only the aggregation results. @@ -264,8 +266,10 @@ defmodule ExlasticSearch.Repo do index_type = query.index_type || :read + doc_types = if doc_type = model.__doc_type__(), do: [doc_type], else: [] + es_url(index_type) - |> Search.search(model.__es_index__(index_type), [model.__doc_type__()], search, size: 0) + |> Search.search(model.__es_index__(index_type), doc_types, search, size: 0) # TODO: figure out how to decode these, it's not trivial to type them |> log_response() end @@ -277,7 +281,7 @@ defmodule ExlasticSearch.Repo do @decorate retry() def delete(%{__struct__: model} = struct, index \\ :index) do es_url(index) - |> Document.delete(model.__es_index__(index), model.__doc_type__(), Indexable.id(struct)) + |> Document.delete(model.__es_index__(index), model.__doc_type__() || "_doc", Indexable.id(struct)) |> log_response() |> mark_failure() end diff --git a/test/exlasticsearch/repo_test.exs b/test/exlasticsearch/repo_test.exs index 470ae4e..09459a9 100644 --- a/test/exlasticsearch/repo_test.exs +++ b/test/exlasticsearch/repo_test.exs @@ -5,6 +5,7 @@ defmodule ExlasticSearch.RepoTest do Repo, TestModel, TestModel2, + TypelessTestModel, Aggregation, Query } @@ -27,6 +28,11 @@ defmodule ExlasticSearch.RepoTest do Repo.delete_index(MVTestModel, :read) Repo.create_index(MVTestModel, :read) Repo.create_mapping(MVTestModel, :read) + + Repo.delete_index(TypelessTestModel) + Repo.create_index(TypelessTestModel) + Repo.create_mapping(TypelessTestModel) + :ok end diff --git a/test/support/test_model.ex b/test/support/test_model.ex index 561f6b7..6b3506b 100644 --- a/test/support/test_model.ex +++ b/test/support/test_model.ex @@ -74,8 +74,24 @@ defmodule ExlasticSearch.MultiVersionTestModel do end end +defmodule ExlasticSearch.TypelessTestModel do + use Ecto.Schema + use ExlasticSearch.Model + + schema "typeless_test_model" do + field(:name, :string) + end + + indexes :typeless_test_model, doc_type: nil do + versions(2) + settings(%{}) + options(%{dynamic: :strict}) + mapping(:name) + end +end + defimpl ExlasticSearch.Indexable, - for: [ExlasticSearch.TestModel, ExlasticSearch.TestModel2, ExlasticSearch.MultiVersionTestModel] do + for: [ExlasticSearch.TestModel, ExlasticSearch.TestModel2, ExlasticSearch.MultiVersionTestModel, ExlasticSearch.TypelessTestModel] do def id(%{id: id}), do: id def document(struct) do From db24793de4b843dd2db4cd3b13b7b828fb584c23 Mon Sep 17 00:00:00 2001 From: Jesse Kelly Date: Tue, 21 May 2024 15:20:34 -0500 Subject: [PATCH 4/5] Added unit tests for es 8+ --- README.md | 12 +- config/config.exs | 7 +- config/dev.exs | 1 - config/prod.exs | 1 - config/test.exs | 14 +- docker-compose.yaml | 24 ++++ elasticsearch.yml | 4 + lib/exlasticsearch.ex | 2 +- lib/exlasticsearch/model.ex | 2 +- lib/exlasticsearch/repo.ex | 6 +- lib/exlasticsearch/retry/decorator.ex | 4 +- test/exlasticsearch/repo_test.exs | 196 +++++++++++++++++++++++++- test/support/test_model.ex | 48 ++++++- 13 files changed, 297 insertions(+), 24 deletions(-) create mode 100644 docker-compose.yaml create mode 100644 elasticsearch.yml diff --git a/README.md b/README.md index e2cedcd..6e358a1 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ defmodule MySchema do use ExlasticSearch.Model indexes :my_index do - settings Application.get_env(:some, :settings) + settings Application.compile_env(:some, :settings) mapping :field mapping :other_field, type: :keyword # ecto derived defaults can be overridden @@ -61,3 +61,13 @@ config :exlasticsearch, :monitoring, ExlasticSearch.Monitoring.Mock config :exlasticsearch, ExlasticSearch.Repo, url: "http://localhost:9200" ``` + +## Testing + +Run integration tests with local ElasticSearch clusters. +Ensure Docker resources include at least 8 GB of memory. + +```sh +docker-compose up -d +mix test +``` diff --git a/config/config.exs b/config/config.exs index bcc4754..38a9422 100644 --- a/config/config.exs +++ b/config/config.exs @@ -1,8 +1,5 @@ -use Mix.Config +import Config config :exlasticsearch, :type_inference, ExlasticSearch.TypeInference -config :exlasticsearch, :monitoring, ExlasticSearch.Monitoring.Mock - -config :exlasticsearch, ExlasticSearch.Repo, - url: "http://localhost:9200" \ No newline at end of file +import_config "#{config_env()}.exs" diff --git a/config/dev.exs b/config/dev.exs index a7d258e..e69de29 100644 --- a/config/dev.exs +++ b/config/dev.exs @@ -1 +0,0 @@ -use Mix.Config \ No newline at end of file diff --git a/config/prod.exs b/config/prod.exs index a7d258e..e69de29 100644 --- a/config/prod.exs +++ b/config/prod.exs @@ -1 +0,0 @@ -use Mix.Config \ No newline at end of file diff --git a/config/test.exs b/config/test.exs index 9671e22..4b30f44 100644 --- a/config/test.exs +++ b/config/test.exs @@ -1,5 +1,15 @@ -use Mix.Config +import Config config :exlasticsearch, :type_mappings, [ {DB.CustomType, :integer} -] \ No newline at end of file +] + +config :exlasticsearch, :monitoring, ExlasticSearch.Monitoring.Mock + +config :exlasticsearch, ExlasticSearch.Repo, + url: "http://localhost:9200", + es8: "http://localhost:9201" + + +config :exlasticsearch, ExlasticSearch.TypelessTestModel, + index_type: :es8 diff --git a/docker-compose.yaml b/docker-compose.yaml new file mode 100644 index 0000000..2e0e653 --- /dev/null +++ b/docker-compose.yaml @@ -0,0 +1,24 @@ +version: "3.7" +volumes: + elasticsearch-7-data: + elasticsearch-8-data: +services: + elasticsearch-7: + image: docker.elastic.co/elasticsearch/elasticsearch:7.17.16 + restart: always + environment: + discovery.type: single-node + volumes: + - elasticsearch-7-data:/usr/share/elasticsearch/data + ports: + - 9200:9200 + elasticsearch-8: + image: docker.elastic.co/elasticsearch/elasticsearch:8.13.4 + restart: always + environment: + discovery.type: single-node + volumes: + - elasticsearch-8-data:/usr/share/elasticsearch/data + - ./elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml:ro + ports: + - 9201:9200 diff --git a/elasticsearch.yml b/elasticsearch.yml new file mode 100644 index 0000000..f3d1379 --- /dev/null +++ b/elasticsearch.yml @@ -0,0 +1,4 @@ +cluster.name: "docker-cluster" +network.host: 0.0.0.0 +xpack.security.enabled: false +xpack.security.enrollment.enabled: false diff --git a/lib/exlasticsearch.ex b/lib/exlasticsearch.ex index 1842ff2..11dcdd6 100644 --- a/lib/exlasticsearch.ex +++ b/lib/exlasticsearch.ex @@ -9,7 +9,7 @@ defmodule ExlasticSearch do use ExlasticSearch.Model indexes :my_index do - settings Application.get_env(:some, :settings) + settings Application.compile_env(:some, :settings) mapping :field mapping :other_field, type: :keyword # ecto derived defaults can be overridden diff --git a/lib/exlasticsearch/model.ex b/lib/exlasticsearch/model.ex index 89d09d4..7dc3491 100644 --- a/lib/exlasticsearch/model.ex +++ b/lib/exlasticsearch/model.ex @@ -13,7 +13,7 @@ defmodule ExlasticSearch.Model do ``` indexes :my_type do - settings Application.get_env(:some, :settings) + settings Application.compile_env(:some, :settings) mapping :column mapping :other_column, type: :keyword diff --git a/lib/exlasticsearch/repo.ex b/lib/exlasticsearch/repo.ex index f3c6091..d4a9d31 100644 --- a/lib/exlasticsearch/repo.ex +++ b/lib/exlasticsearch/repo.ex @@ -18,7 +18,7 @@ defmodule ExlasticSearch.Repo do @chunk_size 2000 @type response :: {:ok, %HTTPoison.Response{}} | {:error, any} - @log_level Application.get_env(:exlasticsearch, __MODULE__, []) + @log_level Application.compile_env(:exlasticsearch, __MODULE__, []) |> Keyword.get(:log_level, :debug) @doc """ @@ -267,9 +267,10 @@ defmodule ExlasticSearch.Repo do index_type = query.index_type || :read doc_types = if doc_type = model.__doc_type__(), do: [doc_type], else: [] + es_index = model.__es_index__(index_type) es_url(index_type) - |> Search.search(model.__es_index__(index_type), doc_types, search, size: 0) + |> Search.search(es_index, doc_types, search, size: 0) # TODO: figure out how to decode these, it's not trivial to type them |> log_response() end @@ -310,7 +311,6 @@ defmodule ExlasticSearch.Repo do do: struct |> Indexable.preload(index) |> Indexable.document(index) defp es_url(index) do - case Application.get_env(:exlasticsearch, __MODULE__)[index] do nil -> Application.get_env(:exlasticsearch, __MODULE__)[:url] diff --git a/lib/exlasticsearch/retry/decorator.ex b/lib/exlasticsearch/retry/decorator.ex index c48cd2c..aba8cb5 100644 --- a/lib/exlasticsearch/retry/decorator.ex +++ b/lib/exlasticsearch/retry/decorator.ex @@ -7,7 +7,7 @@ defmodule ExlasticSearch.Retry.Decorator do ``` """ use Decorator.Define, retry: 0 - @config Application.get_env(:exlasticsearch, :retry, []) + @config Application.compile_env(:exlasticsearch, :retry, []) def retry(body, _ctx) do {strategy, config} = Keyword.pop(@config, :strategy, ExlasticSearch.Retry.ExponentialBackoff) @@ -15,4 +15,4 @@ defmodule ExlasticSearch.Retry.Decorator do unquote(strategy).retry(fn -> unquote(body) end, unquote(config)) end end -end \ No newline at end of file +end diff --git a/test/exlasticsearch/repo_test.exs b/test/exlasticsearch/repo_test.exs index 09459a9..1beddca 100644 --- a/test/exlasticsearch/repo_test.exs +++ b/test/exlasticsearch/repo_test.exs @@ -11,6 +11,7 @@ defmodule ExlasticSearch.RepoTest do } alias ExlasticSearch.MultiVersionTestModel, as: MVTestModel + alias ExlasticSearch.TypelessMultiVersionTestModel, as: TypelessMVTestModel setup_all do Repo.delete_index(TestModel) @@ -29,9 +30,13 @@ defmodule ExlasticSearch.RepoTest do Repo.create_index(MVTestModel, :read) Repo.create_mapping(MVTestModel, :read) - Repo.delete_index(TypelessTestModel) - Repo.create_index(TypelessTestModel) - Repo.create_mapping(TypelessTestModel) + Repo.delete_index(TypelessTestModel, :es8) + Repo.create_index(TypelessTestModel, :es8) + Repo.create_mapping(TypelessTestModel, :es8) + + Repo.delete_index(TypelessMVTestModel, :es8) + Repo.create_index(TypelessMVTestModel, :es8) + Repo.create_mapping(TypelessMVTestModel, :es8) :ok end @@ -43,6 +48,13 @@ defmodule ExlasticSearch.RepoTest do assert exists?(model) end + + test "It will index an element in es 8+" do + model = %ExlasticSearch.TypelessTestModel{id: Ecto.UUID.generate()} + {:ok, _} = Repo.index(model, :es8) + + assert exists?(model, :es8) + end end describe "#update" do @@ -56,6 +68,16 @@ defmodule ExlasticSearch.RepoTest do {:ok, %{_source: data}} = Repo.get(model) assert data.name == "test edited" end + + test "It will fail to update an element in es 8+" do + id = Ecto.UUID.generate() + model = %ExlasticSearch.TypelessTestModel{id: id, name: "test"} + Repo.index(model, :es8) + + {:ok, response} = Repo.update(ExlasticSearch.TypelessTestModel, id, %{doc: %{name: "test edited"}}, :es8) + + assert %{"error" => _} = response.body + end end describe "#bulk" do @@ -66,6 +88,13 @@ defmodule ExlasticSearch.RepoTest do assert exists?(model) end + test "It will bulk index/delete from es 8+" do + model = %ExlasticSearch.TypelessTestModel{id: Ecto.UUID.generate()} + {:ok, _} = Repo.bulk([{:index, model, :es8}], :es8) + + assert exists?(model, :es8) + end + test "It will bulk update from es" do model1 = %ExlasticSearch.TestModel{id: Ecto.UUID.generate(), name: "test 1"} model2 = %ExlasticSearch.TestModel{id: Ecto.UUID.generate(), name: "test 2"} @@ -132,6 +161,15 @@ defmodule ExlasticSearch.RepoTest do assert exists?(model) end + + test "It can deprecate an old index version on es 8+" do + model = %TypelessMVTestModel{id: Ecto.UUID.generate()} + {:ok, _} = Repo.index(model, :es8) + + Repo.rotate(TypelessMVTestModel, :read, :es8) + + assert exists?(model, :es8) + end end describe "#aggregate/2" do @@ -164,6 +202,35 @@ defmodule ExlasticSearch.RepoTest do assert Enum.all?(buckets, &(&1["key"] in [1, 2])) end + test "It can perform terms aggreagtions on es 8+" do + models = + for i <- 1..3, + do: %TypelessTestModel{id: Ecto.UUID.generate(), name: "name #{i}", age: i} + + {:ok, _} = Enum.map(models, &{:index, &1, :es8}) |> Repo.bulk(:es8) + + Repo.refresh(TypelessTestModel, :es8) + + aggregation = Aggregation.new() |> Aggregation.terms(:age, field: :age, size: 2) + + {:ok, + %{ + body: %{ + "aggregations" => %{ + "age" => %{ + "buckets" => buckets + } + } + } + }} = + TypelessTestModel.search_query() + |> Query.must(Query.match(:name, "name")) + |> Repo.aggregate(aggregation) + + assert length(buckets) == 2 + assert Enum.all?(buckets, &(&1["key"] in [1, 2])) + end + test "It can perform top_hits aggregations, even when nested" do models = for i <- 1..3 do @@ -204,6 +271,46 @@ defmodule ExlasticSearch.RepoTest do assert Enum.all?(buckets, &(!Enum.empty?(get_hits(&1)))) end + test "It can perform top_hits aggregations, even when nested, on es 8+" do + models = + for i <- 1..3 do + %TypelessTestModel{ + id: Ecto.UUID.generate(), + name: "name #{i}", + age: i, + group: if(rem(i, 2) == 0, do: "even", else: "odd") + } + end + + {:ok, _} = Enum.map(models, &{:index, &1, :es8}) |> Repo.bulk(:es8) + + Repo.refresh(TypelessTestModel, :es8) + + nested = Aggregation.new() |> Aggregation.top_hits(:hits, %{}) + + aggregation = + Aggregation.new() + |> Aggregation.terms(:group, field: :group) + |> Aggregation.nest(:group, nested) + + {:ok, + %{ + body: %{ + "aggregations" => %{ + "group" => %{ + "buckets" => buckets + } + } + } + }} = + TypelessTestModel.search_query() + |> Query.must(Query.match(:name, "name")) + |> Repo.aggregate(aggregation) + + assert length(buckets) == 2 + assert Enum.all?(buckets, &(!Enum.empty?(get_hits(&1)))) + end + test "It can perform composite aggregations" do models = for i <- 1..3 do @@ -250,6 +357,53 @@ defmodule ExlasticSearch.RepoTest do end) end end + + test "It can perform composite aggregations on es 8+" do + models = + for i <- 1..3 do + %TypelessTestModel{ + id: Ecto.UUID.generate(), + name: "name #{i}", + age: i, + group: if(rem(i, 2) == 0, do: "even", else: "odd") + } + end + + {:ok, _} = Enum.map(models, &{:index, &1, :es8}) |> Repo.bulk(:es8) + + Repo.refresh(TypelessTestModel, :es8) + + sources = [ + Aggregation.composite_source(:group, :terms, field: :group, order: :desc), + Aggregation.composite_source(:age, :terms, field: :age, order: :asc) + ] + + aggregation = Aggregation.new() |> Aggregation.composite(:group, sources) + + {:ok, + %{ + body: %{ + "aggregations" => %{ + "group" => %{ + "buckets" => buckets + } + } + } + }} = + TypelessTestModel.search_query() + |> Query.must(Query.match(:name, "name")) + |> Repo.aggregate(aggregation) + + for i <- 1..3 do + assert Enum.any?(buckets, fn + %{"key" => %{"age" => ^i, "group" => group}} -> + group == if rem(i, 2) == 0, do: "even", else: "odd" + + _ -> + false + end) + end + end end describe "#search/2" do @@ -284,6 +438,38 @@ defmodule ExlasticSearch.RepoTest do assert Enum.find(results, & &1._id == id2) end + test "It will search in a single index in es 8+" do + id1 = Ecto.UUID.generate() + id2 = Ecto.UUID.generate() + id3 = Ecto.UUID.generate() + + rand_name = Ecto.UUID.generate() |> String.replace("-", "") + + model1 = %TypelessTestModel{id: id1, name: rand_name} + model2 = %TypelessTestModel{id: id2, name: rand_name} + model3 = %TypelessTestModel{id: id3, name: "something else"} + + Repo.index(model1, :es8) + Repo.index(model2, :es8) + Repo.index(model3, :es8) + + Repo.refresh(TypelessTestModel, :es8) + + query = %ExlasticSearch.Query{ + queryable: ExlasticSearch.TypelessTestModel, + filter: [ + %{term: %{name: rand_name}} + ], + index_type: :es8 + } + + {:ok, %{hits: %{hits: results}}} = Repo.search(query, []) + + assert length(results) == 2 + assert Enum.find(results, & &1._id == id1) + assert Enum.find(results, & &1._id == id2) + end + test "It will search in a multiple indexes" do id1 = Ecto.UUID.generate() id2 = Ecto.UUID.generate() @@ -323,8 +509,8 @@ defmodule ExlasticSearch.RepoTest do end end - defp exists?(model) do - case Repo.get(model) do + defp exists?(model, index \\ :index) do + case Repo.get(model, index) do {:ok, %{found: true}} -> true _ -> false end diff --git a/test/support/test_model.ex b/test/support/test_model.ex index 6b3506b..a4b59ce 100644 --- a/test/support/test_model.ex +++ b/test/support/test_model.ex @@ -78,8 +78,11 @@ defmodule ExlasticSearch.TypelessTestModel do use Ecto.Schema use ExlasticSearch.Model - schema "typeless_test_model" do + schema "typeless_test_models" do field(:name, :string) + field(:age, :integer, default: 0) + field(:group, :string) + field(:teams, {:array, :map}) end indexes :typeless_test_model, doc_type: nil do @@ -87,11 +90,52 @@ defmodule ExlasticSearch.TypelessTestModel do settings(%{}) options(%{dynamic: :strict}) mapping(:name) + mapping(:age) + mapping(:group, type: :keyword) + + mapping(:user, properties: %{ext_name: %{type: :text}}) + + mapping(:teams, + type: :nested, + properties: %{ + name: %{type: :keyword}, + rating: %{type: :integer} + } + ) + end +end + +defmodule ExlasticSearch.TypelessMultiVersionTestModel do + use Ecto.Schema + use ExlasticSearch.Model + + schema "typeless_mv_models" do + field(:name, :string) + field(:age, :integer, default: 0) + field(:teams, {:array, :map}) + end + + indexes :typeless_multiversion_model, doc_type: nil do + versions({:ignore, 2}) + settings(%{}) + options(%{dynamic: :strict}) + mapping(:name) + mapping(:age) + + mapping(:user, properties: %{ext_name: %{type: :text}}) + + mapping(:teams, + type: :nested, + properties: %{ + name: %{type: :keyword}, + rating: %{type: :integer} + } + ) end end defimpl ExlasticSearch.Indexable, - for: [ExlasticSearch.TestModel, ExlasticSearch.TestModel2, ExlasticSearch.MultiVersionTestModel, ExlasticSearch.TypelessTestModel] do + for: [ExlasticSearch.TestModel, ExlasticSearch.TestModel2, ExlasticSearch.MultiVersionTestModel, ExlasticSearch.TypelessTestModel, ExlasticSearch.TypelessMultiVersionTestModel] do def id(%{id: id}), do: id def document(struct) do From d2f31c7d1604f79a178e4de59f999db49e89bbc9 Mon Sep 17 00:00:00 2001 From: Jesse Kelly Date: Tue, 21 May 2024 15:42:33 -0500 Subject: [PATCH 5/5] Support update in es8 by actually using index --- lib/exlasticsearch/repo.ex | 19 +++++++++++++++---- test/exlasticsearch/repo_test.exs | 7 ++++--- 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/lib/exlasticsearch/repo.ex b/lib/exlasticsearch/repo.ex index d4a9d31..f16fb01 100644 --- a/lib/exlasticsearch/repo.ex +++ b/lib/exlasticsearch/repo.ex @@ -166,10 +166,21 @@ defmodule ExlasticSearch.Repo do """ @decorate retry() def update(model, id, data, index \\ :index) do - es_url(index) - |> Document.update(model.__es_index__(index), model.__doc_type__() || "_update", id, data) - |> log_response() - |> mark_failure() + cond do + doc_type = model.__doc_type__() -> + es_url(index) + |> Document.update(model.__es_index__(index), doc_type, id, data) + |> log_response() + |> mark_failure() + + %{doc: doc} = data -> + model + |> struct(Map.put(doc, :id, id)) + |> index(index) + + true -> + {:error, "ExlasticSearch only supports doc updates for ES 8+"} + end end @doc """ diff --git a/test/exlasticsearch/repo_test.exs b/test/exlasticsearch/repo_test.exs index 1beddca..de076ba 100644 --- a/test/exlasticsearch/repo_test.exs +++ b/test/exlasticsearch/repo_test.exs @@ -69,14 +69,15 @@ defmodule ExlasticSearch.RepoTest do assert data.name == "test edited" end - test "It will fail to update an element in es 8+" do + test "It will update an element in es 8+" do id = Ecto.UUID.generate() model = %ExlasticSearch.TypelessTestModel{id: id, name: "test"} Repo.index(model, :es8) - {:ok, response} = Repo.update(ExlasticSearch.TypelessTestModel, id, %{doc: %{name: "test edited"}}, :es8) + {:ok, _} = Repo.update(ExlasticSearch.TypelessTestModel, id, %{doc: %{name: "test edited"}}, :es8) - assert %{"error" => _} = response.body + {:ok, %{_source: data}} = Repo.get(model, :es8) + assert data.name == "test edited" end end