From 772a4ede75f4fa598d93de9307ff9ffd5d69838c Mon Sep 17 00:00:00 2001 From: Jesse Kelly Date: Mon, 20 May 2024 17:43:36 -0500 Subject: [PATCH 01/13] Fix failing tests --- test/exlasticsearch/repo_test.exs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/exlasticsearch/repo_test.exs b/test/exlasticsearch/repo_test.exs index 86d557d..0e07141 100644 --- a/test/exlasticsearch/repo_test.exs +++ b/test/exlasticsearch/repo_test.exs @@ -175,7 +175,7 @@ defmodule ExlasticSearch.RepoTest do aggregation = Aggregation.new() - |> Aggregation.terms(:group, field: :group) + |> Aggregation.terms(:group, field: String.to_atom("group.keyword")) |> Aggregation.nest(:group, nested) {:ok, @@ -213,7 +213,7 @@ defmodule ExlasticSearch.RepoTest do Repo.refresh(TestModel) sources = [ - Aggregation.composite_source(:group, :terms, field: :group, order: :desc), + Aggregation.composite_source(:group, :terms, field: String.to_atom("group.keyword"), order: :desc), Aggregation.composite_source(:age, :terms, field: :age, order: :asc) ] From 2fde740795d44c114476ba77ab9d9458dcaf9a40 Mon Sep 17 00:00:00 2001 From: Jesse Kelly Date: Mon, 20 May 2024 18:10:38 -0500 Subject: [PATCH 02/13] I think it is ES8 compliant now, need to add unit tests --- lib/exlasticsearch/bulk.ex | 34 ++++++++++++++++---------- lib/exlasticsearch/model.ex | 6 +++-- lib/exlasticsearch/repo.ex | 40 +++++++++++++++---------------- test/exlasticsearch/repo_test.exs | 15 +++++++++++- test/support/test_model.ex | 18 +++++++++++++- 5 files changed, 76 insertions(+), 37 deletions(-) diff --git a/lib/exlasticsearch/bulk.ex b/lib/exlasticsearch/bulk.ex index 5d0b0d8..bef67e4 100644 --- a/lib/exlasticsearch/bulk.ex +++ b/lib/exlasticsearch/bulk.ex @@ -22,14 +22,18 @@ defmodule ExlasticSearch.BulkOperation do def bulk_operation({op_type, struct}), do: bulk_operation_default({op_type, struct, :index}) defp bulk_operation_default({op_type, %{__struct__: model} = struct, index}) do + op = %{ + _id: Indexable.id(struct), + _index: model.__es_index__(index) + } + + op = + if doc_type = model.__doc_type__(), + do: Map.put(op, :_type, doc_type), + else: op + [ - %{ - op_type => %{ - _id: Indexable.id(struct), - _index: model.__es_index__(index), - _type: model.__doc_type__() - } - }, + %{op_type => op}, build_document(struct, index) ] end @@ -42,13 +46,19 @@ defmodule ExlasticSearch.BulkOperation do end defp bulk_operation_delete({:delete, %{__struct__: model} = struct, index}) do + op = %{ + _id: Indexable.id(struct), + _index: model.__es_index__(index) + } + + op = + if doc_type = model.__doc_type__(), + do: Map.put(op, :_type, doc_type), + else: op + [ %{ - delete: %{ - _id: Indexable.id(struct), - _index: model.__es_index__(index), - _type: model.__doc_type__() - } + delete: op } ] end diff --git a/lib/exlasticsearch/model.ex b/lib/exlasticsearch/model.ex index 56ff44f..d922842 100644 --- a/lib/exlasticsearch/model.ex +++ b/lib/exlasticsearch/model.ex @@ -77,13 +77,15 @@ defmodule ExlasticSearch.Model do * `block` - the definition of the index """ - defmacro indexes(type, block) do + defmacro indexes(type, opts \\ [], block) do + doc_type = Keyword.get(opts, :doc_type, type) + quote do Module.register_attribute(__MODULE__, :es_mappings, accumulate: true) @read_version :ignore @index_version :ignore - def __doc_type__, do: unquote(type) + def __doc_type__(), do: unquote(doc_type) unquote(block) diff --git a/lib/exlasticsearch/repo.ex b/lib/exlasticsearch/repo.ex index f022fb8..771c54f 100644 --- a/lib/exlasticsearch/repo.ex +++ b/lib/exlasticsearch/repo.ex @@ -69,9 +69,8 @@ defmodule ExlasticSearch.Repo do """ @spec create_mapping(atom) :: response def create_mapping(model, index \\ :index, opts \\ []) do - index - |> es_url() - |> Mapping.put(model.__es_index__(index), model.__doc_type__(), model.__es_mappings__(), opts) + es_url(index) + |> Mapping.put(model.__es_index__(index), model.__doc_type__() || "", model.__es_mappings__(), opts) end @doc """ @@ -170,9 +169,8 @@ defmodule ExlasticSearch.Repo do id = Indexable.id(struct) document = build_document(struct, index) - index - |> es_url() - |> Document.index(model.__es_index__(index), model.__doc_type__(), id, document) + es_url(index) + |> Document.index(model.__es_index__(index), model.__doc_type__() || "_doc", id, document) |> log_response() |> mark_failure() end @@ -182,9 +180,8 @@ defmodule ExlasticSearch.Repo do """ @decorate retry() def update(model, id, data, index \\ :index) do - index - |> es_url() - |> Document.update(model.__es_index__(index), model.__doc_type__(), id, data) + es_url(index) + |> Document.update(model.__es_index__(index), model.__doc_type__() || "_update", id, data) |> log_response() |> mark_failure() end @@ -232,9 +229,8 @@ defmodule ExlasticSearch.Repo do """ @spec get(struct) :: {:ok, %Response.Record{}} | {:error, any} def get(%{__struct__: model} = struct, index_type \\ :read) do - index_type - |> es_url() - |> Document.get(model.__es_index__(index_type), model.__doc_type__(), Indexable.id(struct)) + es_url(index_type) + |> Document.get(model.__es_index__(index_type), model.__doc_type__() || "_doc", Indexable.id(struct)) |> log_response() |> decode(Response.Record, model) end @@ -268,10 +264,12 @@ defmodule ExlasticSearch.Repo do defp model_to_index(model, index_type), do: model.__es_index__(index_type) defp model_to_doc_types(models) when is_list(models) do - Enum.map(models, & &1.__doc_type__()) + models + |> Enum.flat_map(&model_to_doc_types/1) + end + defp model_to_doc_types(model) do + if doc_type = model.__doc_type__(), do: [doc_type], else: [] end - - defp model_to_doc_types(model), do: [model.__doc_type__()] @doc """ Performs an aggregation against a query, and returns only the aggregation results. @@ -284,9 +282,10 @@ defmodule ExlasticSearch.Repo do index_type = query.index_type || :read - index_type - |> es_url() - |> Search.search(model.__es_index__(index_type), [model.__doc_type__()], search, size: 0) + doc_types = if doc_type = model.__doc_type__(), do: [doc_type], else: [] + + es_url(index_type) + |> Search.search(model.__es_index__(index_type), doc_types, search, size: 0) # TODO: figure out how to decode these, it's not trivial to type them |> log_response() end @@ -297,9 +296,8 @@ defmodule ExlasticSearch.Repo do @spec delete(struct) :: response @decorate retry() def delete(%{__struct__: model} = struct, index \\ :index) do - index - |> es_url() - |> Document.delete(model.__es_index__(index), model.__doc_type__(), Indexable.id(struct)) + es_url(index) + |> Document.delete(model.__es_index__(index), model.__doc_type__() || "_doc", Indexable.id(struct)) |> log_response() |> mark_failure() end diff --git a/test/exlasticsearch/repo_test.exs b/test/exlasticsearch/repo_test.exs index 0e07141..1eb86d4 100644 --- a/test/exlasticsearch/repo_test.exs +++ b/test/exlasticsearch/repo_test.exs @@ -1,7 +1,15 @@ defmodule ExlasticSearch.RepoTest do use ExUnit.Case, async: true - alias ExlasticSearch.Aggregation + alias ExlasticSearch.{ + Repo, + TestModel, + TestModel2, + TypelessTestModel, + Aggregation, + Query + } + alias ExlasticSearch.MultiVersionTestModel, as: MVTestModel alias ExlasticSearch.Query alias ExlasticSearch.Repo @@ -24,6 +32,11 @@ defmodule ExlasticSearch.RepoTest do Repo.delete_index(MVTestModel, :read) Repo.create_index(MVTestModel, :read) Repo.create_mapping(MVTestModel, :read) + + Repo.delete_index(TypelessTestModel) + Repo.create_index(TypelessTestModel) + Repo.create_mapping(TypelessTestModel) + :ok end diff --git a/test/support/test_model.ex b/test/support/test_model.ex index 3b69a9a..665628c 100644 --- a/test/support/test_model.ex +++ b/test/support/test_model.ex @@ -77,8 +77,24 @@ defmodule ExlasticSearch.MultiVersionTestModel do end end +defmodule ExlasticSearch.TypelessTestModel do + use Ecto.Schema + use ExlasticSearch.Model + + schema "typeless_test_model" do + field(:name, :string) + end + + indexes :typeless_test_model, doc_type: nil do + versions(2) + settings(%{}) + options(%{dynamic: :strict}) + mapping(:name) + end +end + defimpl ExlasticSearch.Indexable, - for: [ExlasticSearch.TestModel, ExlasticSearch.TestModel2, ExlasticSearch.MultiVersionTestModel] do + for: [ExlasticSearch.TestModel, ExlasticSearch.TestModel2, ExlasticSearch.MultiVersionTestModel, ExlasticSearch.TypelessTestModel] do def id(%{id: id}), do: id def document(struct) do From 39883736fbaa07677020ee78a642f15c5a4b596a Mon Sep 17 00:00:00 2001 From: Jesse Kelly Date: Tue, 21 May 2024 15:20:34 -0500 Subject: [PATCH 03/13] Added unit tests for es 8+ --- README.md | 16 ++- config/test.exs | 14 +++ docker-compose.yaml | 24 ++++ elasticsearch.yml | 4 + lib/exlasticsearch.ex | 2 +- lib/exlasticsearch/model.ex | 2 +- lib/exlasticsearch/repo.ex | 3 +- test/exlasticsearch/repo_test.exs | 198 ++++++++++++++++++++++++++++-- test/support/test_model.ex | 48 +++++++- 9 files changed, 296 insertions(+), 15 deletions(-) create mode 100644 docker-compose.yaml create mode 100644 elasticsearch.yml diff --git a/README.md b/README.md index bf8b4ea..97c8f39 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,7 @@ defmodule MySchema do use ExlasticSearch.Model indexes :my_index do - settings Application.get_env(:some, :settings) + settings Application.compile_env(:some, :settings) mapping :field mapping :other_field, type: :keyword # ecto derived defaults can be overridden @@ -64,10 +64,20 @@ This library requires [Elastix](https://hex.pm/packages/elastix), an Elixir Elas config :exlasticsearch, :type_inference, ExlasticSearch.TypeInference config :exlasticsearch, ExlasticSearch.Repo, - url: "http://localhost:9200", - json_library: Jason + url: "http://localhost:9200" ``` +## Testing + +Run integration tests with local ElasticSearch clusters. +Ensure Docker resources include at least 8 GB of memory. + +```sh +docker-compose up -d +mix test +``` + + ## Copyright and License Copyright (c) 2025 Adobe/Frame.io diff --git a/config/test.exs b/config/test.exs index 8d6f83d..71d77f3 100644 --- a/config/test.exs +++ b/config/test.exs @@ -3,3 +3,17 @@ import Config config :elastix, json_codec: Jason config :exlasticsearch, json_library: Jason + +config :exlasticsearch, :type_mappings, [ + {DB.CustomType, :integer} +] + +config :exlasticsearch, :monitoring, ExlasticSearch.Monitoring.Mock + +config :exlasticsearch, ExlasticSearch.Repo, + url: "http://localhost:9200", + es8: "http://localhost:9201" + + +config :exlasticsearch, ExlasticSearch.TypelessTestModel, + index_type: :es8 diff --git a/docker-compose.yaml b/docker-compose.yaml new file mode 100644 index 0000000..2e0e653 --- /dev/null +++ b/docker-compose.yaml @@ -0,0 +1,24 @@ +version: "3.7" +volumes: + elasticsearch-7-data: + elasticsearch-8-data: +services: + elasticsearch-7: + image: docker.elastic.co/elasticsearch/elasticsearch:7.17.16 + restart: always + environment: + discovery.type: single-node + volumes: + - elasticsearch-7-data:/usr/share/elasticsearch/data + ports: + - 9200:9200 + elasticsearch-8: + image: docker.elastic.co/elasticsearch/elasticsearch:8.13.4 + restart: always + environment: + discovery.type: single-node + volumes: + - elasticsearch-8-data:/usr/share/elasticsearch/data + - ./elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml:ro + ports: + - 9201:9200 diff --git a/elasticsearch.yml b/elasticsearch.yml new file mode 100644 index 0000000..f3d1379 --- /dev/null +++ b/elasticsearch.yml @@ -0,0 +1,4 @@ +cluster.name: "docker-cluster" +network.host: 0.0.0.0 +xpack.security.enabled: false +xpack.security.enrollment.enabled: false diff --git a/lib/exlasticsearch.ex b/lib/exlasticsearch.ex index 6b2f834..05de361 100644 --- a/lib/exlasticsearch.ex +++ b/lib/exlasticsearch.ex @@ -9,7 +9,7 @@ defmodule ExlasticSearch do use ExlasticSearch.Model indexes :my_index do - settings Application.get_env(:some, :settings) + settings Application.compile_env(:some, :settings) mapping :field mapping :other_field, type: :keyword # ecto derived defaults can be overridden diff --git a/lib/exlasticsearch/model.ex b/lib/exlasticsearch/model.ex index d922842..0540b19 100644 --- a/lib/exlasticsearch/model.ex +++ b/lib/exlasticsearch/model.ex @@ -13,7 +13,7 @@ defmodule ExlasticSearch.Model do The usage is something like this indexes :my_type do - settings Application.get_env(:some, :settings) + settings Application.compile_env(:some, :settings) mapping :column mapping :other_column, type: :keyword diff --git a/lib/exlasticsearch/repo.ex b/lib/exlasticsearch/repo.ex index 771c54f..b8eb5f5 100644 --- a/lib/exlasticsearch/repo.ex +++ b/lib/exlasticsearch/repo.ex @@ -283,9 +283,10 @@ defmodule ExlasticSearch.Repo do index_type = query.index_type || :read doc_types = if doc_type = model.__doc_type__(), do: [doc_type], else: [] + es_index = model.__es_index__(index_type) es_url(index_type) - |> Search.search(model.__es_index__(index_type), doc_types, search, size: 0) + |> Search.search(es_index, doc_types, search, size: 0) # TODO: figure out how to decode these, it's not trivial to type them |> log_response() end diff --git a/test/exlasticsearch/repo_test.exs b/test/exlasticsearch/repo_test.exs index 1eb86d4..d14fcee 100644 --- a/test/exlasticsearch/repo_test.exs +++ b/test/exlasticsearch/repo_test.exs @@ -15,6 +15,7 @@ defmodule ExlasticSearch.RepoTest do alias ExlasticSearch.Repo alias ExlasticSearch.TestModel alias ExlasticSearch.TestModel2 + alias ExlasticSearch.TypelessMultiVersionTestModel, as: TypelessMVTestModel setup_all do Repo.delete_index(TestModel) @@ -33,9 +34,13 @@ defmodule ExlasticSearch.RepoTest do Repo.create_index(MVTestModel, :read) Repo.create_mapping(MVTestModel, :read) - Repo.delete_index(TypelessTestModel) - Repo.create_index(TypelessTestModel) - Repo.create_mapping(TypelessTestModel) + Repo.delete_index(TypelessTestModel, :es8) + Repo.create_index(TypelessTestModel, :es8) + Repo.create_mapping(TypelessTestModel, :es8) + + Repo.delete_index(TypelessMVTestModel, :es8) + Repo.create_index(TypelessMVTestModel, :es8) + Repo.create_mapping(TypelessMVTestModel, :es8) :ok end @@ -47,6 +52,13 @@ defmodule ExlasticSearch.RepoTest do assert exists?(model) end + + test "It will index an element in es 8+" do + model = %ExlasticSearch.TypelessTestModel{id: Ecto.UUID.generate()} + {:ok, _} = Repo.index(model, :es8) + + assert exists?(model, :es8) + end end describe "#update" do @@ -60,6 +72,16 @@ defmodule ExlasticSearch.RepoTest do {:ok, %{_source: data}} = Repo.get(model) assert data.name == "test edited" end + + test "It will fail to update an element in es 8+" do + id = Ecto.UUID.generate() + model = %ExlasticSearch.TypelessTestModel{id: id, name: "test"} + Repo.index(model, :es8) + + {:ok, response} = Repo.update(ExlasticSearch.TypelessTestModel, id, %{doc: %{name: "test edited"}}, :es8) + + assert %{"error" => _} = response.body + end end describe "#bulk" do @@ -70,6 +92,13 @@ defmodule ExlasticSearch.RepoTest do assert exists?(model) end + test "It will bulk index/delete from es 8+" do + model = %ExlasticSearch.TypelessTestModel{id: Ecto.UUID.generate()} + {:ok, _} = Repo.bulk([{:index, model, :es8}], :es8) + + assert exists?(model, :es8) + end + test "It will bulk update from es" do model1 = %ExlasticSearch.TestModel{id: Ecto.UUID.generate(), name: "test 1"} model2 = %ExlasticSearch.TestModel{id: Ecto.UUID.generate(), name: "test 2"} @@ -136,6 +165,15 @@ defmodule ExlasticSearch.RepoTest do assert exists?(model) end + + test "It can deprecate an old index version on es 8+" do + model = %TypelessMVTestModel{id: Ecto.UUID.generate()} + {:ok, _} = Repo.index(model, :es8) + + Repo.rotate(TypelessMVTestModel, :read, :es8) + + assert exists?(model, :es8) + end end describe "#aggregate/2" do @@ -168,7 +206,35 @@ defmodule ExlasticSearch.RepoTest do assert Enum.all?(buckets, &(&1["key"] in [1, 2])) end - @tag :skip + test "It can perform terms aggreagtions on es 8+" do + models = + for i <- 1..3, + do: %TypelessTestModel{id: Ecto.UUID.generate(), name: "name #{i}", age: i} + + {:ok, _} = Enum.map(models, &{:index, &1, :es8}) |> Repo.bulk(:es8) + + Repo.refresh(TypelessTestModel, :es8) + + aggregation = Aggregation.new() |> Aggregation.terms(:age, field: :age, size: 2) + + {:ok, + %{ + body: %{ + "aggregations" => %{ + "age" => %{ + "buckets" => buckets + } + } + } + }} = + TypelessTestModel.search_query() + |> Query.must(Query.match(:name, "name")) + |> Repo.aggregate(aggregation) + + assert length(buckets) == 2 + assert Enum.all?(buckets, &(&1["key"] in [1, 2])) + end + test "It can perform top_hits aggregations, even when nested" do models = for i <- 1..3 do @@ -209,7 +275,46 @@ defmodule ExlasticSearch.RepoTest do assert Enum.all?(buckets, &(!Enum.empty?(get_hits(&1)))) end - @tag :skip + test "It can perform top_hits aggregations, even when nested, on es 8+" do + models = + for i <- 1..3 do + %TypelessTestModel{ + id: Ecto.UUID.generate(), + name: "name #{i}", + age: i, + group: if(rem(i, 2) == 0, do: "even", else: "odd") + } + end + + {:ok, _} = Enum.map(models, &{:index, &1, :es8}) |> Repo.bulk(:es8) + + Repo.refresh(TypelessTestModel, :es8) + + nested = Aggregation.new() |> Aggregation.top_hits(:hits, %{}) + + aggregation = + Aggregation.new() + |> Aggregation.terms(:group, field: :group) + |> Aggregation.nest(:group, nested) + + {:ok, + %{ + body: %{ + "aggregations" => %{ + "group" => %{ + "buckets" => buckets + } + } + } + }} = + TypelessTestModel.search_query() + |> Query.must(Query.match(:name, "name")) + |> Repo.aggregate(aggregation) + + assert length(buckets) == 2 + assert Enum.all?(buckets, &(!Enum.empty?(get_hits(&1)))) + end + test "It can perform composite aggregations" do models = for i <- 1..3 do @@ -256,6 +361,53 @@ defmodule ExlasticSearch.RepoTest do end) end end + + test "It can perform composite aggregations on es 8+" do + models = + for i <- 1..3 do + %TypelessTestModel{ + id: Ecto.UUID.generate(), + name: "name #{i}", + age: i, + group: if(rem(i, 2) == 0, do: "even", else: "odd") + } + end + + {:ok, _} = Enum.map(models, &{:index, &1, :es8}) |> Repo.bulk(:es8) + + Repo.refresh(TypelessTestModel, :es8) + + sources = [ + Aggregation.composite_source(:group, :terms, field: :group, order: :desc), + Aggregation.composite_source(:age, :terms, field: :age, order: :asc) + ] + + aggregation = Aggregation.new() |> Aggregation.composite(:group, sources) + + {:ok, + %{ + body: %{ + "aggregations" => %{ + "group" => %{ + "buckets" => buckets + } + } + } + }} = + TypelessTestModel.search_query() + |> Query.must(Query.match(:name, "name")) + |> Repo.aggregate(aggregation) + + for i <- 1..3 do + assert Enum.any?(buckets, fn + %{"key" => %{"age" => ^i, "group" => group}} -> + group == if rem(i, 2) == 0, do: "even", else: "odd" + + _ -> + false + end) + end + end end describe "#search/2" do @@ -290,6 +442,38 @@ defmodule ExlasticSearch.RepoTest do assert Enum.find(results, &(&1._id == id2)) end + test "It will search in a single index in es 8+" do + id1 = Ecto.UUID.generate() + id2 = Ecto.UUID.generate() + id3 = Ecto.UUID.generate() + + rand_name = Ecto.UUID.generate() |> String.replace("-", "") + + model1 = %TypelessTestModel{id: id1, name: rand_name} + model2 = %TypelessTestModel{id: id2, name: rand_name} + model3 = %TypelessTestModel{id: id3, name: "something else"} + + Repo.index(model1, :es8) + Repo.index(model2, :es8) + Repo.index(model3, :es8) + + Repo.refresh(TypelessTestModel, :es8) + + query = %ExlasticSearch.Query{ + queryable: ExlasticSearch.TypelessTestModel, + filter: [ + %{term: %{name: rand_name}} + ], + index_type: :es8 + } + + {:ok, %{hits: %{hits: results}}} = Repo.search(query, []) + + assert length(results) == 2 + assert Enum.find(results, & &1._id == id1) + assert Enum.find(results, & &1._id == id2) + end + test "It will search in a multiple indexes" do id1 = Ecto.UUID.generate() id2 = Ecto.UUID.generate() @@ -329,8 +513,8 @@ defmodule ExlasticSearch.RepoTest do end end - defp exists?(model) do - case Repo.get(model) do + defp exists?(model, index \\ :index) do + case Repo.get(model, index) do {:ok, %{found: true}} -> true _ -> false end diff --git a/test/support/test_model.ex b/test/support/test_model.ex index 665628c..1e1c07d 100644 --- a/test/support/test_model.ex +++ b/test/support/test_model.ex @@ -81,8 +81,11 @@ defmodule ExlasticSearch.TypelessTestModel do use Ecto.Schema use ExlasticSearch.Model - schema "typeless_test_model" do + schema "typeless_test_models" do field(:name, :string) + field(:age, :integer, default: 0) + field(:group, :string) + field(:teams, {:array, :map}) end indexes :typeless_test_model, doc_type: nil do @@ -90,11 +93,52 @@ defmodule ExlasticSearch.TypelessTestModel do settings(%{}) options(%{dynamic: :strict}) mapping(:name) + mapping(:age) + mapping(:group, type: :keyword) + + mapping(:user, properties: %{ext_name: %{type: :text}}) + + mapping(:teams, + type: :nested, + properties: %{ + name: %{type: :keyword}, + rating: %{type: :integer} + } + ) + end +end + +defmodule ExlasticSearch.TypelessMultiVersionTestModel do + use Ecto.Schema + use ExlasticSearch.Model + + schema "typeless_mv_models" do + field(:name, :string) + field(:age, :integer, default: 0) + field(:teams, {:array, :map}) + end + + indexes :typeless_multiversion_model, doc_type: nil do + versions({:ignore, 2}) + settings(%{}) + options(%{dynamic: :strict}) + mapping(:name) + mapping(:age) + + mapping(:user, properties: %{ext_name: %{type: :text}}) + + mapping(:teams, + type: :nested, + properties: %{ + name: %{type: :keyword}, + rating: %{type: :integer} + } + ) end end defimpl ExlasticSearch.Indexable, - for: [ExlasticSearch.TestModel, ExlasticSearch.TestModel2, ExlasticSearch.MultiVersionTestModel, ExlasticSearch.TypelessTestModel] do + for: [ExlasticSearch.TestModel, ExlasticSearch.TestModel2, ExlasticSearch.MultiVersionTestModel, ExlasticSearch.TypelessTestModel, ExlasticSearch.TypelessMultiVersionTestModel] do def id(%{id: id}), do: id def document(struct) do From b24eb690f2d64a7c842752fb8e1342129d2f48e0 Mon Sep 17 00:00:00 2001 From: Jesse Kelly Date: Tue, 21 May 2024 15:42:33 -0500 Subject: [PATCH 04/13] Support update in es8 by actually using index --- README.md | 1 - lib/exlasticsearch/repo.ex | 19 +++++++++++++++---- test/exlasticsearch/repo_test.exs | 7 ++++--- 3 files changed, 19 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 97c8f39..52f7604 100644 --- a/README.md +++ b/README.md @@ -77,7 +77,6 @@ docker-compose up -d mix test ``` - ## Copyright and License Copyright (c) 2025 Adobe/Frame.io diff --git a/lib/exlasticsearch/repo.ex b/lib/exlasticsearch/repo.ex index b8eb5f5..3728290 100644 --- a/lib/exlasticsearch/repo.ex +++ b/lib/exlasticsearch/repo.ex @@ -180,10 +180,21 @@ defmodule ExlasticSearch.Repo do """ @decorate retry() def update(model, id, data, index \\ :index) do - es_url(index) - |> Document.update(model.__es_index__(index), model.__doc_type__() || "_update", id, data) - |> log_response() - |> mark_failure() + cond do + doc_type = model.__doc_type__() -> + es_url(index) + |> Document.update(model.__es_index__(index), doc_type, id, data) + |> log_response() + |> mark_failure() + + %{doc: doc} = data -> + model + |> struct(Map.put(doc, :id, id)) + |> index(index) + + true -> + {:error, "ExlasticSearch only supports doc updates for ES 8+"} + end end @doc """ diff --git a/test/exlasticsearch/repo_test.exs b/test/exlasticsearch/repo_test.exs index d14fcee..78c1d34 100644 --- a/test/exlasticsearch/repo_test.exs +++ b/test/exlasticsearch/repo_test.exs @@ -73,14 +73,15 @@ defmodule ExlasticSearch.RepoTest do assert data.name == "test edited" end - test "It will fail to update an element in es 8+" do + test "It will update an element in es 8+" do id = Ecto.UUID.generate() model = %ExlasticSearch.TypelessTestModel{id: id, name: "test"} Repo.index(model, :es8) - {:ok, response} = Repo.update(ExlasticSearch.TypelessTestModel, id, %{doc: %{name: "test edited"}}, :es8) + {:ok, _} = Repo.update(ExlasticSearch.TypelessTestModel, id, %{doc: %{name: "test edited"}}, :es8) - assert %{"error" => _} = response.body + {:ok, %{_source: data}} = Repo.get(model, :es8) + assert data.name == "test edited" end end From 8767f9ed5d0bf45ded888db36c3ccdafeba4e347 Mon Sep 17 00:00:00 2001 From: William Gelhar Date: Wed, 19 Mar 2025 13:14:41 -0400 Subject: [PATCH 05/13] mix format --- config/test.exs | 17 +++++++---------- lib/exlasticsearch/model.ex | 2 +- lib/exlasticsearch/repo.ex | 22 ++++++++++++++-------- test/exlasticsearch/repo_test.exs | 29 +++++++++++------------------ test/support/test_model.ex | 10 +++++++++- 5 files changed, 42 insertions(+), 38 deletions(-) diff --git a/config/test.exs b/config/test.exs index 71d77f3..b9d360c 100644 --- a/config/test.exs +++ b/config/test.exs @@ -2,18 +2,15 @@ import Config config :elastix, json_codec: Jason -config :exlasticsearch, json_library: Jason - -config :exlasticsearch, :type_mappings, [ - {DB.CustomType, :integer} -] - -config :exlasticsearch, :monitoring, ExlasticSearch.Monitoring.Mock - config :exlasticsearch, ExlasticSearch.Repo, url: "http://localhost:9200", es8: "http://localhost:9201" +config :exlasticsearch, ExlasticSearch.TypelessTestModel, index_type: :es8 +config :exlasticsearch, :monitoring, ExlasticSearch.Monitoring.Mock -config :exlasticsearch, ExlasticSearch.TypelessTestModel, - index_type: :es8 +config :exlasticsearch, :type_mappings, [ + {DB.CustomType, :integer} +] + +config :exlasticsearch, json_library: Jason diff --git a/lib/exlasticsearch/model.ex b/lib/exlasticsearch/model.ex index 0540b19..fa2eaf5 100644 --- a/lib/exlasticsearch/model.ex +++ b/lib/exlasticsearch/model.ex @@ -85,7 +85,7 @@ defmodule ExlasticSearch.Model do @read_version :ignore @index_version :ignore - def __doc_type__(), do: unquote(doc_type) + def __doc_type__, do: unquote(doc_type) unquote(block) diff --git a/lib/exlasticsearch/repo.ex b/lib/exlasticsearch/repo.ex index 3728290..62fa5d8 100644 --- a/lib/exlasticsearch/repo.ex +++ b/lib/exlasticsearch/repo.ex @@ -69,7 +69,8 @@ defmodule ExlasticSearch.Repo do """ @spec create_mapping(atom) :: response def create_mapping(model, index \\ :index, opts \\ []) do - es_url(index) + index + |> es_url() |> Mapping.put(model.__es_index__(index), model.__doc_type__() || "", model.__es_mappings__(), opts) end @@ -169,7 +170,8 @@ defmodule ExlasticSearch.Repo do id = Indexable.id(struct) document = build_document(struct, index) - es_url(index) + index + |> es_url() |> Document.index(model.__es_index__(index), model.__doc_type__() || "_doc", id, document) |> log_response() |> mark_failure() @@ -182,7 +184,8 @@ defmodule ExlasticSearch.Repo do def update(model, id, data, index \\ :index) do cond do doc_type = model.__doc_type__() -> - es_url(index) + index + |> es_url() |> Document.update(model.__es_index__(index), doc_type, id, data) |> log_response() |> mark_failure() @@ -240,7 +243,8 @@ defmodule ExlasticSearch.Repo do """ @spec get(struct) :: {:ok, %Response.Record{}} | {:error, any} def get(%{__struct__: model} = struct, index_type \\ :read) do - es_url(index_type) + index_type + |> es_url() |> Document.get(model.__es_index__(index_type), model.__doc_type__() || "_doc", Indexable.id(struct)) |> log_response() |> decode(Response.Record, model) @@ -275,9 +279,9 @@ defmodule ExlasticSearch.Repo do defp model_to_index(model, index_type), do: model.__es_index__(index_type) defp model_to_doc_types(models) when is_list(models) do - models - |> Enum.flat_map(&model_to_doc_types/1) + Enum.flat_map(models, &model_to_doc_types/1) end + defp model_to_doc_types(model) do if doc_type = model.__doc_type__(), do: [doc_type], else: [] end @@ -296,7 +300,8 @@ defmodule ExlasticSearch.Repo do doc_types = if doc_type = model.__doc_type__(), do: [doc_type], else: [] es_index = model.__es_index__(index_type) - es_url(index_type) + index_type + |> es_url() |> Search.search(es_index, doc_types, search, size: 0) # TODO: figure out how to decode these, it's not trivial to type them |> log_response() @@ -308,7 +313,8 @@ defmodule ExlasticSearch.Repo do @spec delete(struct) :: response @decorate retry() def delete(%{__struct__: model} = struct, index \\ :index) do - es_url(index) + index + |> es_url() |> Document.delete(model.__es_index__(index), model.__doc_type__() || "_doc", Indexable.id(struct)) |> log_response() |> mark_failure() diff --git a/test/exlasticsearch/repo_test.exs b/test/exlasticsearch/repo_test.exs index 78c1d34..2bc5f7a 100644 --- a/test/exlasticsearch/repo_test.exs +++ b/test/exlasticsearch/repo_test.exs @@ -1,21 +1,14 @@ defmodule ExlasticSearch.RepoTest do use ExUnit.Case, async: true - alias ExlasticSearch.{ - Repo, - TestModel, - TestModel2, - TypelessTestModel, - Aggregation, - Query - } - + alias ExlasticSearch.Aggregation alias ExlasticSearch.MultiVersionTestModel, as: MVTestModel alias ExlasticSearch.Query alias ExlasticSearch.Repo alias ExlasticSearch.TestModel alias ExlasticSearch.TestModel2 alias ExlasticSearch.TypelessMultiVersionTestModel, as: TypelessMVTestModel + alias ExlasticSearch.TypelessTestModel setup_all do Repo.delete_index(TestModel) @@ -212,11 +205,11 @@ defmodule ExlasticSearch.RepoTest do for i <- 1..3, do: %TypelessTestModel{id: Ecto.UUID.generate(), name: "name #{i}", age: i} - {:ok, _} = Enum.map(models, &{:index, &1, :es8}) |> Repo.bulk(:es8) + {:ok, _} = models |> Enum.map(&{:index, &1, :es8}) |> Repo.bulk(:es8) Repo.refresh(TypelessTestModel, :es8) - aggregation = Aggregation.new() |> Aggregation.terms(:age, field: :age, size: 2) + aggregation = Aggregation.terms(Aggregation.new(), :age, field: :age, size: 2) {:ok, %{ @@ -287,11 +280,11 @@ defmodule ExlasticSearch.RepoTest do } end - {:ok, _} = Enum.map(models, &{:index, &1, :es8}) |> Repo.bulk(:es8) + {:ok, _} = models |> Enum.map(&{:index, &1, :es8}) |> Repo.bulk(:es8) Repo.refresh(TypelessTestModel, :es8) - nested = Aggregation.new() |> Aggregation.top_hits(:hits, %{}) + nested = Aggregation.top_hits(Aggregation.new(), :hits, %{}) aggregation = Aggregation.new() @@ -374,7 +367,7 @@ defmodule ExlasticSearch.RepoTest do } end - {:ok, _} = Enum.map(models, &{:index, &1, :es8}) |> Repo.bulk(:es8) + {:ok, _} = models |> Enum.map(&{:index, &1, :es8}) |> Repo.bulk(:es8) Repo.refresh(TypelessTestModel, :es8) @@ -383,7 +376,7 @@ defmodule ExlasticSearch.RepoTest do Aggregation.composite_source(:age, :terms, field: :age, order: :asc) ] - aggregation = Aggregation.new() |> Aggregation.composite(:group, sources) + aggregation = Aggregation.composite(Aggregation.new(), :group, sources) {:ok, %{ @@ -448,7 +441,7 @@ defmodule ExlasticSearch.RepoTest do id2 = Ecto.UUID.generate() id3 = Ecto.UUID.generate() - rand_name = Ecto.UUID.generate() |> String.replace("-", "") + rand_name = String.replace(Ecto.UUID.generate(), "-", "") model1 = %TypelessTestModel{id: id1, name: rand_name} model2 = %TypelessTestModel{id: id2, name: rand_name} @@ -471,8 +464,8 @@ defmodule ExlasticSearch.RepoTest do {:ok, %{hits: %{hits: results}}} = Repo.search(query, []) assert length(results) == 2 - assert Enum.find(results, & &1._id == id1) - assert Enum.find(results, & &1._id == id2) + assert Enum.find(results, &(&1._id == id1)) + assert Enum.find(results, &(&1._id == id2)) end test "It will search in a multiple indexes" do diff --git a/test/support/test_model.ex b/test/support/test_model.ex index 1e1c07d..c99f3ba 100644 --- a/test/support/test_model.ex +++ b/test/support/test_model.ex @@ -78,6 +78,7 @@ defmodule ExlasticSearch.MultiVersionTestModel do end defmodule ExlasticSearch.TypelessTestModel do + @moduledoc false use Ecto.Schema use ExlasticSearch.Model @@ -109,6 +110,7 @@ defmodule ExlasticSearch.TypelessTestModel do end defmodule ExlasticSearch.TypelessMultiVersionTestModel do + @moduledoc false use Ecto.Schema use ExlasticSearch.Model @@ -138,7 +140,13 @@ defmodule ExlasticSearch.TypelessMultiVersionTestModel do end defimpl ExlasticSearch.Indexable, - for: [ExlasticSearch.TestModel, ExlasticSearch.TestModel2, ExlasticSearch.MultiVersionTestModel, ExlasticSearch.TypelessTestModel, ExlasticSearch.TypelessMultiVersionTestModel] do + for: [ + ExlasticSearch.TestModel, + ExlasticSearch.TestModel2, + ExlasticSearch.MultiVersionTestModel, + ExlasticSearch.TypelessTestModel, + ExlasticSearch.TypelessMultiVersionTestModel + ] do def id(%{id: id}), do: id def document(struct) do From fda6b17d27901e6ff378f92e3ed4f6c20f186143 Mon Sep 17 00:00:00 2001 From: William Gelhar Date: Thu, 20 Mar 2025 13:10:14 -0400 Subject: [PATCH 06/13] cleanup --- README.md | 3 ++- lib/exlasticsearch/repo.ex | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 52f7604..4207e8b 100644 --- a/README.md +++ b/README.md @@ -64,7 +64,8 @@ This library requires [Elastix](https://hex.pm/packages/elastix), an Elixir Elas config :exlasticsearch, :type_inference, ExlasticSearch.TypeInference config :exlasticsearch, ExlasticSearch.Repo, - url: "http://localhost:9200" + url: "http://localhost:9200", + json_library: Jason ``` ## Testing diff --git a/lib/exlasticsearch/repo.ex b/lib/exlasticsearch/repo.ex index 62fa5d8..70ad272 100644 --- a/lib/exlasticsearch/repo.ex +++ b/lib/exlasticsearch/repo.ex @@ -190,9 +190,9 @@ defmodule ExlasticSearch.Repo do |> log_response() |> mark_failure() - %{doc: doc} = data -> + match?(%{doc: _}, data) -> model - |> struct(Map.put(doc, :id, id)) + |> struct(Map.put(data.doc, :id, id)) |> index(index) true -> From b3a97fe0f38ca30f037882fb702531e9b8a1eabc Mon Sep 17 00:00:00 2001 From: William Gelhar Date: Thu, 20 Mar 2025 13:17:01 -0400 Subject: [PATCH 07/13] my long-winded fight for running es8 tests has concluded, an es7 and es8 instance are included in our github action matrices, and tests are passing in es7 and es8 (except ones relying on mapping types, which aren't supported by es8) cleanup (+15 squashed commits) Squashed commits: [3fb4418] start excluding mapping types in es8 tests again [4dc2703] alright separating and cleaning [34590cc] I want to check something cheeky [2fec4a1] flip reverse it so only es7 runs mapping type tests [ce87d3d] getting weird with it [fc17334] ok next level [3ce0564] first pass on a matrix [7a88a3a] bringing back es8 [bb819f6] bringing the port back [224e686] skipping es8 tests, not running its action [0e46124] whatever I'll just skip those tests again [c0afa77] do I get a different error at least [5842e27] I call this throwing stuff at the wall and hoping for the best [364c0e1] container name overlap [eb6d57a] es8 in gh actions --- .github/workflows/elixir.yml | 13 +- README.md | 10 - config/test.exs | 5 +- docker-compose.yaml | 24 -- .../repo/mapping_types_test.exs | 325 ++++++++++++++++++ test/exlasticsearch/repo_test.exs | 312 +---------------- 6 files changed, 351 insertions(+), 338 deletions(-) delete mode 100644 docker-compose.yaml create mode 100644 test/exlasticsearch/repo/mapping_types_test.exs diff --git a/.github/workflows/elixir.yml b/.github/workflows/elixir.yml index 791efaf..de010cf 100644 --- a/.github/workflows/elixir.yml +++ b/.github/workflows/elixir.yml @@ -23,9 +23,18 @@ jobs: include: - elixir: "1.15" otp: "25" + elasticsearch: "7.17.16" + include_tags: "" - elixir: "1.18" otp: "27" lint: true + elasticsearch: "7.17.16" + include_tags: "" + - elixir: "1.18" + otp: "27" + lint: true + elasticsearch: "8.13.4" + include_tags: "--exclude mapping_types" steps: - name: Configure sysctl limits run: | @@ -37,7 +46,7 @@ jobs: - name: Runs Elasticsearch uses: elastic/elastic-github-actions/elasticsearch@master with: - stack-version: 7.17.16 + stack-version: ${{matrix.elasticsearch}} security-enabled: false - name: Checkout code @@ -63,7 +72,7 @@ jobs: run: mix compile --warnings-as-errors - name: Run tests - run: mix test + run: mix test ${{matrix.include_tags}} - name: checks that the mix.lock file has no unused deps run: mix deps.unlock --check-unused diff --git a/README.md b/README.md index 4207e8b..cda3410 100644 --- a/README.md +++ b/README.md @@ -68,16 +68,6 @@ config :exlasticsearch, ExlasticSearch.Repo, json_library: Jason ``` -## Testing - -Run integration tests with local ElasticSearch clusters. -Ensure Docker resources include at least 8 GB of memory. - -```sh -docker-compose up -d -mix test -``` - ## Copyright and License Copyright (c) 2025 Adobe/Frame.io diff --git a/config/test.exs b/config/test.exs index b9d360c..29fa7db 100644 --- a/config/test.exs +++ b/config/test.exs @@ -2,10 +2,7 @@ import Config config :elastix, json_codec: Jason -config :exlasticsearch, ExlasticSearch.Repo, - url: "http://localhost:9200", - es8: "http://localhost:9201" - +config :exlasticsearch, ExlasticSearch.Repo, url: "http://localhost:9200" config :exlasticsearch, ExlasticSearch.TypelessTestModel, index_type: :es8 config :exlasticsearch, :monitoring, ExlasticSearch.Monitoring.Mock diff --git a/docker-compose.yaml b/docker-compose.yaml deleted file mode 100644 index 2e0e653..0000000 --- a/docker-compose.yaml +++ /dev/null @@ -1,24 +0,0 @@ -version: "3.7" -volumes: - elasticsearch-7-data: - elasticsearch-8-data: -services: - elasticsearch-7: - image: docker.elastic.co/elasticsearch/elasticsearch:7.17.16 - restart: always - environment: - discovery.type: single-node - volumes: - - elasticsearch-7-data:/usr/share/elasticsearch/data - ports: - - 9200:9200 - elasticsearch-8: - image: docker.elastic.co/elasticsearch/elasticsearch:8.13.4 - restart: always - environment: - discovery.type: single-node - volumes: - - elasticsearch-8-data:/usr/share/elasticsearch/data - - ./elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml:ro - ports: - - 9201:9200 diff --git a/test/exlasticsearch/repo/mapping_types_test.exs b/test/exlasticsearch/repo/mapping_types_test.exs new file mode 100644 index 0000000..8a5896e --- /dev/null +++ b/test/exlasticsearch/repo/mapping_types_test.exs @@ -0,0 +1,325 @@ +defmodule ExlasticSearch.Repo.MappingTypesTest do + use ExUnit.Case, async: true + + alias ExlasticSearch.Aggregation + alias ExlasticSearch.MultiVersionTestModel, as: MVTestModel + alias ExlasticSearch.Query + alias ExlasticSearch.Repo + alias ExlasticSearch.TestModel + alias ExlasticSearch.TestModel2 + + require Logger + + setup_all do + Repo.delete_index(TestModel) + Repo.create_index(TestModel) + {:ok, %{status_code: 200}} = Repo.create_mapping(TestModel, :index, include_type_name: true) + + Repo.delete_index(TestModel2) + Repo.create_index(TestModel2) + {:ok, %{status_code: 200}} = Repo.create_mapping(TestModel2, :index, include_type_name: true) + + Repo.delete_index(MVTestModel) + Repo.create_index(MVTestModel) + {:ok, %{status_code: 200}} = Repo.create_mapping(MVTestModel, :index, include_type_name: true) + + Repo.delete_index(MVTestModel, :read) + Repo.create_index(MVTestModel, :read) + {:ok, %{status_code: 200}} = Repo.create_mapping(MVTestModel, :read, include_type_name: true) + + :ok + end + + describe "Elasticsearch 7 Mapping Types" do + @describetag :mapping_types + + test "It will index an element in es" do + model = %ExlasticSearch.TestModel{id: Ecto.UUID.generate()} + {:ok, _} = Repo.index(model) + + assert exists?(model) + end + + test "It will update an element in es" do + id = Ecto.UUID.generate() + model = %ExlasticSearch.TestModel{id: id, name: "test"} + Repo.index(model) + + {:ok, _} = Repo.update(ExlasticSearch.TestModel, id, %{doc: %{name: "test edited"}}) + + {:ok, %{_source: data}} = Repo.get(model) + assert data.name == "test edited" + end + + test "It will bulk index/delete from es" do + model = %ExlasticSearch.TestModel{id: Ecto.UUID.generate()} + {:ok, _} = Repo.bulk([{:index, model}]) + + assert exists?(model) + end + + test "It will bulk update from es" do + model1 = %ExlasticSearch.TestModel{id: Ecto.UUID.generate(), name: "test 1"} + model2 = %ExlasticSearch.TestModel{id: Ecto.UUID.generate(), name: "test 2"} + + Repo.index(model1) + Repo.index(model2) + + {:ok, _} = + Repo.bulk([ + {:update, ExlasticSearch.TestModel, model1.id, %{doc: %{name: "test 1 edited"}}}, + {:update, ExlasticSearch.TestModel, model2.id, %{doc: %{name: "test 2 edited"}}} + ]) + + {:ok, %{_source: data1}} = Repo.get(model1) + {:ok, %{_source: data2}} = Repo.get(model2) + + assert data1.name == "test 1 edited" + assert data2.name == "test 2 edited" + end + + test "It will bulk update nested from es" do + model1 = %ExlasticSearch.TestModel{ + id: Ecto.UUID.generate(), + teams: [%{name: "arsenal", rating: 100}] + } + + model2 = %ExlasticSearch.TestModel{ + id: Ecto.UUID.generate(), + teams: [%{name: "tottenham", rating: 0}] + } + + Repo.index(model1) + Repo.index(model2) + + source = + "ctx._source.teams.find(cf -> cf.name == params.data.name).rating = params.data.rating" + + data1 = %{script: %{source: source, params: %{data: %{name: "arsenal", rating: 1000}}}} + data2 = %{script: %{source: source, params: %{data: %{name: "tottenham", rating: -1}}}} + + {:ok, _} = + Repo.bulk([ + {:update, ExlasticSearch.TestModel, model1.id, data1}, + {:update, ExlasticSearch.TestModel, model2.id, data2} + ]) + + {:ok, %{_source: %{teams: [team1]}}} = Repo.get(model1) + {:ok, %{_source: %{teams: [team2]}}} = Repo.get(model2) + + assert team1.name == "arsenal" + assert team1.rating == 1000 + + assert team2.name == "tottenham" + assert team2.rating == -1 + end + + test "It can deprecate an old index version" do + model = %MVTestModel{id: Ecto.UUID.generate()} + {:ok, _} = Repo.index(model) + + Repo.rotate(MVTestModel) + + assert exists?(model) + end + + test "It can perform terms aggregations" do + models = + for i <- 1..3, + do: %TestModel{id: Ecto.UUID.generate(), name: "name #{i}", age: i} + + {:ok, _} = models |> Enum.map(&{:index, &1}) |> Repo.bulk() + + Repo.refresh(TestModel) + + aggregation = Aggregation.terms(Aggregation.new(), :age, field: :age, size: 2) + + {:ok, + %{ + body: %{ + "aggregations" => %{ + "age" => %{ + "buckets" => buckets + } + } + } + }} = + TestModel.search_query() + |> Query.must(Query.match(:name, "name")) + |> Repo.aggregate(aggregation) + + assert length(buckets) == 2 + assert Enum.all?(buckets, &(&1["key"] in [1, 2])) + end + + # can we unskip? fails from mapping types + @tag :skip + test "It can perform top_hits aggregations, even when nested" do + models = + for i <- 1..3 do + %TestModel{ + id: Ecto.UUID.generate(), + name: "name #{i}", + age: i, + group: if(rem(i, 2) == 0, do: "even", else: "odd") + } + end + + {:ok, _} = models |> Enum.map(&{:index, &1}) |> Repo.bulk() + + Repo.refresh(TestModel) + + nested = Aggregation.top_hits(Aggregation.new(), :hits, %{}) + + aggregation = + Aggregation.new() + |> Aggregation.terms(:group, field: String.to_atom("group.keyword")) + |> Aggregation.nest(:group, nested) + + {:ok, + %{ + body: %{ + "aggregations" => %{ + "group" => %{ + "buckets" => buckets + } + } + } + }} = + TestModel.search_query() + |> Query.must(Query.match(:name, "name")) + |> Repo.aggregate(aggregation) + + assert length(buckets) == 2 + assert Enum.all?(buckets, &(!Enum.empty?(get_hits(&1)))) + end + + # can we unskip? fails from mapping types + @tag :skip + test "It can perform composite aggregations" do + models = + for i <- 1..3 do + %TestModel{ + id: Ecto.UUID.generate(), + name: "name #{i}", + age: i, + group: if(rem(i, 2) == 0, do: "even", else: "odd") + } + end + + {:ok, _} = models |> Enum.map(&{:index, &1}) |> Repo.bulk() + + Repo.refresh(TestModel) + + sources = [ + Aggregation.composite_source(:group, :terms, field: String.to_atom("group.keyword"), order: :desc), + Aggregation.composite_source(:age, :terms, field: :age, order: :asc) + ] + + aggregation = Aggregation.composite(Aggregation.new(), :group, sources) + + {:ok, + %{ + body: %{ + "aggregations" => %{ + "group" => %{ + "buckets" => buckets + } + } + } + }} = + TestModel.search_query() + |> Query.must(Query.match(:name, "name")) + |> Repo.aggregate(aggregation) + + for i <- 1..3 do + assert Enum.any?(buckets, fn + %{"key" => %{"age" => ^i, "group" => group}} -> + group == if rem(i, 2) == 0, do: "even", else: "odd" + + _ -> + false + end) + end + end + + @tag :mapping_types + test "It will search in a single index" do + id1 = Ecto.UUID.generate() + id2 = Ecto.UUID.generate() + id3 = Ecto.UUID.generate() + + rand_name = String.replace(Ecto.UUID.generate(), "-", "") + + model1 = %TestModel{id: id1, name: rand_name} + model2 = %TestModel{id: id2, name: rand_name} + model3 = %TestModel{id: id3, name: "something else"} + + Repo.index(model1) + Repo.index(model2) + Repo.index(model3) + + Repo.refresh(TestModel) + + query = %ExlasticSearch.Query{ + queryable: ExlasticSearch.TestModel, + filter: [ + %{term: %{name: rand_name}} + ] + } + + {:ok, %{hits: %{hits: results}}} = Repo.search(query, []) + + assert length(results) == 2 + assert Enum.find(results, &(&1._id == id1)) + assert Enum.find(results, &(&1._id == id2)) + end + + test "It will search in a multiple indexes" do + id1 = Ecto.UUID.generate() + id2 = Ecto.UUID.generate() + id3 = Ecto.UUID.generate() + id4 = Ecto.UUID.generate() + + rand_name = String.replace(Ecto.UUID.generate(), "-", "") + + model1 = %TestModel{id: id1, name: rand_name} + model2 = %TestModel{id: id2, name: rand_name} + model3 = %TestModel{id: id3, name: "something else"} + + model4 = %TestModel2{id: id4, name: rand_name} + + Repo.index(model1) + Repo.index(model2) + Repo.index(model3) + + Repo.index(model4) + + Repo.refresh(TestModel) + Repo.refresh(TestModel2) + + query = %ExlasticSearch.Query{ + queryable: [ExlasticSearch.TestModel, ExlasticSearch.TestModel2], + filter: [ + %{term: %{name: rand_name}} + ] + } + + {:ok, %{hits: %{hits: results}}} = Repo.search(query, []) + + assert length(results) == 3 + assert Enum.find(results, &(&1._id == id1)) + assert Enum.find(results, &(&1._id == id2)) + assert Enum.find(results, &(&1._id == id4)) + end + end + + defp exists?(model, index \\ :index) do + case Repo.get(model, index) do + {:ok, %{found: true}} -> true + _ -> false + end + end + + defp get_hits(%{"hits" => %{"hits" => %{"hits" => hits}}}), do: hits +end diff --git a/test/exlasticsearch/repo_test.exs b/test/exlasticsearch/repo_test.exs index 2bc5f7a..d260bda 100644 --- a/test/exlasticsearch/repo_test.exs +++ b/test/exlasticsearch/repo_test.exs @@ -2,50 +2,27 @@ defmodule ExlasticSearch.RepoTest do use ExUnit.Case, async: true alias ExlasticSearch.Aggregation - alias ExlasticSearch.MultiVersionTestModel, as: MVTestModel alias ExlasticSearch.Query alias ExlasticSearch.Repo - alias ExlasticSearch.TestModel - alias ExlasticSearch.TestModel2 alias ExlasticSearch.TypelessMultiVersionTestModel, as: TypelessMVTestModel alias ExlasticSearch.TypelessTestModel - setup_all do - Repo.delete_index(TestModel) - Repo.create_index(TestModel) - Repo.create_mapping(TestModel) - - Repo.delete_index(TestModel2) - Repo.create_index(TestModel2) - Repo.create_mapping(TestModel2) - - Repo.delete_index(MVTestModel) - Repo.create_index(MVTestModel) - Repo.create_mapping(MVTestModel) - - Repo.delete_index(MVTestModel, :read) - Repo.create_index(MVTestModel, :read) - Repo.create_mapping(MVTestModel, :read) + require Logger + setup_all do Repo.delete_index(TypelessTestModel, :es8) Repo.create_index(TypelessTestModel, :es8) - Repo.create_mapping(TypelessTestModel, :es8) + {:ok, %{status_code: 200}} = Repo.create_mapping(TypelessTestModel, :es8) Repo.delete_index(TypelessMVTestModel, :es8) Repo.create_index(TypelessMVTestModel, :es8) - Repo.create_mapping(TypelessMVTestModel, :es8) + {:ok, %{status_code: 200}} = Repo.create_mapping(TypelessMVTestModel, :es8) :ok end describe "#index" do - test "It will index an element in es" do - model = %ExlasticSearch.TestModel{id: Ecto.UUID.generate()} - {:ok, _} = Repo.index(model) - - assert exists?(model) - end - + @tag :elasticsearch8 test "It will index an element in es 8+" do model = %ExlasticSearch.TypelessTestModel{id: Ecto.UUID.generate()} {:ok, _} = Repo.index(model, :es8) @@ -55,17 +32,7 @@ defmodule ExlasticSearch.RepoTest do end describe "#update" do - test "It will update an element in es" do - id = Ecto.UUID.generate() - model = %ExlasticSearch.TestModel{id: id, name: "test"} - Repo.index(model) - - {:ok, _} = Repo.update(ExlasticSearch.TestModel, id, %{doc: %{name: "test edited"}}) - - {:ok, %{_source: data}} = Repo.get(model) - assert data.name == "test edited" - end - + @tag :elasticsearch8 test "It will update an element in es 8+" do id = Ecto.UUID.generate() model = %ExlasticSearch.TypelessTestModel{id: id, name: "test"} @@ -79,87 +46,17 @@ defmodule ExlasticSearch.RepoTest do end describe "#bulk" do - test "It will bulk index/delete from es" do - model = %ExlasticSearch.TestModel{id: Ecto.UUID.generate()} - {:ok, _} = Repo.bulk([{:index, model}]) - - assert exists?(model) - end - + @tag :elasticsearch8 test "It will bulk index/delete from es 8+" do model = %ExlasticSearch.TypelessTestModel{id: Ecto.UUID.generate()} {:ok, _} = Repo.bulk([{:index, model, :es8}], :es8) assert exists?(model, :es8) end - - test "It will bulk update from es" do - model1 = %ExlasticSearch.TestModel{id: Ecto.UUID.generate(), name: "test 1"} - model2 = %ExlasticSearch.TestModel{id: Ecto.UUID.generate(), name: "test 2"} - - Repo.index(model1) - Repo.index(model2) - - {:ok, _} = - Repo.bulk([ - {:update, ExlasticSearch.TestModel, model1.id, %{doc: %{name: "test 1 edited"}}}, - {:update, ExlasticSearch.TestModel, model2.id, %{doc: %{name: "test 2 edited"}}} - ]) - - {:ok, %{_source: data1}} = Repo.get(model1) - {:ok, %{_source: data2}} = Repo.get(model2) - - assert data1.name == "test 1 edited" - assert data2.name == "test 2 edited" - end - - test "It will bulk update nested from es" do - model1 = %ExlasticSearch.TestModel{ - id: Ecto.UUID.generate(), - teams: [%{name: "arsenal", rating: 100}] - } - - model2 = %ExlasticSearch.TestModel{ - id: Ecto.UUID.generate(), - teams: [%{name: "tottenham", rating: 0}] - } - - Repo.index(model1) - Repo.index(model2) - - source = - "ctx._source.teams.find(cf -> cf.name == params.data.name).rating = params.data.rating" - - data1 = %{script: %{source: source, params: %{data: %{name: "arsenal", rating: 1000}}}} - data2 = %{script: %{source: source, params: %{data: %{name: "tottenham", rating: -1}}}} - - {:ok, _} = - Repo.bulk([ - {:update, ExlasticSearch.TestModel, model1.id, data1}, - {:update, ExlasticSearch.TestModel, model2.id, data2} - ]) - - {:ok, %{_source: %{teams: [team1]}}} = Repo.get(model1) - {:ok, %{_source: %{teams: [team2]}}} = Repo.get(model2) - - assert team1.name == "arsenal" - assert team1.rating == 1000 - - assert team2.name == "tottenham" - assert team2.rating == -1 - end end describe "#rotate" do - test "It can deprecate an old index version" do - model = %MVTestModel{id: Ecto.UUID.generate()} - {:ok, _} = Repo.index(model) - - Repo.rotate(MVTestModel) - - assert exists?(model) - end - + @tag :elasticsearch8 test "It can deprecate an old index version on es 8+" do model = %TypelessMVTestModel{id: Ecto.UUID.generate()} {:ok, _} = Repo.index(model, :es8) @@ -171,36 +68,8 @@ defmodule ExlasticSearch.RepoTest do end describe "#aggregate/2" do - test "It can perform terms aggreagtions" do - models = - for i <- 1..3, - do: %TestModel{id: Ecto.UUID.generate(), name: "name #{i}", age: i} - - {:ok, _} = models |> Enum.map(&{:index, &1}) |> Repo.bulk() - - Repo.refresh(TestModel) - - aggregation = Aggregation.terms(Aggregation.new(), :age, field: :age, size: 2) - - {:ok, - %{ - body: %{ - "aggregations" => %{ - "age" => %{ - "buckets" => buckets - } - } - } - }} = - TestModel.search_query() - |> Query.must(Query.match(:name, "name")) - |> Repo.aggregate(aggregation) - - assert length(buckets) == 2 - assert Enum.all?(buckets, &(&1["key"] in [1, 2])) - end - - test "It can perform terms aggreagtions on es 8+" do + @tag :elasticsearch8 + test "It can perform terms aggregations on es 8+" do models = for i <- 1..3, do: %TypelessTestModel{id: Ecto.UUID.generate(), name: "name #{i}", age: i} @@ -229,46 +98,7 @@ defmodule ExlasticSearch.RepoTest do assert Enum.all?(buckets, &(&1["key"] in [1, 2])) end - test "It can perform top_hits aggregations, even when nested" do - models = - for i <- 1..3 do - %TestModel{ - id: Ecto.UUID.generate(), - name: "name #{i}", - age: i, - group: if(rem(i, 2) == 0, do: "even", else: "odd") - } - end - - {:ok, _} = models |> Enum.map(&{:index, &1}) |> Repo.bulk() - - Repo.refresh(TestModel) - - nested = Aggregation.top_hits(Aggregation.new(), :hits, %{}) - - aggregation = - Aggregation.new() - |> Aggregation.terms(:group, field: String.to_atom("group.keyword")) - |> Aggregation.nest(:group, nested) - - {:ok, - %{ - body: %{ - "aggregations" => %{ - "group" => %{ - "buckets" => buckets - } - } - } - }} = - TestModel.search_query() - |> Query.must(Query.match(:name, "name")) - |> Repo.aggregate(aggregation) - - assert length(buckets) == 2 - assert Enum.all?(buckets, &(!Enum.empty?(get_hits(&1)))) - end - + @tag :elasticsearch8 test "It can perform top_hits aggregations, even when nested, on es 8+" do models = for i <- 1..3 do @@ -309,53 +139,7 @@ defmodule ExlasticSearch.RepoTest do assert Enum.all?(buckets, &(!Enum.empty?(get_hits(&1)))) end - test "It can perform composite aggregations" do - models = - for i <- 1..3 do - %TestModel{ - id: Ecto.UUID.generate(), - name: "name #{i}", - age: i, - group: if(rem(i, 2) == 0, do: "even", else: "odd") - } - end - - {:ok, _} = models |> Enum.map(&{:index, &1}) |> Repo.bulk() - - Repo.refresh(TestModel) - - sources = [ - Aggregation.composite_source(:group, :terms, field: String.to_atom("group.keyword"), order: :desc), - Aggregation.composite_source(:age, :terms, field: :age, order: :asc) - ] - - aggregation = Aggregation.composite(Aggregation.new(), :group, sources) - - {:ok, - %{ - body: %{ - "aggregations" => %{ - "group" => %{ - "buckets" => buckets - } - } - } - }} = - TestModel.search_query() - |> Query.must(Query.match(:name, "name")) - |> Repo.aggregate(aggregation) - - for i <- 1..3 do - assert Enum.any?(buckets, fn - %{"key" => %{"age" => ^i, "group" => group}} -> - group == if rem(i, 2) == 0, do: "even", else: "odd" - - _ -> - false - end) - end - end - + @tag :elasticsearch8 test "It can perform composite aggregations on es 8+" do models = for i <- 1..3 do @@ -405,37 +189,7 @@ defmodule ExlasticSearch.RepoTest do end describe "#search/2" do - test "It will search in a single index" do - id1 = Ecto.UUID.generate() - id2 = Ecto.UUID.generate() - id3 = Ecto.UUID.generate() - - rand_name = String.replace(Ecto.UUID.generate(), "-", "") - - model1 = %TestModel{id: id1, name: rand_name} - model2 = %TestModel{id: id2, name: rand_name} - model3 = %TestModel{id: id3, name: "something else"} - - Repo.index(model1) - Repo.index(model2) - Repo.index(model3) - - Repo.refresh(TestModel) - - query = %ExlasticSearch.Query{ - queryable: ExlasticSearch.TestModel, - filter: [ - %{term: %{name: rand_name}} - ] - } - - {:ok, %{hits: %{hits: results}}} = Repo.search(query, []) - - assert length(results) == 2 - assert Enum.find(results, &(&1._id == id1)) - assert Enum.find(results, &(&1._id == id2)) - end - + @tag :elasticsearch8 test "It will search in a single index in es 8+" do id1 = Ecto.UUID.generate() id2 = Ecto.UUID.generate() @@ -467,47 +221,9 @@ defmodule ExlasticSearch.RepoTest do assert Enum.find(results, &(&1._id == id1)) assert Enum.find(results, &(&1._id == id2)) end - - test "It will search in a multiple indexes" do - id1 = Ecto.UUID.generate() - id2 = Ecto.UUID.generate() - id3 = Ecto.UUID.generate() - id4 = Ecto.UUID.generate() - - rand_name = String.replace(Ecto.UUID.generate(), "-", "") - - model1 = %TestModel{id: id1, name: rand_name} - model2 = %TestModel{id: id2, name: rand_name} - model3 = %TestModel{id: id3, name: "something else"} - - model4 = %TestModel2{id: id4, name: rand_name} - - Repo.index(model1) - Repo.index(model2) - Repo.index(model3) - - Repo.index(model4) - - Repo.refresh(TestModel) - Repo.refresh(TestModel2) - - query = %ExlasticSearch.Query{ - queryable: [ExlasticSearch.TestModel, ExlasticSearch.TestModel2], - filter: [ - %{term: %{name: rand_name}} - ] - } - - {:ok, %{hits: %{hits: results}}} = Repo.search(query, []) - - assert length(results) == 3 - assert Enum.find(results, &(&1._id == id1)) - assert Enum.find(results, &(&1._id == id2)) - assert Enum.find(results, &(&1._id == id4)) - end end - defp exists?(model, index \\ :index) do + defp exists?(model, index) do case Repo.get(model, index) do {:ok, %{found: true}} -> true _ -> false From 05e50d45028c04a17f02b8ff8ac21bed3ad18571 Mon Sep 17 00:00:00 2001 From: William Gelhar Date: Wed, 26 Mar 2025 15:39:24 -0400 Subject: [PATCH 08/13] removing get_env -> compile_env --- README.md | 2 +- lib/exlasticsearch.ex | 2 +- lib/exlasticsearch/model.ex | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index cda3410..bf8b4ea 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,7 @@ defmodule MySchema do use ExlasticSearch.Model indexes :my_index do - settings Application.compile_env(:some, :settings) + settings Application.get_env(:some, :settings) mapping :field mapping :other_field, type: :keyword # ecto derived defaults can be overridden diff --git a/lib/exlasticsearch.ex b/lib/exlasticsearch.ex index 05de361..6b2f834 100644 --- a/lib/exlasticsearch.ex +++ b/lib/exlasticsearch.ex @@ -9,7 +9,7 @@ defmodule ExlasticSearch do use ExlasticSearch.Model indexes :my_index do - settings Application.compile_env(:some, :settings) + settings Application.get_env(:some, :settings) mapping :field mapping :other_field, type: :keyword # ecto derived defaults can be overridden diff --git a/lib/exlasticsearch/model.ex b/lib/exlasticsearch/model.ex index fa2eaf5..a81c571 100644 --- a/lib/exlasticsearch/model.ex +++ b/lib/exlasticsearch/model.ex @@ -13,7 +13,7 @@ defmodule ExlasticSearch.Model do The usage is something like this indexes :my_type do - settings Application.compile_env(:some, :settings) + settings Application.get_env(:some, :settings) mapping :column mapping :other_column, type: :keyword From 105979f9f2a2c35391f4d986e277938c0eff4869 Mon Sep 17 00:00:00 2001 From: William Gelhar Date: Wed, 26 Mar 2025 15:40:22 -0400 Subject: [PATCH 09/13] removing elasticsearch 8 tags --- config/test.exs | 4 ---- elasticsearch.yml | 4 ---- test/exlasticsearch/repo_test.exs | 8 -------- 3 files changed, 16 deletions(-) delete mode 100644 elasticsearch.yml diff --git a/config/test.exs b/config/test.exs index 29fa7db..02f5099 100644 --- a/config/test.exs +++ b/config/test.exs @@ -6,8 +6,4 @@ config :exlasticsearch, ExlasticSearch.Repo, url: "http://localhost:9200" config :exlasticsearch, ExlasticSearch.TypelessTestModel, index_type: :es8 config :exlasticsearch, :monitoring, ExlasticSearch.Monitoring.Mock -config :exlasticsearch, :type_mappings, [ - {DB.CustomType, :integer} -] - config :exlasticsearch, json_library: Jason diff --git a/elasticsearch.yml b/elasticsearch.yml deleted file mode 100644 index f3d1379..0000000 --- a/elasticsearch.yml +++ /dev/null @@ -1,4 +0,0 @@ -cluster.name: "docker-cluster" -network.host: 0.0.0.0 -xpack.security.enabled: false -xpack.security.enrollment.enabled: false diff --git a/test/exlasticsearch/repo_test.exs b/test/exlasticsearch/repo_test.exs index d260bda..bf12f68 100644 --- a/test/exlasticsearch/repo_test.exs +++ b/test/exlasticsearch/repo_test.exs @@ -22,7 +22,6 @@ defmodule ExlasticSearch.RepoTest do end describe "#index" do - @tag :elasticsearch8 test "It will index an element in es 8+" do model = %ExlasticSearch.TypelessTestModel{id: Ecto.UUID.generate()} {:ok, _} = Repo.index(model, :es8) @@ -32,7 +31,6 @@ defmodule ExlasticSearch.RepoTest do end describe "#update" do - @tag :elasticsearch8 test "It will update an element in es 8+" do id = Ecto.UUID.generate() model = %ExlasticSearch.TypelessTestModel{id: id, name: "test"} @@ -46,7 +44,6 @@ defmodule ExlasticSearch.RepoTest do end describe "#bulk" do - @tag :elasticsearch8 test "It will bulk index/delete from es 8+" do model = %ExlasticSearch.TypelessTestModel{id: Ecto.UUID.generate()} {:ok, _} = Repo.bulk([{:index, model, :es8}], :es8) @@ -56,7 +53,6 @@ defmodule ExlasticSearch.RepoTest do end describe "#rotate" do - @tag :elasticsearch8 test "It can deprecate an old index version on es 8+" do model = %TypelessMVTestModel{id: Ecto.UUID.generate()} {:ok, _} = Repo.index(model, :es8) @@ -68,7 +64,6 @@ defmodule ExlasticSearch.RepoTest do end describe "#aggregate/2" do - @tag :elasticsearch8 test "It can perform terms aggregations on es 8+" do models = for i <- 1..3, @@ -98,7 +93,6 @@ defmodule ExlasticSearch.RepoTest do assert Enum.all?(buckets, &(&1["key"] in [1, 2])) end - @tag :elasticsearch8 test "It can perform top_hits aggregations, even when nested, on es 8+" do models = for i <- 1..3 do @@ -139,7 +133,6 @@ defmodule ExlasticSearch.RepoTest do assert Enum.all?(buckets, &(!Enum.empty?(get_hits(&1)))) end - @tag :elasticsearch8 test "It can perform composite aggregations on es 8+" do models = for i <- 1..3 do @@ -189,7 +182,6 @@ defmodule ExlasticSearch.RepoTest do end describe "#search/2" do - @tag :elasticsearch8 test "It will search in a single index in es 8+" do id1 = Ecto.UUID.generate() id2 = Ecto.UUID.generate() From 72439cbec62e1067f3abdff3f7a41f6a9702bd95 Mon Sep 17 00:00:00 2001 From: William Gelhar Date: Wed, 26 Mar 2025 16:16:12 -0400 Subject: [PATCH 10/13] restoring tests that weren't covered for non-es7 before --- config/test.exs | 1 - .../repo/mapping_types_test.exs | 2 +- test/exlasticsearch/repo_test.exs | 99 +++++++++++++++++++ test/support/test_model.ex | 18 ++++ 4 files changed, 118 insertions(+), 2 deletions(-) diff --git a/config/test.exs b/config/test.exs index 02f5099..21f588f 100644 --- a/config/test.exs +++ b/config/test.exs @@ -5,5 +5,4 @@ config :elastix, json_codec: Jason config :exlasticsearch, ExlasticSearch.Repo, url: "http://localhost:9200" config :exlasticsearch, ExlasticSearch.TypelessTestModel, index_type: :es8 config :exlasticsearch, :monitoring, ExlasticSearch.Monitoring.Mock - config :exlasticsearch, json_library: Jason diff --git a/test/exlasticsearch/repo/mapping_types_test.exs b/test/exlasticsearch/repo/mapping_types_test.exs index 8a5896e..b7381fe 100644 --- a/test/exlasticsearch/repo/mapping_types_test.exs +++ b/test/exlasticsearch/repo/mapping_types_test.exs @@ -275,7 +275,7 @@ defmodule ExlasticSearch.Repo.MappingTypesTest do assert Enum.find(results, &(&1._id == id2)) end - test "It will search in a multiple indexes" do + test "It will search in multiple indexes" do id1 = Ecto.UUID.generate() id2 = Ecto.UUID.generate() id3 = Ecto.UUID.generate() diff --git a/test/exlasticsearch/repo_test.exs b/test/exlasticsearch/repo_test.exs index bf12f68..c822a4c 100644 --- a/test/exlasticsearch/repo_test.exs +++ b/test/exlasticsearch/repo_test.exs @@ -6,6 +6,7 @@ defmodule ExlasticSearch.RepoTest do alias ExlasticSearch.Repo alias ExlasticSearch.TypelessMultiVersionTestModel, as: TypelessMVTestModel alias ExlasticSearch.TypelessTestModel + alias ExlasticSearch.TypelessTestModel2 require Logger @@ -14,6 +15,10 @@ defmodule ExlasticSearch.RepoTest do Repo.create_index(TypelessTestModel, :es8) {:ok, %{status_code: 200}} = Repo.create_mapping(TypelessTestModel, :es8) + Repo.delete_index(TypelessTestModel2, :es8) + Repo.create_index(TypelessTestModel2, :es8) + {:ok, %{status_code: 200}} = Repo.create_mapping(TypelessTestModel2, :es8) + Repo.delete_index(TypelessMVTestModel, :es8) Repo.create_index(TypelessMVTestModel, :es8) {:ok, %{status_code: 200}} = Repo.create_mapping(TypelessMVTestModel, :es8) @@ -44,6 +49,62 @@ defmodule ExlasticSearch.RepoTest do end describe "#bulk" do + test "It will bulk update from es" do + model1 = %ExlasticSearch.TypelessTestModel{id: Ecto.UUID.generate(), name: "test 1"} + model2 = %ExlasticSearch.TypelessTestModel{id: Ecto.UUID.generate(), name: "test 2"} + + Repo.index(model1) + Repo.index(model2) + + {:ok, _} = + Repo.bulk([ + {:update, ExlasticSearch.TypelessTestModel, model1.id, %{doc: %{name: "test 1 edited"}}}, + {:update, ExlasticSearch.TypelessTestModel, model2.id, %{doc: %{name: "test 2 edited"}}} + ]) + + {:ok, %{_source: data1}} = Repo.get(model1) + {:ok, %{_source: data2}} = Repo.get(model2) + + assert data1.name == "test 1 edited" + assert data2.name == "test 2 edited" + end + + test "It will bulk update nested from es" do + model1 = %ExlasticSearch.TypelessTestModel{ + id: Ecto.UUID.generate(), + teams: [%{name: "arsenal", rating: 100}] + } + + model2 = %ExlasticSearch.TypelessTestModel{ + id: Ecto.UUID.generate(), + teams: [%{name: "tottenham", rating: 0}] + } + + Repo.index(model1) + Repo.index(model2) + + source = + "ctx._source.teams.find(cf -> cf.name == params.data.name).rating = params.data.rating" + + data1 = %{script: %{source: source, params: %{data: %{name: "arsenal", rating: 1000}}}} + data2 = %{script: %{source: source, params: %{data: %{name: "tottenham", rating: -1}}}} + + {:ok, _} = + Repo.bulk([ + {:update, ExlasticSearch.TypelessTestModel, model1.id, data1}, + {:update, ExlasticSearch.TypelessTestModel, model2.id, data2} + ]) + + {:ok, %{_source: %{teams: [team1]}}} = Repo.get(model1) + {:ok, %{_source: %{teams: [team2]}}} = Repo.get(model2) + + assert team1.name == "arsenal" + assert team1.rating == 1000 + + assert team2.name == "tottenham" + assert team2.rating == -1 + end + test "It will bulk index/delete from es 8+" do model = %ExlasticSearch.TypelessTestModel{id: Ecto.UUID.generate()} {:ok, _} = Repo.bulk([{:index, model, :es8}], :es8) @@ -213,6 +274,44 @@ defmodule ExlasticSearch.RepoTest do assert Enum.find(results, &(&1._id == id1)) assert Enum.find(results, &(&1._id == id2)) end + + test "It will search in multiple indexes" do + id1 = Ecto.UUID.generate() + id2 = Ecto.UUID.generate() + id3 = Ecto.UUID.generate() + id4 = Ecto.UUID.generate() + + rand_name = String.replace(Ecto.UUID.generate(), "-", "") + + model1 = %TypelessTestModel{id: id1, name: rand_name} + model2 = %TypelessTestModel{id: id2, name: rand_name} + model3 = %TypelessTestModel{id: id3, name: "something else"} + + model4 = %TypelessTestModel2{id: id4, name: rand_name} + + Repo.index(model1) + Repo.index(model2) + Repo.index(model3) + + Repo.index(model4) + + Repo.refresh(TypelessTestModel) + Repo.refresh(TypelessTestModel2) + + query = %ExlasticSearch.Query{ + queryable: [TypelessTestModel, TypelessTestModel2], + filter: [ + %{term: %{name: rand_name}} + ] + } + + {:ok, %{hits: %{hits: results}}} = Repo.search(query, []) + + assert length(results) == 3 + assert Enum.find(results, &(&1._id == id1)) + assert Enum.find(results, &(&1._id == id2)) + assert Enum.find(results, &(&1._id == id4)) + end end defp exists?(model, index) do diff --git a/test/support/test_model.ex b/test/support/test_model.ex index c99f3ba..6b5b0b7 100644 --- a/test/support/test_model.ex +++ b/test/support/test_model.ex @@ -109,6 +109,23 @@ defmodule ExlasticSearch.TypelessTestModel do end end +defmodule ExlasticSearch.TypelessTestModel2 do + @moduledoc false + use Ecto.Schema + use ExlasticSearch.Model + + schema "typeless_test_models2" do + field(:name, :string) + end + + indexes :typeless_test_model2, doc_type: nil do + versions(2) + settings(%{}) + options(%{dynamic: :strict}) + mapping(:name) + end +end + defmodule ExlasticSearch.TypelessMultiVersionTestModel do @moduledoc false use Ecto.Schema @@ -145,6 +162,7 @@ defimpl ExlasticSearch.Indexable, ExlasticSearch.TestModel2, ExlasticSearch.MultiVersionTestModel, ExlasticSearch.TypelessTestModel, + ExlasticSearch.TypelessTestModel2, ExlasticSearch.TypelessMultiVersionTestModel ] do def id(%{id: id}), do: id From 9585c8e453b43eb15c3da140371db0fa2d78edf5 Mon Sep 17 00:00:00 2001 From: William Gelhar Date: Wed, 26 Mar 2025 16:44:04 -0400 Subject: [PATCH 11/13] reverting the change to Repo.update, updating bulk operation instead --- lib/exlasticsearch/bulk.ex | 11 +++++++---- lib/exlasticsearch/repo.ex | 21 +++++---------------- test/exlasticsearch/repo_test.exs | 2 +- 3 files changed, 13 insertions(+), 21 deletions(-) diff --git a/lib/exlasticsearch/bulk.ex b/lib/exlasticsearch/bulk.ex index bef67e4..e30d1b8 100644 --- a/lib/exlasticsearch/bulk.ex +++ b/lib/exlasticsearch/bulk.ex @@ -39,10 +39,13 @@ defmodule ExlasticSearch.BulkOperation do end defp bulk_operation_update({:update, struct, id, data, index}) do - [ - %{update: %{_id: id, _index: struct.__es_index__(index), _type: struct.__doc_type__()}}, - data - ] + op = %{_id: id, _index: struct.__es_index__(index)} + + if doc_type = struct.__doc_type__(), + do: Map.put(op, :_type, doc_type), + else: op + + [%{update: op}, data] end defp bulk_operation_delete({:delete, %{__struct__: model} = struct, index}) do diff --git a/lib/exlasticsearch/repo.ex b/lib/exlasticsearch/repo.ex index 70ad272..97d92a6 100644 --- a/lib/exlasticsearch/repo.ex +++ b/lib/exlasticsearch/repo.ex @@ -182,22 +182,11 @@ defmodule ExlasticSearch.Repo do """ @decorate retry() def update(model, id, data, index \\ :index) do - cond do - doc_type = model.__doc_type__() -> - index - |> es_url() - |> Document.update(model.__es_index__(index), doc_type, id, data) - |> log_response() - |> mark_failure() - - match?(%{doc: _}, data) -> - model - |> struct(Map.put(data.doc, :id, id)) - |> index(index) - - true -> - {:error, "ExlasticSearch only supports doc updates for ES 8+"} - end + index + |> es_url() + |> Document.update(model.__es_index__(index), model.__doc_type__(), id, data) + |> log_response() + |> mark_failure() end @doc """ diff --git a/test/exlasticsearch/repo_test.exs b/test/exlasticsearch/repo_test.exs index c822a4c..78a6cfd 100644 --- a/test/exlasticsearch/repo_test.exs +++ b/test/exlasticsearch/repo_test.exs @@ -89,7 +89,7 @@ defmodule ExlasticSearch.RepoTest do data1 = %{script: %{source: source, params: %{data: %{name: "arsenal", rating: 1000}}}} data2 = %{script: %{source: source, params: %{data: %{name: "tottenham", rating: -1}}}} - {:ok, _} = + {:ok, %{status_code: 200}} = Repo.bulk([ {:update, ExlasticSearch.TypelessTestModel, model1.id, data1}, {:update, ExlasticSearch.TypelessTestModel, model2.id, data2} From ff39af70592db844e4faf9111a3b1d519d63c082 Mon Sep 17 00:00:00 2001 From: William Gelhar Date: Wed, 26 Mar 2025 16:52:53 -0400 Subject: [PATCH 12/13] matching against status code in tests because its way too easy to break stuff without knowing just because the http request succeeds but with an errorful status code --- test/exlasticsearch/repo_test.exs | 34 +++++++++++++++---------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/test/exlasticsearch/repo_test.exs b/test/exlasticsearch/repo_test.exs index 78a6cfd..c9de87a 100644 --- a/test/exlasticsearch/repo_test.exs +++ b/test/exlasticsearch/repo_test.exs @@ -29,7 +29,7 @@ defmodule ExlasticSearch.RepoTest do describe "#index" do test "It will index an element in es 8+" do model = %ExlasticSearch.TypelessTestModel{id: Ecto.UUID.generate()} - {:ok, _} = Repo.index(model, :es8) + {:ok, %{status_code: 201}} = Repo.index(model, :es8) assert exists?(model, :es8) end @@ -41,7 +41,7 @@ defmodule ExlasticSearch.RepoTest do model = %ExlasticSearch.TypelessTestModel{id: id, name: "test"} Repo.index(model, :es8) - {:ok, _} = Repo.update(ExlasticSearch.TypelessTestModel, id, %{doc: %{name: "test edited"}}, :es8) + {:ok, %{status_code: 200}} = Repo.update(ExlasticSearch.TypelessTestModel, id, %{doc: %{name: "test edited"}}, :es8) {:ok, %{_source: data}} = Repo.get(model, :es8) assert data.name == "test edited" @@ -56,7 +56,7 @@ defmodule ExlasticSearch.RepoTest do Repo.index(model1) Repo.index(model2) - {:ok, _} = + {:ok, %{status_code: 200}} = Repo.bulk([ {:update, ExlasticSearch.TypelessTestModel, model1.id, %{doc: %{name: "test 1 edited"}}}, {:update, ExlasticSearch.TypelessTestModel, model2.id, %{doc: %{name: "test 2 edited"}}} @@ -107,7 +107,7 @@ defmodule ExlasticSearch.RepoTest do test "It will bulk index/delete from es 8+" do model = %ExlasticSearch.TypelessTestModel{id: Ecto.UUID.generate()} - {:ok, _} = Repo.bulk([{:index, model, :es8}], :es8) + {:ok, %{status_code: 200}} = Repo.bulk([{:index, model, :es8}], :es8) assert exists?(model, :es8) end @@ -116,7 +116,7 @@ defmodule ExlasticSearch.RepoTest do describe "#rotate" do test "It can deprecate an old index version on es 8+" do model = %TypelessMVTestModel{id: Ecto.UUID.generate()} - {:ok, _} = Repo.index(model, :es8) + {:ok, %{status_code: 201}} = Repo.index(model, :es8) Repo.rotate(TypelessMVTestModel, :read, :es8) @@ -130,7 +130,7 @@ defmodule ExlasticSearch.RepoTest do for i <- 1..3, do: %TypelessTestModel{id: Ecto.UUID.generate(), name: "name #{i}", age: i} - {:ok, _} = models |> Enum.map(&{:index, &1, :es8}) |> Repo.bulk(:es8) + {:ok, %{status_code: 200}} = models |> Enum.map(&{:index, &1, :es8}) |> Repo.bulk(:es8) Repo.refresh(TypelessTestModel, :es8) @@ -165,7 +165,7 @@ defmodule ExlasticSearch.RepoTest do } end - {:ok, _} = models |> Enum.map(&{:index, &1, :es8}) |> Repo.bulk(:es8) + {:ok, %{status_code: 200}} = models |> Enum.map(&{:index, &1, :es8}) |> Repo.bulk(:es8) Repo.refresh(TypelessTestModel, :es8) @@ -205,7 +205,7 @@ defmodule ExlasticSearch.RepoTest do } end - {:ok, _} = models |> Enum.map(&{:index, &1, :es8}) |> Repo.bulk(:es8) + {:ok, %{status_code: 200}} = models |> Enum.map(&{:index, &1, :es8}) |> Repo.bulk(:es8) Repo.refresh(TypelessTestModel, :es8) @@ -254,9 +254,9 @@ defmodule ExlasticSearch.RepoTest do model2 = %TypelessTestModel{id: id2, name: rand_name} model3 = %TypelessTestModel{id: id3, name: "something else"} - Repo.index(model1, :es8) - Repo.index(model2, :es8) - Repo.index(model3, :es8) + {:ok, %{status_code: 201}} = Repo.index(model1, :es8) + {:ok, %{status_code: 201}} = Repo.index(model2, :es8) + {:ok, %{status_code: 201}} = Repo.index(model3, :es8) Repo.refresh(TypelessTestModel, :es8) @@ -289,14 +289,14 @@ defmodule ExlasticSearch.RepoTest do model4 = %TypelessTestModel2{id: id4, name: rand_name} - Repo.index(model1) - Repo.index(model2) - Repo.index(model3) + {:ok, %{status_code: 201}} = Repo.index(model1) + {:ok, %{status_code: 201}} = Repo.index(model2) + {:ok, %{status_code: 201}} = Repo.index(model3) - Repo.index(model4) + {:ok, %{status_code: 201}} = Repo.index(model4) - Repo.refresh(TypelessTestModel) - Repo.refresh(TypelessTestModel2) + {:ok, %{status_code: 200}} = Repo.refresh(TypelessTestModel) + {:ok, %{status_code: 200}} = Repo.refresh(TypelessTestModel2) query = %ExlasticSearch.Query{ queryable: [TypelessTestModel, TypelessTestModel2], From 3295aca4898c0716bc8db265b5d6406b6aeb68b3 Mon Sep 17 00:00:00 2001 From: William Gelhar Date: Wed, 26 Mar 2025 16:55:24 -0400 Subject: [PATCH 13/13] removing that test ssh --- test/exlasticsearch/repo_test.exs | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/test/exlasticsearch/repo_test.exs b/test/exlasticsearch/repo_test.exs index c9de87a..9d0701c 100644 --- a/test/exlasticsearch/repo_test.exs +++ b/test/exlasticsearch/repo_test.exs @@ -35,19 +35,6 @@ defmodule ExlasticSearch.RepoTest do end end - describe "#update" do - test "It will update an element in es 8+" do - id = Ecto.UUID.generate() - model = %ExlasticSearch.TypelessTestModel{id: id, name: "test"} - Repo.index(model, :es8) - - {:ok, %{status_code: 200}} = Repo.update(ExlasticSearch.TypelessTestModel, id, %{doc: %{name: "test edited"}}, :es8) - - {:ok, %{_source: data}} = Repo.get(model, :es8) - assert data.name == "test edited" - end - end - describe "#bulk" do test "It will bulk update from es" do model1 = %ExlasticSearch.TypelessTestModel{id: Ecto.UUID.generate(), name: "test 1"}