diff --git a/.github/workflows/elixir.yml b/.github/workflows/elixir.yml index 791efaf..de010cf 100644 --- a/.github/workflows/elixir.yml +++ b/.github/workflows/elixir.yml @@ -23,9 +23,18 @@ jobs: include: - elixir: "1.15" otp: "25" + elasticsearch: "7.17.16" + include_tags: "" - elixir: "1.18" otp: "27" lint: true + elasticsearch: "7.17.16" + include_tags: "" + - elixir: "1.18" + otp: "27" + lint: true + elasticsearch: "8.13.4" + include_tags: "--exclude mapping_types" steps: - name: Configure sysctl limits run: | @@ -37,7 +46,7 @@ jobs: - name: Runs Elasticsearch uses: elastic/elastic-github-actions/elasticsearch@master with: - stack-version: 7.17.16 + stack-version: ${{matrix.elasticsearch}} security-enabled: false - name: Checkout code @@ -63,7 +72,7 @@ jobs: run: mix compile --warnings-as-errors - name: Run tests - run: mix test + run: mix test ${{matrix.include_tags}} - name: checks that the mix.lock file has no unused deps run: mix deps.unlock --check-unused diff --git a/config/test.exs b/config/test.exs index 8d6f83d..21f588f 100644 --- a/config/test.exs +++ b/config/test.exs @@ -2,4 +2,7 @@ import Config config :elastix, json_codec: Jason +config :exlasticsearch, ExlasticSearch.Repo, url: "http://localhost:9200" +config :exlasticsearch, ExlasticSearch.TypelessTestModel, index_type: :es8 +config :exlasticsearch, :monitoring, ExlasticSearch.Monitoring.Mock config :exlasticsearch, json_library: Jason diff --git a/lib/exlasticsearch/bulk.ex b/lib/exlasticsearch/bulk.ex index 5d0b0d8..e30d1b8 100644 --- a/lib/exlasticsearch/bulk.ex +++ b/lib/exlasticsearch/bulk.ex @@ -22,33 +22,46 @@ defmodule ExlasticSearch.BulkOperation do def bulk_operation({op_type, struct}), do: bulk_operation_default({op_type, struct, :index}) defp bulk_operation_default({op_type, %{__struct__: model} = struct, index}) do + op = %{ + _id: Indexable.id(struct), + _index: model.__es_index__(index) + } + + op = + if doc_type = model.__doc_type__(), + do: Map.put(op, :_type, doc_type), + else: op + [ - %{ - op_type => %{ - _id: Indexable.id(struct), - _index: model.__es_index__(index), - _type: model.__doc_type__() - } - }, + %{op_type => op}, build_document(struct, index) ] end defp bulk_operation_update({:update, struct, id, data, index}) do - [ - %{update: %{_id: id, _index: struct.__es_index__(index), _type: struct.__doc_type__()}}, - data - ] + op = %{_id: id, _index: struct.__es_index__(index)} + + if doc_type = struct.__doc_type__(), + do: Map.put(op, :_type, doc_type), + else: op + + [%{update: op}, data] end defp bulk_operation_delete({:delete, %{__struct__: model} = struct, index}) do + op = %{ + _id: Indexable.id(struct), + _index: model.__es_index__(index) + } + + op = + if doc_type = model.__doc_type__(), + do: Map.put(op, :_type, doc_type), + else: op + [ %{ - delete: %{ - _id: Indexable.id(struct), - _index: model.__es_index__(index), - _type: model.__doc_type__() - } + delete: op } ] end diff --git a/lib/exlasticsearch/model.ex b/lib/exlasticsearch/model.ex index 56ff44f..a81c571 100644 --- a/lib/exlasticsearch/model.ex +++ b/lib/exlasticsearch/model.ex @@ -77,13 +77,15 @@ defmodule ExlasticSearch.Model do * `block` - the definition of the index """ - defmacro indexes(type, block) do + defmacro indexes(type, opts \\ [], block) do + doc_type = Keyword.get(opts, :doc_type, type) + quote do Module.register_attribute(__MODULE__, :es_mappings, accumulate: true) @read_version :ignore @index_version :ignore - def __doc_type__, do: unquote(type) + def __doc_type__, do: unquote(doc_type) unquote(block) diff --git a/lib/exlasticsearch/repo.ex b/lib/exlasticsearch/repo.ex index f022fb8..97d92a6 100644 --- a/lib/exlasticsearch/repo.ex +++ b/lib/exlasticsearch/repo.ex @@ -71,7 +71,7 @@ defmodule ExlasticSearch.Repo do def create_mapping(model, index \\ :index, opts \\ []) do index |> es_url() - |> Mapping.put(model.__es_index__(index), model.__doc_type__(), model.__es_mappings__(), opts) + |> Mapping.put(model.__es_index__(index), model.__doc_type__() || "", model.__es_mappings__(), opts) end @doc """ @@ -172,7 +172,7 @@ defmodule ExlasticSearch.Repo do index |> es_url() - |> Document.index(model.__es_index__(index), model.__doc_type__(), id, document) + |> Document.index(model.__es_index__(index), model.__doc_type__() || "_doc", id, document) |> log_response() |> mark_failure() end @@ -234,7 +234,7 @@ defmodule ExlasticSearch.Repo do def get(%{__struct__: model} = struct, index_type \\ :read) do index_type |> es_url() - |> Document.get(model.__es_index__(index_type), model.__doc_type__(), Indexable.id(struct)) + |> Document.get(model.__es_index__(index_type), model.__doc_type__() || "_doc", Indexable.id(struct)) |> log_response() |> decode(Response.Record, model) end @@ -268,10 +268,12 @@ defmodule ExlasticSearch.Repo do defp model_to_index(model, index_type), do: model.__es_index__(index_type) defp model_to_doc_types(models) when is_list(models) do - Enum.map(models, & &1.__doc_type__()) + Enum.flat_map(models, &model_to_doc_types/1) end - defp model_to_doc_types(model), do: [model.__doc_type__()] + defp model_to_doc_types(model) do + if doc_type = model.__doc_type__(), do: [doc_type], else: [] + end @doc """ Performs an aggregation against a query, and returns only the aggregation results. @@ -284,9 +286,12 @@ defmodule ExlasticSearch.Repo do index_type = query.index_type || :read + doc_types = if doc_type = model.__doc_type__(), do: [doc_type], else: [] + es_index = model.__es_index__(index_type) + index_type |> es_url() - |> Search.search(model.__es_index__(index_type), [model.__doc_type__()], search, size: 0) + |> Search.search(es_index, doc_types, search, size: 0) # TODO: figure out how to decode these, it's not trivial to type them |> log_response() end @@ -299,7 +304,7 @@ defmodule ExlasticSearch.Repo do def delete(%{__struct__: model} = struct, index \\ :index) do index |> es_url() - |> Document.delete(model.__es_index__(index), model.__doc_type__(), Indexable.id(struct)) + |> Document.delete(model.__es_index__(index), model.__doc_type__() || "_doc", Indexable.id(struct)) |> log_response() |> mark_failure() end diff --git a/test/exlasticsearch/repo/mapping_types_test.exs b/test/exlasticsearch/repo/mapping_types_test.exs new file mode 100644 index 0000000..b7381fe --- /dev/null +++ b/test/exlasticsearch/repo/mapping_types_test.exs @@ -0,0 +1,325 @@ +defmodule ExlasticSearch.Repo.MappingTypesTest do + use ExUnit.Case, async: true + + alias ExlasticSearch.Aggregation + alias ExlasticSearch.MultiVersionTestModel, as: MVTestModel + alias ExlasticSearch.Query + alias ExlasticSearch.Repo + alias ExlasticSearch.TestModel + alias ExlasticSearch.TestModel2 + + require Logger + + setup_all do + Repo.delete_index(TestModel) + Repo.create_index(TestModel) + {:ok, %{status_code: 200}} = Repo.create_mapping(TestModel, :index, include_type_name: true) + + Repo.delete_index(TestModel2) + Repo.create_index(TestModel2) + {:ok, %{status_code: 200}} = Repo.create_mapping(TestModel2, :index, include_type_name: true) + + Repo.delete_index(MVTestModel) + Repo.create_index(MVTestModel) + {:ok, %{status_code: 200}} = Repo.create_mapping(MVTestModel, :index, include_type_name: true) + + Repo.delete_index(MVTestModel, :read) + Repo.create_index(MVTestModel, :read) + {:ok, %{status_code: 200}} = Repo.create_mapping(MVTestModel, :read, include_type_name: true) + + :ok + end + + describe "Elasticsearch 7 Mapping Types" do + @describetag :mapping_types + + test "It will index an element in es" do + model = %ExlasticSearch.TestModel{id: Ecto.UUID.generate()} + {:ok, _} = Repo.index(model) + + assert exists?(model) + end + + test "It will update an element in es" do + id = Ecto.UUID.generate() + model = %ExlasticSearch.TestModel{id: id, name: "test"} + Repo.index(model) + + {:ok, _} = Repo.update(ExlasticSearch.TestModel, id, %{doc: %{name: "test edited"}}) + + {:ok, %{_source: data}} = Repo.get(model) + assert data.name == "test edited" + end + + test "It will bulk index/delete from es" do + model = %ExlasticSearch.TestModel{id: Ecto.UUID.generate()} + {:ok, _} = Repo.bulk([{:index, model}]) + + assert exists?(model) + end + + test "It will bulk update from es" do + model1 = %ExlasticSearch.TestModel{id: Ecto.UUID.generate(), name: "test 1"} + model2 = %ExlasticSearch.TestModel{id: Ecto.UUID.generate(), name: "test 2"} + + Repo.index(model1) + Repo.index(model2) + + {:ok, _} = + Repo.bulk([ + {:update, ExlasticSearch.TestModel, model1.id, %{doc: %{name: "test 1 edited"}}}, + {:update, ExlasticSearch.TestModel, model2.id, %{doc: %{name: "test 2 edited"}}} + ]) + + {:ok, %{_source: data1}} = Repo.get(model1) + {:ok, %{_source: data2}} = Repo.get(model2) + + assert data1.name == "test 1 edited" + assert data2.name == "test 2 edited" + end + + test "It will bulk update nested from es" do + model1 = %ExlasticSearch.TestModel{ + id: Ecto.UUID.generate(), + teams: [%{name: "arsenal", rating: 100}] + } + + model2 = %ExlasticSearch.TestModel{ + id: Ecto.UUID.generate(), + teams: [%{name: "tottenham", rating: 0}] + } + + Repo.index(model1) + Repo.index(model2) + + source = + "ctx._source.teams.find(cf -> cf.name == params.data.name).rating = params.data.rating" + + data1 = %{script: %{source: source, params: %{data: %{name: "arsenal", rating: 1000}}}} + data2 = %{script: %{source: source, params: %{data: %{name: "tottenham", rating: -1}}}} + + {:ok, _} = + Repo.bulk([ + {:update, ExlasticSearch.TestModel, model1.id, data1}, + {:update, ExlasticSearch.TestModel, model2.id, data2} + ]) + + {:ok, %{_source: %{teams: [team1]}}} = Repo.get(model1) + {:ok, %{_source: %{teams: [team2]}}} = Repo.get(model2) + + assert team1.name == "arsenal" + assert team1.rating == 1000 + + assert team2.name == "tottenham" + assert team2.rating == -1 + end + + test "It can deprecate an old index version" do + model = %MVTestModel{id: Ecto.UUID.generate()} + {:ok, _} = Repo.index(model) + + Repo.rotate(MVTestModel) + + assert exists?(model) + end + + test "It can perform terms aggregations" do + models = + for i <- 1..3, + do: %TestModel{id: Ecto.UUID.generate(), name: "name #{i}", age: i} + + {:ok, _} = models |> Enum.map(&{:index, &1}) |> Repo.bulk() + + Repo.refresh(TestModel) + + aggregation = Aggregation.terms(Aggregation.new(), :age, field: :age, size: 2) + + {:ok, + %{ + body: %{ + "aggregations" => %{ + "age" => %{ + "buckets" => buckets + } + } + } + }} = + TestModel.search_query() + |> Query.must(Query.match(:name, "name")) + |> Repo.aggregate(aggregation) + + assert length(buckets) == 2 + assert Enum.all?(buckets, &(&1["key"] in [1, 2])) + end + + # can we unskip? fails from mapping types + @tag :skip + test "It can perform top_hits aggregations, even when nested" do + models = + for i <- 1..3 do + %TestModel{ + id: Ecto.UUID.generate(), + name: "name #{i}", + age: i, + group: if(rem(i, 2) == 0, do: "even", else: "odd") + } + end + + {:ok, _} = models |> Enum.map(&{:index, &1}) |> Repo.bulk() + + Repo.refresh(TestModel) + + nested = Aggregation.top_hits(Aggregation.new(), :hits, %{}) + + aggregation = + Aggregation.new() + |> Aggregation.terms(:group, field: String.to_atom("group.keyword")) + |> Aggregation.nest(:group, nested) + + {:ok, + %{ + body: %{ + "aggregations" => %{ + "group" => %{ + "buckets" => buckets + } + } + } + }} = + TestModel.search_query() + |> Query.must(Query.match(:name, "name")) + |> Repo.aggregate(aggregation) + + assert length(buckets) == 2 + assert Enum.all?(buckets, &(!Enum.empty?(get_hits(&1)))) + end + + # can we unskip? fails from mapping types + @tag :skip + test "It can perform composite aggregations" do + models = + for i <- 1..3 do + %TestModel{ + id: Ecto.UUID.generate(), + name: "name #{i}", + age: i, + group: if(rem(i, 2) == 0, do: "even", else: "odd") + } + end + + {:ok, _} = models |> Enum.map(&{:index, &1}) |> Repo.bulk() + + Repo.refresh(TestModel) + + sources = [ + Aggregation.composite_source(:group, :terms, field: String.to_atom("group.keyword"), order: :desc), + Aggregation.composite_source(:age, :terms, field: :age, order: :asc) + ] + + aggregation = Aggregation.composite(Aggregation.new(), :group, sources) + + {:ok, + %{ + body: %{ + "aggregations" => %{ + "group" => %{ + "buckets" => buckets + } + } + } + }} = + TestModel.search_query() + |> Query.must(Query.match(:name, "name")) + |> Repo.aggregate(aggregation) + + for i <- 1..3 do + assert Enum.any?(buckets, fn + %{"key" => %{"age" => ^i, "group" => group}} -> + group == if rem(i, 2) == 0, do: "even", else: "odd" + + _ -> + false + end) + end + end + + @tag :mapping_types + test "It will search in a single index" do + id1 = Ecto.UUID.generate() + id2 = Ecto.UUID.generate() + id3 = Ecto.UUID.generate() + + rand_name = String.replace(Ecto.UUID.generate(), "-", "") + + model1 = %TestModel{id: id1, name: rand_name} + model2 = %TestModel{id: id2, name: rand_name} + model3 = %TestModel{id: id3, name: "something else"} + + Repo.index(model1) + Repo.index(model2) + Repo.index(model3) + + Repo.refresh(TestModel) + + query = %ExlasticSearch.Query{ + queryable: ExlasticSearch.TestModel, + filter: [ + %{term: %{name: rand_name}} + ] + } + + {:ok, %{hits: %{hits: results}}} = Repo.search(query, []) + + assert length(results) == 2 + assert Enum.find(results, &(&1._id == id1)) + assert Enum.find(results, &(&1._id == id2)) + end + + test "It will search in multiple indexes" do + id1 = Ecto.UUID.generate() + id2 = Ecto.UUID.generate() + id3 = Ecto.UUID.generate() + id4 = Ecto.UUID.generate() + + rand_name = String.replace(Ecto.UUID.generate(), "-", "") + + model1 = %TestModel{id: id1, name: rand_name} + model2 = %TestModel{id: id2, name: rand_name} + model3 = %TestModel{id: id3, name: "something else"} + + model4 = %TestModel2{id: id4, name: rand_name} + + Repo.index(model1) + Repo.index(model2) + Repo.index(model3) + + Repo.index(model4) + + Repo.refresh(TestModel) + Repo.refresh(TestModel2) + + query = %ExlasticSearch.Query{ + queryable: [ExlasticSearch.TestModel, ExlasticSearch.TestModel2], + filter: [ + %{term: %{name: rand_name}} + ] + } + + {:ok, %{hits: %{hits: results}}} = Repo.search(query, []) + + assert length(results) == 3 + assert Enum.find(results, &(&1._id == id1)) + assert Enum.find(results, &(&1._id == id2)) + assert Enum.find(results, &(&1._id == id4)) + end + end + + defp exists?(model, index \\ :index) do + case Repo.get(model, index) do + {:ok, %{found: true}} -> true + _ -> false + end + end + + defp get_hits(%{"hits" => %{"hits" => %{"hits" => hits}}}), do: hits +end diff --git a/test/exlasticsearch/repo_test.exs b/test/exlasticsearch/repo_test.exs index 86d557d..9d0701c 100644 --- a/test/exlasticsearch/repo_test.exs +++ b/test/exlasticsearch/repo_test.exs @@ -2,72 +2,51 @@ defmodule ExlasticSearch.RepoTest do use ExUnit.Case, async: true alias ExlasticSearch.Aggregation - alias ExlasticSearch.MultiVersionTestModel, as: MVTestModel alias ExlasticSearch.Query alias ExlasticSearch.Repo - alias ExlasticSearch.TestModel - alias ExlasticSearch.TestModel2 + alias ExlasticSearch.TypelessMultiVersionTestModel, as: TypelessMVTestModel + alias ExlasticSearch.TypelessTestModel + alias ExlasticSearch.TypelessTestModel2 + + require Logger setup_all do - Repo.delete_index(TestModel) - Repo.create_index(TestModel) - Repo.create_mapping(TestModel) + Repo.delete_index(TypelessTestModel, :es8) + Repo.create_index(TypelessTestModel, :es8) + {:ok, %{status_code: 200}} = Repo.create_mapping(TypelessTestModel, :es8) - Repo.delete_index(TestModel2) - Repo.create_index(TestModel2) - Repo.create_mapping(TestModel2) + Repo.delete_index(TypelessTestModel2, :es8) + Repo.create_index(TypelessTestModel2, :es8) + {:ok, %{status_code: 200}} = Repo.create_mapping(TypelessTestModel2, :es8) - Repo.delete_index(MVTestModel) - Repo.create_index(MVTestModel) - Repo.create_mapping(MVTestModel) + Repo.delete_index(TypelessMVTestModel, :es8) + Repo.create_index(TypelessMVTestModel, :es8) + {:ok, %{status_code: 200}} = Repo.create_mapping(TypelessMVTestModel, :es8) - Repo.delete_index(MVTestModel, :read) - Repo.create_index(MVTestModel, :read) - Repo.create_mapping(MVTestModel, :read) :ok end describe "#index" do - test "It will index an element in es" do - model = %ExlasticSearch.TestModel{id: Ecto.UUID.generate()} - {:ok, _} = Repo.index(model) + test "It will index an element in es 8+" do + model = %ExlasticSearch.TypelessTestModel{id: Ecto.UUID.generate()} + {:ok, %{status_code: 201}} = Repo.index(model, :es8) - assert exists?(model) - end - end - - describe "#update" do - test "It will update an element in es" do - id = Ecto.UUID.generate() - model = %ExlasticSearch.TestModel{id: id, name: "test"} - Repo.index(model) - - {:ok, _} = Repo.update(ExlasticSearch.TestModel, id, %{doc: %{name: "test edited"}}) - - {:ok, %{_source: data}} = Repo.get(model) - assert data.name == "test edited" + assert exists?(model, :es8) end end describe "#bulk" do - test "It will bulk index/delete from es" do - model = %ExlasticSearch.TestModel{id: Ecto.UUID.generate()} - {:ok, _} = Repo.bulk([{:index, model}]) - - assert exists?(model) - end - test "It will bulk update from es" do - model1 = %ExlasticSearch.TestModel{id: Ecto.UUID.generate(), name: "test 1"} - model2 = %ExlasticSearch.TestModel{id: Ecto.UUID.generate(), name: "test 2"} + model1 = %ExlasticSearch.TypelessTestModel{id: Ecto.UUID.generate(), name: "test 1"} + model2 = %ExlasticSearch.TypelessTestModel{id: Ecto.UUID.generate(), name: "test 2"} Repo.index(model1) Repo.index(model2) - {:ok, _} = + {:ok, %{status_code: 200}} = Repo.bulk([ - {:update, ExlasticSearch.TestModel, model1.id, %{doc: %{name: "test 1 edited"}}}, - {:update, ExlasticSearch.TestModel, model2.id, %{doc: %{name: "test 2 edited"}}} + {:update, ExlasticSearch.TypelessTestModel, model1.id, %{doc: %{name: "test 1 edited"}}}, + {:update, ExlasticSearch.TypelessTestModel, model2.id, %{doc: %{name: "test 2 edited"}}} ]) {:ok, %{_source: data1}} = Repo.get(model1) @@ -78,12 +57,12 @@ defmodule ExlasticSearch.RepoTest do end test "It will bulk update nested from es" do - model1 = %ExlasticSearch.TestModel{ + model1 = %ExlasticSearch.TypelessTestModel{ id: Ecto.UUID.generate(), teams: [%{name: "arsenal", rating: 100}] } - model2 = %ExlasticSearch.TestModel{ + model2 = %ExlasticSearch.TypelessTestModel{ id: Ecto.UUID.generate(), teams: [%{name: "tottenham", rating: 0}] } @@ -97,10 +76,10 @@ defmodule ExlasticSearch.RepoTest do data1 = %{script: %{source: source, params: %{data: %{name: "arsenal", rating: 1000}}}} data2 = %{script: %{source: source, params: %{data: %{name: "tottenham", rating: -1}}}} - {:ok, _} = + {:ok, %{status_code: 200}} = Repo.bulk([ - {:update, ExlasticSearch.TestModel, model1.id, data1}, - {:update, ExlasticSearch.TestModel, model2.id, data2} + {:update, ExlasticSearch.TypelessTestModel, model1.id, data1}, + {:update, ExlasticSearch.TypelessTestModel, model2.id, data2} ]) {:ok, %{_source: %{teams: [team1]}}} = Repo.get(model1) @@ -112,28 +91,35 @@ defmodule ExlasticSearch.RepoTest do assert team2.name == "tottenham" assert team2.rating == -1 end + + test "It will bulk index/delete from es 8+" do + model = %ExlasticSearch.TypelessTestModel{id: Ecto.UUID.generate()} + {:ok, %{status_code: 200}} = Repo.bulk([{:index, model, :es8}], :es8) + + assert exists?(model, :es8) + end end describe "#rotate" do - test "It can deprecate an old index version" do - model = %MVTestModel{id: Ecto.UUID.generate()} - {:ok, _} = Repo.index(model) + test "It can deprecate an old index version on es 8+" do + model = %TypelessMVTestModel{id: Ecto.UUID.generate()} + {:ok, %{status_code: 201}} = Repo.index(model, :es8) - Repo.rotate(MVTestModel) + Repo.rotate(TypelessMVTestModel, :read, :es8) - assert exists?(model) + assert exists?(model, :es8) end end describe "#aggregate/2" do - test "It can perform terms aggreagtions" do + test "It can perform terms aggregations on es 8+" do models = for i <- 1..3, - do: %TestModel{id: Ecto.UUID.generate(), name: "name #{i}", age: i} + do: %TypelessTestModel{id: Ecto.UUID.generate(), name: "name #{i}", age: i} - {:ok, _} = models |> Enum.map(&{:index, &1}) |> Repo.bulk() + {:ok, %{status_code: 200}} = models |> Enum.map(&{:index, &1, :es8}) |> Repo.bulk(:es8) - Repo.refresh(TestModel) + Repo.refresh(TypelessTestModel, :es8) aggregation = Aggregation.terms(Aggregation.new(), :age, field: :age, size: 2) @@ -147,7 +133,7 @@ defmodule ExlasticSearch.RepoTest do } } }} = - TestModel.search_query() + TypelessTestModel.search_query() |> Query.must(Query.match(:name, "name")) |> Repo.aggregate(aggregation) @@ -155,11 +141,10 @@ defmodule ExlasticSearch.RepoTest do assert Enum.all?(buckets, &(&1["key"] in [1, 2])) end - @tag :skip - test "It can perform top_hits aggregations, even when nested" do + test "It can perform top_hits aggregations, even when nested, on es 8+" do models = for i <- 1..3 do - %TestModel{ + %TypelessTestModel{ id: Ecto.UUID.generate(), name: "name #{i}", age: i, @@ -167,9 +152,9 @@ defmodule ExlasticSearch.RepoTest do } end - {:ok, _} = models |> Enum.map(&{:index, &1}) |> Repo.bulk() + {:ok, %{status_code: 200}} = models |> Enum.map(&{:index, &1, :es8}) |> Repo.bulk(:es8) - Repo.refresh(TestModel) + Repo.refresh(TypelessTestModel, :es8) nested = Aggregation.top_hits(Aggregation.new(), :hits, %{}) @@ -188,7 +173,7 @@ defmodule ExlasticSearch.RepoTest do } } }} = - TestModel.search_query() + TypelessTestModel.search_query() |> Query.must(Query.match(:name, "name")) |> Repo.aggregate(aggregation) @@ -196,11 +181,10 @@ defmodule ExlasticSearch.RepoTest do assert Enum.all?(buckets, &(!Enum.empty?(get_hits(&1)))) end - @tag :skip - test "It can perform composite aggregations" do + test "It can perform composite aggregations on es 8+" do models = for i <- 1..3 do - %TestModel{ + %TypelessTestModel{ id: Ecto.UUID.generate(), name: "name #{i}", age: i, @@ -208,9 +192,9 @@ defmodule ExlasticSearch.RepoTest do } end - {:ok, _} = models |> Enum.map(&{:index, &1}) |> Repo.bulk() + {:ok, %{status_code: 200}} = models |> Enum.map(&{:index, &1, :es8}) |> Repo.bulk(:es8) - Repo.refresh(TestModel) + Repo.refresh(TypelessTestModel, :es8) sources = [ Aggregation.composite_source(:group, :terms, field: :group, order: :desc), @@ -229,7 +213,7 @@ defmodule ExlasticSearch.RepoTest do } } }} = - TestModel.search_query() + TypelessTestModel.search_query() |> Query.must(Query.match(:name, "name")) |> Repo.aggregate(aggregation) @@ -246,28 +230,29 @@ defmodule ExlasticSearch.RepoTest do end describe "#search/2" do - test "It will search in a single index" do + test "It will search in a single index in es 8+" do id1 = Ecto.UUID.generate() id2 = Ecto.UUID.generate() id3 = Ecto.UUID.generate() rand_name = String.replace(Ecto.UUID.generate(), "-", "") - model1 = %TestModel{id: id1, name: rand_name} - model2 = %TestModel{id: id2, name: rand_name} - model3 = %TestModel{id: id3, name: "something else"} + model1 = %TypelessTestModel{id: id1, name: rand_name} + model2 = %TypelessTestModel{id: id2, name: rand_name} + model3 = %TypelessTestModel{id: id3, name: "something else"} - Repo.index(model1) - Repo.index(model2) - Repo.index(model3) + {:ok, %{status_code: 201}} = Repo.index(model1, :es8) + {:ok, %{status_code: 201}} = Repo.index(model2, :es8) + {:ok, %{status_code: 201}} = Repo.index(model3, :es8) - Repo.refresh(TestModel) + Repo.refresh(TypelessTestModel, :es8) query = %ExlasticSearch.Query{ - queryable: ExlasticSearch.TestModel, + queryable: ExlasticSearch.TypelessTestModel, filter: [ %{term: %{name: rand_name}} - ] + ], + index_type: :es8 } {:ok, %{hits: %{hits: results}}} = Repo.search(query, []) @@ -277,7 +262,7 @@ defmodule ExlasticSearch.RepoTest do assert Enum.find(results, &(&1._id == id2)) end - test "It will search in a multiple indexes" do + test "It will search in multiple indexes" do id1 = Ecto.UUID.generate() id2 = Ecto.UUID.generate() id3 = Ecto.UUID.generate() @@ -285,23 +270,23 @@ defmodule ExlasticSearch.RepoTest do rand_name = String.replace(Ecto.UUID.generate(), "-", "") - model1 = %TestModel{id: id1, name: rand_name} - model2 = %TestModel{id: id2, name: rand_name} - model3 = %TestModel{id: id3, name: "something else"} + model1 = %TypelessTestModel{id: id1, name: rand_name} + model2 = %TypelessTestModel{id: id2, name: rand_name} + model3 = %TypelessTestModel{id: id3, name: "something else"} - model4 = %TestModel2{id: id4, name: rand_name} + model4 = %TypelessTestModel2{id: id4, name: rand_name} - Repo.index(model1) - Repo.index(model2) - Repo.index(model3) + {:ok, %{status_code: 201}} = Repo.index(model1) + {:ok, %{status_code: 201}} = Repo.index(model2) + {:ok, %{status_code: 201}} = Repo.index(model3) - Repo.index(model4) + {:ok, %{status_code: 201}} = Repo.index(model4) - Repo.refresh(TestModel) - Repo.refresh(TestModel2) + {:ok, %{status_code: 200}} = Repo.refresh(TypelessTestModel) + {:ok, %{status_code: 200}} = Repo.refresh(TypelessTestModel2) query = %ExlasticSearch.Query{ - queryable: [ExlasticSearch.TestModel, ExlasticSearch.TestModel2], + queryable: [TypelessTestModel, TypelessTestModel2], filter: [ %{term: %{name: rand_name}} ] @@ -316,8 +301,8 @@ defmodule ExlasticSearch.RepoTest do end end - defp exists?(model) do - case Repo.get(model) do + defp exists?(model, index) do + case Repo.get(model, index) do {:ok, %{found: true}} -> true _ -> false end diff --git a/test/support/test_model.ex b/test/support/test_model.ex index 3b69a9a..6b5b0b7 100644 --- a/test/support/test_model.ex +++ b/test/support/test_model.ex @@ -77,8 +77,94 @@ defmodule ExlasticSearch.MultiVersionTestModel do end end +defmodule ExlasticSearch.TypelessTestModel do + @moduledoc false + use Ecto.Schema + use ExlasticSearch.Model + + schema "typeless_test_models" do + field(:name, :string) + field(:age, :integer, default: 0) + field(:group, :string) + field(:teams, {:array, :map}) + end + + indexes :typeless_test_model, doc_type: nil do + versions(2) + settings(%{}) + options(%{dynamic: :strict}) + mapping(:name) + mapping(:age) + mapping(:group, type: :keyword) + + mapping(:user, properties: %{ext_name: %{type: :text}}) + + mapping(:teams, + type: :nested, + properties: %{ + name: %{type: :keyword}, + rating: %{type: :integer} + } + ) + end +end + +defmodule ExlasticSearch.TypelessTestModel2 do + @moduledoc false + use Ecto.Schema + use ExlasticSearch.Model + + schema "typeless_test_models2" do + field(:name, :string) + end + + indexes :typeless_test_model2, doc_type: nil do + versions(2) + settings(%{}) + options(%{dynamic: :strict}) + mapping(:name) + end +end + +defmodule ExlasticSearch.TypelessMultiVersionTestModel do + @moduledoc false + use Ecto.Schema + use ExlasticSearch.Model + + schema "typeless_mv_models" do + field(:name, :string) + field(:age, :integer, default: 0) + field(:teams, {:array, :map}) + end + + indexes :typeless_multiversion_model, doc_type: nil do + versions({:ignore, 2}) + settings(%{}) + options(%{dynamic: :strict}) + mapping(:name) + mapping(:age) + + mapping(:user, properties: %{ext_name: %{type: :text}}) + + mapping(:teams, + type: :nested, + properties: %{ + name: %{type: :keyword}, + rating: %{type: :integer} + } + ) + end +end + defimpl ExlasticSearch.Indexable, - for: [ExlasticSearch.TestModel, ExlasticSearch.TestModel2, ExlasticSearch.MultiVersionTestModel] do + for: [ + ExlasticSearch.TestModel, + ExlasticSearch.TestModel2, + ExlasticSearch.MultiVersionTestModel, + ExlasticSearch.TypelessTestModel, + ExlasticSearch.TypelessTestModel2, + ExlasticSearch.TypelessMultiVersionTestModel + ] do def id(%{id: id}), do: id def document(struct) do