From 498140560090d0714d5696a49f99a4cb4f9e0cb5 Mon Sep 17 00:00:00 2001 From: Dan Lysiak Date: Tue, 10 Feb 2026 20:18:42 +0000 Subject: [PATCH 1/2] Allow all native ES queries in JestToESConverter --- .../elasticsearch/JestToESConverter.java | 90 +++++++------------ .../ElasticsearchCaseSearchOperationTest.java | 5 +- .../ElasticsearchQueryHelperTest.java | 22 +++++ .../elasticsearch/JestToESConverterTest.java | 75 ++++++++++++++++ 4 files changed, 132 insertions(+), 60 deletions(-) diff --git a/src/main/java/uk/gov/hmcts/ccd/domain/service/search/elasticsearch/JestToESConverter.java b/src/main/java/uk/gov/hmcts/ccd/domain/service/search/elasticsearch/JestToESConverter.java index d45b30b456..bd698d46cb 100644 --- a/src/main/java/uk/gov/hmcts/ccd/domain/service/search/elasticsearch/JestToESConverter.java +++ b/src/main/java/uk/gov/hmcts/ccd/domain/service/search/elasticsearch/JestToESConverter.java @@ -1,18 +1,18 @@ package uk.gov.hmcts.ccd.domain.service.search.elasticsearch; -import co.elastic.clients.elasticsearch._types.query_dsl.Query; import co.elastic.clients.elasticsearch.core.MsearchRequest; import co.elastic.clients.elasticsearch.core.msearch.RequestItem; import co.elastic.clients.elasticsearch.core.search.SearchRequestBody; -import co.elastic.clients.elasticsearch.core.search.SourceConfig; import co.elastic.clients.json.JsonpMapper; import co.elastic.clients.json.jackson.JacksonJsonpMapper; import com.google.gson.Gson; import io.searchbox.core.Search; import jakarta.json.Json; +import jakarta.json.JsonArray; import jakarta.json.JsonObject; import jakarta.json.JsonObjectBuilder; import jakarta.json.JsonReader; +import jakarta.json.JsonValue; import java.io.StringReader; import java.util.List; @@ -21,9 +21,8 @@ public class JestToESConverter { private static final JsonpMapper mapper = new JacksonJsonpMapper(); private static final Gson GSON = new Gson(); - private static final String FALSE = "false"; + private static final String INCLUDES = "includes"; private static final String SOURCE = "_source"; - private static final String TRUE = "true"; public static MsearchRequest fromJest(List jestSearches) { List items = jestSearches.stream() @@ -55,69 +54,44 @@ private static RequestItem toRequestItem(Search jestSearch) { } private static SearchRequestBody parseBody(String json) { - String queryJson = normalizeRequest(json); - SourceConfig sourceConfig = parseSourceJson(extractSource(json)); + String normalizedBody = normalizeSourceSyntax(json); try { - try (JsonReader reader = Json.createReader(new StringReader(queryJson))) { - JsonObject jsonObject = reader.readObject(); - SearchRequestBody requestBody = SearchRequestBody._DESERIALIZER.deserialize( - mapper.jsonProvider().createParser(new StringReader(jsonObject.toString())), - mapper - ); - return SearchRequestBody.of(b -> b - .query(requestBody.query()) - .sort(requestBody.sort()) - .from(requestBody.from()) - .size(requestBody.size()) - .source(sourceConfig)); - } - } catch (Exception e) { - Query query = Query.of(b -> b - .withJson(new StringReader(queryJson)) + return SearchRequestBody._DESERIALIZER.deserialize( + mapper.jsonProvider().createParser(new StringReader(normalizedBody)), + mapper ); - - return SearchRequestBody.of(b -> b - .query(query) - .source(sourceConfig)); - } - } - - private static String normalizeRequest(String json) { - try (jakarta.json.JsonReader reader = jakarta.json.Json.createReader(new StringReader(json))) { - jakarta.json.JsonObject obj = reader.readObject(); - JsonObjectBuilder builder = Json.createObjectBuilder(); - obj.forEach((k, v) -> { - if (!SOURCE.equals(k)) { - builder.add(k, v); - } - }); - return builder.build().toString(); + } catch (Exception e) { + throw new IllegalArgumentException("Failed to parse Jest Search body", e); } } - private static String extractSource(String json) { + private static String normalizeSourceSyntax(String json) { try (JsonReader reader = Json.createReader(new StringReader(json))) { - JsonObject obj = reader.readObject(); - if (obj.containsKey(SOURCE)) { - return obj.get(SOURCE).toString(); + JsonObject body = reader.readObject(); + if (!body.containsKey(SOURCE)) { + return json; } - } - return "true"; - } - private static SourceConfig parseSourceJson(String src) { - if (src.equals(TRUE) || src.equals(FALSE)) { - return SourceConfig.of(s -> s.fetch(Boolean.valueOf(src))); - } - if (src.startsWith("[")) { - return SourceConfig.of(b -> b.filter(f -> f.includes(parseArray(src)))); - } - return SourceConfig.of(s -> s.fetch(Boolean.TRUE)); - } + JsonValue source = body.get(SOURCE); + if (source.getValueType() != JsonValue.ValueType.ARRAY) { + return json; + } + + JsonArray sourceIncludes = source.asJsonArray(); + JsonObject normalizedSource = Json.createObjectBuilder() + .add(INCLUDES, sourceIncludes) + .build(); + + JsonObjectBuilder normalizedBody = Json.createObjectBuilder(); + body.forEach((key, value) -> { + if (SOURCE.equals(key)) { + normalizedBody.add(key, normalizedSource); + } else { + normalizedBody.add(key, value); + } + }); - private static List parseArray(String src) { - try (JsonReader reader = Json.createReader(new StringReader(src))) { - return reader.readArray().getValuesAs(v -> v.toString().replace("\"", "")); + return normalizedBody.build().toString(); } } } diff --git a/src/test/java/uk/gov/hmcts/ccd/domain/service/search/elasticsearch/ElasticsearchCaseSearchOperationTest.java b/src/test/java/uk/gov/hmcts/ccd/domain/service/search/elasticsearch/ElasticsearchCaseSearchOperationTest.java index ebe5a3e98a..e0a73e5e93 100644 --- a/src/test/java/uk/gov/hmcts/ccd/domain/service/search/elasticsearch/ElasticsearchCaseSearchOperationTest.java +++ b/src/test/java/uk/gov/hmcts/ccd/domain/service/search/elasticsearch/ElasticsearchCaseSearchOperationTest.java @@ -12,7 +12,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.JsonNodeFactory; import com.fasterxml.jackson.databind.node.ObjectNode; -import com.fasterxml.jackson.databind.node.TextNode; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Nested; @@ -318,7 +317,9 @@ void setUp() { when(applicationParams.getCasesIndexNameCaseTypeIdGroup()).thenReturn("(.+)(_cases.*)"); when(applicationParams.getCasesIndexNameCaseTypeIdGroupPosition()).thenReturn(1); when(applicationParams.getCasesIndexType()).thenReturn(INDEX_TYPE); - searchRequestJsonNode.set(QUERY, new TextNode("queryVal")); + ObjectNode matchAllQuery = JsonNodeFactory.instance.objectNode(); + matchAllQuery.set("match_all", JsonNodeFactory.instance.objectNode()); + searchRequestJsonNode.set(QUERY, matchAllQuery); searchOperation = new ElasticsearchCaseSearchOperation( elasticsearchClient, diff --git a/src/test/java/uk/gov/hmcts/ccd/domain/service/search/elasticsearch/ElasticsearchQueryHelperTest.java b/src/test/java/uk/gov/hmcts/ccd/domain/service/search/elasticsearch/ElasticsearchQueryHelperTest.java index 3a9913b042..02b23c4540 100644 --- a/src/test/java/uk/gov/hmcts/ccd/domain/service/search/elasticsearch/ElasticsearchQueryHelperTest.java +++ b/src/test/java/uk/gov/hmcts/ccd/domain/service/search/elasticsearch/ElasticsearchQueryHelperTest.java @@ -217,6 +217,28 @@ void shouldConvertNativeQueryToElasticSearchRequest() { ); } + @Test + void shouldPreserveSearchAfterInConvertedElasticSearchRequest() { + String searchRequest = """ + { + "query": {"match_all": {}}, + "sort": [{"id": {"order": "asc"}}], + "search_after": [123456789, "case-2"] + } + """; + + ElasticsearchRequest elasticsearchRequest + = elasticsearchQueryHelper.validateAndConvertRequest(searchRequest); + + JsonNode searchAfterNode = elasticsearchRequest.getNativeSearchRequest().get("search_after"); + + assertAll( + () -> assertTrue(elasticsearchRequest.getNativeSearchRequest().has("search_after")), + () -> assertThat(searchAfterNode.get(0).asLong(), is(123456789L)), + () -> assertThat(searchAfterNode.get(1).asText(), is("case-2")) + ); + } + @Test void shouldConvertWrappedQueryToElasticSearchRequest() { String searchRequest diff --git a/src/test/java/uk/gov/hmcts/ccd/domain/service/search/elasticsearch/JestToESConverterTest.java b/src/test/java/uk/gov/hmcts/ccd/domain/service/search/elasticsearch/JestToESConverterTest.java index 6145d8133f..1e580d4e8d 100644 --- a/src/test/java/uk/gov/hmcts/ccd/domain/service/search/elasticsearch/JestToESConverterTest.java +++ b/src/test/java/uk/gov/hmcts/ccd/domain/service/search/elasticsearch/JestToESConverterTest.java @@ -1,18 +1,26 @@ package uk.gov.hmcts.ccd.domain.service.search.elasticsearch; +import co.elastic.clients.json.jackson.JacksonJsonpMapper; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; import co.elastic.clients.elasticsearch.core.MsearchRequest; import co.elastic.clients.elasticsearch.core.msearch.RequestItem; import io.searchbox.core.Search; import org.junit.jupiter.api.Test; +import java.io.StringWriter; import java.util.List; +import static org.junit.jupiter.api.Assertions.assertAll; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; class JestToESConverterTest { + private final ObjectMapper objectMapper = new ObjectMapper(); + @Test void fromJest_shouldConvertSingleSearch() { Search jestSearch = new Search.Builder("{\"query\":{\"match_all\":{}},\"size\":10}") @@ -40,4 +48,71 @@ void toRequestItem_shouldThrowOnEmptyBody() { JestToESConverter.fromJest(List.of(jestSearch)) ); } + + @Test + void fromJest_shouldPreserveSearchAfterInConvertedBody() throws Exception { + Search jestSearch = new Search.Builder(""" + { + "query": {"match_all": {}}, + "sort": [{"id": {"order": "asc"}}], + "search_after": [123456789, "case-2"], + "size": 25 + } + """) + .addIndex("cases") + .build(); + + MsearchRequest request = JestToESConverter.fromJest(List.of(jestSearch)); + RequestItem item = request.searches().get(0); + + StringWriter writer = new StringWriter(); + var mapper = new JacksonJsonpMapper(); + var generator = mapper.jsonProvider().createGenerator(writer); + item.body().serialize(generator, mapper); + generator.close(); + + JsonNode bodyJson = objectMapper.readTree(writer.toString()); + + assertAll( + () -> assertTrue(bodyJson.has("query")), + () -> assertTrue(bodyJson.has("sort")), + () -> assertTrue(bodyJson.has("size")), + () -> assertTrue(bodyJson.has("search_after")), + () -> assertEquals(123456789L, bodyJson.get("search_after").get(0).asLong()), + () -> assertEquals("case-2", bodyJson.get("search_after").get(1).asText()) + ); + } + + @Test + void fromJest_shouldPreserveSearchAfterWhenSourceIsArray() throws Exception { + Search jestSearch = new Search.Builder(""" + { + "_source": ["data", "reference"], + "query": {"match_all": {}}, + "sort": [{"id": {"order": "asc"}}], + "search_after": [987654321, "case-3"], + "size": 25 + } + """) + .addIndex("cases") + .build(); + + MsearchRequest request = JestToESConverter.fromJest(List.of(jestSearch)); + RequestItem item = request.searches().get(0); + + StringWriter writer = new StringWriter(); + var mapper = new JacksonJsonpMapper(); + var generator = mapper.jsonProvider().createGenerator(writer); + item.body().serialize(generator, mapper); + generator.close(); + + JsonNode bodyJson = objectMapper.readTree(writer.toString()); + + assertAll( + () -> assertTrue(bodyJson.has("_source")), + () -> assertTrue(bodyJson.has("search_after")), + () -> assertEquals(987654321L, bodyJson.get("search_after").get(0).asLong()), + () -> assertEquals("case-3", bodyJson.get("search_after").get(1).asText()) + ); + } } From bb2457adc29b2e1e9f810077a0f364bcf7c60116 Mon Sep 17 00:00:00 2001 From: Dan Lysiak Date: Tue, 10 Feb 2026 20:47:21 +0000 Subject: [PATCH 2/2] Remove Jest --- build.gradle | 1 - .../resources/application.properties | 2 - .../ElasticsearchCaseSearchOperation.java | 43 +++---- ...> ElasticsearchMsearchRequestBuilder.java} | 34 ++--- .../elasticsearch/JestSearchResult.java | 30 ----- src/main/resources/application.properties | 2 - .../ElasticsearchCaseSearchOperationTest.java | 63 ++++++++++ ...lasticsearchMsearchRequestBuilderTest.java | 101 +++++++++++++++ .../elasticsearch/JestToESConverterTest.java | 118 ------------------ 9 files changed, 193 insertions(+), 201 deletions(-) rename src/main/java/uk/gov/hmcts/ccd/domain/service/search/elasticsearch/{JestToESConverter.java => ElasticsearchMsearchRequestBuilder.java} (70%) delete mode 100644 src/main/java/uk/gov/hmcts/ccd/domain/service/search/elasticsearch/JestSearchResult.java create mode 100644 src/test/java/uk/gov/hmcts/ccd/domain/service/search/elasticsearch/ElasticsearchMsearchRequestBuilderTest.java delete mode 100644 src/test/java/uk/gov/hmcts/ccd/domain/service/search/elasticsearch/JestToESConverterTest.java diff --git a/build.gradle b/build.gradle index 47a12ead3c..a8b82fc8a3 100644 --- a/build.gradle +++ b/build.gradle @@ -257,7 +257,6 @@ dependencies { implementation group: 'io.jsonwebtoken', name: 'jjwt-api', version: '0.13.0' implementation group: 'io.jsonwebtoken', name: 'jjwt-impl', version: '0.13.0' implementation group: 'io.jsonwebtoken', name: 'jjwt-jackson', version: '0.13.0' - implementation group: 'io.searchbox', name: 'jest', version: '6.3.1' implementation group: 'io.vavr', name: 'vavr', version: '0.11.0' implementation group: 'jakarta.validation', name: 'jakarta.validation-api', version: '3.1.1' implementation group: 'org.apache.commons', name: 'commons-jexl3', version: '3.6.1' diff --git a/src/contractTest/resources/application.properties b/src/contractTest/resources/application.properties index 33aff8980d..a953b78f9d 100644 --- a/src/contractTest/resources/application.properties +++ b/src/contractTest/resources/application.properties @@ -192,8 +192,6 @@ search.cases.index.name.type=${ELASTIC_SEARCH_CASE_INDEX_TYPE:_doc} search.elastic.nodes.discovery.enabled=${ELASTIC_SEARCH_NODES_DISCOVERY_ENABLED:false} search.elastic.nodes.discovery.frequency.millis=${ELASTIC_SEARCH_NODES_DISCOVERY_FREQUENCY_MILLIS:5000} search.elastic.nodes.discovery.filter=${ELASTIC_SEARCH_NODES_DISCOVERY_FILTER:_all} -spring.elasticsearch.jest.uris=${ELASTIC_SEARCH_HOSTS:http://localhost:9200} -spring.elasticsearch.jest.read-timeout=10000ms search.global.index.name=${GLOBAL_SEARCH_INDEX_NAME:global_search} search.global.index.type=${GLOBAL_SEARCH_INDEX_TYPE:_doc} search.internal.case-access-metadata.enabled=${INTERNAL_SEARCH_CAM_ENABLED:false} diff --git a/src/main/java/uk/gov/hmcts/ccd/domain/service/search/elasticsearch/ElasticsearchCaseSearchOperation.java b/src/main/java/uk/gov/hmcts/ccd/domain/service/search/elasticsearch/ElasticsearchCaseSearchOperation.java index f3f0b60e00..26835636ec 100644 --- a/src/main/java/uk/gov/hmcts/ccd/domain/service/search/elasticsearch/ElasticsearchCaseSearchOperation.java +++ b/src/main/java/uk/gov/hmcts/ccd/domain/service/search/elasticsearch/ElasticsearchCaseSearchOperation.java @@ -4,10 +4,10 @@ import co.elastic.clients.elasticsearch.core.MsearchRequest; import co.elastic.clients.elasticsearch.core.MsearchResponse; import co.elastic.clients.elasticsearch.core.msearch.MultiSearchResponseItem; +import co.elastic.clients.elasticsearch.core.msearch.RequestItem; import co.elastic.clients.elasticsearch.core.search.Hit; import co.elastic.clients.elasticsearch.core.search.TotalHits; import com.fasterxml.jackson.databind.ObjectMapper; -import io.searchbox.core.Search; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; @@ -32,7 +32,6 @@ import java.util.stream.Collectors; import static java.lang.String.format; -import static java.util.stream.Collectors.toList; @Service @Qualifier(ElasticsearchCaseSearchOperation.QUALIFIER) @@ -80,38 +79,38 @@ private MsearchResponse search(CrossCaseTypeSearchR } private MsearchRequest secureAndTransformSearchRequest(CrossCaseTypeSearchRequest request) { - final List securedSearches = request.getSearchIndex() - .map(searchIndex -> List.of(createSecuredSearch(searchIndex, request))) - .orElseGet(() -> buildSearchesByCaseType(request)); - return JestToESConverter.fromJest(securedSearches); + final List securedSearches = request.getSearchIndex() + .map(searchIndex -> List.of(createSecuredRequestItem(searchIndex, request))) + .orElseGet(() -> buildRequestItemsByCaseType(request)); + return new MsearchRequest.Builder().searches(securedSearches).build(); } - private List buildSearchesByCaseType(final CrossCaseTypeSearchRequest request) { + private List buildRequestItemsByCaseType(final CrossCaseTypeSearchRequest request) { return request.getCaseTypeIds() .stream() - .map(caseTypeId -> createSecuredSearch(caseTypeId, request)) - .collect(toList()); + .map(caseTypeId -> createSecuredRequestItem(caseTypeId, request)) + .toList(); } - private Search createSecuredSearch(final SearchIndex searchIndex, final CrossCaseTypeSearchRequest request) { + private RequestItem createSecuredRequestItem(final SearchIndex searchIndex, + final CrossCaseTypeSearchRequest request) { final CrossCaseTypeSearchRequest securedSearchRequest = caseSearchRequestSecurity.createSecuredSearchRequest(request); final ElasticsearchRequest elasticSearchRequest = securedSearchRequest.getElasticSearchRequest(); - - return new Search.Builder(elasticSearchRequest.toFinalRequest()) - .addIndex(searchIndex.getIndexName()) - .addType(searchIndex.getIndexType()) - .build(); + return ElasticsearchMsearchRequestBuilder.createRequestItem( + searchIndex.getIndexName(), + elasticSearchRequest.toFinalRequest() + ); } - private Search createSecuredSearch(String caseTypeId, CrossCaseTypeSearchRequest request) { + private RequestItem createSecuredRequestItem(String caseTypeId, CrossCaseTypeSearchRequest request) { CaseSearchRequest securedSearchRequest = caseSearchRequestSecurity.createSecuredSearchRequest( new CaseSearchRequest(caseTypeId, request.getElasticSearchRequest())); - return new Search.Builder(securedSearchRequest.toJsonString()) - .addIndex(getCaseIndexName(caseTypeId)) - .addType(getCaseIndexType()) - .build(); + return ElasticsearchMsearchRequestBuilder.createRequestItem( + getCaseIndexName(caseTypeId), + securedSearchRequest.toJsonString() + ); } private CaseSearchResult toCaseDetailsSearchResult(MsearchResponse multiSearchResult, @@ -189,8 +188,4 @@ private String getCaseIndexName(String caseTypeId) { return format(applicationParams.getCasesIndexNameFormat(), caseTypeId.toLowerCase()); } - private String getCaseIndexType() { - return applicationParams.getCasesIndexType(); - } - } diff --git a/src/main/java/uk/gov/hmcts/ccd/domain/service/search/elasticsearch/JestToESConverter.java b/src/main/java/uk/gov/hmcts/ccd/domain/service/search/elasticsearch/ElasticsearchMsearchRequestBuilder.java similarity index 70% rename from src/main/java/uk/gov/hmcts/ccd/domain/service/search/elasticsearch/JestToESConverter.java rename to src/main/java/uk/gov/hmcts/ccd/domain/service/search/elasticsearch/ElasticsearchMsearchRequestBuilder.java index bd698d46cb..7c8aa91d29 100644 --- a/src/main/java/uk/gov/hmcts/ccd/domain/service/search/elasticsearch/JestToESConverter.java +++ b/src/main/java/uk/gov/hmcts/ccd/domain/service/search/elasticsearch/ElasticsearchMsearchRequestBuilder.java @@ -1,12 +1,9 @@ package uk.gov.hmcts.ccd.domain.service.search.elasticsearch; -import co.elastic.clients.elasticsearch.core.MsearchRequest; import co.elastic.clients.elasticsearch.core.msearch.RequestItem; import co.elastic.clients.elasticsearch.core.search.SearchRequestBody; import co.elastic.clients.json.JsonpMapper; import co.elastic.clients.json.jackson.JacksonJsonpMapper; -import com.google.gson.Gson; -import io.searchbox.core.Search; import jakarta.json.Json; import jakarta.json.JsonArray; import jakarta.json.JsonObject; @@ -15,37 +12,26 @@ import jakarta.json.JsonValue; import java.io.StringReader; -import java.util.List; -public class JestToESConverter { +public final class ElasticsearchMsearchRequestBuilder { - private static final JsonpMapper mapper = new JacksonJsonpMapper(); - private static final Gson GSON = new Gson(); private static final String INCLUDES = "includes"; + private static final JsonpMapper MAPPER = new JacksonJsonpMapper(); private static final String SOURCE = "_source"; - public static MsearchRequest fromJest(List jestSearches) { - List items = jestSearches.stream() - .map(JestToESConverter::toRequestItem) - .toList(); - - return new MsearchRequest.Builder() - .searches(items) - .build(); + private ElasticsearchMsearchRequestBuilder() { } - private static RequestItem toRequestItem(Search jestSearch) { - String bodyJson = jestSearch.getData(GSON); + public static RequestItem createRequestItem(String indexName, String bodyJson) { if (bodyJson == null || bodyJson.isBlank()) { - throw new IllegalArgumentException("Jest Search body is empty"); + throw new IllegalArgumentException("Search request body is empty"); } SearchRequestBody body = parseBody(bodyJson); - return new RequestItem.Builder() .header(h -> { - if (jestSearch.getIndex() != null) { - h.index(jestSearch.getIndex()); + if (indexName != null && !indexName.isBlank()) { + h.index(indexName); } return h; }) @@ -57,11 +43,11 @@ private static SearchRequestBody parseBody(String json) { String normalizedBody = normalizeSourceSyntax(json); try { return SearchRequestBody._DESERIALIZER.deserialize( - mapper.jsonProvider().createParser(new StringReader(normalizedBody)), - mapper + MAPPER.jsonProvider().createParser(new StringReader(normalizedBody)), + MAPPER ); } catch (Exception e) { - throw new IllegalArgumentException("Failed to parse Jest Search body", e); + throw new IllegalArgumentException("Failed to parse search request body", e); } } diff --git a/src/main/java/uk/gov/hmcts/ccd/domain/service/search/elasticsearch/JestSearchResult.java b/src/main/java/uk/gov/hmcts/ccd/domain/service/search/elasticsearch/JestSearchResult.java deleted file mode 100644 index 6df1743367..0000000000 --- a/src/main/java/uk/gov/hmcts/ccd/domain/service/search/elasticsearch/JestSearchResult.java +++ /dev/null @@ -1,30 +0,0 @@ -package uk.gov.hmcts.ccd.domain.service.search.elasticsearch; - -import com.google.gson.JsonElement; -import com.google.gson.JsonObject; -import com.google.gson.JsonPrimitive; -import io.searchbox.core.SearchResult; - -public class JestSearchResult extends SearchResult { - - public JestSearchResult(SearchResult searchResult) { - super(searchResult); - } - - //Compatible with multiple versions （for es5,es6,es7） - // TODO: remove ASAP under RDM-8901 - //https://github.com/searchbox-io/Jest/issues/656 - @Override - public Long getTotal() { - Long total = null; - JsonElement element = getPath(PATH_TO_TOTAL); - if (element != null) { - if (element instanceof JsonPrimitive) { - return (element).getAsLong(); - } else if (element instanceof JsonObject) { - total = ((JsonObject) element).getAsJsonPrimitive("value").getAsLong(); - } - } - return total; - } -} diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index 594640355c..c951fa9e38 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -218,8 +218,6 @@ search.cases.index.name.type=${ELASTIC_SEARCH_CASE_INDEX_TYPE:_doc} search.elastic.nodes.discovery.enabled=${ELASTIC_SEARCH_NODES_DISCOVERY_ENABLED:false} search.elastic.nodes.discovery.frequency.millis=${ELASTIC_SEARCH_NODES_DISCOVERY_FREQUENCY_MILLIS:5000} search.elastic.nodes.discovery.filter=${ELASTIC_SEARCH_NODES_DISCOVERY_FILTER:_all} -spring.elasticsearch.jest.uris=${ELASTIC_SEARCH_HOSTS:http://localhost:9200} -spring.elasticsearch.jest.read-timeout=10000ms search.global.index.name=${GLOBAL_SEARCH_INDEX_NAME:global_search} search.global.index.type=${GLOBAL_SEARCH_INDEX_TYPE:_doc} search.internal.case-access-metadata.enabled=${INTERNAL_SEARCH_CAM_ENABLED:false} diff --git a/src/test/java/uk/gov/hmcts/ccd/domain/service/search/elasticsearch/ElasticsearchCaseSearchOperationTest.java b/src/test/java/uk/gov/hmcts/ccd/domain/service/search/elasticsearch/ElasticsearchCaseSearchOperationTest.java index e0a73e5e93..6fedeb94b1 100644 --- a/src/test/java/uk/gov/hmcts/ccd/domain/service/search/elasticsearch/ElasticsearchCaseSearchOperationTest.java +++ b/src/test/java/uk/gov/hmcts/ccd/domain/service/search/elasticsearch/ElasticsearchCaseSearchOperationTest.java @@ -5,17 +5,22 @@ import co.elastic.clients.elasticsearch.core.MsearchResponse; import co.elastic.clients.elasticsearch.core.msearch.MultiSearchResponseItem; import co.elastic.clients.elasticsearch.core.search.Hit; +import co.elastic.clients.elasticsearch.core.search.SearchRequestBody; import co.elastic.clients.elasticsearch.core.search.TotalHits; import co.elastic.clients.elasticsearch.core.search.TotalHitsRelation; import co.elastic.clients.json.JsonpMapper; +import co.elastic.clients.json.jackson.JacksonJsonpMapper; import co.elastic.clients.transport.ElasticsearchTransport; +import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.JsonNodeFactory; import com.fasterxml.jackson.databind.node.ObjectNode; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; import org.mockito.Mock; import org.mockito.MockitoAnnotations; import uk.gov.hmcts.ccd.ApplicationParams; @@ -29,6 +34,7 @@ import uk.gov.hmcts.ccd.endpoint.exceptions.ServiceException; import java.io.IOException; +import java.io.StringWriter; import java.util.List; import java.util.Map; @@ -380,6 +386,54 @@ void testSearchNamedIndex() throws IOException { .createSecuredSearchRequest(any(CrossCaseTypeSearchRequest.class)) ); } + + @Test + @DisplayName("should preserve search_after in msearch request body") + void shouldPreserveSearchAfterInMsearchRequestBody() throws Exception { + final String indexName = "global_search"; + + ObjectNode searchRequestNode = JsonNodeFactory.instance.objectNode(); + searchRequestNode.set(QUERY, JsonNodeFactory.instance.objectNode() + .set("match_all", JsonNodeFactory.instance.objectNode())); + + ArrayNode sort = JsonNodeFactory.instance.arrayNode(); + sort.add(JsonNodeFactory.instance.objectNode() + .set("id", JsonNodeFactory.instance.objectNode().put("order", "asc"))); + searchRequestNode.set("sort", sort); + + ArrayNode searchAfter = JsonNodeFactory.instance.arrayNode(); + searchAfter.add(123456789L); + searchAfter.add("case-2"); + searchRequestNode.set("search_after", searchAfter); + + ElasticsearchRequest requestWithSearchAfter = new ElasticsearchRequest(searchRequestNode); + CrossCaseTypeSearchRequest crossCaseTypeSearchRequest = new CrossCaseTypeSearchRequest.Builder() + .withCaseTypes(List.of(CASE_TYPE_ID_1, CASE_TYPE_ID_2)) + .withSearchRequest(requestWithSearchAfter) + .withSearchIndex(new SearchIndex(indexName, "_doc")) + .build(); + + when(caseSearchRequestSecurity.createSecuredSearchRequest(any(CrossCaseTypeSearchRequest.class))) + .thenReturn(crossCaseTypeSearchRequest); + + MultiSearchResponseItem responseItem = createSuccessItemWithNoHits(indexName); + MsearchResponse msearchResponse = createMsearchResponse(List.of(responseItem)); + when(elasticsearchClient.msearch(any(MsearchRequest.class), eq(ElasticSearchCaseDetailsDTO.class))) + .thenReturn(msearchResponse); + + searchOperation.execute(crossCaseTypeSearchRequest, true); + + ArgumentCaptor requestCaptor = ArgumentCaptor.forClass(MsearchRequest.class); + verify(elasticsearchClient).msearch(requestCaptor.capture(), eq(ElasticSearchCaseDetailsDTO.class)); + + JsonNode requestBody = toJson(requestCaptor.getValue().searches().getFirst().body()); + + assertAll( + () -> assertThat(requestBody.has("search_after")).isTrue(), + () -> assertThat(requestBody.get("search_after").get(0).asLong()).isEqualTo(123456789L), + () -> assertThat(requestBody.get("search_after").get(1).asText()).isEqualTo("case-2") + ); + } } @Nested @@ -823,4 +877,13 @@ void searchShouldReturnBadSearchRequestOnResponseError() throws IOException { } + private JsonNode toJson(SearchRequestBody body) throws Exception { + StringWriter writer = new StringWriter(); + var mapper = new JacksonJsonpMapper(); + var generator = mapper.jsonProvider().createGenerator(writer); + body.serialize(generator, mapper); + generator.close(); + return new ObjectMapper().readTree(writer.toString()); + } + } diff --git a/src/test/java/uk/gov/hmcts/ccd/domain/service/search/elasticsearch/ElasticsearchMsearchRequestBuilderTest.java b/src/test/java/uk/gov/hmcts/ccd/domain/service/search/elasticsearch/ElasticsearchMsearchRequestBuilderTest.java new file mode 100644 index 0000000000..a4d4a433e8 --- /dev/null +++ b/src/test/java/uk/gov/hmcts/ccd/domain/service/search/elasticsearch/ElasticsearchMsearchRequestBuilderTest.java @@ -0,0 +1,101 @@ +package uk.gov.hmcts.ccd.domain.service.search.elasticsearch; + +import co.elastic.clients.elasticsearch.core.msearch.RequestItem; +import co.elastic.clients.json.jackson.JacksonJsonpMapper; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.Test; + +import java.io.StringWriter; + +import static org.junit.jupiter.api.Assertions.assertAll; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class ElasticsearchMsearchRequestBuilderTest { + + private final ObjectMapper objectMapper = new ObjectMapper(); + + @Test + void createRequestItem_shouldConvertSingleSearchBody() { + RequestItem item = ElasticsearchMsearchRequestBuilder.createRequestItem( + "cases", + "{\"query\":{\"match_all\":{}},\"size\":10}" + ); + + assertNotNull(item); + assertEquals("cases", item.header().index().getFirst()); + assertNotNull(item.body()); + assertEquals(10, item.body().size()); + assertNotNull(item.body().query()); + } + + @Test + void createRequestItem_shouldThrowOnEmptyBody() { + assertThrows(IllegalArgumentException.class, () -> + ElasticsearchMsearchRequestBuilder.createRequestItem("cases", "") + ); + } + + @Test + void createRequestItem_shouldPreserveSearchAfterInConvertedBody() throws Exception { + RequestItem item = ElasticsearchMsearchRequestBuilder.createRequestItem( + "cases", + """ + { + "query": {"match_all": {}}, + "sort": [{"id": {"order": "asc"}}], + "search_after": [123456789, "case-2"], + "size": 25 + } + """ + ); + + JsonNode bodyJson = toBodyJson(item); + + assertAll( + () -> assertTrue(bodyJson.has("query")), + () -> assertTrue(bodyJson.has("sort")), + () -> assertTrue(bodyJson.has("size")), + () -> assertTrue(bodyJson.has("search_after")), + () -> assertEquals(123456789L, bodyJson.get("search_after").get(0).asLong()), + () -> assertEquals("case-2", bodyJson.get("search_after").get(1).asText()) + ); + } + + @Test + void createRequestItem_shouldPreserveSearchAfterWhenSourceIsArray() throws Exception { + RequestItem item = ElasticsearchMsearchRequestBuilder.createRequestItem( + "cases", + """ + { + "_source": ["data", "reference"], + "query": {"match_all": {}}, + "sort": [{"id": {"order": "asc"}}], + "search_after": [987654321, "case-3"], + "size": 25 + } + """ + ); + + JsonNode bodyJson = toBodyJson(item); + + assertAll( + () -> assertTrue(bodyJson.has("_source")), + () -> assertTrue(bodyJson.has("search_after")), + () -> assertEquals(987654321L, bodyJson.get("search_after").get(0).asLong()), + () -> assertEquals("case-3", bodyJson.get("search_after").get(1).asText()) + ); + } + + private JsonNode toBodyJson(RequestItem item) throws Exception { + StringWriter writer = new StringWriter(); + var mapper = new JacksonJsonpMapper(); + var generator = mapper.jsonProvider().createGenerator(writer); + item.body().serialize(generator, mapper); + generator.close(); + return objectMapper.readTree(writer.toString()); + } +} diff --git a/src/test/java/uk/gov/hmcts/ccd/domain/service/search/elasticsearch/JestToESConverterTest.java b/src/test/java/uk/gov/hmcts/ccd/domain/service/search/elasticsearch/JestToESConverterTest.java deleted file mode 100644 index 1e580d4e8d..0000000000 --- a/src/test/java/uk/gov/hmcts/ccd/domain/service/search/elasticsearch/JestToESConverterTest.java +++ /dev/null @@ -1,118 +0,0 @@ -package uk.gov.hmcts.ccd.domain.service.search.elasticsearch; - -import co.elastic.clients.json.jackson.JacksonJsonpMapper; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import co.elastic.clients.elasticsearch.core.MsearchRequest; -import co.elastic.clients.elasticsearch.core.msearch.RequestItem; -import io.searchbox.core.Search; -import org.junit.jupiter.api.Test; - -import java.io.StringWriter; -import java.util.List; - -import static org.junit.jupiter.api.Assertions.assertAll; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; - -class JestToESConverterTest { - - private final ObjectMapper objectMapper = new ObjectMapper(); - - @Test - void fromJest_shouldConvertSingleSearch() { - Search jestSearch = new Search.Builder("{\"query\":{\"match_all\":{}},\"size\":10}") - .addIndex("cases") - .build(); - - MsearchRequest request = JestToESConverter.fromJest(List.of(jestSearch)); - - assertNotNull(request); - assertEquals(1, request.searches().size()); - - RequestItem item = request.searches().get(0); - assertEquals("cases", item.header().index().getFirst()); - assertNotNull(item.body()); - assertEquals(10, item.body().size()); - assertNotNull(item.body().query()); - } - - @Test - void toRequestItem_shouldThrowOnEmptyBody() { - Search jestSearch = new Search.Builder("") - .build(); - - assertThrows(IllegalArgumentException.class, () -> - JestToESConverter.fromJest(List.of(jestSearch)) - ); - } - - @Test - void fromJest_shouldPreserveSearchAfterInConvertedBody() throws Exception { - Search jestSearch = new Search.Builder(""" - { - "query": {"match_all": {}}, - "sort": [{"id": {"order": "asc"}}], - "search_after": [123456789, "case-2"], - "size": 25 - } - """) - .addIndex("cases") - .build(); - - MsearchRequest request = JestToESConverter.fromJest(List.of(jestSearch)); - RequestItem item = request.searches().get(0); - - StringWriter writer = new StringWriter(); - var mapper = new JacksonJsonpMapper(); - var generator = mapper.jsonProvider().createGenerator(writer); - item.body().serialize(generator, mapper); - generator.close(); - - JsonNode bodyJson = objectMapper.readTree(writer.toString()); - - assertAll( - () -> assertTrue(bodyJson.has("query")), - () -> assertTrue(bodyJson.has("sort")), - () -> assertTrue(bodyJson.has("size")), - () -> assertTrue(bodyJson.has("search_after")), - () -> assertEquals(123456789L, bodyJson.get("search_after").get(0).asLong()), - () -> assertEquals("case-2", bodyJson.get("search_after").get(1).asText()) - ); - } - - @Test - void fromJest_shouldPreserveSearchAfterWhenSourceIsArray() throws Exception { - Search jestSearch = new Search.Builder(""" - { - "_source": ["data", "reference"], - "query": {"match_all": {}}, - "sort": [{"id": {"order": "asc"}}], - "search_after": [987654321, "case-3"], - "size": 25 - } - """) - .addIndex("cases") - .build(); - - MsearchRequest request = JestToESConverter.fromJest(List.of(jestSearch)); - RequestItem item = request.searches().get(0); - - StringWriter writer = new StringWriter(); - var mapper = new JacksonJsonpMapper(); - var generator = mapper.jsonProvider().createGenerator(writer); - item.body().serialize(generator, mapper); - generator.close(); - - JsonNode bodyJson = objectMapper.readTree(writer.toString()); - - assertAll( - () -> assertTrue(bodyJson.has("_source")), - () -> assertTrue(bodyJson.has("search_after")), - () -> assertEquals(987654321L, bodyJson.get("search_after").get(0).asLong()), - () -> assertEquals("case-3", bodyJson.get("search_after").get(1).asText()) - ); - } -}