diff --git a/search-api/search-actors/src/main/java/org/sunbird/actors/SearchActor.java b/search-api/search-actors/src/main/java/org/sunbird/actors/SearchActor.java index 68f175363..63ad3d453 100644 --- a/search-api/search-actors/src/main/java/org/sunbird/actors/SearchActor.java +++ b/search-api/search-actors/src/main/java/org/sunbird/actors/SearchActor.java @@ -13,6 +13,7 @@ import org.sunbird.common.dto.Response; import org.sunbird.common.exception.ClientException; import org.sunbird.common.exception.ResponseCode; +import org.sunbird.search.client.ElasticSearchUtil; import org.sunbird.search.dto.SearchDTO; import org.sunbird.search.processor.SearchProcessor; import org.sunbird.search.util.DefinitionUtil; @@ -22,6 +23,7 @@ import scala.concurrent.Future; import scala.concurrent.duration.Duration; +import javax.naming.directory.SearchResult; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; @@ -42,7 +44,8 @@ public Future onReceive(Request request) throws Throwable { try{ if (StringUtils.equalsIgnoreCase("INDEX_SEARCH", operation)) { SearchDTO searchDTO = getSearchDTO(request); - Future> searchResult = processor.processSearch(searchDTO, true); + boolean fuzzyEnabled=false; + Future> searchResult = processor.processSearch(searchDTO, true, fuzzyEnabled); return searchResult.map(new Mapper, Response>() { @Override public Response apply(Map lstResult) { @@ -594,7 +597,7 @@ private Map getCompositeSearchResponse(Map searc String objectType = ((String) map.getOrDefault("objectType", "")).replaceAll("Image", ""); if(StringUtils.equalsIgnoreCase("Collection", objectType) || StringUtils.equalsIgnoreCase("Asset", objectType)) map.replace("objectType", "Content"); - else + else map.replace("objectType", objectType); if (StringUtils.isNotBlank(objectType)) { String key = getResultParamKey(objectType); @@ -777,4 +780,4 @@ private void setImplicitFilters(Map filters, SearchDTO searchObj searchObj.setImplicitFilterProperties(implicitFilterProps); } } -} +} \ No newline at end of file diff --git a/search-api/search-core/pom.xml b/search-api/search-core/pom.xml index a68a641e4..3c0b36dde 100644 --- a/search-api/search-core/pom.xml +++ b/search-api/search-core/pom.xml @@ -80,6 +80,11 @@ 4.13.1 test + + org.elasticsearch + elasticsearch + 6.3.2 + diff --git a/search-api/search-core/src/main/java/org/sunbird/search/client/ElasticSearchUtil.java b/search-api/search-core/src/main/java/org/sunbird/search/client/ElasticSearchUtil.java index a25be9575..7d18df88b 100644 --- a/search-api/search-core/src/main/java/org/sunbird/search/client/ElasticSearchUtil.java +++ b/search-api/search-core/src/main/java/org/sunbird/search/client/ElasticSearchUtil.java @@ -727,4 +727,5 @@ private static Map checkDocStringLength(Map doc) return doc; } -} \ No newline at end of file + +} diff --git a/search-api/search-core/src/main/java/org/sunbird/search/processor/SearchProcessor.java b/search-api/search-core/src/main/java/org/sunbird/search/processor/SearchProcessor.java index 3a46c3ddb..c547d39de 100644 --- a/search-api/search-core/src/main/java/org/sunbird/search/processor/SearchProcessor.java +++ b/search-api/search-core/src/main/java/org/sunbird/search/processor/SearchProcessor.java @@ -5,6 +5,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang.StringUtils; +import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.index.query.*; import org.elasticsearch.index.query.MultiMatchQueryBuilder.Type; @@ -29,776 +30,936 @@ import scala.concurrent.ExecutionContext; import scala.concurrent.Future; +import java.io.IOException; import java.util.*; import java.util.stream.Collectors; public class SearchProcessor { - private ObjectMapper mapper = new ObjectMapper(); - private static final String ASC_ORDER = "asc"; - private static final String AND = "AND"; - private boolean relevanceSort = false; - - public SearchProcessor() { - ElasticSearchUtil.initialiseESClient(SearchConstants.COMPOSITE_SEARCH_INDEX, - Platform.config.getString("search.es_conn_info")); - } - - public SearchProcessor(String indexName) { - } - - @SuppressWarnings({ "unchecked", "rawtypes" }) - public Future> processSearch(SearchDTO searchDTO, boolean includeResults) - throws Exception { - List> groupByFinalList = new ArrayList>(); - SearchSourceBuilder query = processSearchQuery(searchDTO, groupByFinalList, true); - Future searchResponse = ElasticSearchUtil.search( - SearchConstants.COMPOSITE_SEARCH_INDEX, - query); - - return searchResponse.map(new Mapper>() { - public Map apply(SearchResponse searchResult) { - Map resp = new HashMap<>(); - if (includeResults) { - if (searchDTO.isFuzzySearch()) { - List results = ElasticSearchUtil.getDocumentsFromSearchResultWithScore(searchResult); - resp.put("results", results); - } else { - List results = ElasticSearchUtil.getDocumentsFromSearchResult(searchResult, Map.class); - resp.put("results", results); - } - } - Aggregations aggregations = searchResult.getAggregations(); - if (null != aggregations) { - AggregationsResultTransformer transformer = new AggregationsResultTransformer(); - if(CollectionUtils.isNotEmpty(searchDTO.getFacets())) { - resp.put("facets", (List>) ElasticSearchUtil - .getCountFromAggregation(aggregations, groupByFinalList, transformer)); - } else if(CollectionUtils.isNotEmpty(searchDTO.getAggregations())){ - resp.put("aggregations", aggregateResult(aggregations)); - } - - } - resp.put("count", (int) searchResult.getHits().getTotalHits()); - return resp; - } - }, ExecutionContext.Implicits$.MODULE$.global()); - } - - public Map processCount(SearchDTO searchDTO) throws Exception { - Map response = new HashMap(); - SearchSourceBuilder searchSourceBuilder = processSearchQuery(searchDTO, null, false); - searchSourceBuilder.from(searchDTO.getOffset()).size(0); - int countResult = ElasticSearchUtil.count(SearchConstants.COMPOSITE_SEARCH_INDEX, - searchSourceBuilder); - response.put("count", countResult); - - return response; - } - - /** - * Returns the list of words which are synonyms of the synsetIds passed in the - * request - * - * @param synsetIds - * @return - * @throws Exception - */ - @SuppressWarnings({ "unchecked", "rawtypes", "unused" }) - public Map multiWordDocSearch(List synsetIds) throws Exception { - Map response = new HashMap(); - Map translations = new HashMap(); - Map synsets = new HashMap(); - if (synsetIds != null && synsetIds.size() > 0) { - List resultList = ElasticSearchUtil.getMultiDocumentAsStringByIdList( - SearchConstants.COMPOSITE_SEARCH_INDEX, - SearchConstants.COMPOSITE_SEARCH_INDEX_TYPE, synsetIds); - for (String synsetDoc : resultList) { - Map wordTranslationList = new HashMap(); - Map indexDocument = new HashMap(); - if (synsetDoc != null && !synsetDoc.isEmpty()) { - indexDocument = mapper.readValue(synsetDoc, new TypeReference>() { - }); - Object words = indexDocument.get("synonyms"); - String identifier = (String) indexDocument.get("identifier"); - String gloss = (String) indexDocument.get("gloss"); - wordTranslationList.put("gloss", gloss); - if (words != null) { - List wordIdList = (List) words; - if (wordIdList != null && wordIdList.size() > 0) { - List wordResultList = ElasticSearchUtil.getMultiDocumentAsStringByIdList( - SearchConstants.COMPOSITE_SEARCH_INDEX, - SearchConstants.COMPOSITE_SEARCH_INDEX_TYPE, wordIdList); - for (String wordDoc : wordResultList) { - List synsetWordLangList = new ArrayList(); - Map indexWordDocument = new HashMap(); - indexWordDocument = mapper.readValue(wordDoc, new TypeReference>() { - }); - String wordId = (String) indexWordDocument.get("identifier"); - String graphId = (String) indexWordDocument.get("graph_id"); - if (wordTranslationList.containsKey(graphId)) { - synsetWordLangList = (List) wordTranslationList.get(graphId); - } - String lemma = (String) indexWordDocument.get("lemma"); - String status = (String) indexWordDocument.get("status"); - if (!StringUtils.equalsIgnoreCase(status, "Retired")) { - Map wordMap = new HashMap(); - wordMap.put("id", wordId); - wordMap.put("lemma", lemma); - synsetWordLangList.add(wordMap); - } - wordTranslationList.put(graphId, synsetWordLangList); - } - - } - } - synsets.put(identifier, wordTranslationList); - } - - } - response.put("translations", synsets); - } - - return response; - } - - /** - * Returns list of synsetsIds which has valid documents in composite index - * - * @param synsetIds - * @return - * @throws Exception - */ - public Map multiSynsetDocSearch(List synsetIds) throws Exception { - Map synsetDocList = new HashMap(); - List identifierList = new ArrayList(); - if (synsetIds != null && synsetIds.size() > 0) { - List resultList = ElasticSearchUtil.getMultiDocumentAsStringByIdList( - SearchConstants.COMPOSITE_SEARCH_INDEX, - SearchConstants.COMPOSITE_SEARCH_INDEX_TYPE, synsetIds); - for (String synsetDoc : resultList) { - Map indexDocument = new HashMap(); - if (synsetDoc != null && !synsetDoc.isEmpty()) { - indexDocument = mapper.readValue(synsetDoc, new TypeReference>() { - }); - String identifier = (String) indexDocument.get("identifier"); - identifierList.add(identifier); - - } - - } - } - synsetDocList.put("synsets", identifierList); - - return synsetDocList; - } - - public void destroy() { - // ElasticSearchUtil.cleanESClient(); - } - - /** - * @param searchDTO - * @param groupByFinalList - * @return - */ - private SearchSourceBuilder processSearchQuery(SearchDTO searchDTO, List> groupByFinalList, - boolean sortBy) { - - SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); - List fields = searchDTO.getFields(); - if (null != fields && !fields.isEmpty()) { - fields.add("objectType"); - fields.add("identifier"); - searchSourceBuilder.fetchSource(fields.toArray(new String[fields.size()]), null); - } - - if (searchDTO.getFacets() != null && groupByFinalList != null) { - for (String facet : searchDTO.getFacets()) { - Map groupByMap = new HashMap(); - groupByMap.put("groupByParent", facet); - groupByFinalList.add(groupByMap); - } - } - - searchSourceBuilder.size(searchDTO.getLimit()); - searchSourceBuilder.from(searchDTO.getOffset()); - QueryBuilder query = getSearchQuery(searchDTO); - if (searchDTO.isFuzzySearch()) - relevanceSort = true; - - searchSourceBuilder.query(query); - - if (sortBy && !relevanceSort - && (null == searchDTO.getSoftConstraints() || searchDTO.getSoftConstraints().isEmpty())) { - Map sorting = searchDTO.getSortBy(); - if (sorting == null || sorting.isEmpty()) { - sorting = new HashMap(); - sorting.put("name", "asc"); - sorting.put("lastUpdatedOn", "desc"); - } - for (String key : sorting.keySet()){ - if(key.contains(".")){ - String nestedPath = key.split("\\.")[0]; - searchSourceBuilder.sort(SortBuilders.fieldSort(key + SearchConstants.RAW_FIELD_EXTENSION).order(getSortOrder(sorting.get(key))).setNestedSort(new NestedSortBuilder(nestedPath))); - } else{ - searchSourceBuilder.sort(key + SearchConstants.RAW_FIELD_EXTENSION, - getSortOrder(sorting.get(key))); - } - } - } - setAggregations(groupByFinalList, searchSourceBuilder); - setAggregations(searchSourceBuilder, searchDTO.getAggregations()); - searchSourceBuilder.trackScores(true); - return searchSourceBuilder; - } - - /** - * @param groupByList - * @param searchSourceBuilder - * @return - */ - @SuppressWarnings("unchecked") - private void setAggregations(List> groupByList, - SearchSourceBuilder searchSourceBuilder) { - TermsAggregationBuilder termBuilder = null; - if (groupByList != null && !groupByList.isEmpty()) { - HashMap> nestedAggregation = new HashMap<>(); - for (Map groupByMap : groupByList) { - String groupByParent = (String) groupByMap.get("groupByParent"); - if (!groupByParent.contains(".")) { - termBuilder = AggregationBuilders.terms(groupByParent) - .field(groupByParent + SearchConstants.RAW_FIELD_EXTENSION) - .size(ElasticSearchUtil.defaultResultLimit); - List groupByChildList = (List) groupByMap.get("groupByChildList"); - if (groupByChildList != null && !groupByChildList.isEmpty()) { - for (String childGroupBy : groupByChildList) { - termBuilder.subAggregation(AggregationBuilders.terms(childGroupBy) - .field(childGroupBy + SearchConstants.RAW_FIELD_EXTENSION) - .size(ElasticSearchUtil.defaultResultLimit)); - } - } - searchSourceBuilder.aggregation(termBuilder); - } else { - if (nestedAggregation.get(groupByParent.split("\\.")[0]) != null) { - nestedAggregation.get(groupByParent.split("\\.")[0]).add(groupByParent.split("\\.")[1]); - } else { - List nestedAggrList = new ArrayList<>(); - nestedAggrList.add(groupByParent.split("\\.")[1]); - nestedAggregation.put(groupByParent.split("\\.")[0], nestedAggrList); - } - } - } - if (!nestedAggregation.isEmpty()) { - for (Map.Entry> mapData : nestedAggregation.entrySet()) { - AggregationBuilder nestedAggregationBuilder = AggregationBuilders.nested(mapData.getKey(), mapData.getKey()); - for (String nestedValue : mapData.getValue()) { - termBuilder = AggregationBuilders.terms(nestedValue) - .field(mapData.getKey() + "." + nestedValue + SearchConstants.RAW_FIELD_EXTENSION) - .size(ElasticSearchUtil.defaultResultLimit); - nestedAggregationBuilder.subAggregation(termBuilder); - } - searchSourceBuilder.aggregation(nestedAggregationBuilder); - } - } - } - } - - /** - * @param searchDTO - * @return - */ - @SuppressWarnings({ "unchecked", "rawtypes" }) - private QueryBuilder prepareSearchQuery(SearchDTO searchDTO) { - BoolQueryBuilder boolQuery = new BoolQueryBuilder(); - QueryBuilder queryBuilder = null; - String totalOperation = searchDTO.getOperation(); - List properties = searchDTO.getProperties(); - formQuery(properties, queryBuilder, boolQuery, totalOperation, searchDTO.isFuzzySearch()); - if(searchDTO.getMultiFilterProperties() != null) { - formQuery(searchDTO.getMultiFilterProperties(), queryBuilder, boolQuery, SearchConstants.SEARCH_OPERATION_OR, searchDTO.isFuzzySearch()); - } - - Map softConstraints = searchDTO.getSoftConstraints(); - if (null != softConstraints && !softConstraints.isEmpty()) { - boolQuery.should(getSoftConstraintQuery(softConstraints)); - searchDTO.setSortBy(null); - // relevanceSort = true; - } - return boolQuery; - } - - private void formQuery(List properties, QueryBuilder queryBuilder, BoolQueryBuilder boolQuery, String operation, Boolean fuzzy) { - for (Map property : properties) { - String opertation = (String) property.get("operation"); - - List values; - try { - values = (List) property.get("values"); - } catch (Exception e) { - values = Arrays.asList(property.get("values")); - } - values = values.stream().filter(value -> (null != value)).collect(Collectors.toList()); - - - String propertyName = (String) property.get("propertyName"); - if (propertyName.equals("*")) { - relevanceSort = true; - propertyName = "all_fields"; - queryBuilder = getAllFieldsPropertyQuery(values, fuzzy); - boolQuery.must(queryBuilder); - continue; - } - - propertyName = propertyName + SearchConstants.RAW_FIELD_EXTENSION; - - switch (opertation) { - case SearchConstants.SEARCH_OPERATION_EQUAL: { - queryBuilder = getMustTermQuery(propertyName, values, true); - queryBuilder = checkNestedProperty(queryBuilder, propertyName); - break; - } - case SearchConstants.SEARCH_OPERATION_NOT_EQUAL: { - queryBuilder = getMustTermQuery(propertyName, values, false); - queryBuilder = checkNestedProperty(queryBuilder, propertyName); - break; - } - case SearchConstants.SEARCH_OPERATION_NOT_IN: { - queryBuilder = getNotInQuery(propertyName, values); - queryBuilder = checkNestedProperty(queryBuilder, propertyName); - break; - } - case SearchConstants.SEARCH_OPERATION_ENDS_WITH: { - queryBuilder = getRegexQuery(propertyName, values); - queryBuilder = checkNestedProperty(queryBuilder, propertyName); - break; - } - case SearchConstants.SEARCH_OPERATION_LIKE: - case SearchConstants.SEARCH_OPERATION_CONTAINS: { - queryBuilder = getMatchPhraseQuery(propertyName, values, true); - queryBuilder = checkNestedProperty(queryBuilder, propertyName); - break; - } - case SearchConstants.SEARCH_OPERATION_NOT_LIKE: { - queryBuilder = getMatchPhraseQuery(propertyName, values, false); - queryBuilder = checkNestedProperty(queryBuilder, propertyName); - break; - } - case SearchConstants.SEARCH_OPERATION_STARTS_WITH: { - queryBuilder = getMatchPhrasePrefixQuery(propertyName, values); - queryBuilder = checkNestedProperty(queryBuilder, propertyName); - break; - } - case SearchConstants.SEARCH_OPERATION_EXISTS: { - queryBuilder = getExistsQuery(propertyName, values, true); - queryBuilder = checkNestedProperty(queryBuilder, propertyName); - break; - } - case SearchConstants.SEARCH_OPERATION_NOT_EXISTS: { - queryBuilder = getExistsQuery(propertyName, values, false); - queryBuilder = checkNestedProperty(queryBuilder, propertyName); - break; - } - case SearchConstants.SEARCH_OPERATION_GREATER_THAN: { - queryBuilder = getRangeQuery(propertyName, values, - SearchConstants.SEARCH_OPERATION_GREATER_THAN); - queryBuilder = checkNestedProperty(queryBuilder, propertyName); - break; - } - case SearchConstants.SEARCH_OPERATION_GREATER_THAN_EQUALS: { - queryBuilder = getRangeQuery(propertyName, values, - SearchConstants.SEARCH_OPERATION_GREATER_THAN_EQUALS); - queryBuilder = checkNestedProperty(queryBuilder, propertyName); - break; - } - case SearchConstants.SEARCH_OPERATION_LESS_THAN: { - queryBuilder = getRangeQuery(propertyName, values, SearchConstants.SEARCH_OPERATION_LESS_THAN); - queryBuilder = checkNestedProperty(queryBuilder, propertyName); - break; - } - case SearchConstants.SEARCH_OPERATION_LESS_THAN_EQUALS: { - queryBuilder = getRangeQuery(propertyName, values, - SearchConstants.SEARCH_OPERATION_LESS_THAN_EQUALS); - queryBuilder = checkNestedProperty(queryBuilder, propertyName); - break; - } - case SearchConstants.SEARCH_OPERATION_RANGE: { - queryBuilder = getRangeQuery(propertyName, values); - queryBuilder = checkNestedProperty(queryBuilder, propertyName); - break; - } - case SearchConstants.SEARCH_OPERATION_AND: { - queryBuilder = getAndQuery(propertyName, values); - queryBuilder = checkNestedProperty(queryBuilder, propertyName); - break; - } - } - if (operation.equalsIgnoreCase(AND)) { - boolQuery.must(queryBuilder); - } else { - boolQuery.should(queryBuilder); - } - - } - } - - private QueryBuilder checkNestedProperty(QueryBuilder queryBuilder, String propertyName) { - if(propertyName.replaceAll(SearchConstants.RAW_FIELD_EXTENSION, "").contains(".")) { - queryBuilder = QueryBuilders.nestedQuery(propertyName.split("\\.")[0], queryBuilder, org.apache.lucene.search.join.ScoreMode.None); - } - return queryBuilder; - } - - - private QueryBuilder getAndQuery(String propertyName, List values) { - BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery(); - for (Object value : values) { - queryBuilder.must( - QueryBuilders.matchQuery(propertyName, value).operator(Operator.AND).fuzzyTranspositions(false)); - } - return queryBuilder; - } - - /** - * @param querySearchFeilds - * @param propertyName - * @return - */ - private float getweight(List querySearchFeilds, String propertyName) { - float weight = 1.0F; - if (querySearchFeilds.contains(propertyName)) { - for (String field : querySearchFeilds) { - if (field.contains(propertyName)) { - weight = Float - .parseFloat((StringUtils.isNotBlank(field.split("^")[1])) ? field.split("^")[1] : "1.0"); - } - } - } - return weight; - } - - /** - * @param values - * @return - */ - private QueryBuilder getAllFieldsPropertyQuery(List values, Boolean fuzzy) { - List queryFields = ElasticSearchUtil.getQuerySearchFields(); - Map queryFieldsMap = new HashMap<>(); - for (String field : queryFields) { - if (field.contains("^")) - queryFieldsMap.put(field.split("\\^")[0], Float.valueOf(field.split("\\^")[1])); - else - queryFieldsMap.put(field, 1.0f); - } - BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery(); - for (Object value : values) { - if (fuzzy) { - queryBuilder - .should(QueryBuilders.multiMatchQuery(value).fields(queryFieldsMap) - .operator(Operator.AND).fuzziness("AUTO").lenient(true)); - } else { - queryBuilder - .should(QueryBuilders.multiMatchQuery(value).fields(queryFieldsMap) - .operator(Operator.AND).type(Type.CROSS_FIELDS).fuzzyTranspositions(false).lenient(true)); - } - } - - return queryBuilder; - } - - /** - * @param softConstraints - * @return - */ - @SuppressWarnings("unchecked") - private QueryBuilder getSoftConstraintQuery(Map softConstraints) { - BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery(); - for (String key : softConstraints.keySet()) { - List data = (List) softConstraints.get(key); - if(data.get(1) instanceof List) { - List dataList = (List) data.get(1); - for(Object value: dataList) { - queryBuilder - .should(QueryBuilders.matchQuery(key + SearchConstants.RAW_FIELD_EXTENSION, value) - .boost(Integer.valueOf((int) data.get(0)).floatValue()).fuzzyTranspositions(false)); - } - } - else { - queryBuilder.should( - QueryBuilders.matchQuery(key + SearchConstants.RAW_FIELD_EXTENSION, data.get(1)) - .boost(Integer.valueOf((int) data.get(0)).floatValue()).fuzzyTranspositions(false)); - } - } - return queryBuilder; - } - - /** - * @param propertyName - * @param values - * @return - */ - private QueryBuilder getRangeQuery(String propertyName, List values, String operation) { - BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery(); - for (Object value : values) { - switch (operation) { - case SearchConstants.SEARCH_OPERATION_GREATER_THAN: { - queryBuilder.should(QueryBuilders - .rangeQuery(propertyName).gt(value)); - break; - } - case SearchConstants.SEARCH_OPERATION_GREATER_THAN_EQUALS: { - queryBuilder.should(QueryBuilders - .rangeQuery(propertyName).gte(value)); - break; - } - case SearchConstants.SEARCH_OPERATION_LESS_THAN: { - queryBuilder.should(QueryBuilders - .rangeQuery(propertyName).lt(value)); - break; - } - case SearchConstants.SEARCH_OPERATION_LESS_THAN_EQUALS: { - queryBuilder.should(QueryBuilders - .rangeQuery(propertyName).lte(value)); - break; - } - } - } - - return queryBuilder; - } - - /** - * @param propertyName - * @param values - * @return - */ - private QueryBuilder getExistsQuery(String propertyName, List values, boolean exists) { - BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery(); - for (Object value : values) { - if (exists) { - queryBuilder.should(QueryBuilders.existsQuery(String.valueOf(value))); - } else { - queryBuilder.mustNot(QueryBuilders.existsQuery(String.valueOf(value))); - } - } - return queryBuilder; - } - - /** - * @param propertyName - * @param values - * @return - */ - private QueryBuilder getNotInQuery(String propertyName, List values) { - BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery(); - queryBuilder - .mustNot(QueryBuilders.termsQuery(propertyName, values)); - return queryBuilder; - } - - /** - * @param propertyName - * @param values - * @return - */ - private QueryBuilder getMatchPhrasePrefixQuery(String propertyName, List values) { - BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery(); - for (Object value : values) { - queryBuilder.should(QueryBuilders.prefixQuery( - propertyName, ((String) value).toLowerCase())); - } - return queryBuilder; - } - - /** - * @param propertyName - * @param values - * @param match - * @return - */ - private QueryBuilder getMatchPhraseQuery(String propertyName, List values, boolean match) { - BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery(); - for (Object value : values) { - String stringValue = String.valueOf(value); - if (match) { - queryBuilder.should(QueryBuilders - .regexpQuery(propertyName, - ".*" + stringValue.toLowerCase() + ".*")); - } else { - queryBuilder.mustNot(QueryBuilders - .regexpQuery(propertyName, - ".*" + stringValue.toLowerCase() + ".*")); - } - } - return queryBuilder; - } - - /** - * @param propertyName - * @param values - * @return - */ - private QueryBuilder getRegexQuery(String propertyName, List values) { - BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery(); - for (Object value : values) { - String stringValue = String.valueOf(value); - queryBuilder.should(QueryBuilders.regexpQuery(propertyName, - ".*" + stringValue.toLowerCase())); - } - return queryBuilder; - } - - /** - * @param propertyName - * @param values - * @param match - * @return - */ - private QueryBuilder getMustTermQuery(String propertyName, List values, boolean match) { - BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery(); - for (Object value : values) { - if (match) { - queryBuilder.should( - QueryBuilders.matchQuery(propertyName, value).fuzzyTranspositions(false)); - } else { - queryBuilder.mustNot( - QueryBuilders.matchQuery(propertyName, value).fuzzyTranspositions(false)); - } - } - - return queryBuilder; - } - - /** - * @param propertyName - * @param values - * @return - */ - @SuppressWarnings("unchecked") - private QueryBuilder getRangeQuery(String propertyName, List values) { - RangeQueryBuilder queryBuilder = new RangeQueryBuilder(propertyName); - for (Object value : values) { - Map rangeMap = (Map) value; - if (!rangeMap.isEmpty()) { - for (String key : rangeMap.keySet()) { - switch (key) { - case SearchConstants.SEARCH_OPERATION_RANGE_GTE: { - queryBuilder.from(rangeMap.get(key)); - break; - } - case SearchConstants.SEARCH_OPERATION_RANGE_LTE: { - queryBuilder.to(rangeMap.get(key)); - break; - } - } - } - } - } - return queryBuilder; - } - - /** - * @param value - * @return - */ - private SortOrder getSortOrder(String value) { - return value.equalsIgnoreCase(ASC_ORDER) ? SortOrder.ASC : SortOrder.DESC; - } - - public Future> processSearchQuery(SearchDTO searchDTO, boolean includeResults, String index) - throws Exception { - return processSearchQuery(searchDTO, includeResults, index, true); - } - - public Future> processSearchQuery(SearchDTO searchDTO, boolean includeResults, String index, - boolean sort) - throws Exception { - List> groupByFinalList = new ArrayList>(); - if (searchDTO.getLimit() == 0) - searchDTO.setLimit(ElasticSearchUtil.defaultResultLimit); - SearchSourceBuilder query = processSearchQuery(searchDTO, groupByFinalList, sort); - TelemetryManager.log(" search query: " + query); - Future searchResponse = ElasticSearchUtil.search(index, query); - - return searchResponse.map(new Mapper>() { - public List apply(SearchResponse searchResult) { - List response = new ArrayList(); - TelemetryManager.log("search result from elastic search" + searchResult); - SearchHits resultMap = searchResult.getHits(); - SearchHit[] result = resultMap.getHits(); - for (SearchHit hit : result) { - response.add(hit.getSourceAsMap()); - } - TelemetryManager.log("search response size: " + response.size()); - return response; - } - }, ExecutionContext.Implicits$.MODULE$.global()); - - } - - public Future processSearchQueryWithSearchResult(SearchDTO searchDTO, boolean includeResults, - String index, - boolean sort) throws Exception { - List> groupByFinalList = new ArrayList>(); - if (searchDTO.getLimit() == 0) - searchDTO.setLimit(ElasticSearchUtil.defaultResultLimit); - SearchSourceBuilder query = processSearchQuery(searchDTO, groupByFinalList, sort); - TelemetryManager.log(" search query: " + query); - Future searchResult = ElasticSearchUtil.search(index, query); - TelemetryManager.log("search result from elastic search" + searchResult); - return searchResult; - } - - private void setAggregations(SearchSourceBuilder searchSourceBuilder, List> aggregations) { - if(CollectionUtils.isNotEmpty(aggregations)){ - for(Map aggregate: aggregations){ - TermsAggregationBuilder termBuilder = AggregationBuilders.terms((String)aggregate.get("l1")) - .field(aggregate.get("l1") + SearchConstants.RAW_FIELD_EXTENSION) - .size(ElasticSearchUtil.defaultResultLimit); - int level = 2; - termBuilder.subAggregation(getNextLevelAggregation(aggregate, level)); - searchSourceBuilder.aggregation(termBuilder); - } - } - } - - private AggregationBuilder getNextLevelAggregation(Map aggregate, int level) { - TermsAggregationBuilder termBuilder = AggregationBuilders.terms((String)aggregate.get("l" + level)) + private ObjectMapper mapper = new ObjectMapper(); + private static final String ASC_ORDER = "asc"; + private static final String AND = "AND"; + private boolean relevanceSort = false; + + public SearchProcessor() { + ElasticSearchUtil.initialiseESClient(SearchConstants.COMPOSITE_SEARCH_INDEX, + Platform.config.getString("search.es_conn_info")); + } + + public SearchProcessor(String indexName) { + } + + @SuppressWarnings({ "unchecked", "rawtypes" }) + public Future> processSearch(SearchDTO searchDTO, boolean includeResults) + throws Exception { + List> groupByFinalList = new ArrayList>(); + SearchSourceBuilder query = processSearchQuery(searchDTO, groupByFinalList, true); + Future searchResponse = ElasticSearchUtil.search( + SearchConstants.COMPOSITE_SEARCH_INDEX, + query); + + return searchResponse.map(new Mapper>() { + public Map apply(SearchResponse searchResult) { + Map resp = new HashMap<>(); + if (includeResults) { + if (searchDTO.isFuzzySearch()) { + List results = ElasticSearchUtil.getDocumentsFromSearchResultWithScore(searchResult); + resp.put("results", results); + } else { + List results = ElasticSearchUtil.getDocumentsFromSearchResult(searchResult, Map.class); + resp.put("results", results); + } + } + Aggregations aggregations = searchResult.getAggregations(); + if (null != aggregations) { + AggregationsResultTransformer transformer = new AggregationsResultTransformer(); + if (CollectionUtils.isNotEmpty(searchDTO.getFacets())) { + resp.put("facets", (List>) ElasticSearchUtil + .getCountFromAggregation(aggregations, groupByFinalList, transformer)); + } else if (CollectionUtils.isNotEmpty(searchDTO.getAggregations())) { + resp.put("aggregations", aggregateResult(aggregations)); + } + + } + resp.put("count", (int) searchResult.getHits().getTotalHits()); + return resp; + } + }, ExecutionContext.Implicits$.MODULE$.global()); + } + public Future> processSearch(SearchDTO searchDTO, boolean includeResults, boolean fuzzyEnabled) + throws Exception { + List> groupByFinalList = new ArrayList>(); + SearchSourceBuilder query = processSearchQuery(searchDTO, groupByFinalList, true, fuzzyEnabled); + Future searchResponse = ElasticSearchUtil.search( + SearchConstants.COMPOSITE_SEARCH_INDEX, + query); + + int exactMatchCount = ElasticSearchUtil.count(SearchConstants.COMPOSITE_SEARCH_INDEX, + query); + + if(exactMatchCount == 0){ + query = processSearchQuery(searchDTO, groupByFinalList, true, true); + searchResponse = ElasticSearchUtil.search( + SearchConstants.COMPOSITE_SEARCH_INDEX, + query); + } + + return searchResponse.map(new Mapper>() { + public Map apply(SearchResponse searchResult) { + Map resp = new HashMap<>(); + + if (includeResults) { + if (searchDTO.isFuzzySearch()) { + List results = ElasticSearchUtil.getDocumentsFromSearchResultWithScore(searchResult); + resp.put("results", results); + } else { + List results = ElasticSearchUtil.getDocumentsFromSearchResult(searchResult, Map.class); + resp.put("results", results); + } + } + Aggregations aggregations = searchResult.getAggregations(); + if (null != aggregations) { + AggregationsResultTransformer transformer = new AggregationsResultTransformer(); + if (CollectionUtils.isNotEmpty(searchDTO.getFacets())) { + resp.put("facets", (List>) ElasticSearchUtil + .getCountFromAggregation(aggregations, groupByFinalList, transformer)); + } else if (CollectionUtils.isNotEmpty(searchDTO.getAggregations())) { + resp.put("aggregations", aggregateResult(aggregations)); + } + + } + + + + resp.put("count", (int) searchResult.getHits().getTotalHits()); + return resp; + } + }, ExecutionContext.Implicits$.MODULE$.global()); + + + + + + } + + private Map getMatchCount(Future searchResponse) { + + return (Map) searchResponse.map(new Mapper>() { + public Map apply(SearchResponse searchResult) { + Map resp = new HashMap<>(); + resp.put("hits",searchResult.getHits().getTotalHits()); + return resp; + } + }, ExecutionContext.Implicits$.MODULE$.global()); + + } + + private Future exactMatchExists(Future searchResponse) { + return searchResponse.map(new Mapper() { + public Boolean apply(SearchResponse searchResult) { + Map resp = new HashMap<>(); + long matches = searchResult.getHits().getTotalHits(); + if(matches>0) + return true; + else + return false; + + } + }, ExecutionContext.Implicits$.MODULE$.global()); + + } + public Map processCount(SearchDTO searchDTO) throws Exception { + Map response = new HashMap(); + SearchSourceBuilder searchSourceBuilder = processSearchQuery(searchDTO, null, false); + searchSourceBuilder.from(searchDTO.getOffset()).size(0); + int countResult = ElasticSearchUtil.count(SearchConstants.COMPOSITE_SEARCH_INDEX, + searchSourceBuilder); + response.put("count", countResult); + + return response; + } + + /** + * Returns the list of words which are synonyms of the synsetIds passed in the + * request + * + * @param synsetIds + * @return + * @throws Exception + */ + @SuppressWarnings({ "unchecked", "rawtypes", "unused" }) + public Map multiWordDocSearch(List synsetIds) throws Exception { + Map response = new HashMap(); + Map translations = new HashMap(); + Map synsets = new HashMap(); + if (synsetIds != null && synsetIds.size() > 0) { + List resultList = ElasticSearchUtil.getMultiDocumentAsStringByIdList( + SearchConstants.COMPOSITE_SEARCH_INDEX, + SearchConstants.COMPOSITE_SEARCH_INDEX_TYPE, synsetIds); + for (String synsetDoc : resultList) { + Map wordTranslationList = new HashMap(); + Map indexDocument = new HashMap(); + if (synsetDoc != null && !synsetDoc.isEmpty()) { + indexDocument = mapper.readValue(synsetDoc, new TypeReference>() { + }); + Object words = indexDocument.get("synonyms"); + String identifier = (String) indexDocument.get("identifier"); + String gloss = (String) indexDocument.get("gloss"); + wordTranslationList.put("gloss", gloss); + if (words != null) { + List wordIdList = (List) words; + if (wordIdList != null && wordIdList.size() > 0) { + List wordResultList = ElasticSearchUtil.getMultiDocumentAsStringByIdList( + SearchConstants.COMPOSITE_SEARCH_INDEX, + SearchConstants.COMPOSITE_SEARCH_INDEX_TYPE, wordIdList); + for (String wordDoc : wordResultList) { + List synsetWordLangList = new ArrayList(); + Map indexWordDocument = new HashMap(); + indexWordDocument = mapper.readValue(wordDoc, new TypeReference>() { + }); + String wordId = (String) indexWordDocument.get("identifier"); + String graphId = (String) indexWordDocument.get("graph_id"); + if (wordTranslationList.containsKey(graphId)) { + synsetWordLangList = (List) wordTranslationList.get(graphId); + } + String lemma = (String) indexWordDocument.get("lemma"); + String status = (String) indexWordDocument.get("status"); + if (!StringUtils.equalsIgnoreCase(status, "Retired")) { + Map wordMap = new HashMap(); + wordMap.put("id", wordId); + wordMap.put("lemma", lemma); + synsetWordLangList.add(wordMap); + } + wordTranslationList.put(graphId, synsetWordLangList); + } + + } + } + synsets.put(identifier, wordTranslationList); + } + + } + response.put("translations", synsets); + } + + return response; + } + + /** + * Returns list of synsetsIds which has valid documents in composite index + * + * @param synsetIds + * @return + * @throws Exception + */ + public Map multiSynsetDocSearch(List synsetIds) throws Exception { + Map synsetDocList = new HashMap(); + List identifierList = new ArrayList(); + if (synsetIds != null && synsetIds.size() > 0) { + List resultList = ElasticSearchUtil.getMultiDocumentAsStringByIdList( + SearchConstants.COMPOSITE_SEARCH_INDEX, + SearchConstants.COMPOSITE_SEARCH_INDEX_TYPE, synsetIds); + for (String synsetDoc : resultList) { + Map indexDocument = new HashMap(); + if (synsetDoc != null && !synsetDoc.isEmpty()) { + indexDocument = mapper.readValue(synsetDoc, new TypeReference>() { + }); + String identifier = (String) indexDocument.get("identifier"); + identifierList.add(identifier); + + } + + } + } + synsetDocList.put("synsets", identifierList); + + return synsetDocList; + } + + public void destroy() { + // ElasticSearchUtil.cleanESClient(); + } + + /** + * @param searchDTO + * @param groupByFinalList + * @return + */ + private SearchSourceBuilder processSearchQuery(SearchDTO searchDTO, List> groupByFinalList, + boolean sortBy) { + + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + List fields = searchDTO.getFields(); + if (null != fields && !fields.isEmpty()) { + fields.add("objectType"); + fields.add("identifier"); + searchSourceBuilder.fetchSource(fields.toArray(new String[fields.size()]), null); + } + + if (searchDTO.getFacets() != null && groupByFinalList != null) { + for (String facet : searchDTO.getFacets()) { + Map groupByMap = new HashMap(); + groupByMap.put("groupByParent", facet); + groupByFinalList.add(groupByMap); + } + } + + searchSourceBuilder.size(searchDTO.getLimit()); + searchSourceBuilder.from(searchDTO.getOffset()); + QueryBuilder query = getSearchQuery(searchDTO); + if (searchDTO.isFuzzySearch()) + relevanceSort = true; + + searchSourceBuilder.query(query); + + if (sortBy && !relevanceSort + && (null == searchDTO.getSoftConstraints() || searchDTO.getSoftConstraints().isEmpty())) { + Map sorting = searchDTO.getSortBy(); + if (sorting == null || sorting.isEmpty()) { + sorting = new HashMap(); + sorting.put("name", "asc"); + sorting.put("lastUpdatedOn", "desc"); + } + for (String key : sorting.keySet()) { + if (key.contains(".")) { + String nestedPath = key.split("\\.")[0]; + searchSourceBuilder.sort(SortBuilders.fieldSort(key + SearchConstants.RAW_FIELD_EXTENSION) + .order(getSortOrder(sorting.get(key))).setNestedSort(new NestedSortBuilder(nestedPath))); + } else { + searchSourceBuilder.sort(key + SearchConstants.RAW_FIELD_EXTENSION, + getSortOrder(sorting.get(key))); + } + } + } + setAggregations(groupByFinalList, searchSourceBuilder); + setAggregations(searchSourceBuilder, searchDTO.getAggregations()); + searchSourceBuilder.trackScores(true); + return searchSourceBuilder; + } + + private SearchSourceBuilder processSearchQuery(SearchDTO searchDTO, List> groupByFinalList, + boolean sortBy, boolean fuzzyEnabled) { + + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + List fields = searchDTO.getFields(); + if (null != fields && !fields.isEmpty()) { + fields.add("objectType"); + fields.add("identifier"); + searchSourceBuilder.fetchSource(fields.toArray(new String[fields.size()]), null); + } + + if (searchDTO.getFacets() != null && groupByFinalList != null && !fuzzyEnabled) { + for (String facet : searchDTO.getFacets()) { + Map groupByMap = new HashMap(); + groupByMap.put("groupByParent", facet); + groupByFinalList.add(groupByMap); + } + } + + searchSourceBuilder.size(searchDTO.getLimit()); + searchSourceBuilder.from(searchDTO.getOffset()); + QueryBuilder query = getSearchQuery(searchDTO, fuzzyEnabled); + if (searchDTO.isFuzzySearch()) + relevanceSort = true; + + searchSourceBuilder.query(query); + + if (sortBy && !relevanceSort + && (null == searchDTO.getSoftConstraints() || searchDTO.getSoftConstraints().isEmpty())) { + Map sorting = searchDTO.getSortBy(); + if (sorting == null || sorting.isEmpty()) { + sorting = new HashMap(); + sorting.put("name", "asc"); + sorting.put("lastUpdatedOn", "desc"); + } + for (String key : sorting.keySet()) { + if (key.contains(".")) { + String nestedPath = key.split("\\.")[0]; + searchSourceBuilder.sort(SortBuilders.fieldSort(key + SearchConstants.RAW_FIELD_EXTENSION) + .order(getSortOrder(sorting.get(key))).setNestedSort(new NestedSortBuilder(nestedPath))); + } else { + searchSourceBuilder.sort(key + SearchConstants.RAW_FIELD_EXTENSION, + getSortOrder(sorting.get(key))); + } + } + } + setAggregations(groupByFinalList, searchSourceBuilder); + setAggregations(searchSourceBuilder, searchDTO.getAggregations()); + searchSourceBuilder.trackScores(true); + return searchSourceBuilder; + } + + /** + * @param groupByList + * @param searchSourceBuilder + * @return + */ + @SuppressWarnings("unchecked") + private void setAggregations(List> groupByList, + SearchSourceBuilder searchSourceBuilder) { + TermsAggregationBuilder termBuilder = null; + if (groupByList != null && !groupByList.isEmpty()) { + HashMap> nestedAggregation = new HashMap<>(); + for (Map groupByMap : groupByList) { + String groupByParent = (String) groupByMap.get("groupByParent"); + if (!groupByParent.contains(".")) { + termBuilder = AggregationBuilders.terms(groupByParent) + .field(groupByParent + SearchConstants.RAW_FIELD_EXTENSION) + .size(ElasticSearchUtil.defaultResultLimit); + List groupByChildList = (List) groupByMap.get("groupByChildList"); + if (groupByChildList != null && !groupByChildList.isEmpty()) { + for (String childGroupBy : groupByChildList) { + termBuilder.subAggregation(AggregationBuilders.terms(childGroupBy) + .field(childGroupBy + SearchConstants.RAW_FIELD_EXTENSION) + .size(ElasticSearchUtil.defaultResultLimit)); + } + } + searchSourceBuilder.aggregation(termBuilder); + } else { + if (nestedAggregation.get(groupByParent.split("\\.")[0]) != null) { + nestedAggregation.get(groupByParent.split("\\.")[0]).add(groupByParent.split("\\.")[1]); + } else { + List nestedAggrList = new ArrayList<>(); + nestedAggrList.add(groupByParent.split("\\.")[1]); + nestedAggregation.put(groupByParent.split("\\.")[0], nestedAggrList); + } + } + } + if (!nestedAggregation.isEmpty()) { + for (Map.Entry> mapData : nestedAggregation.entrySet()) { + AggregationBuilder nestedAggregationBuilder = AggregationBuilders.nested(mapData.getKey(), + mapData.getKey()); + for (String nestedValue : mapData.getValue()) { + termBuilder = AggregationBuilders.terms(nestedValue) + .field(mapData.getKey() + "." + nestedValue + SearchConstants.RAW_FIELD_EXTENSION) + .size(ElasticSearchUtil.defaultResultLimit); + nestedAggregationBuilder.subAggregation(termBuilder); + } + searchSourceBuilder.aggregation(nestedAggregationBuilder); + } + } + } + } + + /** + * @param searchDTO + * @return + */ + @SuppressWarnings({ "unchecked", "rawtypes" }) + private QueryBuilder prepareSearchQuery(SearchDTO searchDTO) { + BoolQueryBuilder boolQuery = new BoolQueryBuilder(); + QueryBuilder queryBuilder = null; + String totalOperation = searchDTO.getOperation(); + List properties = searchDTO.getProperties(); + formQuery(properties, queryBuilder, boolQuery, totalOperation, searchDTO.isFuzzySearch()); + if (searchDTO.getMultiFilterProperties() != null) { + formQuery(searchDTO.getMultiFilterProperties(), queryBuilder, boolQuery, + SearchConstants.SEARCH_OPERATION_OR, searchDTO.isFuzzySearch()); + } + + Map softConstraints = searchDTO.getSoftConstraints(); + if (null != softConstraints && !softConstraints.isEmpty()) { + boolQuery.should(getSoftConstraintQuery(softConstraints)); + searchDTO.setSortBy(null); + // relevanceSort = true; + } + return boolQuery; + } + + private QueryBuilder prepareSearchQuery(SearchDTO searchDTO, boolean fuzzyEnabled) { + BoolQueryBuilder boolQuery = new BoolQueryBuilder(); + QueryBuilder queryBuilder = null; + String totalOperation = searchDTO.getOperation(); + List properties = searchDTO.getProperties(); + formQuery(properties, queryBuilder, boolQuery, totalOperation, fuzzyEnabled); + if (searchDTO.getMultiFilterProperties() != null) { + formQuery(searchDTO.getMultiFilterProperties(), queryBuilder, boolQuery, + SearchConstants.SEARCH_OPERATION_OR, fuzzyEnabled); + } + + Map softConstraints = searchDTO.getSoftConstraints(); + if (null != softConstraints && !softConstraints.isEmpty()) { + boolQuery.should(getSoftConstraintQuery(softConstraints)); + searchDTO.setSortBy(null); + // relevanceSort = true; + } + return boolQuery; + } + + private void formQuery(List properties, QueryBuilder queryBuilder, BoolQueryBuilder boolQuery, + String operation, boolean fuzzy) { + for (Map property : properties) { + String opertation = (String) property.get("operation"); + + List values; + try { + values = (List) property.get("values"); + } catch (Exception e) { + values = Arrays.asList(property.get("values")); + } + values = values.stream().filter(value -> (null != value)).collect(Collectors.toList()); + + String propertyName = (String) property.get("propertyName"); + if (propertyName.equals("*")) { + relevanceSort = true; + propertyName = "all_fields"; + queryBuilder = getAllFieldsPropertyQuery(values, fuzzy); + boolQuery.must(queryBuilder); + continue; + } + + propertyName = propertyName + SearchConstants.RAW_FIELD_EXTENSION; + + switch (opertation) { + case SearchConstants.SEARCH_OPERATION_EQUAL: { + queryBuilder = getMustTermQuery(propertyName, values, true); + queryBuilder = checkNestedProperty(queryBuilder, propertyName); + break; + } + case SearchConstants.SEARCH_OPERATION_NOT_EQUAL: { + queryBuilder = getMustTermQuery(propertyName, values, false); + queryBuilder = checkNestedProperty(queryBuilder, propertyName); + break; + } + case SearchConstants.SEARCH_OPERATION_NOT_IN: { + queryBuilder = getNotInQuery(propertyName, values); + queryBuilder = checkNestedProperty(queryBuilder, propertyName); + break; + } + case SearchConstants.SEARCH_OPERATION_ENDS_WITH: { + queryBuilder = getRegexQuery(propertyName, values); + queryBuilder = checkNestedProperty(queryBuilder, propertyName); + break; + } + case SearchConstants.SEARCH_OPERATION_LIKE: + case SearchConstants.SEARCH_OPERATION_CONTAINS: { + queryBuilder = getMatchPhraseQuery(propertyName, values, true); + queryBuilder = checkNestedProperty(queryBuilder, propertyName); + break; + } + case SearchConstants.SEARCH_OPERATION_NOT_LIKE: { + queryBuilder = getMatchPhraseQuery(propertyName, values, false); + queryBuilder = checkNestedProperty(queryBuilder, propertyName); + break; + } + case SearchConstants.SEARCH_OPERATION_STARTS_WITH: { + queryBuilder = getMatchPhrasePrefixQuery(propertyName, values); + queryBuilder = checkNestedProperty(queryBuilder, propertyName); + break; + } + case SearchConstants.SEARCH_OPERATION_EXISTS: { + queryBuilder = getExistsQuery(propertyName, values, true); + queryBuilder = checkNestedProperty(queryBuilder, propertyName); + break; + } + case SearchConstants.SEARCH_OPERATION_NOT_EXISTS: { + queryBuilder = getExistsQuery(propertyName, values, false); + queryBuilder = checkNestedProperty(queryBuilder, propertyName); + break; + } + case SearchConstants.SEARCH_OPERATION_GREATER_THAN: { + queryBuilder = getRangeQuery(propertyName, values, + SearchConstants.SEARCH_OPERATION_GREATER_THAN); + queryBuilder = checkNestedProperty(queryBuilder, propertyName); + break; + } + case SearchConstants.SEARCH_OPERATION_GREATER_THAN_EQUALS: { + queryBuilder = getRangeQuery(propertyName, values, + SearchConstants.SEARCH_OPERATION_GREATER_THAN_EQUALS); + queryBuilder = checkNestedProperty(queryBuilder, propertyName); + break; + } + case SearchConstants.SEARCH_OPERATION_LESS_THAN: { + queryBuilder = getRangeQuery(propertyName, values, SearchConstants.SEARCH_OPERATION_LESS_THAN); + queryBuilder = checkNestedProperty(queryBuilder, propertyName); + break; + } + case SearchConstants.SEARCH_OPERATION_LESS_THAN_EQUALS: { + queryBuilder = getRangeQuery(propertyName, values, + SearchConstants.SEARCH_OPERATION_LESS_THAN_EQUALS); + queryBuilder = checkNestedProperty(queryBuilder, propertyName); + break; + } + case SearchConstants.SEARCH_OPERATION_RANGE: { + queryBuilder = getRangeQuery(propertyName, values); + queryBuilder = checkNestedProperty(queryBuilder, propertyName); + break; + } + case SearchConstants.SEARCH_OPERATION_AND: { + queryBuilder = getAndQuery(propertyName, values); + queryBuilder = checkNestedProperty(queryBuilder, propertyName); + break; + } + } + if (operation.equalsIgnoreCase(AND)) { + boolQuery.must(queryBuilder); + } else { + boolQuery.should(queryBuilder); + } + + } + } + + private QueryBuilder checkNestedProperty(QueryBuilder queryBuilder, String propertyName) { + if (propertyName.replaceAll(SearchConstants.RAW_FIELD_EXTENSION, "").contains(".")) { + queryBuilder = QueryBuilders.nestedQuery(propertyName.split("\\.")[0], queryBuilder, + org.apache.lucene.search.join.ScoreMode.None); + } + return queryBuilder; + } + + private QueryBuilder getAndQuery(String propertyName, List values) { + BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery(); + for (Object value : values) { + queryBuilder.must( + QueryBuilders.matchQuery(propertyName, value).operator(Operator.AND).fuzzyTranspositions(false)); + } + return queryBuilder; + } + + /** + * @param querySearchFeilds + * @param propertyName + * @return + */ + private float getweight(List querySearchFeilds, String propertyName) { + float weight = 1.0F; + if (querySearchFeilds.contains(propertyName)) { + for (String field : querySearchFeilds) { + if (field.contains(propertyName)) { + weight = Float + .parseFloat((StringUtils.isNotBlank(field.split("^")[1])) ? field.split("^")[1] : "1.0"); + } + } + } + return weight; + } + + /** + * @param values + * @return + */ + private QueryBuilder getAllFieldsPropertyQuery(List values, boolean fuzzy) { + List queryFields = ElasticSearchUtil.getQuerySearchFields(); + Map queryFieldsMap = new HashMap<>(); + for (String field : queryFields) { + if (field.contains("^")) + queryFieldsMap.put(field.split("\\^")[0], Float.valueOf(field.split("\\^")[1])); + else + queryFieldsMap.put(field, 1.0f); + } + BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery(); + for (Object value : values) { + if (fuzzy) { + queryBuilder + .should(QueryBuilders.multiMatchQuery(value).fields(queryFieldsMap) + .operator(Operator.AND).fuzziness("AUTO").lenient(true)); + } else { + queryBuilder + .should(QueryBuilders.multiMatchQuery(value).fields(queryFieldsMap) + .operator(Operator.AND).type(Type.CROSS_FIELDS).fuzzyTranspositions(false) + .lenient(true)); + } + } + + return queryBuilder; + } + + /** + * @param softConstraints + * @return + */ + @SuppressWarnings("unchecked") + private QueryBuilder getSoftConstraintQuery(Map softConstraints) { + BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery(); + for (String key : softConstraints.keySet()) { + List data = (List) softConstraints.get(key); + if (data.get(1) instanceof List) { + List dataList = (List) data.get(1); + for (Object value : dataList) { + queryBuilder + .should(QueryBuilders.matchQuery(key + SearchConstants.RAW_FIELD_EXTENSION, value) + .boost(Integer.valueOf((int) data.get(0)).floatValue()).fuzzyTranspositions(false)); + } + } else { + queryBuilder.should( + QueryBuilders.matchQuery(key + SearchConstants.RAW_FIELD_EXTENSION, data.get(1)) + .boost(Integer.valueOf((int) data.get(0)).floatValue()).fuzzyTranspositions(false)); + } + } + return queryBuilder; + } + + /** + * @param propertyName + * @param values + * @return + */ + private QueryBuilder getRangeQuery(String propertyName, List values, String operation) { + BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery(); + for (Object value : values) { + switch (operation) { + case SearchConstants.SEARCH_OPERATION_GREATER_THAN: { + queryBuilder.should(QueryBuilders + .rangeQuery(propertyName).gt(value)); + break; + } + case SearchConstants.SEARCH_OPERATION_GREATER_THAN_EQUALS: { + queryBuilder.should(QueryBuilders + .rangeQuery(propertyName).gte(value)); + break; + } + case SearchConstants.SEARCH_OPERATION_LESS_THAN: { + queryBuilder.should(QueryBuilders + .rangeQuery(propertyName).lt(value)); + break; + } + case SearchConstants.SEARCH_OPERATION_LESS_THAN_EQUALS: { + queryBuilder.should(QueryBuilders + .rangeQuery(propertyName).lte(value)); + break; + } + } + } + + return queryBuilder; + } + + /** + * @param propertyName + * @param values + * @return + */ + private QueryBuilder getExistsQuery(String propertyName, List values, boolean exists) { + BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery(); + for (Object value : values) { + if (exists) { + queryBuilder.should(QueryBuilders.existsQuery(String.valueOf(value))); + } else { + queryBuilder.mustNot(QueryBuilders.existsQuery(String.valueOf(value))); + } + } + return queryBuilder; + } + + /** + * @param propertyName + * @param values + * @return + */ + private QueryBuilder getNotInQuery(String propertyName, List values) { + BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery(); + queryBuilder + .mustNot(QueryBuilders.termsQuery(propertyName, values)); + return queryBuilder; + } + + /** + * @param propertyName + * @param values + * @return + */ + private QueryBuilder getMatchPhrasePrefixQuery(String propertyName, List values) { + BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery(); + for (Object value : values) { + queryBuilder.should(QueryBuilders.prefixQuery( + propertyName, ((String) value).toLowerCase())); + } + return queryBuilder; + } + + /** + * @param propertyName + * @param values + * @param match + * @return + */ + private QueryBuilder getMatchPhraseQuery(String propertyName, List values, boolean match) { + BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery(); + for (Object value : values) { + String stringValue = String.valueOf(value); + if (match) { + queryBuilder.should(QueryBuilders + .regexpQuery(propertyName, + ".*" + stringValue.toLowerCase() + ".*")); + } else { + queryBuilder.mustNot(QueryBuilders + .regexpQuery(propertyName, + ".*" + stringValue.toLowerCase() + ".*")); + } + } + return queryBuilder; + } + + /** + * @param propertyName + * @param values + * @return + */ + private QueryBuilder getRegexQuery(String propertyName, List values) { + BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery(); + for (Object value : values) { + String stringValue = String.valueOf(value); + queryBuilder.should(QueryBuilders.regexpQuery(propertyName, + ".*" + stringValue.toLowerCase())); + } + return queryBuilder; + } + + /** + * @param propertyName + * @param values + * @param match + * @return + */ + private QueryBuilder getMustTermQuery(String propertyName, List values, boolean match) { + BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery(); + for (Object value : values) { + if (match) { + queryBuilder.should( + QueryBuilders.matchQuery(propertyName, value).fuzzyTranspositions(false)); + } else { + queryBuilder.mustNot( + QueryBuilders.matchQuery(propertyName, value).fuzzyTranspositions(false)); + } + } + + return queryBuilder; + } + + /** + * @param propertyName + * @param values + * @return + */ + @SuppressWarnings("unchecked") + private QueryBuilder getRangeQuery(String propertyName, List values) { + RangeQueryBuilder queryBuilder = new RangeQueryBuilder(propertyName); + for (Object value : values) { + Map rangeMap = (Map) value; + if (!rangeMap.isEmpty()) { + for (String key : rangeMap.keySet()) { + switch (key) { + case SearchConstants.SEARCH_OPERATION_RANGE_GTE: { + queryBuilder.from(rangeMap.get(key)); + break; + } + case SearchConstants.SEARCH_OPERATION_RANGE_LTE: { + queryBuilder.to(rangeMap.get(key)); + break; + } + } + } + } + } + return queryBuilder; + } + + /** + * @param value + * @return + */ + private SortOrder getSortOrder(String value) { + return value.equalsIgnoreCase(ASC_ORDER) ? SortOrder.ASC : SortOrder.DESC; + } + + public Future> processSearchQuery(SearchDTO searchDTO, boolean includeResults, String index) + throws Exception { + return processSearchQuery(searchDTO, includeResults, index, true); + } + + public Future> processSearchQuery(SearchDTO searchDTO, boolean includeResults, String index, + boolean sort) + throws Exception { + List> groupByFinalList = new ArrayList>(); + if (searchDTO.getLimit() == 0) + searchDTO.setLimit(ElasticSearchUtil.defaultResultLimit); + SearchSourceBuilder query = processSearchQuery(searchDTO, groupByFinalList, sort); + TelemetryManager.log(" search query: " + query); + Future searchResponse = ElasticSearchUtil.search(index, query); + + return searchResponse.map(new Mapper>() { + public List apply(SearchResponse searchResult) { + List response = new ArrayList(); + TelemetryManager.log("search result from elastic search" + searchResult); + SearchHits resultMap = searchResult.getHits(); + SearchHit[] result = resultMap.getHits(); + for (SearchHit hit : result) { + response.add(hit.getSourceAsMap()); + } + TelemetryManager.log("search response size: " + response.size()); + return response; + } + }, ExecutionContext.Implicits$.MODULE$.global()); + + } + + public Future processSearchQueryWithSearchResult(SearchDTO searchDTO, boolean includeResults, + String index, + boolean sort) throws Exception { + List> groupByFinalList = new ArrayList>(); + if (searchDTO.getLimit() == 0) + searchDTO.setLimit(ElasticSearchUtil.defaultResultLimit); + SearchSourceBuilder query = processSearchQuery(searchDTO, groupByFinalList, sort); + TelemetryManager.log(" search query: " + query); + Future searchResult = ElasticSearchUtil.search(index, query); + TelemetryManager.log("search result from elastic search" + searchResult); + return searchResult; + } + + private void setAggregations(SearchSourceBuilder searchSourceBuilder, List> aggregations) { + if (CollectionUtils.isNotEmpty(aggregations)) { + for (Map aggregate : aggregations) { + TermsAggregationBuilder termBuilder = AggregationBuilders.terms((String) aggregate.get("l1")) + .field(aggregate.get("l1") + SearchConstants.RAW_FIELD_EXTENSION) + .size(ElasticSearchUtil.defaultResultLimit); + int level = 2; + termBuilder.subAggregation(getNextLevelAggregation(aggregate, level)); + searchSourceBuilder.aggregation(termBuilder); + } + } + } + + private AggregationBuilder getNextLevelAggregation(Map aggregate, int level) { + TermsAggregationBuilder termBuilder = AggregationBuilders.terms((String) aggregate.get("l" + level)) .field(aggregate.get("l" + level) + SearchConstants.RAW_FIELD_EXTENSION) .size(ElasticSearchUtil.defaultResultLimit); + if (level == aggregate.keySet().size()) { + return termBuilder; + } else { + level += 1; + return termBuilder.subAggregation(getNextLevelAggregation(aggregate, level)); + } + } - if(level == aggregate.keySet().size()){ - return termBuilder; - }else { - level += 1; - return termBuilder.subAggregation(getNextLevelAggregation(aggregate, level)); - } - } - - private List> aggregateResult(Aggregations aggregations) { + private List> aggregateResult(Aggregations aggregations) { List> aggregationList = new ArrayList<>(); - if(null != aggregations){ + if (null != aggregations) { Map aggregationMap = aggregations.getAsMap(); - for(String key: aggregationMap.keySet()){ + for (String key : aggregationMap.keySet()) { Terms terms = (Terms) aggregationMap.get(key); List buckets = (List) terms.getBuckets(); List> values = new ArrayList<>(); - if(CollectionUtils.isNotEmpty(buckets)) { - for(Terms.Bucket bucket: buckets) { - Map termBucket = new HashMap() {{ - put("count", bucket.getDocCount()); - put("name", bucket.getKey()); - List> subAggregations = aggregateResult(bucket.getAggregations()); - if(CollectionUtils.isNotEmpty(subAggregations)) - put("aggregations", subAggregations); - }}; + if (CollectionUtils.isNotEmpty(buckets)) { + for (Terms.Bucket bucket : buckets) { + Map termBucket = new HashMap() { + { + put("count", bucket.getDocCount()); + put("name", bucket.getKey()); + List> subAggregations = aggregateResult(bucket.getAggregations()); + if (CollectionUtils.isNotEmpty(subAggregations)) + put("aggregations", subAggregations); + } + }; values.add(termBucket); } - aggregationList.add(new HashMap(){{ - put("values", values); - put("name", key); - }}); + aggregationList.add(new HashMap() { + { + put("values", values); + put("name", key); + } + }); } } @@ -806,27 +967,48 @@ private List> aggregateResult(Aggregations aggregations) { return aggregationList; } - private QueryBuilder getSearchQuery(SearchDTO searchDTO) { - BoolQueryBuilder boolQuery = new BoolQueryBuilder(); - QueryBuilder origFilterQry = getQuery(searchDTO); - QueryBuilder implFilterQuery = null; - - if (CollectionUtils.isNotEmpty(searchDTO.getImplicitFilterProperties())) { - List properties = searchDTO.getProperties(); - searchDTO.setProperties(searchDTO.getImplicitFilterProperties()); - implFilterQuery = getQuery(searchDTO); - searchDTO.setProperties(properties); - boolQuery.should(origFilterQry); - boolQuery.should(implFilterQuery); - return boolQuery; - } else { - return origFilterQry; - } - } - - private QueryBuilder getQuery(SearchDTO searchDTO) { - return prepareSearchQuery(searchDTO); - } + private QueryBuilder getSearchQuery(SearchDTO searchDTO) { + BoolQueryBuilder boolQuery = new BoolQueryBuilder(); + QueryBuilder origFilterQry = getQuery(searchDTO); + QueryBuilder implFilterQuery = null; + + if (CollectionUtils.isNotEmpty(searchDTO.getImplicitFilterProperties())) { + List properties = searchDTO.getProperties(); + searchDTO.setProperties(searchDTO.getImplicitFilterProperties()); + implFilterQuery = getQuery(searchDTO); + searchDTO.setProperties(properties); + boolQuery.should(origFilterQry); + boolQuery.should(implFilterQuery); + return boolQuery; + } else { + return origFilterQry; + } + } + private QueryBuilder getSearchQuery(SearchDTO searchDTO, boolean fuzzyEnabled) { + BoolQueryBuilder boolQuery = new BoolQueryBuilder(); + QueryBuilder origFilterQry = getQuery(searchDTO, fuzzyEnabled); + QueryBuilder implFilterQuery = null; + + if (CollectionUtils.isNotEmpty(searchDTO.getImplicitFilterProperties())) { + List properties = searchDTO.getProperties(); + searchDTO.setProperties(searchDTO.getImplicitFilterProperties()); + implFilterQuery = getQuery(searchDTO); + searchDTO.setProperties(properties); + boolQuery.should(origFilterQry); + boolQuery.should(implFilterQuery); + return boolQuery; + } else { + return origFilterQry; + } + } + + private QueryBuilder getQuery(SearchDTO searchDTO) { + return prepareSearchQuery(searchDTO); + } + + private QueryBuilder getQuery(SearchDTO searchDTO, boolean fuzzyEnabled) { + return prepareSearchQuery(searchDTO, fuzzyEnabled); + } } \ No newline at end of file diff --git a/search-api/search-service/conf/application.conf b/search-api/search-service/conf/application.conf index a803688db..a2f9d3aca 100644 --- a/search-api/search-service/conf/application.conf +++ b/search-api/search-service/conf/application.conf @@ -293,7 +293,7 @@ search.config.path=/home/learning/platform/search cache.type="redis" search.es_conn_info="localhost:9200" -search.fields.query=["name^100","title^100","lemma^100","code^100","domain","subject","description^10","keywords^100","ageGroup^10","filter^10","theme^10","genre^10","objects^25","contentType^100","language^200","teachingMode^25","skills^10","learningObjective^10","curriculum^100","gradeLevel^100","developer^100","attributions^10","identifier^100","IL_UNIQUE_ID^100","owner^50","board^100","relatedBoards^100","creator^100", "dialcodes^100","text","words","releaseNotes"] +search.fields.query=["name^100.0","keywords^60.0","competencies_v3.name^60.0","systemTopics^60","purpose^60", "source^50.0","description^5.0","language^5.0"] search.fields.date=["lastUpdatedOn","createdOn","versionDate","lastSubmittedOn","lastPublishedOn"] search.fields.mode_collection=["identifier","name","objectType","contentType","mimeType","size","childNodes","board","subject","medium","gradeLevel","appIcon", "origin", "originData"] search.batch.size=500