From 76dacc8e3b399f60ee67c0d3de2e12979e1b0a32 Mon Sep 17 00:00:00 2001 From: ti-lap-563 Date: Mon, 14 Nov 2022 15:45:31 +0530 Subject: [PATCH 1/5] Modified changes for course search api --- .../java/org/sunbird/actors/SearchActor.java | 7 +- .../search/processor/SearchProcessor.java | 465 ++++++++++++------ .../search-service/conf/application.conf | 2 +- 3 files changed, 308 insertions(+), 166 deletions(-) diff --git a/search-api/search-actors/src/main/java/org/sunbird/actors/SearchActor.java b/search-api/search-actors/src/main/java/org/sunbird/actors/SearchActor.java index 68f175363..521c2dd05 100644 --- a/search-api/search-actors/src/main/java/org/sunbird/actors/SearchActor.java +++ b/search-api/search-actors/src/main/java/org/sunbird/actors/SearchActor.java @@ -42,7 +42,8 @@ public Future onReceive(Request request) throws Throwable { try{ if (StringUtils.equalsIgnoreCase("INDEX_SEARCH", operation)) { SearchDTO searchDTO = getSearchDTO(request); - Future> searchResult = processor.processSearch(searchDTO, true); + int pass= 1; + Future> searchResult = processor.processSearch(searchDTO, true, pass); return searchResult.map(new Mapper, Response>() { @Override public Response apply(Map lstResult) { @@ -594,7 +595,7 @@ private Map getCompositeSearchResponse(Map searc String objectType = ((String) map.getOrDefault("objectType", "")).replaceAll("Image", ""); if(StringUtils.equalsIgnoreCase("Collection", objectType) || StringUtils.equalsIgnoreCase("Asset", objectType)) map.replace("objectType", "Content"); - else + else map.replace("objectType", objectType); if (StringUtils.isNotBlank(objectType)) { String key = getResultParamKey(objectType); @@ -777,4 +778,4 @@ private void setImplicitFilters(Map filters, SearchDTO searchObj searchObj.setImplicitFilterProperties(implicitFilterProps); } } -} +} \ No newline at end of file diff --git a/search-api/search-core/src/main/java/org/sunbird/search/processor/SearchProcessor.java b/search-api/search-core/src/main/java/org/sunbird/search/processor/SearchProcessor.java index 3a46c3ddb..776bf478e 100644 --- a/search-api/search-core/src/main/java/org/sunbird/search/processor/SearchProcessor.java +++ b/search-api/search-core/src/main/java/org/sunbird/search/processor/SearchProcessor.java @@ -43,7 +43,7 @@ public SearchProcessor() { ElasticSearchUtil.initialiseESClient(SearchConstants.COMPOSITE_SEARCH_INDEX, Platform.config.getString("search.es_conn_info")); } - + public SearchProcessor(String indexName) { } @@ -85,6 +85,51 @@ public Map apply(SearchResponse searchResult) { }, ExecutionContext.Implicits$.MODULE$.global()); } + public Future> processSearch(SearchDTO searchDTO, boolean includeResults, int pass) + throws Exception { + List> groupByFinalList = new ArrayList>(); + SearchSourceBuilder query = processSearchQuery(searchDTO, groupByFinalList, true,pass); + Future searchResponse = ElasticSearchUtil.search( + SearchConstants.COMPOSITE_SEARCH_INDEX, + query); + + //System.out.println(" SEARCH RESPONSE : "+searchResponse); + if(((SearchResponse) searchResponse).getHits().getTotalHits() == 0 && pass == 1) + { + query = processSearchQuery(searchDTO, groupByFinalList, true,++pass); + searchResponse = ElasticSearchUtil.search( + SearchConstants.COMPOSITE_SEARCH_INDEX, + query); + } + return searchResponse.map(new Mapper>() { + public Map apply(SearchResponse searchResult) { + Map resp = new HashMap<>(); + if (includeResults) { + if (searchDTO.isFuzzySearch()) { + List results = ElasticSearchUtil.getDocumentsFromSearchResultWithScore(searchResult); + resp.put("results", results); + } else { + List results = ElasticSearchUtil.getDocumentsFromSearchResult(searchResult, Map.class); + resp.put("results", results); + } + } + Aggregations aggregations = searchResult.getAggregations(); + if (null != aggregations) { + AggregationsResultTransformer transformer = new AggregationsResultTransformer(); + if(CollectionUtils.isNotEmpty(searchDTO.getFacets())) { + resp.put("facets", (List>) ElasticSearchUtil + .getCountFromAggregation(aggregations, groupByFinalList, transformer)); + } else if(CollectionUtils.isNotEmpty(searchDTO.getAggregations())){ + resp.put("aggregations", aggregateResult(aggregations)); + } + + } + + resp.put("count", (int) searchResult.getHits().getTotalHits()); + return resp; + } + }, ExecutionContext.Implicits$.MODULE$.global()); + } public Map processCount(SearchDTO searchDTO) throws Exception { Map response = new HashMap(); SearchSourceBuilder searchSourceBuilder = processSearchQuery(searchDTO, null, false); @@ -99,7 +144,7 @@ public Map processCount(SearchDTO searchDTO) throws Exception { /** * Returns the list of words which are synonyms of the synsetIds passed in the * request - * + * * @param synsetIds * @return * @throws Exception @@ -164,7 +209,7 @@ public Map multiWordDocSearch(List synsetIds) throws Exc /** * Returns list of synsetsIds which has valid documents in composite index - * + * * @param synsetIds * @return * @throws Exception @@ -203,7 +248,7 @@ public void destroy() { * @return */ private SearchSourceBuilder processSearchQuery(SearchDTO searchDTO, List> groupByFinalList, - boolean sortBy) { + boolean sortBy) { SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); List fields = searchDTO.getFields(); @@ -253,6 +298,56 @@ private SearchSourceBuilder processSearchQuery(SearchDTO searchDTO, List> groupByFinalList, + boolean sortBy, int pass) { + + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + List fields = searchDTO.getFields(); + if (null != fields && !fields.isEmpty()) { + fields.add("objectType"); + fields.add("identifier"); + searchSourceBuilder.fetchSource(fields.toArray(new String[fields.size()]), null); + } + + if (searchDTO.getFacets() != null && groupByFinalList != null) { + for (String facet : searchDTO.getFacets()) { + Map groupByMap = new HashMap(); + groupByMap.put("groupByParent", facet); + groupByFinalList.add(groupByMap); + } + } + + searchSourceBuilder.size(searchDTO.getLimit()); + searchSourceBuilder.from(searchDTO.getOffset()); + QueryBuilder query = getSearchQuery(searchDTO,pass); + if (searchDTO.isFuzzySearch()) + relevanceSort = true; + + searchSourceBuilder.query(query); + + if (sortBy && !relevanceSort + && (null == searchDTO.getSoftConstraints() || searchDTO.getSoftConstraints().isEmpty())) { + Map sorting = searchDTO.getSortBy(); + if (sorting == null || sorting.isEmpty()) { + sorting = new HashMap(); + sorting.put("name", "asc"); + sorting.put("lastUpdatedOn", "desc"); + } + for (String key : sorting.keySet()){ + if(key.contains(".")){ + String nestedPath = key.split("\\.")[0]; + searchSourceBuilder.sort(SortBuilders.fieldSort(key + SearchConstants.RAW_FIELD_EXTENSION).order(getSortOrder(sorting.get(key))).setNestedSort(new NestedSortBuilder(nestedPath))); + } else{ + searchSourceBuilder.sort(key + SearchConstants.RAW_FIELD_EXTENSION, + getSortOrder(sorting.get(key))); + } + } + } + setAggregations(groupByFinalList, searchSourceBuilder); + setAggregations(searchSourceBuilder, searchDTO.getAggregations()); + searchSourceBuilder.trackScores(true); + return searchSourceBuilder; + } /** * @param groupByList * @param searchSourceBuilder @@ -260,7 +355,7 @@ private SearchSourceBuilder processSearchQuery(SearchDTO searchDTO, List> groupByList, - SearchSourceBuilder searchSourceBuilder) { + SearchSourceBuilder searchSourceBuilder) { TermsAggregationBuilder termBuilder = null; if (groupByList != null && !groupByList.isEmpty()) { HashMap> nestedAggregation = new HashMap<>(); @@ -268,17 +363,17 @@ private void setAggregations(List> groupByList, String groupByParent = (String) groupByMap.get("groupByParent"); if (!groupByParent.contains(".")) { termBuilder = AggregationBuilders.terms(groupByParent) - .field(groupByParent + SearchConstants.RAW_FIELD_EXTENSION) - .size(ElasticSearchUtil.defaultResultLimit); - List groupByChildList = (List) groupByMap.get("groupByChildList"); - if (groupByChildList != null && !groupByChildList.isEmpty()) { - for (String childGroupBy : groupByChildList) { - termBuilder.subAggregation(AggregationBuilders.terms(childGroupBy) - .field(childGroupBy + SearchConstants.RAW_FIELD_EXTENSION) - .size(ElasticSearchUtil.defaultResultLimit)); + .field(groupByParent + SearchConstants.RAW_FIELD_EXTENSION) + .size(ElasticSearchUtil.defaultResultLimit); + List groupByChildList = (List) groupByMap.get("groupByChildList"); + if (groupByChildList != null && !groupByChildList.isEmpty()) { + for (String childGroupBy : groupByChildList) { + termBuilder.subAggregation(AggregationBuilders.terms(childGroupBy) + .field(childGroupBy + SearchConstants.RAW_FIELD_EXTENSION) + .size(ElasticSearchUtil.defaultResultLimit)); + } } - } - searchSourceBuilder.aggregation(termBuilder); + searchSourceBuilder.aggregation(termBuilder); } else { if (nestedAggregation.get(groupByParent.split("\\.")[0]) != null) { nestedAggregation.get(groupByParent.split("\\.")[0]).add(groupByParent.split("\\.")[1]); @@ -328,6 +423,29 @@ private QueryBuilder prepareSearchQuery(SearchDTO searchDTO) { return boolQuery; } + private QueryBuilder prepareSearchQuery(SearchDTO searchDTO, int pass) { + BoolQueryBuilder boolQuery = new BoolQueryBuilder(); + QueryBuilder queryBuilder = null; + String totalOperation = searchDTO.getOperation(); + List properties = searchDTO.getProperties(); + if(!searchDTO.isFuzzySearch() || pass == 1) + formQuery(properties, queryBuilder, boolQuery, totalOperation, false); + else + formQuery(properties, queryBuilder, boolQuery, totalOperation, searchDTO.isFuzzySearch()); + if(searchDTO.getMultiFilterProperties() != null && (!searchDTO.isFuzzySearch() || pass == 1)) { + formQuery(searchDTO.getMultiFilterProperties(), queryBuilder, boolQuery, SearchConstants.SEARCH_OPERATION_OR, false); + } + else + formQuery(searchDTO.getMultiFilterProperties(), queryBuilder, boolQuery, SearchConstants.SEARCH_OPERATION_OR, searchDTO.isFuzzySearch()); + + Map softConstraints = searchDTO.getSoftConstraints(); + if (null != softConstraints && !softConstraints.isEmpty()) { + boolQuery.should(getSoftConstraintQuery(softConstraints)); + searchDTO.setSortBy(null); + // relevanceSort = true; + } + return boolQuery; + } private void formQuery(List properties, QueryBuilder queryBuilder, BoolQueryBuilder boolQuery, String operation, Boolean fuzzy) { for (Map property : properties) { String opertation = (String) property.get("operation"); @@ -353,85 +471,85 @@ private void formQuery(List properties, QueryBuilder queryBuilder, BoolQuer propertyName = propertyName + SearchConstants.RAW_FIELD_EXTENSION; switch (opertation) { - case SearchConstants.SEARCH_OPERATION_EQUAL: { - queryBuilder = getMustTermQuery(propertyName, values, true); - queryBuilder = checkNestedProperty(queryBuilder, propertyName); - break; - } - case SearchConstants.SEARCH_OPERATION_NOT_EQUAL: { - queryBuilder = getMustTermQuery(propertyName, values, false); - queryBuilder = checkNestedProperty(queryBuilder, propertyName); - break; - } - case SearchConstants.SEARCH_OPERATION_NOT_IN: { - queryBuilder = getNotInQuery(propertyName, values); - queryBuilder = checkNestedProperty(queryBuilder, propertyName); - break; - } - case SearchConstants.SEARCH_OPERATION_ENDS_WITH: { - queryBuilder = getRegexQuery(propertyName, values); - queryBuilder = checkNestedProperty(queryBuilder, propertyName); - break; - } - case SearchConstants.SEARCH_OPERATION_LIKE: - case SearchConstants.SEARCH_OPERATION_CONTAINS: { - queryBuilder = getMatchPhraseQuery(propertyName, values, true); - queryBuilder = checkNestedProperty(queryBuilder, propertyName); - break; - } - case SearchConstants.SEARCH_OPERATION_NOT_LIKE: { - queryBuilder = getMatchPhraseQuery(propertyName, values, false); - queryBuilder = checkNestedProperty(queryBuilder, propertyName); - break; - } - case SearchConstants.SEARCH_OPERATION_STARTS_WITH: { - queryBuilder = getMatchPhrasePrefixQuery(propertyName, values); - queryBuilder = checkNestedProperty(queryBuilder, propertyName); - break; - } - case SearchConstants.SEARCH_OPERATION_EXISTS: { - queryBuilder = getExistsQuery(propertyName, values, true); - queryBuilder = checkNestedProperty(queryBuilder, propertyName); - break; - } - case SearchConstants.SEARCH_OPERATION_NOT_EXISTS: { - queryBuilder = getExistsQuery(propertyName, values, false); - queryBuilder = checkNestedProperty(queryBuilder, propertyName); - break; - } - case SearchConstants.SEARCH_OPERATION_GREATER_THAN: { - queryBuilder = getRangeQuery(propertyName, values, - SearchConstants.SEARCH_OPERATION_GREATER_THAN); - queryBuilder = checkNestedProperty(queryBuilder, propertyName); - break; - } - case SearchConstants.SEARCH_OPERATION_GREATER_THAN_EQUALS: { - queryBuilder = getRangeQuery(propertyName, values, - SearchConstants.SEARCH_OPERATION_GREATER_THAN_EQUALS); - queryBuilder = checkNestedProperty(queryBuilder, propertyName); - break; - } - case SearchConstants.SEARCH_OPERATION_LESS_THAN: { - queryBuilder = getRangeQuery(propertyName, values, SearchConstants.SEARCH_OPERATION_LESS_THAN); - queryBuilder = checkNestedProperty(queryBuilder, propertyName); - break; - } - case SearchConstants.SEARCH_OPERATION_LESS_THAN_EQUALS: { - queryBuilder = getRangeQuery(propertyName, values, - SearchConstants.SEARCH_OPERATION_LESS_THAN_EQUALS); - queryBuilder = checkNestedProperty(queryBuilder, propertyName); - break; - } - case SearchConstants.SEARCH_OPERATION_RANGE: { - queryBuilder = getRangeQuery(propertyName, values); - queryBuilder = checkNestedProperty(queryBuilder, propertyName); - break; - } - case SearchConstants.SEARCH_OPERATION_AND: { - queryBuilder = getAndQuery(propertyName, values); - queryBuilder = checkNestedProperty(queryBuilder, propertyName); - break; - } + case SearchConstants.SEARCH_OPERATION_EQUAL: { + queryBuilder = getMustTermQuery(propertyName, values, true); + queryBuilder = checkNestedProperty(queryBuilder, propertyName); + break; + } + case SearchConstants.SEARCH_OPERATION_NOT_EQUAL: { + queryBuilder = getMustTermQuery(propertyName, values, false); + queryBuilder = checkNestedProperty(queryBuilder, propertyName); + break; + } + case SearchConstants.SEARCH_OPERATION_NOT_IN: { + queryBuilder = getNotInQuery(propertyName, values); + queryBuilder = checkNestedProperty(queryBuilder, propertyName); + break; + } + case SearchConstants.SEARCH_OPERATION_ENDS_WITH: { + queryBuilder = getRegexQuery(propertyName, values); + queryBuilder = checkNestedProperty(queryBuilder, propertyName); + break; + } + case SearchConstants.SEARCH_OPERATION_LIKE: + case SearchConstants.SEARCH_OPERATION_CONTAINS: { + queryBuilder = getMatchPhraseQuery(propertyName, values, true); + queryBuilder = checkNestedProperty(queryBuilder, propertyName); + break; + } + case SearchConstants.SEARCH_OPERATION_NOT_LIKE: { + queryBuilder = getMatchPhraseQuery(propertyName, values, false); + queryBuilder = checkNestedProperty(queryBuilder, propertyName); + break; + } + case SearchConstants.SEARCH_OPERATION_STARTS_WITH: { + queryBuilder = getMatchPhrasePrefixQuery(propertyName, values); + queryBuilder = checkNestedProperty(queryBuilder, propertyName); + break; + } + case SearchConstants.SEARCH_OPERATION_EXISTS: { + queryBuilder = getExistsQuery(propertyName, values, true); + queryBuilder = checkNestedProperty(queryBuilder, propertyName); + break; + } + case SearchConstants.SEARCH_OPERATION_NOT_EXISTS: { + queryBuilder = getExistsQuery(propertyName, values, false); + queryBuilder = checkNestedProperty(queryBuilder, propertyName); + break; + } + case SearchConstants.SEARCH_OPERATION_GREATER_THAN: { + queryBuilder = getRangeQuery(propertyName, values, + SearchConstants.SEARCH_OPERATION_GREATER_THAN); + queryBuilder = checkNestedProperty(queryBuilder, propertyName); + break; + } + case SearchConstants.SEARCH_OPERATION_GREATER_THAN_EQUALS: { + queryBuilder = getRangeQuery(propertyName, values, + SearchConstants.SEARCH_OPERATION_GREATER_THAN_EQUALS); + queryBuilder = checkNestedProperty(queryBuilder, propertyName); + break; + } + case SearchConstants.SEARCH_OPERATION_LESS_THAN: { + queryBuilder = getRangeQuery(propertyName, values, SearchConstants.SEARCH_OPERATION_LESS_THAN); + queryBuilder = checkNestedProperty(queryBuilder, propertyName); + break; + } + case SearchConstants.SEARCH_OPERATION_LESS_THAN_EQUALS: { + queryBuilder = getRangeQuery(propertyName, values, + SearchConstants.SEARCH_OPERATION_LESS_THAN_EQUALS); + queryBuilder = checkNestedProperty(queryBuilder, propertyName); + break; + } + case SearchConstants.SEARCH_OPERATION_RANGE: { + queryBuilder = getRangeQuery(propertyName, values); + queryBuilder = checkNestedProperty(queryBuilder, propertyName); + break; + } + case SearchConstants.SEARCH_OPERATION_AND: { + queryBuilder = getAndQuery(propertyName, values); + queryBuilder = checkNestedProperty(queryBuilder, propertyName); + break; + } } if (operation.equalsIgnoreCase(AND)) { boolQuery.must(queryBuilder); @@ -453,8 +571,8 @@ private QueryBuilder checkNestedProperty(QueryBuilder queryBuilder, String prope private QueryBuilder getAndQuery(String propertyName, List values) { BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery(); for (Object value : values) { - queryBuilder.must( - QueryBuilders.matchQuery(propertyName, value).operator(Operator.AND).fuzzyTranspositions(false)); + queryBuilder.must( + QueryBuilders.matchQuery(propertyName, value).operator(Operator.AND).fuzzyTranspositions(false)); } return queryBuilder; } @@ -541,26 +659,26 @@ private QueryBuilder getRangeQuery(String propertyName, List values, Str BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery(); for (Object value : values) { switch (operation) { - case SearchConstants.SEARCH_OPERATION_GREATER_THAN: { - queryBuilder.should(QueryBuilders - .rangeQuery(propertyName).gt(value)); - break; - } - case SearchConstants.SEARCH_OPERATION_GREATER_THAN_EQUALS: { - queryBuilder.should(QueryBuilders - .rangeQuery(propertyName).gte(value)); - break; - } - case SearchConstants.SEARCH_OPERATION_LESS_THAN: { - queryBuilder.should(QueryBuilders - .rangeQuery(propertyName).lt(value)); - break; - } - case SearchConstants.SEARCH_OPERATION_LESS_THAN_EQUALS: { - queryBuilder.should(QueryBuilders - .rangeQuery(propertyName).lte(value)); - break; - } + case SearchConstants.SEARCH_OPERATION_GREATER_THAN: { + queryBuilder.should(QueryBuilders + .rangeQuery(propertyName).gt(value)); + break; + } + case SearchConstants.SEARCH_OPERATION_GREATER_THAN_EQUALS: { + queryBuilder.should(QueryBuilders + .rangeQuery(propertyName).gte(value)); + break; + } + case SearchConstants.SEARCH_OPERATION_LESS_THAN: { + queryBuilder.should(QueryBuilders + .rangeQuery(propertyName).lt(value)); + break; + } + case SearchConstants.SEARCH_OPERATION_LESS_THAN_EQUALS: { + queryBuilder.should(QueryBuilders + .rangeQuery(propertyName).lte(value)); + break; + } } } @@ -682,14 +800,14 @@ private QueryBuilder getRangeQuery(String propertyName, List values) { if (!rangeMap.isEmpty()) { for (String key : rangeMap.keySet()) { switch (key) { - case SearchConstants.SEARCH_OPERATION_RANGE_GTE: { - queryBuilder.from(rangeMap.get(key)); - break; - } - case SearchConstants.SEARCH_OPERATION_RANGE_LTE: { - queryBuilder.to(rangeMap.get(key)); - break; - } + case SearchConstants.SEARCH_OPERATION_RANGE_GTE: { + queryBuilder.from(rangeMap.get(key)); + break; + } + case SearchConstants.SEARCH_OPERATION_RANGE_LTE: { + queryBuilder.to(rangeMap.get(key)); + break; + } } } } @@ -711,7 +829,7 @@ public Future> processSearchQuery(SearchDTO searchDTO, boolean incl } public Future> processSearchQuery(SearchDTO searchDTO, boolean includeResults, String index, - boolean sort) + boolean sort) throws Exception { List> groupByFinalList = new ArrayList>(); if (searchDTO.getLimit() == 0) @@ -719,7 +837,7 @@ public Future> processSearchQuery(SearchDTO searchDTO, boolean incl SearchSourceBuilder query = processSearchQuery(searchDTO, groupByFinalList, sort); TelemetryManager.log(" search query: " + query); Future searchResponse = ElasticSearchUtil.search(index, query); - + return searchResponse.map(new Mapper>() { public List apply(SearchResponse searchResult) { List response = new ArrayList(); @@ -733,12 +851,12 @@ public List apply(SearchResponse searchResult) { return response; } }, ExecutionContext.Implicits$.MODULE$.global()); - + } public Future processSearchQueryWithSearchResult(SearchDTO searchDTO, boolean includeResults, - String index, - boolean sort) throws Exception { + String index, + boolean sort) throws Exception { List> groupByFinalList = new ArrayList>(); if (searchDTO.getLimit() == 0) searchDTO.setLimit(ElasticSearchUtil.defaultResultLimit); @@ -763,48 +881,48 @@ private void setAggregations(SearchSourceBuilder searchSourceBuilder, List aggregate, int level) { - TermsAggregationBuilder termBuilder = AggregationBuilders.terms((String)aggregate.get("l" + level)) - .field(aggregate.get("l" + level) + SearchConstants.RAW_FIELD_EXTENSION) - .size(ElasticSearchUtil.defaultResultLimit); + TermsAggregationBuilder termBuilder = AggregationBuilders.terms((String)aggregate.get("l" + level)) + .field(aggregate.get("l" + level) + SearchConstants.RAW_FIELD_EXTENSION) + .size(ElasticSearchUtil.defaultResultLimit); if(level == aggregate.keySet().size()){ return termBuilder; }else { - level += 1; + level += 1; return termBuilder.subAggregation(getNextLevelAggregation(aggregate, level)); } } - private List> aggregateResult(Aggregations aggregations) { - List> aggregationList = new ArrayList<>(); - if(null != aggregations){ - Map aggregationMap = aggregations.getAsMap(); - for(String key: aggregationMap.keySet()){ - Terms terms = (Terms) aggregationMap.get(key); - List buckets = (List) terms.getBuckets(); - List> values = new ArrayList<>(); - if(CollectionUtils.isNotEmpty(buckets)) { - for(Terms.Bucket bucket: buckets) { - Map termBucket = new HashMap() {{ - put("count", bucket.getDocCount()); - put("name", bucket.getKey()); - List> subAggregations = aggregateResult(bucket.getAggregations()); - if(CollectionUtils.isNotEmpty(subAggregations)) - put("aggregations", subAggregations); - }}; - values.add(termBucket); - } - aggregationList.add(new HashMap(){{ - put("values", values); - put("name", key); - }}); - } - } - - } - return aggregationList; - } + private List> aggregateResult(Aggregations aggregations) { + List> aggregationList = new ArrayList<>(); + if(null != aggregations){ + Map aggregationMap = aggregations.getAsMap(); + for(String key: aggregationMap.keySet()){ + Terms terms = (Terms) aggregationMap.get(key); + List buckets = (List) terms.getBuckets(); + List> values = new ArrayList<>(); + if(CollectionUtils.isNotEmpty(buckets)) { + for(Terms.Bucket bucket: buckets) { + Map termBucket = new HashMap() {{ + put("count", bucket.getDocCount()); + put("name", bucket.getKey()); + List> subAggregations = aggregateResult(bucket.getAggregations()); + if(CollectionUtils.isNotEmpty(subAggregations)) + put("aggregations", subAggregations); + }}; + values.add(termBucket); + } + aggregationList.add(new HashMap(){{ + put("values", values); + put("name", key); + }}); + } + } + + } + return aggregationList; + } private QueryBuilder getSearchQuery(SearchDTO searchDTO) { BoolQueryBuilder boolQuery = new BoolQueryBuilder(); @@ -824,9 +942,32 @@ private QueryBuilder getSearchQuery(SearchDTO searchDTO) { } } + private QueryBuilder getSearchQuery(SearchDTO searchDTO, int pass) { + BoolQueryBuilder boolQuery = new BoolQueryBuilder(); + QueryBuilder origFilterQry = getQuery(searchDTO, pass); + QueryBuilder implFilterQuery = null; + + if (CollectionUtils.isNotEmpty(searchDTO.getImplicitFilterProperties())) { + List properties = searchDTO.getProperties(); + searchDTO.setProperties(searchDTO.getImplicitFilterProperties()); + implFilterQuery = getQuery(searchDTO); + searchDTO.setProperties(properties); + boolQuery.should(origFilterQry); + boolQuery.should(implFilterQuery); + return boolQuery; + } else { + return origFilterQry; + } + } + + private QueryBuilder getQuery(SearchDTO searchDTO) { return prepareSearchQuery(searchDTO); } + private QueryBuilder getQuery(SearchDTO searchDTO, int pass) { + return prepareSearchQuery(searchDTO,pass); + } + } \ No newline at end of file diff --git a/search-api/search-service/conf/application.conf b/search-api/search-service/conf/application.conf index a803688db..63dd77bd9 100644 --- a/search-api/search-service/conf/application.conf +++ b/search-api/search-service/conf/application.conf @@ -293,7 +293,7 @@ search.config.path=/home/learning/platform/search cache.type="redis" search.es_conn_info="localhost:9200" -search.fields.query=["name^100","title^100","lemma^100","code^100","domain","subject","description^10","keywords^100","ageGroup^10","filter^10","theme^10","genre^10","objects^25","contentType^100","language^200","teachingMode^25","skills^10","learningObjective^10","curriculum^100","gradeLevel^100","developer^100","attributions^10","identifier^100","IL_UNIQUE_ID^100","owner^50","board^100","relatedBoards^100","creator^100", "dialcodes^100","text","words","releaseNotes"] +search.fields.query=["name^100.0","keywords^60.0","competencies_v3.name^60.0","systemTopics^60","purpose^60","source^50.0","description^5.0","language^5.0"] search.fields.date=["lastUpdatedOn","createdOn","versionDate","lastSubmittedOn","lastPublishedOn"] search.fields.mode_collection=["identifier","name","objectType","contentType","mimeType","size","childNodes","board","subject","medium","gradeLevel","appIcon", "origin", "originData"] search.batch.size=500 From cea977c7192dfa1c10219e5bcfda5bbd3030b628 Mon Sep 17 00:00:00 2001 From: ti-lap-563 Date: Wed, 16 Nov 2022 17:34:03 +0530 Subject: [PATCH 2/5] Updated the getQuery() --- .../search/client/ElasticSearchUtil.java | 13 ++ .../search/processor/SearchProcessor.java | 179 +++++++++++------- 2 files changed, 125 insertions(+), 67 deletions(-) diff --git a/search-api/search-core/src/main/java/org/sunbird/search/client/ElasticSearchUtil.java b/search-api/search-core/src/main/java/org/sunbird/search/client/ElasticSearchUtil.java index a25be9575..9f2d27037 100644 --- a/search-api/search-core/src/main/java/org/sunbird/search/client/ElasticSearchUtil.java +++ b/search-api/search-core/src/main/java/org/sunbird/search/client/ElasticSearchUtil.java @@ -726,5 +726,18 @@ private static Map checkDocStringLength(Map doc) } return doc; } + public static boolean checkMatchExist(String indexName, SearchSourceBuilder searchSourceBuilder) throws IOException { + SearchResponse response = getClient(indexName) + .search(new SearchRequest().indices(indexName).source(searchSourceBuilder)); + + try{ + SearchHit hitAt0 = response.getHits().getAt(0); + return true; + } + catch (ArrayIndexOutOfBoundsException e){ + return false; + } + + } } \ No newline at end of file diff --git a/search-api/search-core/src/main/java/org/sunbird/search/processor/SearchProcessor.java b/search-api/search-core/src/main/java/org/sunbird/search/processor/SearchProcessor.java index 776bf478e..34c87cf58 100644 --- a/search-api/search-core/src/main/java/org/sunbird/search/processor/SearchProcessor.java +++ b/search-api/search-core/src/main/java/org/sunbird/search/processor/SearchProcessor.java @@ -5,6 +5,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang.StringUtils; +import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.index.query.*; import org.elasticsearch.index.query.MultiMatchQueryBuilder.Type; @@ -29,6 +30,7 @@ import scala.concurrent.ExecutionContext; import scala.concurrent.Future; +import java.io.IOException; import java.util.*; import java.util.stream.Collectors; @@ -71,10 +73,10 @@ public Map apply(SearchResponse searchResult) { Aggregations aggregations = searchResult.getAggregations(); if (null != aggregations) { AggregationsResultTransformer transformer = new AggregationsResultTransformer(); - if(CollectionUtils.isNotEmpty(searchDTO.getFacets())) { + if (CollectionUtils.isNotEmpty(searchDTO.getFacets())) { resp.put("facets", (List>) ElasticSearchUtil .getCountFromAggregation(aggregations, groupByFinalList, transformer)); - } else if(CollectionUtils.isNotEmpty(searchDTO.getAggregations())){ + } else if (CollectionUtils.isNotEmpty(searchDTO.getAggregations())) { resp.put("aggregations", aggregateResult(aggregations)); } @@ -85,25 +87,31 @@ public Map apply(SearchResponse searchResult) { }, ExecutionContext.Implicits$.MODULE$.global()); } + + public Future> processSearch(SearchDTO searchDTO, boolean includeResults, int pass) throws Exception { List> groupByFinalList = new ArrayList>(); - SearchSourceBuilder query = processSearchQuery(searchDTO, groupByFinalList, true,pass); + SearchSourceBuilder query = processSearchQuery(searchDTO, groupByFinalList, true, pass); Future searchResponse = ElasticSearchUtil.search( SearchConstants.COMPOSITE_SEARCH_INDEX, query); - //System.out.println(" SEARCH RESPONSE : "+searchResponse); - if(((SearchResponse) searchResponse).getHits().getTotalHits() == 0 && pass == 1) - { - query = processSearchQuery(searchDTO, groupByFinalList, true,++pass); + boolean exactMatchExists=ElasticSearchUtil.checkMatchExist(SearchConstants.COMPOSITE_SEARCH_INDEX, + query); + + if(!exactMatchExists){ + query = processSearchQuery(searchDTO, groupByFinalList, true, ++pass); searchResponse = ElasticSearchUtil.search( SearchConstants.COMPOSITE_SEARCH_INDEX, query); } + return searchResponse.map(new Mapper>() { public Map apply(SearchResponse searchResult) { Map resp = new HashMap<>(); + + if (includeResults) { if (searchDTO.isFuzzySearch()) { List results = ElasticSearchUtil.getDocumentsFromSearchResultWithScore(searchResult); @@ -116,19 +124,53 @@ public Map apply(SearchResponse searchResult) { Aggregations aggregations = searchResult.getAggregations(); if (null != aggregations) { AggregationsResultTransformer transformer = new AggregationsResultTransformer(); - if(CollectionUtils.isNotEmpty(searchDTO.getFacets())) { + if (CollectionUtils.isNotEmpty(searchDTO.getFacets())) { resp.put("facets", (List>) ElasticSearchUtil .getCountFromAggregation(aggregations, groupByFinalList, transformer)); - } else if(CollectionUtils.isNotEmpty(searchDTO.getAggregations())){ + } else if (CollectionUtils.isNotEmpty(searchDTO.getAggregations())) { resp.put("aggregations", aggregateResult(aggregations)); } } + + resp.put("count", (int) searchResult.getHits().getTotalHits()); return resp; } }, ExecutionContext.Implicits$.MODULE$.global()); + + + + + + } + + private Map getMatchCount(Future searchResponse) { + + return (Map) searchResponse.map(new Mapper>() { + public Map apply(SearchResponse searchResult) { + Map resp = new HashMap<>(); + resp.put("hits",searchResult.getHits().getTotalHits()); + return resp; + } + }, ExecutionContext.Implicits$.MODULE$.global()); + + } + + private Future exactMatchExists(Future searchResponse) { + return searchResponse.map(new Mapper() { + public Boolean apply(SearchResponse searchResult) { + Map resp = new HashMap<>(); + long matches = searchResult.getHits().getTotalHits(); + if(matches>0) + return true; + else + return false; + + } + }, ExecutionContext.Implicits$.MODULE$.global()); + } public Map processCount(SearchDTO searchDTO) throws Exception { Map response = new HashMap(); @@ -282,11 +324,12 @@ private SearchSourceBuilder processSearchQuery(SearchDTO searchDTO, List groupByMap = new HashMap(); groupByMap.put("groupByParent", facet); @@ -319,7 +362,7 @@ private SearchSourceBuilder processSearchQuery(SearchDTO searchDTO, List> groupByList, } if (!nestedAggregation.isEmpty()) { for (Map.Entry> mapData : nestedAggregation.entrySet()) { - AggregationBuilder nestedAggregationBuilder = AggregationBuilders.nested(mapData.getKey(), mapData.getKey()); + AggregationBuilder nestedAggregationBuilder = AggregationBuilders.nested(mapData.getKey(), + mapData.getKey()); for (String nestedValue : mapData.getValue()) { termBuilder = AggregationBuilders.terms(nestedValue) .field(mapData.getKey() + "." + nestedValue + SearchConstants.RAW_FIELD_EXTENSION) @@ -410,8 +456,9 @@ private QueryBuilder prepareSearchQuery(SearchDTO searchDTO) { String totalOperation = searchDTO.getOperation(); List properties = searchDTO.getProperties(); formQuery(properties, queryBuilder, boolQuery, totalOperation, searchDTO.isFuzzySearch()); - if(searchDTO.getMultiFilterProperties() != null) { - formQuery(searchDTO.getMultiFilterProperties(), queryBuilder, boolQuery, SearchConstants.SEARCH_OPERATION_OR, searchDTO.isFuzzySearch()); + if (searchDTO.getMultiFilterProperties() != null) { + formQuery(searchDTO.getMultiFilterProperties(), queryBuilder, boolQuery, + SearchConstants.SEARCH_OPERATION_OR, searchDTO.isFuzzySearch()); } Map softConstraints = searchDTO.getSoftConstraints(); @@ -428,15 +475,11 @@ private QueryBuilder prepareSearchQuery(SearchDTO searchDTO, int pass) { QueryBuilder queryBuilder = null; String totalOperation = searchDTO.getOperation(); List properties = searchDTO.getProperties(); - if(!searchDTO.isFuzzySearch() || pass == 1) - formQuery(properties, queryBuilder, boolQuery, totalOperation, false); - else - formQuery(properties, queryBuilder, boolQuery, totalOperation, searchDTO.isFuzzySearch()); - if(searchDTO.getMultiFilterProperties() != null && (!searchDTO.isFuzzySearch() || pass == 1)) { - formQuery(searchDTO.getMultiFilterProperties(), queryBuilder, boolQuery, SearchConstants.SEARCH_OPERATION_OR, false); + formQuery(properties, queryBuilder, boolQuery, totalOperation, pass==1? false : searchDTO.isFuzzySearch()); + if (searchDTO.getMultiFilterProperties() != null) { + formQuery(searchDTO.getMultiFilterProperties(), queryBuilder, boolQuery, + SearchConstants.SEARCH_OPERATION_OR, pass==1? false : searchDTO.isFuzzySearch()); } - else - formQuery(searchDTO.getMultiFilterProperties(), queryBuilder, boolQuery, SearchConstants.SEARCH_OPERATION_OR, searchDTO.isFuzzySearch()); Map softConstraints = searchDTO.getSoftConstraints(); if (null != softConstraints && !softConstraints.isEmpty()) { @@ -446,7 +489,9 @@ private QueryBuilder prepareSearchQuery(SearchDTO searchDTO, int pass) { } return boolQuery; } - private void formQuery(List properties, QueryBuilder queryBuilder, BoolQueryBuilder boolQuery, String operation, Boolean fuzzy) { + + private void formQuery(List properties, QueryBuilder queryBuilder, BoolQueryBuilder boolQuery, + String operation, boolean fuzzy) { for (Map property : properties) { String opertation = (String) property.get("operation"); @@ -458,7 +503,6 @@ private void formQuery(List properties, QueryBuilder queryBuilder, BoolQuer } values = values.stream().filter(value -> (null != value)).collect(Collectors.toList()); - String propertyName = (String) property.get("propertyName"); if (propertyName.equals("*")) { relevanceSort = true; @@ -561,13 +605,13 @@ private void formQuery(List properties, QueryBuilder queryBuilder, BoolQuer } private QueryBuilder checkNestedProperty(QueryBuilder queryBuilder, String propertyName) { - if(propertyName.replaceAll(SearchConstants.RAW_FIELD_EXTENSION, "").contains(".")) { - queryBuilder = QueryBuilders.nestedQuery(propertyName.split("\\.")[0], queryBuilder, org.apache.lucene.search.join.ScoreMode.None); + if (propertyName.replaceAll(SearchConstants.RAW_FIELD_EXTENSION, "").contains(".")) { + queryBuilder = QueryBuilders.nestedQuery(propertyName.split("\\.")[0], queryBuilder, + org.apache.lucene.search.join.ScoreMode.None); } return queryBuilder; } - private QueryBuilder getAndQuery(String propertyName, List values) { BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery(); for (Object value : values) { @@ -599,7 +643,7 @@ private float getweight(List querySearchFeilds, String propertyName) { * @param values * @return */ - private QueryBuilder getAllFieldsPropertyQuery(List values, Boolean fuzzy) { + private QueryBuilder getAllFieldsPropertyQuery(List values, boolean fuzzy) { List queryFields = ElasticSearchUtil.getQuerySearchFields(); Map queryFieldsMap = new HashMap<>(); for (String field : queryFields) { @@ -617,7 +661,8 @@ private QueryBuilder getAllFieldsPropertyQuery(List values, Boolean fuzz } else { queryBuilder .should(QueryBuilders.multiMatchQuery(value).fields(queryFieldsMap) - .operator(Operator.AND).type(Type.CROSS_FIELDS).fuzzyTranspositions(false).lenient(true)); + .operator(Operator.AND).type(Type.CROSS_FIELDS).fuzzyTranspositions(false) + .lenient(true)); } } @@ -633,15 +678,14 @@ private QueryBuilder getSoftConstraintQuery(Map softConstraints) BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery(); for (String key : softConstraints.keySet()) { List data = (List) softConstraints.get(key); - if(data.get(1) instanceof List) { + if (data.get(1) instanceof List) { List dataList = (List) data.get(1); - for(Object value: dataList) { + for (Object value : dataList) { queryBuilder .should(QueryBuilders.matchQuery(key + SearchConstants.RAW_FIELD_EXTENSION, value) .boost(Integer.valueOf((int) data.get(0)).floatValue()).fuzzyTranspositions(false)); } - } - else { + } else { queryBuilder.should( QueryBuilders.matchQuery(key + SearchConstants.RAW_FIELD_EXTENSION, data.get(1)) .boost(Integer.valueOf((int) data.get(0)).floatValue()).fuzzyTranspositions(false)); @@ -868,9 +912,9 @@ public Future processSearchQueryWithSearchResult(SearchDTO searc } private void setAggregations(SearchSourceBuilder searchSourceBuilder, List> aggregations) { - if(CollectionUtils.isNotEmpty(aggregations)){ - for(Map aggregate: aggregations){ - TermsAggregationBuilder termBuilder = AggregationBuilders.terms((String)aggregate.get("l1")) + if (CollectionUtils.isNotEmpty(aggregations)) { + for (Map aggregate : aggregations) { + TermsAggregationBuilder termBuilder = AggregationBuilders.terms((String) aggregate.get("l1")) .field(aggregate.get("l1") + SearchConstants.RAW_FIELD_EXTENSION) .size(ElasticSearchUtil.defaultResultLimit); int level = 2; @@ -881,42 +925,45 @@ private void setAggregations(SearchSourceBuilder searchSourceBuilder, List aggregate, int level) { - TermsAggregationBuilder termBuilder = AggregationBuilders.terms((String)aggregate.get("l" + level)) + TermsAggregationBuilder termBuilder = AggregationBuilders.terms((String) aggregate.get("l" + level)) .field(aggregate.get("l" + level) + SearchConstants.RAW_FIELD_EXTENSION) .size(ElasticSearchUtil.defaultResultLimit); - - if(level == aggregate.keySet().size()){ + if (level == aggregate.keySet().size()) { return termBuilder; - }else { + } else { level += 1; return termBuilder.subAggregation(getNextLevelAggregation(aggregate, level)); } } - private List> aggregateResult(Aggregations aggregations) { + private List> aggregateResult(Aggregations aggregations) { List> aggregationList = new ArrayList<>(); - if(null != aggregations){ + if (null != aggregations) { Map aggregationMap = aggregations.getAsMap(); - for(String key: aggregationMap.keySet()){ + for (String key : aggregationMap.keySet()) { Terms terms = (Terms) aggregationMap.get(key); List buckets = (List) terms.getBuckets(); List> values = new ArrayList<>(); - if(CollectionUtils.isNotEmpty(buckets)) { - for(Terms.Bucket bucket: buckets) { - Map termBucket = new HashMap() {{ - put("count", bucket.getDocCount()); - put("name", bucket.getKey()); - List> subAggregations = aggregateResult(bucket.getAggregations()); - if(CollectionUtils.isNotEmpty(subAggregations)) - put("aggregations", subAggregations); - }}; + if (CollectionUtils.isNotEmpty(buckets)) { + for (Terms.Bucket bucket : buckets) { + Map termBucket = new HashMap() { + { + put("count", bucket.getDocCount()); + put("name", bucket.getKey()); + List> subAggregations = aggregateResult(bucket.getAggregations()); + if (CollectionUtils.isNotEmpty(subAggregations)) + put("aggregations", subAggregations); + } + }; values.add(termBucket); } - aggregationList.add(new HashMap(){{ - put("values", values); - put("name", key); - }}); + aggregationList.add(new HashMap() { + { + put("values", values); + put("name", key); + } + }); } } @@ -960,14 +1007,12 @@ private QueryBuilder getSearchQuery(SearchDTO searchDTO, int pass) { } } - private QueryBuilder getQuery(SearchDTO searchDTO) { return prepareSearchQuery(searchDTO); } private QueryBuilder getQuery(SearchDTO searchDTO, int pass) { - return prepareSearchQuery(searchDTO,pass); + return prepareSearchQuery(searchDTO, pass); } - } \ No newline at end of file From e50542f78abad48c3a2126ab8d2e1b7581a6c37f Mon Sep 17 00:00:00 2001 From: ti-lap-563 Date: Wed, 23 Nov 2022 16:06:27 +0530 Subject: [PATCH 3/5] Removed extra methods in SearchProcessor --- .../java/org/sunbird/actors/SearchActor.java | 6 +- search-api/search-core/pom.xml | 5 + .../search/processor/SearchProcessor.java | 1952 ++++++++--------- .../app/controllers/SearchController.scala | 4 +- .../search-service/conf/application.conf | 2 +- 5 files changed, 986 insertions(+), 983 deletions(-) diff --git a/search-api/search-actors/src/main/java/org/sunbird/actors/SearchActor.java b/search-api/search-actors/src/main/java/org/sunbird/actors/SearchActor.java index 521c2dd05..63ad3d453 100644 --- a/search-api/search-actors/src/main/java/org/sunbird/actors/SearchActor.java +++ b/search-api/search-actors/src/main/java/org/sunbird/actors/SearchActor.java @@ -13,6 +13,7 @@ import org.sunbird.common.dto.Response; import org.sunbird.common.exception.ClientException; import org.sunbird.common.exception.ResponseCode; +import org.sunbird.search.client.ElasticSearchUtil; import org.sunbird.search.dto.SearchDTO; import org.sunbird.search.processor.SearchProcessor; import org.sunbird.search.util.DefinitionUtil; @@ -22,6 +23,7 @@ import scala.concurrent.Future; import scala.concurrent.duration.Duration; +import javax.naming.directory.SearchResult; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; @@ -42,8 +44,8 @@ public Future onReceive(Request request) throws Throwable { try{ if (StringUtils.equalsIgnoreCase("INDEX_SEARCH", operation)) { SearchDTO searchDTO = getSearchDTO(request); - int pass= 1; - Future> searchResult = processor.processSearch(searchDTO, true, pass); + boolean fuzzyEnabled=false; + Future> searchResult = processor.processSearch(searchDTO, true, fuzzyEnabled); return searchResult.map(new Mapper, Response>() { @Override public Response apply(Map lstResult) { diff --git a/search-api/search-core/pom.xml b/search-api/search-core/pom.xml index a68a641e4..3c0b36dde 100644 --- a/search-api/search-core/pom.xml +++ b/search-api/search-core/pom.xml @@ -80,6 +80,11 @@ 4.13.1 test + + org.elasticsearch + elasticsearch + 6.3.2 + diff --git a/search-api/search-core/src/main/java/org/sunbird/search/processor/SearchProcessor.java b/search-api/search-core/src/main/java/org/sunbird/search/processor/SearchProcessor.java index 34c87cf58..c547d39de 100644 --- a/search-api/search-core/src/main/java/org/sunbird/search/processor/SearchProcessor.java +++ b/search-api/search-core/src/main/java/org/sunbird/search/processor/SearchProcessor.java @@ -36,983 +36,979 @@ public class SearchProcessor { - private ObjectMapper mapper = new ObjectMapper(); - private static final String ASC_ORDER = "asc"; - private static final String AND = "AND"; - private boolean relevanceSort = false; - - public SearchProcessor() { - ElasticSearchUtil.initialiseESClient(SearchConstants.COMPOSITE_SEARCH_INDEX, - Platform.config.getString("search.es_conn_info")); - } - - public SearchProcessor(String indexName) { - } - - @SuppressWarnings({ "unchecked", "rawtypes" }) - public Future> processSearch(SearchDTO searchDTO, boolean includeResults) - throws Exception { - List> groupByFinalList = new ArrayList>(); - SearchSourceBuilder query = processSearchQuery(searchDTO, groupByFinalList, true); - Future searchResponse = ElasticSearchUtil.search( - SearchConstants.COMPOSITE_SEARCH_INDEX, - query); - - return searchResponse.map(new Mapper>() { - public Map apply(SearchResponse searchResult) { - Map resp = new HashMap<>(); - if (includeResults) { - if (searchDTO.isFuzzySearch()) { - List results = ElasticSearchUtil.getDocumentsFromSearchResultWithScore(searchResult); - resp.put("results", results); - } else { - List results = ElasticSearchUtil.getDocumentsFromSearchResult(searchResult, Map.class); - resp.put("results", results); - } - } - Aggregations aggregations = searchResult.getAggregations(); - if (null != aggregations) { - AggregationsResultTransformer transformer = new AggregationsResultTransformer(); - if (CollectionUtils.isNotEmpty(searchDTO.getFacets())) { - resp.put("facets", (List>) ElasticSearchUtil - .getCountFromAggregation(aggregations, groupByFinalList, transformer)); - } else if (CollectionUtils.isNotEmpty(searchDTO.getAggregations())) { - resp.put("aggregations", aggregateResult(aggregations)); - } - - } - resp.put("count", (int) searchResult.getHits().getTotalHits()); - return resp; - } - }, ExecutionContext.Implicits$.MODULE$.global()); - } - - - - public Future> processSearch(SearchDTO searchDTO, boolean includeResults, int pass) - throws Exception { - List> groupByFinalList = new ArrayList>(); - SearchSourceBuilder query = processSearchQuery(searchDTO, groupByFinalList, true, pass); - Future searchResponse = ElasticSearchUtil.search( - SearchConstants.COMPOSITE_SEARCH_INDEX, - query); - - boolean exactMatchExists=ElasticSearchUtil.checkMatchExist(SearchConstants.COMPOSITE_SEARCH_INDEX, - query); - - if(!exactMatchExists){ - query = processSearchQuery(searchDTO, groupByFinalList, true, ++pass); - searchResponse = ElasticSearchUtil.search( - SearchConstants.COMPOSITE_SEARCH_INDEX, - query); - } - - return searchResponse.map(new Mapper>() { - public Map apply(SearchResponse searchResult) { - Map resp = new HashMap<>(); - - - if (includeResults) { - if (searchDTO.isFuzzySearch()) { - List results = ElasticSearchUtil.getDocumentsFromSearchResultWithScore(searchResult); - resp.put("results", results); - } else { - List results = ElasticSearchUtil.getDocumentsFromSearchResult(searchResult, Map.class); - resp.put("results", results); - } - } - Aggregations aggregations = searchResult.getAggregations(); - if (null != aggregations) { - AggregationsResultTransformer transformer = new AggregationsResultTransformer(); - if (CollectionUtils.isNotEmpty(searchDTO.getFacets())) { - resp.put("facets", (List>) ElasticSearchUtil - .getCountFromAggregation(aggregations, groupByFinalList, transformer)); - } else if (CollectionUtils.isNotEmpty(searchDTO.getAggregations())) { - resp.put("aggregations", aggregateResult(aggregations)); - } - - } - - - - resp.put("count", (int) searchResult.getHits().getTotalHits()); - return resp; - } - }, ExecutionContext.Implicits$.MODULE$.global()); - - - - - - } - - private Map getMatchCount(Future searchResponse) { - - return (Map) searchResponse.map(new Mapper>() { - public Map apply(SearchResponse searchResult) { - Map resp = new HashMap<>(); - resp.put("hits",searchResult.getHits().getTotalHits()); - return resp; - } - }, ExecutionContext.Implicits$.MODULE$.global()); - - } - - private Future exactMatchExists(Future searchResponse) { - return searchResponse.map(new Mapper() { - public Boolean apply(SearchResponse searchResult) { - Map resp = new HashMap<>(); - long matches = searchResult.getHits().getTotalHits(); - if(matches>0) - return true; - else - return false; - - } - }, ExecutionContext.Implicits$.MODULE$.global()); - - } - public Map processCount(SearchDTO searchDTO) throws Exception { - Map response = new HashMap(); - SearchSourceBuilder searchSourceBuilder = processSearchQuery(searchDTO, null, false); - searchSourceBuilder.from(searchDTO.getOffset()).size(0); - int countResult = ElasticSearchUtil.count(SearchConstants.COMPOSITE_SEARCH_INDEX, - searchSourceBuilder); - response.put("count", countResult); - - return response; - } - - /** - * Returns the list of words which are synonyms of the synsetIds passed in the - * request - * - * @param synsetIds - * @return - * @throws Exception - */ - @SuppressWarnings({ "unchecked", "rawtypes", "unused" }) - public Map multiWordDocSearch(List synsetIds) throws Exception { - Map response = new HashMap(); - Map translations = new HashMap(); - Map synsets = new HashMap(); - if (synsetIds != null && synsetIds.size() > 0) { - List resultList = ElasticSearchUtil.getMultiDocumentAsStringByIdList( - SearchConstants.COMPOSITE_SEARCH_INDEX, - SearchConstants.COMPOSITE_SEARCH_INDEX_TYPE, synsetIds); - for (String synsetDoc : resultList) { - Map wordTranslationList = new HashMap(); - Map indexDocument = new HashMap(); - if (synsetDoc != null && !synsetDoc.isEmpty()) { - indexDocument = mapper.readValue(synsetDoc, new TypeReference>() { - }); - Object words = indexDocument.get("synonyms"); - String identifier = (String) indexDocument.get("identifier"); - String gloss = (String) indexDocument.get("gloss"); - wordTranslationList.put("gloss", gloss); - if (words != null) { - List wordIdList = (List) words; - if (wordIdList != null && wordIdList.size() > 0) { - List wordResultList = ElasticSearchUtil.getMultiDocumentAsStringByIdList( - SearchConstants.COMPOSITE_SEARCH_INDEX, - SearchConstants.COMPOSITE_SEARCH_INDEX_TYPE, wordIdList); - for (String wordDoc : wordResultList) { - List synsetWordLangList = new ArrayList(); - Map indexWordDocument = new HashMap(); - indexWordDocument = mapper.readValue(wordDoc, new TypeReference>() { - }); - String wordId = (String) indexWordDocument.get("identifier"); - String graphId = (String) indexWordDocument.get("graph_id"); - if (wordTranslationList.containsKey(graphId)) { - synsetWordLangList = (List) wordTranslationList.get(graphId); - } - String lemma = (String) indexWordDocument.get("lemma"); - String status = (String) indexWordDocument.get("status"); - if (!StringUtils.equalsIgnoreCase(status, "Retired")) { - Map wordMap = new HashMap(); - wordMap.put("id", wordId); - wordMap.put("lemma", lemma); - synsetWordLangList.add(wordMap); - } - wordTranslationList.put(graphId, synsetWordLangList); - } - - } - } - synsets.put(identifier, wordTranslationList); - } - - } - response.put("translations", synsets); - } - - return response; - } - - /** - * Returns list of synsetsIds which has valid documents in composite index - * - * @param synsetIds - * @return - * @throws Exception - */ - public Map multiSynsetDocSearch(List synsetIds) throws Exception { - Map synsetDocList = new HashMap(); - List identifierList = new ArrayList(); - if (synsetIds != null && synsetIds.size() > 0) { - List resultList = ElasticSearchUtil.getMultiDocumentAsStringByIdList( - SearchConstants.COMPOSITE_SEARCH_INDEX, - SearchConstants.COMPOSITE_SEARCH_INDEX_TYPE, synsetIds); - for (String synsetDoc : resultList) { - Map indexDocument = new HashMap(); - if (synsetDoc != null && !synsetDoc.isEmpty()) { - indexDocument = mapper.readValue(synsetDoc, new TypeReference>() { - }); - String identifier = (String) indexDocument.get("identifier"); - identifierList.add(identifier); - - } - - } - } - synsetDocList.put("synsets", identifierList); - - return synsetDocList; - } - - public void destroy() { - // ElasticSearchUtil.cleanESClient(); - } - - /** - * @param searchDTO - * @param groupByFinalList - * @return - */ - private SearchSourceBuilder processSearchQuery(SearchDTO searchDTO, List> groupByFinalList, - boolean sortBy) { - - SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); - List fields = searchDTO.getFields(); - if (null != fields && !fields.isEmpty()) { - fields.add("objectType"); - fields.add("identifier"); - searchSourceBuilder.fetchSource(fields.toArray(new String[fields.size()]), null); - } - - if (searchDTO.getFacets() != null && groupByFinalList != null) { - for (String facet : searchDTO.getFacets()) { - Map groupByMap = new HashMap(); - groupByMap.put("groupByParent", facet); - groupByFinalList.add(groupByMap); - } - } - - searchSourceBuilder.size(searchDTO.getLimit()); - searchSourceBuilder.from(searchDTO.getOffset()); - QueryBuilder query = getSearchQuery(searchDTO); - if (searchDTO.isFuzzySearch()) - relevanceSort = true; - - searchSourceBuilder.query(query); - - if (sortBy && !relevanceSort - && (null == searchDTO.getSoftConstraints() || searchDTO.getSoftConstraints().isEmpty())) { - Map sorting = searchDTO.getSortBy(); - if (sorting == null || sorting.isEmpty()) { - sorting = new HashMap(); - sorting.put("name", "asc"); - sorting.put("lastUpdatedOn", "desc"); - } - for (String key : sorting.keySet()) { - if (key.contains(".")) { - String nestedPath = key.split("\\.")[0]; - searchSourceBuilder.sort(SortBuilders.fieldSort(key + SearchConstants.RAW_FIELD_EXTENSION) - .order(getSortOrder(sorting.get(key))).setNestedSort(new NestedSortBuilder(nestedPath))); - } else { - searchSourceBuilder.sort(key + SearchConstants.RAW_FIELD_EXTENSION, - getSortOrder(sorting.get(key))); - } - } - } - setAggregations(groupByFinalList, searchSourceBuilder); - setAggregations(searchSourceBuilder, searchDTO.getAggregations()); - searchSourceBuilder.trackScores(true); - return searchSourceBuilder; - } - - private SearchSourceBuilder processSearchQuery(SearchDTO searchDTO, List> groupByFinalList, - boolean sortBy, int pass) { - - SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); - List fields = searchDTO.getFields(); - if (null != fields && !fields.isEmpty()) { - fields.add("objectType"); - fields.add("identifier"); - searchSourceBuilder.fetchSource(fields.toArray(new String[fields.size()]), null); - } - - if (searchDTO.getFacets() != null && groupByFinalList != null && pass == 1) { - for (String facet : searchDTO.getFacets()) { - Map groupByMap = new HashMap(); - groupByMap.put("groupByParent", facet); - groupByFinalList.add(groupByMap); - } - } - - searchSourceBuilder.size(searchDTO.getLimit()); - searchSourceBuilder.from(searchDTO.getOffset()); - QueryBuilder query = getSearchQuery(searchDTO, pass); - if (searchDTO.isFuzzySearch()) - relevanceSort = true; - - searchSourceBuilder.query(query); - - if (sortBy && !relevanceSort - && (null == searchDTO.getSoftConstraints() || searchDTO.getSoftConstraints().isEmpty())) { - Map sorting = searchDTO.getSortBy(); - if (sorting == null || sorting.isEmpty()) { - sorting = new HashMap(); - sorting.put("name", "asc"); - sorting.put("lastUpdatedOn", "desc"); - } - for (String key : sorting.keySet()) { - if (key.contains(".")) { - String nestedPath = key.split("\\.")[0]; - searchSourceBuilder.sort(SortBuilders.fieldSort(key + SearchConstants.RAW_FIELD_EXTENSION) - .order(getSortOrder(sorting.get(key))).setNestedSort(new NestedSortBuilder(nestedPath))); - } else { - searchSourceBuilder.sort(key + SearchConstants.RAW_FIELD_EXTENSION, - getSortOrder(sorting.get(key))); - } - } - } - setAggregations(groupByFinalList, searchSourceBuilder); - setAggregations(searchSourceBuilder, searchDTO.getAggregations()); - searchSourceBuilder.trackScores(true); - return searchSourceBuilder; - } - - /** - * @param groupByList - * @param searchSourceBuilder - * @return - */ - @SuppressWarnings("unchecked") - private void setAggregations(List> groupByList, - SearchSourceBuilder searchSourceBuilder) { - TermsAggregationBuilder termBuilder = null; - if (groupByList != null && !groupByList.isEmpty()) { - HashMap> nestedAggregation = new HashMap<>(); - for (Map groupByMap : groupByList) { - String groupByParent = (String) groupByMap.get("groupByParent"); - if (!groupByParent.contains(".")) { - termBuilder = AggregationBuilders.terms(groupByParent) - .field(groupByParent + SearchConstants.RAW_FIELD_EXTENSION) - .size(ElasticSearchUtil.defaultResultLimit); - List groupByChildList = (List) groupByMap.get("groupByChildList"); - if (groupByChildList != null && !groupByChildList.isEmpty()) { - for (String childGroupBy : groupByChildList) { - termBuilder.subAggregation(AggregationBuilders.terms(childGroupBy) - .field(childGroupBy + SearchConstants.RAW_FIELD_EXTENSION) - .size(ElasticSearchUtil.defaultResultLimit)); - } - } - searchSourceBuilder.aggregation(termBuilder); - } else { - if (nestedAggregation.get(groupByParent.split("\\.")[0]) != null) { - nestedAggregation.get(groupByParent.split("\\.")[0]).add(groupByParent.split("\\.")[1]); - } else { - List nestedAggrList = new ArrayList<>(); - nestedAggrList.add(groupByParent.split("\\.")[1]); - nestedAggregation.put(groupByParent.split("\\.")[0], nestedAggrList); - } - } - } - if (!nestedAggregation.isEmpty()) { - for (Map.Entry> mapData : nestedAggregation.entrySet()) { - AggregationBuilder nestedAggregationBuilder = AggregationBuilders.nested(mapData.getKey(), - mapData.getKey()); - for (String nestedValue : mapData.getValue()) { - termBuilder = AggregationBuilders.terms(nestedValue) - .field(mapData.getKey() + "." + nestedValue + SearchConstants.RAW_FIELD_EXTENSION) - .size(ElasticSearchUtil.defaultResultLimit); - nestedAggregationBuilder.subAggregation(termBuilder); - } - searchSourceBuilder.aggregation(nestedAggregationBuilder); - } - } - } - } - - /** - * @param searchDTO - * @return - */ - @SuppressWarnings({ "unchecked", "rawtypes" }) - private QueryBuilder prepareSearchQuery(SearchDTO searchDTO) { - BoolQueryBuilder boolQuery = new BoolQueryBuilder(); - QueryBuilder queryBuilder = null; - String totalOperation = searchDTO.getOperation(); - List properties = searchDTO.getProperties(); - formQuery(properties, queryBuilder, boolQuery, totalOperation, searchDTO.isFuzzySearch()); - if (searchDTO.getMultiFilterProperties() != null) { - formQuery(searchDTO.getMultiFilterProperties(), queryBuilder, boolQuery, - SearchConstants.SEARCH_OPERATION_OR, searchDTO.isFuzzySearch()); - } - - Map softConstraints = searchDTO.getSoftConstraints(); - if (null != softConstraints && !softConstraints.isEmpty()) { - boolQuery.should(getSoftConstraintQuery(softConstraints)); - searchDTO.setSortBy(null); - // relevanceSort = true; - } - return boolQuery; - } - - private QueryBuilder prepareSearchQuery(SearchDTO searchDTO, int pass) { - BoolQueryBuilder boolQuery = new BoolQueryBuilder(); - QueryBuilder queryBuilder = null; - String totalOperation = searchDTO.getOperation(); - List properties = searchDTO.getProperties(); - formQuery(properties, queryBuilder, boolQuery, totalOperation, pass==1? false : searchDTO.isFuzzySearch()); - if (searchDTO.getMultiFilterProperties() != null) { - formQuery(searchDTO.getMultiFilterProperties(), queryBuilder, boolQuery, - SearchConstants.SEARCH_OPERATION_OR, pass==1? false : searchDTO.isFuzzySearch()); - } - - Map softConstraints = searchDTO.getSoftConstraints(); - if (null != softConstraints && !softConstraints.isEmpty()) { - boolQuery.should(getSoftConstraintQuery(softConstraints)); - searchDTO.setSortBy(null); - // relevanceSort = true; - } - return boolQuery; - } - - private void formQuery(List properties, QueryBuilder queryBuilder, BoolQueryBuilder boolQuery, - String operation, boolean fuzzy) { - for (Map property : properties) { - String opertation = (String) property.get("operation"); - - List values; - try { - values = (List) property.get("values"); - } catch (Exception e) { - values = Arrays.asList(property.get("values")); - } - values = values.stream().filter(value -> (null != value)).collect(Collectors.toList()); - - String propertyName = (String) property.get("propertyName"); - if (propertyName.equals("*")) { - relevanceSort = true; - propertyName = "all_fields"; - queryBuilder = getAllFieldsPropertyQuery(values, fuzzy); - boolQuery.must(queryBuilder); - continue; - } - - propertyName = propertyName + SearchConstants.RAW_FIELD_EXTENSION; - - switch (opertation) { - case SearchConstants.SEARCH_OPERATION_EQUAL: { - queryBuilder = getMustTermQuery(propertyName, values, true); - queryBuilder = checkNestedProperty(queryBuilder, propertyName); - break; - } - case SearchConstants.SEARCH_OPERATION_NOT_EQUAL: { - queryBuilder = getMustTermQuery(propertyName, values, false); - queryBuilder = checkNestedProperty(queryBuilder, propertyName); - break; - } - case SearchConstants.SEARCH_OPERATION_NOT_IN: { - queryBuilder = getNotInQuery(propertyName, values); - queryBuilder = checkNestedProperty(queryBuilder, propertyName); - break; - } - case SearchConstants.SEARCH_OPERATION_ENDS_WITH: { - queryBuilder = getRegexQuery(propertyName, values); - queryBuilder = checkNestedProperty(queryBuilder, propertyName); - break; - } - case SearchConstants.SEARCH_OPERATION_LIKE: - case SearchConstants.SEARCH_OPERATION_CONTAINS: { - queryBuilder = getMatchPhraseQuery(propertyName, values, true); - queryBuilder = checkNestedProperty(queryBuilder, propertyName); - break; - } - case SearchConstants.SEARCH_OPERATION_NOT_LIKE: { - queryBuilder = getMatchPhraseQuery(propertyName, values, false); - queryBuilder = checkNestedProperty(queryBuilder, propertyName); - break; - } - case SearchConstants.SEARCH_OPERATION_STARTS_WITH: { - queryBuilder = getMatchPhrasePrefixQuery(propertyName, values); - queryBuilder = checkNestedProperty(queryBuilder, propertyName); - break; - } - case SearchConstants.SEARCH_OPERATION_EXISTS: { - queryBuilder = getExistsQuery(propertyName, values, true); - queryBuilder = checkNestedProperty(queryBuilder, propertyName); - break; - } - case SearchConstants.SEARCH_OPERATION_NOT_EXISTS: { - queryBuilder = getExistsQuery(propertyName, values, false); - queryBuilder = checkNestedProperty(queryBuilder, propertyName); - break; - } - case SearchConstants.SEARCH_OPERATION_GREATER_THAN: { - queryBuilder = getRangeQuery(propertyName, values, - SearchConstants.SEARCH_OPERATION_GREATER_THAN); - queryBuilder = checkNestedProperty(queryBuilder, propertyName); - break; - } - case SearchConstants.SEARCH_OPERATION_GREATER_THAN_EQUALS: { - queryBuilder = getRangeQuery(propertyName, values, - SearchConstants.SEARCH_OPERATION_GREATER_THAN_EQUALS); - queryBuilder = checkNestedProperty(queryBuilder, propertyName); - break; - } - case SearchConstants.SEARCH_OPERATION_LESS_THAN: { - queryBuilder = getRangeQuery(propertyName, values, SearchConstants.SEARCH_OPERATION_LESS_THAN); - queryBuilder = checkNestedProperty(queryBuilder, propertyName); - break; - } - case SearchConstants.SEARCH_OPERATION_LESS_THAN_EQUALS: { - queryBuilder = getRangeQuery(propertyName, values, - SearchConstants.SEARCH_OPERATION_LESS_THAN_EQUALS); - queryBuilder = checkNestedProperty(queryBuilder, propertyName); - break; - } - case SearchConstants.SEARCH_OPERATION_RANGE: { - queryBuilder = getRangeQuery(propertyName, values); - queryBuilder = checkNestedProperty(queryBuilder, propertyName); - break; - } - case SearchConstants.SEARCH_OPERATION_AND: { - queryBuilder = getAndQuery(propertyName, values); - queryBuilder = checkNestedProperty(queryBuilder, propertyName); - break; - } - } - if (operation.equalsIgnoreCase(AND)) { - boolQuery.must(queryBuilder); - } else { - boolQuery.should(queryBuilder); - } - - } - } - - private QueryBuilder checkNestedProperty(QueryBuilder queryBuilder, String propertyName) { - if (propertyName.replaceAll(SearchConstants.RAW_FIELD_EXTENSION, "").contains(".")) { - queryBuilder = QueryBuilders.nestedQuery(propertyName.split("\\.")[0], queryBuilder, - org.apache.lucene.search.join.ScoreMode.None); - } - return queryBuilder; - } - - private QueryBuilder getAndQuery(String propertyName, List values) { - BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery(); - for (Object value : values) { - queryBuilder.must( - QueryBuilders.matchQuery(propertyName, value).operator(Operator.AND).fuzzyTranspositions(false)); - } - return queryBuilder; - } - - /** - * @param querySearchFeilds - * @param propertyName - * @return - */ - private float getweight(List querySearchFeilds, String propertyName) { - float weight = 1.0F; - if (querySearchFeilds.contains(propertyName)) { - for (String field : querySearchFeilds) { - if (field.contains(propertyName)) { - weight = Float - .parseFloat((StringUtils.isNotBlank(field.split("^")[1])) ? field.split("^")[1] : "1.0"); - } - } - } - return weight; - } - - /** - * @param values - * @return - */ - private QueryBuilder getAllFieldsPropertyQuery(List values, boolean fuzzy) { - List queryFields = ElasticSearchUtil.getQuerySearchFields(); - Map queryFieldsMap = new HashMap<>(); - for (String field : queryFields) { - if (field.contains("^")) - queryFieldsMap.put(field.split("\\^")[0], Float.valueOf(field.split("\\^")[1])); - else - queryFieldsMap.put(field, 1.0f); - } - BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery(); - for (Object value : values) { - if (fuzzy) { - queryBuilder - .should(QueryBuilders.multiMatchQuery(value).fields(queryFieldsMap) - .operator(Operator.AND).fuzziness("AUTO").lenient(true)); - } else { - queryBuilder - .should(QueryBuilders.multiMatchQuery(value).fields(queryFieldsMap) - .operator(Operator.AND).type(Type.CROSS_FIELDS).fuzzyTranspositions(false) - .lenient(true)); - } - } - - return queryBuilder; - } - - /** - * @param softConstraints - * @return - */ - @SuppressWarnings("unchecked") - private QueryBuilder getSoftConstraintQuery(Map softConstraints) { - BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery(); - for (String key : softConstraints.keySet()) { - List data = (List) softConstraints.get(key); - if (data.get(1) instanceof List) { - List dataList = (List) data.get(1); - for (Object value : dataList) { - queryBuilder - .should(QueryBuilders.matchQuery(key + SearchConstants.RAW_FIELD_EXTENSION, value) - .boost(Integer.valueOf((int) data.get(0)).floatValue()).fuzzyTranspositions(false)); - } - } else { - queryBuilder.should( - QueryBuilders.matchQuery(key + SearchConstants.RAW_FIELD_EXTENSION, data.get(1)) - .boost(Integer.valueOf((int) data.get(0)).floatValue()).fuzzyTranspositions(false)); - } - } - return queryBuilder; - } - - /** - * @param propertyName - * @param values - * @return - */ - private QueryBuilder getRangeQuery(String propertyName, List values, String operation) { - BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery(); - for (Object value : values) { - switch (operation) { - case SearchConstants.SEARCH_OPERATION_GREATER_THAN: { - queryBuilder.should(QueryBuilders - .rangeQuery(propertyName).gt(value)); - break; - } - case SearchConstants.SEARCH_OPERATION_GREATER_THAN_EQUALS: { - queryBuilder.should(QueryBuilders - .rangeQuery(propertyName).gte(value)); - break; - } - case SearchConstants.SEARCH_OPERATION_LESS_THAN: { - queryBuilder.should(QueryBuilders - .rangeQuery(propertyName).lt(value)); - break; - } - case SearchConstants.SEARCH_OPERATION_LESS_THAN_EQUALS: { - queryBuilder.should(QueryBuilders - .rangeQuery(propertyName).lte(value)); - break; - } - } - } - - return queryBuilder; - } - - /** - * @param propertyName - * @param values - * @return - */ - private QueryBuilder getExistsQuery(String propertyName, List values, boolean exists) { - BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery(); - for (Object value : values) { - if (exists) { - queryBuilder.should(QueryBuilders.existsQuery(String.valueOf(value))); - } else { - queryBuilder.mustNot(QueryBuilders.existsQuery(String.valueOf(value))); - } - } - return queryBuilder; - } - - /** - * @param propertyName - * @param values - * @return - */ - private QueryBuilder getNotInQuery(String propertyName, List values) { - BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery(); - queryBuilder - .mustNot(QueryBuilders.termsQuery(propertyName, values)); - return queryBuilder; - } - - /** - * @param propertyName - * @param values - * @return - */ - private QueryBuilder getMatchPhrasePrefixQuery(String propertyName, List values) { - BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery(); - for (Object value : values) { - queryBuilder.should(QueryBuilders.prefixQuery( - propertyName, ((String) value).toLowerCase())); - } - return queryBuilder; - } - - /** - * @param propertyName - * @param values - * @param match - * @return - */ - private QueryBuilder getMatchPhraseQuery(String propertyName, List values, boolean match) { - BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery(); - for (Object value : values) { - String stringValue = String.valueOf(value); - if (match) { - queryBuilder.should(QueryBuilders - .regexpQuery(propertyName, - ".*" + stringValue.toLowerCase() + ".*")); - } else { - queryBuilder.mustNot(QueryBuilders - .regexpQuery(propertyName, - ".*" + stringValue.toLowerCase() + ".*")); - } - } - return queryBuilder; - } - - /** - * @param propertyName - * @param values - * @return - */ - private QueryBuilder getRegexQuery(String propertyName, List values) { - BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery(); - for (Object value : values) { - String stringValue = String.valueOf(value); - queryBuilder.should(QueryBuilders.regexpQuery(propertyName, - ".*" + stringValue.toLowerCase())); - } - return queryBuilder; - } - - /** - * @param propertyName - * @param values - * @param match - * @return - */ - private QueryBuilder getMustTermQuery(String propertyName, List values, boolean match) { - BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery(); - for (Object value : values) { - if (match) { - queryBuilder.should( - QueryBuilders.matchQuery(propertyName, value).fuzzyTranspositions(false)); - } else { - queryBuilder.mustNot( - QueryBuilders.matchQuery(propertyName, value).fuzzyTranspositions(false)); - } - } - - return queryBuilder; - } - - /** - * @param propertyName - * @param values - * @return - */ - @SuppressWarnings("unchecked") - private QueryBuilder getRangeQuery(String propertyName, List values) { - RangeQueryBuilder queryBuilder = new RangeQueryBuilder(propertyName); - for (Object value : values) { - Map rangeMap = (Map) value; - if (!rangeMap.isEmpty()) { - for (String key : rangeMap.keySet()) { - switch (key) { - case SearchConstants.SEARCH_OPERATION_RANGE_GTE: { - queryBuilder.from(rangeMap.get(key)); - break; - } - case SearchConstants.SEARCH_OPERATION_RANGE_LTE: { - queryBuilder.to(rangeMap.get(key)); - break; - } - } - } - } - } - return queryBuilder; - } - - /** - * @param value - * @return - */ - private SortOrder getSortOrder(String value) { - return value.equalsIgnoreCase(ASC_ORDER) ? SortOrder.ASC : SortOrder.DESC; - } - - public Future> processSearchQuery(SearchDTO searchDTO, boolean includeResults, String index) - throws Exception { - return processSearchQuery(searchDTO, includeResults, index, true); - } - - public Future> processSearchQuery(SearchDTO searchDTO, boolean includeResults, String index, - boolean sort) - throws Exception { - List> groupByFinalList = new ArrayList>(); - if (searchDTO.getLimit() == 0) - searchDTO.setLimit(ElasticSearchUtil.defaultResultLimit); - SearchSourceBuilder query = processSearchQuery(searchDTO, groupByFinalList, sort); - TelemetryManager.log(" search query: " + query); - Future searchResponse = ElasticSearchUtil.search(index, query); - - return searchResponse.map(new Mapper>() { - public List apply(SearchResponse searchResult) { - List response = new ArrayList(); - TelemetryManager.log("search result from elastic search" + searchResult); - SearchHits resultMap = searchResult.getHits(); - SearchHit[] result = resultMap.getHits(); - for (SearchHit hit : result) { - response.add(hit.getSourceAsMap()); - } - TelemetryManager.log("search response size: " + response.size()); - return response; - } - }, ExecutionContext.Implicits$.MODULE$.global()); - - } - - public Future processSearchQueryWithSearchResult(SearchDTO searchDTO, boolean includeResults, - String index, - boolean sort) throws Exception { - List> groupByFinalList = new ArrayList>(); - if (searchDTO.getLimit() == 0) - searchDTO.setLimit(ElasticSearchUtil.defaultResultLimit); - SearchSourceBuilder query = processSearchQuery(searchDTO, groupByFinalList, sort); - TelemetryManager.log(" search query: " + query); - Future searchResult = ElasticSearchUtil.search(index, query); - TelemetryManager.log("search result from elastic search" + searchResult); - return searchResult; - } - - private void setAggregations(SearchSourceBuilder searchSourceBuilder, List> aggregations) { - if (CollectionUtils.isNotEmpty(aggregations)) { - for (Map aggregate : aggregations) { - TermsAggregationBuilder termBuilder = AggregationBuilders.terms((String) aggregate.get("l1")) - .field(aggregate.get("l1") + SearchConstants.RAW_FIELD_EXTENSION) - .size(ElasticSearchUtil.defaultResultLimit); - int level = 2; - termBuilder.subAggregation(getNextLevelAggregation(aggregate, level)); - searchSourceBuilder.aggregation(termBuilder); - } - } - } - - private AggregationBuilder getNextLevelAggregation(Map aggregate, int level) { - TermsAggregationBuilder termBuilder = AggregationBuilders.terms((String) aggregate.get("l" + level)) - .field(aggregate.get("l" + level) + SearchConstants.RAW_FIELD_EXTENSION) - .size(ElasticSearchUtil.defaultResultLimit); - - if (level == aggregate.keySet().size()) { - return termBuilder; - } else { - level += 1; - return termBuilder.subAggregation(getNextLevelAggregation(aggregate, level)); - } - } - - private List> aggregateResult(Aggregations aggregations) { - List> aggregationList = new ArrayList<>(); - if (null != aggregations) { - Map aggregationMap = aggregations.getAsMap(); - for (String key : aggregationMap.keySet()) { - Terms terms = (Terms) aggregationMap.get(key); - List buckets = (List) terms.getBuckets(); - List> values = new ArrayList<>(); - if (CollectionUtils.isNotEmpty(buckets)) { - for (Terms.Bucket bucket : buckets) { - Map termBucket = new HashMap() { - { - put("count", bucket.getDocCount()); - put("name", bucket.getKey()); - List> subAggregations = aggregateResult(bucket.getAggregations()); - if (CollectionUtils.isNotEmpty(subAggregations)) - put("aggregations", subAggregations); - } - }; - values.add(termBucket); - } - aggregationList.add(new HashMap() { - { - put("values", values); - put("name", key); - } - }); - } - } - - } - return aggregationList; - } - - private QueryBuilder getSearchQuery(SearchDTO searchDTO) { - BoolQueryBuilder boolQuery = new BoolQueryBuilder(); - QueryBuilder origFilterQry = getQuery(searchDTO); - QueryBuilder implFilterQuery = null; - - if (CollectionUtils.isNotEmpty(searchDTO.getImplicitFilterProperties())) { - List properties = searchDTO.getProperties(); - searchDTO.setProperties(searchDTO.getImplicitFilterProperties()); - implFilterQuery = getQuery(searchDTO); - searchDTO.setProperties(properties); - boolQuery.should(origFilterQry); - boolQuery.should(implFilterQuery); - return boolQuery; - } else { - return origFilterQry; - } - } - - private QueryBuilder getSearchQuery(SearchDTO searchDTO, int pass) { - BoolQueryBuilder boolQuery = new BoolQueryBuilder(); - QueryBuilder origFilterQry = getQuery(searchDTO, pass); - QueryBuilder implFilterQuery = null; - - if (CollectionUtils.isNotEmpty(searchDTO.getImplicitFilterProperties())) { - List properties = searchDTO.getProperties(); - searchDTO.setProperties(searchDTO.getImplicitFilterProperties()); - implFilterQuery = getQuery(searchDTO); - searchDTO.setProperties(properties); - boolQuery.should(origFilterQry); - boolQuery.should(implFilterQuery); - return boolQuery; - } else { - return origFilterQry; - } - } - - private QueryBuilder getQuery(SearchDTO searchDTO) { - return prepareSearchQuery(searchDTO); - } - - private QueryBuilder getQuery(SearchDTO searchDTO, int pass) { - return prepareSearchQuery(searchDTO, pass); - } + private ObjectMapper mapper = new ObjectMapper(); + private static final String ASC_ORDER = "asc"; + private static final String AND = "AND"; + private boolean relevanceSort = false; + + public SearchProcessor() { + ElasticSearchUtil.initialiseESClient(SearchConstants.COMPOSITE_SEARCH_INDEX, + Platform.config.getString("search.es_conn_info")); + } + + public SearchProcessor(String indexName) { + } + + @SuppressWarnings({ "unchecked", "rawtypes" }) + public Future> processSearch(SearchDTO searchDTO, boolean includeResults) + throws Exception { + List> groupByFinalList = new ArrayList>(); + SearchSourceBuilder query = processSearchQuery(searchDTO, groupByFinalList, true); + Future searchResponse = ElasticSearchUtil.search( + SearchConstants.COMPOSITE_SEARCH_INDEX, + query); + + return searchResponse.map(new Mapper>() { + public Map apply(SearchResponse searchResult) { + Map resp = new HashMap<>(); + if (includeResults) { + if (searchDTO.isFuzzySearch()) { + List results = ElasticSearchUtil.getDocumentsFromSearchResultWithScore(searchResult); + resp.put("results", results); + } else { + List results = ElasticSearchUtil.getDocumentsFromSearchResult(searchResult, Map.class); + resp.put("results", results); + } + } + Aggregations aggregations = searchResult.getAggregations(); + if (null != aggregations) { + AggregationsResultTransformer transformer = new AggregationsResultTransformer(); + if (CollectionUtils.isNotEmpty(searchDTO.getFacets())) { + resp.put("facets", (List>) ElasticSearchUtil + .getCountFromAggregation(aggregations, groupByFinalList, transformer)); + } else if (CollectionUtils.isNotEmpty(searchDTO.getAggregations())) { + resp.put("aggregations", aggregateResult(aggregations)); + } + + } + resp.put("count", (int) searchResult.getHits().getTotalHits()); + return resp; + } + }, ExecutionContext.Implicits$.MODULE$.global()); + } + public Future> processSearch(SearchDTO searchDTO, boolean includeResults, boolean fuzzyEnabled) + throws Exception { + List> groupByFinalList = new ArrayList>(); + SearchSourceBuilder query = processSearchQuery(searchDTO, groupByFinalList, true, fuzzyEnabled); + Future searchResponse = ElasticSearchUtil.search( + SearchConstants.COMPOSITE_SEARCH_INDEX, + query); + + int exactMatchCount = ElasticSearchUtil.count(SearchConstants.COMPOSITE_SEARCH_INDEX, + query); + + if(exactMatchCount == 0){ + query = processSearchQuery(searchDTO, groupByFinalList, true, true); + searchResponse = ElasticSearchUtil.search( + SearchConstants.COMPOSITE_SEARCH_INDEX, + query); + } + + return searchResponse.map(new Mapper>() { + public Map apply(SearchResponse searchResult) { + Map resp = new HashMap<>(); + + if (includeResults) { + if (searchDTO.isFuzzySearch()) { + List results = ElasticSearchUtil.getDocumentsFromSearchResultWithScore(searchResult); + resp.put("results", results); + } else { + List results = ElasticSearchUtil.getDocumentsFromSearchResult(searchResult, Map.class); + resp.put("results", results); + } + } + Aggregations aggregations = searchResult.getAggregations(); + if (null != aggregations) { + AggregationsResultTransformer transformer = new AggregationsResultTransformer(); + if (CollectionUtils.isNotEmpty(searchDTO.getFacets())) { + resp.put("facets", (List>) ElasticSearchUtil + .getCountFromAggregation(aggregations, groupByFinalList, transformer)); + } else if (CollectionUtils.isNotEmpty(searchDTO.getAggregations())) { + resp.put("aggregations", aggregateResult(aggregations)); + } + + } + + + + resp.put("count", (int) searchResult.getHits().getTotalHits()); + return resp; + } + }, ExecutionContext.Implicits$.MODULE$.global()); + + + + + + } + + private Map getMatchCount(Future searchResponse) { + + return (Map) searchResponse.map(new Mapper>() { + public Map apply(SearchResponse searchResult) { + Map resp = new HashMap<>(); + resp.put("hits",searchResult.getHits().getTotalHits()); + return resp; + } + }, ExecutionContext.Implicits$.MODULE$.global()); + + } + + private Future exactMatchExists(Future searchResponse) { + return searchResponse.map(new Mapper() { + public Boolean apply(SearchResponse searchResult) { + Map resp = new HashMap<>(); + long matches = searchResult.getHits().getTotalHits(); + if(matches>0) + return true; + else + return false; + + } + }, ExecutionContext.Implicits$.MODULE$.global()); + + } + public Map processCount(SearchDTO searchDTO) throws Exception { + Map response = new HashMap(); + SearchSourceBuilder searchSourceBuilder = processSearchQuery(searchDTO, null, false); + searchSourceBuilder.from(searchDTO.getOffset()).size(0); + int countResult = ElasticSearchUtil.count(SearchConstants.COMPOSITE_SEARCH_INDEX, + searchSourceBuilder); + response.put("count", countResult); + + return response; + } + + /** + * Returns the list of words which are synonyms of the synsetIds passed in the + * request + * + * @param synsetIds + * @return + * @throws Exception + */ + @SuppressWarnings({ "unchecked", "rawtypes", "unused" }) + public Map multiWordDocSearch(List synsetIds) throws Exception { + Map response = new HashMap(); + Map translations = new HashMap(); + Map synsets = new HashMap(); + if (synsetIds != null && synsetIds.size() > 0) { + List resultList = ElasticSearchUtil.getMultiDocumentAsStringByIdList( + SearchConstants.COMPOSITE_SEARCH_INDEX, + SearchConstants.COMPOSITE_SEARCH_INDEX_TYPE, synsetIds); + for (String synsetDoc : resultList) { + Map wordTranslationList = new HashMap(); + Map indexDocument = new HashMap(); + if (synsetDoc != null && !synsetDoc.isEmpty()) { + indexDocument = mapper.readValue(synsetDoc, new TypeReference>() { + }); + Object words = indexDocument.get("synonyms"); + String identifier = (String) indexDocument.get("identifier"); + String gloss = (String) indexDocument.get("gloss"); + wordTranslationList.put("gloss", gloss); + if (words != null) { + List wordIdList = (List) words; + if (wordIdList != null && wordIdList.size() > 0) { + List wordResultList = ElasticSearchUtil.getMultiDocumentAsStringByIdList( + SearchConstants.COMPOSITE_SEARCH_INDEX, + SearchConstants.COMPOSITE_SEARCH_INDEX_TYPE, wordIdList); + for (String wordDoc : wordResultList) { + List synsetWordLangList = new ArrayList(); + Map indexWordDocument = new HashMap(); + indexWordDocument = mapper.readValue(wordDoc, new TypeReference>() { + }); + String wordId = (String) indexWordDocument.get("identifier"); + String graphId = (String) indexWordDocument.get("graph_id"); + if (wordTranslationList.containsKey(graphId)) { + synsetWordLangList = (List) wordTranslationList.get(graphId); + } + String lemma = (String) indexWordDocument.get("lemma"); + String status = (String) indexWordDocument.get("status"); + if (!StringUtils.equalsIgnoreCase(status, "Retired")) { + Map wordMap = new HashMap(); + wordMap.put("id", wordId); + wordMap.put("lemma", lemma); + synsetWordLangList.add(wordMap); + } + wordTranslationList.put(graphId, synsetWordLangList); + } + + } + } + synsets.put(identifier, wordTranslationList); + } + + } + response.put("translations", synsets); + } + + return response; + } + + /** + * Returns list of synsetsIds which has valid documents in composite index + * + * @param synsetIds + * @return + * @throws Exception + */ + public Map multiSynsetDocSearch(List synsetIds) throws Exception { + Map synsetDocList = new HashMap(); + List identifierList = new ArrayList(); + if (synsetIds != null && synsetIds.size() > 0) { + List resultList = ElasticSearchUtil.getMultiDocumentAsStringByIdList( + SearchConstants.COMPOSITE_SEARCH_INDEX, + SearchConstants.COMPOSITE_SEARCH_INDEX_TYPE, synsetIds); + for (String synsetDoc : resultList) { + Map indexDocument = new HashMap(); + if (synsetDoc != null && !synsetDoc.isEmpty()) { + indexDocument = mapper.readValue(synsetDoc, new TypeReference>() { + }); + String identifier = (String) indexDocument.get("identifier"); + identifierList.add(identifier); + + } + + } + } + synsetDocList.put("synsets", identifierList); + + return synsetDocList; + } + + public void destroy() { + // ElasticSearchUtil.cleanESClient(); + } + + /** + * @param searchDTO + * @param groupByFinalList + * @return + */ + private SearchSourceBuilder processSearchQuery(SearchDTO searchDTO, List> groupByFinalList, + boolean sortBy) { + + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + List fields = searchDTO.getFields(); + if (null != fields && !fields.isEmpty()) { + fields.add("objectType"); + fields.add("identifier"); + searchSourceBuilder.fetchSource(fields.toArray(new String[fields.size()]), null); + } + + if (searchDTO.getFacets() != null && groupByFinalList != null) { + for (String facet : searchDTO.getFacets()) { + Map groupByMap = new HashMap(); + groupByMap.put("groupByParent", facet); + groupByFinalList.add(groupByMap); + } + } + + searchSourceBuilder.size(searchDTO.getLimit()); + searchSourceBuilder.from(searchDTO.getOffset()); + QueryBuilder query = getSearchQuery(searchDTO); + if (searchDTO.isFuzzySearch()) + relevanceSort = true; + + searchSourceBuilder.query(query); + + if (sortBy && !relevanceSort + && (null == searchDTO.getSoftConstraints() || searchDTO.getSoftConstraints().isEmpty())) { + Map sorting = searchDTO.getSortBy(); + if (sorting == null || sorting.isEmpty()) { + sorting = new HashMap(); + sorting.put("name", "asc"); + sorting.put("lastUpdatedOn", "desc"); + } + for (String key : sorting.keySet()) { + if (key.contains(".")) { + String nestedPath = key.split("\\.")[0]; + searchSourceBuilder.sort(SortBuilders.fieldSort(key + SearchConstants.RAW_FIELD_EXTENSION) + .order(getSortOrder(sorting.get(key))).setNestedSort(new NestedSortBuilder(nestedPath))); + } else { + searchSourceBuilder.sort(key + SearchConstants.RAW_FIELD_EXTENSION, + getSortOrder(sorting.get(key))); + } + } + } + setAggregations(groupByFinalList, searchSourceBuilder); + setAggregations(searchSourceBuilder, searchDTO.getAggregations()); + searchSourceBuilder.trackScores(true); + return searchSourceBuilder; + } + + private SearchSourceBuilder processSearchQuery(SearchDTO searchDTO, List> groupByFinalList, + boolean sortBy, boolean fuzzyEnabled) { + + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + List fields = searchDTO.getFields(); + if (null != fields && !fields.isEmpty()) { + fields.add("objectType"); + fields.add("identifier"); + searchSourceBuilder.fetchSource(fields.toArray(new String[fields.size()]), null); + } + + if (searchDTO.getFacets() != null && groupByFinalList != null && !fuzzyEnabled) { + for (String facet : searchDTO.getFacets()) { + Map groupByMap = new HashMap(); + groupByMap.put("groupByParent", facet); + groupByFinalList.add(groupByMap); + } + } + + searchSourceBuilder.size(searchDTO.getLimit()); + searchSourceBuilder.from(searchDTO.getOffset()); + QueryBuilder query = getSearchQuery(searchDTO, fuzzyEnabled); + if (searchDTO.isFuzzySearch()) + relevanceSort = true; + + searchSourceBuilder.query(query); + + if (sortBy && !relevanceSort + && (null == searchDTO.getSoftConstraints() || searchDTO.getSoftConstraints().isEmpty())) { + Map sorting = searchDTO.getSortBy(); + if (sorting == null || sorting.isEmpty()) { + sorting = new HashMap(); + sorting.put("name", "asc"); + sorting.put("lastUpdatedOn", "desc"); + } + for (String key : sorting.keySet()) { + if (key.contains(".")) { + String nestedPath = key.split("\\.")[0]; + searchSourceBuilder.sort(SortBuilders.fieldSort(key + SearchConstants.RAW_FIELD_EXTENSION) + .order(getSortOrder(sorting.get(key))).setNestedSort(new NestedSortBuilder(nestedPath))); + } else { + searchSourceBuilder.sort(key + SearchConstants.RAW_FIELD_EXTENSION, + getSortOrder(sorting.get(key))); + } + } + } + setAggregations(groupByFinalList, searchSourceBuilder); + setAggregations(searchSourceBuilder, searchDTO.getAggregations()); + searchSourceBuilder.trackScores(true); + return searchSourceBuilder; + } + + /** + * @param groupByList + * @param searchSourceBuilder + * @return + */ + @SuppressWarnings("unchecked") + private void setAggregations(List> groupByList, + SearchSourceBuilder searchSourceBuilder) { + TermsAggregationBuilder termBuilder = null; + if (groupByList != null && !groupByList.isEmpty()) { + HashMap> nestedAggregation = new HashMap<>(); + for (Map groupByMap : groupByList) { + String groupByParent = (String) groupByMap.get("groupByParent"); + if (!groupByParent.contains(".")) { + termBuilder = AggregationBuilders.terms(groupByParent) + .field(groupByParent + SearchConstants.RAW_FIELD_EXTENSION) + .size(ElasticSearchUtil.defaultResultLimit); + List groupByChildList = (List) groupByMap.get("groupByChildList"); + if (groupByChildList != null && !groupByChildList.isEmpty()) { + for (String childGroupBy : groupByChildList) { + termBuilder.subAggregation(AggregationBuilders.terms(childGroupBy) + .field(childGroupBy + SearchConstants.RAW_FIELD_EXTENSION) + .size(ElasticSearchUtil.defaultResultLimit)); + } + } + searchSourceBuilder.aggregation(termBuilder); + } else { + if (nestedAggregation.get(groupByParent.split("\\.")[0]) != null) { + nestedAggregation.get(groupByParent.split("\\.")[0]).add(groupByParent.split("\\.")[1]); + } else { + List nestedAggrList = new ArrayList<>(); + nestedAggrList.add(groupByParent.split("\\.")[1]); + nestedAggregation.put(groupByParent.split("\\.")[0], nestedAggrList); + } + } + } + if (!nestedAggregation.isEmpty()) { + for (Map.Entry> mapData : nestedAggregation.entrySet()) { + AggregationBuilder nestedAggregationBuilder = AggregationBuilders.nested(mapData.getKey(), + mapData.getKey()); + for (String nestedValue : mapData.getValue()) { + termBuilder = AggregationBuilders.terms(nestedValue) + .field(mapData.getKey() + "." + nestedValue + SearchConstants.RAW_FIELD_EXTENSION) + .size(ElasticSearchUtil.defaultResultLimit); + nestedAggregationBuilder.subAggregation(termBuilder); + } + searchSourceBuilder.aggregation(nestedAggregationBuilder); + } + } + } + } + + /** + * @param searchDTO + * @return + */ + @SuppressWarnings({ "unchecked", "rawtypes" }) + private QueryBuilder prepareSearchQuery(SearchDTO searchDTO) { + BoolQueryBuilder boolQuery = new BoolQueryBuilder(); + QueryBuilder queryBuilder = null; + String totalOperation = searchDTO.getOperation(); + List properties = searchDTO.getProperties(); + formQuery(properties, queryBuilder, boolQuery, totalOperation, searchDTO.isFuzzySearch()); + if (searchDTO.getMultiFilterProperties() != null) { + formQuery(searchDTO.getMultiFilterProperties(), queryBuilder, boolQuery, + SearchConstants.SEARCH_OPERATION_OR, searchDTO.isFuzzySearch()); + } + + Map softConstraints = searchDTO.getSoftConstraints(); + if (null != softConstraints && !softConstraints.isEmpty()) { + boolQuery.should(getSoftConstraintQuery(softConstraints)); + searchDTO.setSortBy(null); + // relevanceSort = true; + } + return boolQuery; + } + + private QueryBuilder prepareSearchQuery(SearchDTO searchDTO, boolean fuzzyEnabled) { + BoolQueryBuilder boolQuery = new BoolQueryBuilder(); + QueryBuilder queryBuilder = null; + String totalOperation = searchDTO.getOperation(); + List properties = searchDTO.getProperties(); + formQuery(properties, queryBuilder, boolQuery, totalOperation, fuzzyEnabled); + if (searchDTO.getMultiFilterProperties() != null) { + formQuery(searchDTO.getMultiFilterProperties(), queryBuilder, boolQuery, + SearchConstants.SEARCH_OPERATION_OR, fuzzyEnabled); + } + + Map softConstraints = searchDTO.getSoftConstraints(); + if (null != softConstraints && !softConstraints.isEmpty()) { + boolQuery.should(getSoftConstraintQuery(softConstraints)); + searchDTO.setSortBy(null); + // relevanceSort = true; + } + return boolQuery; + } + + private void formQuery(List properties, QueryBuilder queryBuilder, BoolQueryBuilder boolQuery, + String operation, boolean fuzzy) { + for (Map property : properties) { + String opertation = (String) property.get("operation"); + + List values; + try { + values = (List) property.get("values"); + } catch (Exception e) { + values = Arrays.asList(property.get("values")); + } + values = values.stream().filter(value -> (null != value)).collect(Collectors.toList()); + + String propertyName = (String) property.get("propertyName"); + if (propertyName.equals("*")) { + relevanceSort = true; + propertyName = "all_fields"; + queryBuilder = getAllFieldsPropertyQuery(values, fuzzy); + boolQuery.must(queryBuilder); + continue; + } + + propertyName = propertyName + SearchConstants.RAW_FIELD_EXTENSION; + + switch (opertation) { + case SearchConstants.SEARCH_OPERATION_EQUAL: { + queryBuilder = getMustTermQuery(propertyName, values, true); + queryBuilder = checkNestedProperty(queryBuilder, propertyName); + break; + } + case SearchConstants.SEARCH_OPERATION_NOT_EQUAL: { + queryBuilder = getMustTermQuery(propertyName, values, false); + queryBuilder = checkNestedProperty(queryBuilder, propertyName); + break; + } + case SearchConstants.SEARCH_OPERATION_NOT_IN: { + queryBuilder = getNotInQuery(propertyName, values); + queryBuilder = checkNestedProperty(queryBuilder, propertyName); + break; + } + case SearchConstants.SEARCH_OPERATION_ENDS_WITH: { + queryBuilder = getRegexQuery(propertyName, values); + queryBuilder = checkNestedProperty(queryBuilder, propertyName); + break; + } + case SearchConstants.SEARCH_OPERATION_LIKE: + case SearchConstants.SEARCH_OPERATION_CONTAINS: { + queryBuilder = getMatchPhraseQuery(propertyName, values, true); + queryBuilder = checkNestedProperty(queryBuilder, propertyName); + break; + } + case SearchConstants.SEARCH_OPERATION_NOT_LIKE: { + queryBuilder = getMatchPhraseQuery(propertyName, values, false); + queryBuilder = checkNestedProperty(queryBuilder, propertyName); + break; + } + case SearchConstants.SEARCH_OPERATION_STARTS_WITH: { + queryBuilder = getMatchPhrasePrefixQuery(propertyName, values); + queryBuilder = checkNestedProperty(queryBuilder, propertyName); + break; + } + case SearchConstants.SEARCH_OPERATION_EXISTS: { + queryBuilder = getExistsQuery(propertyName, values, true); + queryBuilder = checkNestedProperty(queryBuilder, propertyName); + break; + } + case SearchConstants.SEARCH_OPERATION_NOT_EXISTS: { + queryBuilder = getExistsQuery(propertyName, values, false); + queryBuilder = checkNestedProperty(queryBuilder, propertyName); + break; + } + case SearchConstants.SEARCH_OPERATION_GREATER_THAN: { + queryBuilder = getRangeQuery(propertyName, values, + SearchConstants.SEARCH_OPERATION_GREATER_THAN); + queryBuilder = checkNestedProperty(queryBuilder, propertyName); + break; + } + case SearchConstants.SEARCH_OPERATION_GREATER_THAN_EQUALS: { + queryBuilder = getRangeQuery(propertyName, values, + SearchConstants.SEARCH_OPERATION_GREATER_THAN_EQUALS); + queryBuilder = checkNestedProperty(queryBuilder, propertyName); + break; + } + case SearchConstants.SEARCH_OPERATION_LESS_THAN: { + queryBuilder = getRangeQuery(propertyName, values, SearchConstants.SEARCH_OPERATION_LESS_THAN); + queryBuilder = checkNestedProperty(queryBuilder, propertyName); + break; + } + case SearchConstants.SEARCH_OPERATION_LESS_THAN_EQUALS: { + queryBuilder = getRangeQuery(propertyName, values, + SearchConstants.SEARCH_OPERATION_LESS_THAN_EQUALS); + queryBuilder = checkNestedProperty(queryBuilder, propertyName); + break; + } + case SearchConstants.SEARCH_OPERATION_RANGE: { + queryBuilder = getRangeQuery(propertyName, values); + queryBuilder = checkNestedProperty(queryBuilder, propertyName); + break; + } + case SearchConstants.SEARCH_OPERATION_AND: { + queryBuilder = getAndQuery(propertyName, values); + queryBuilder = checkNestedProperty(queryBuilder, propertyName); + break; + } + } + if (operation.equalsIgnoreCase(AND)) { + boolQuery.must(queryBuilder); + } else { + boolQuery.should(queryBuilder); + } + + } + } + + private QueryBuilder checkNestedProperty(QueryBuilder queryBuilder, String propertyName) { + if (propertyName.replaceAll(SearchConstants.RAW_FIELD_EXTENSION, "").contains(".")) { + queryBuilder = QueryBuilders.nestedQuery(propertyName.split("\\.")[0], queryBuilder, + org.apache.lucene.search.join.ScoreMode.None); + } + return queryBuilder; + } + + private QueryBuilder getAndQuery(String propertyName, List values) { + BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery(); + for (Object value : values) { + queryBuilder.must( + QueryBuilders.matchQuery(propertyName, value).operator(Operator.AND).fuzzyTranspositions(false)); + } + return queryBuilder; + } + + /** + * @param querySearchFeilds + * @param propertyName + * @return + */ + private float getweight(List querySearchFeilds, String propertyName) { + float weight = 1.0F; + if (querySearchFeilds.contains(propertyName)) { + for (String field : querySearchFeilds) { + if (field.contains(propertyName)) { + weight = Float + .parseFloat((StringUtils.isNotBlank(field.split("^")[1])) ? field.split("^")[1] : "1.0"); + } + } + } + return weight; + } + + /** + * @param values + * @return + */ + private QueryBuilder getAllFieldsPropertyQuery(List values, boolean fuzzy) { + List queryFields = ElasticSearchUtil.getQuerySearchFields(); + Map queryFieldsMap = new HashMap<>(); + for (String field : queryFields) { + if (field.contains("^")) + queryFieldsMap.put(field.split("\\^")[0], Float.valueOf(field.split("\\^")[1])); + else + queryFieldsMap.put(field, 1.0f); + } + BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery(); + for (Object value : values) { + if (fuzzy) { + queryBuilder + .should(QueryBuilders.multiMatchQuery(value).fields(queryFieldsMap) + .operator(Operator.AND).fuzziness("AUTO").lenient(true)); + } else { + queryBuilder + .should(QueryBuilders.multiMatchQuery(value).fields(queryFieldsMap) + .operator(Operator.AND).type(Type.CROSS_FIELDS).fuzzyTranspositions(false) + .lenient(true)); + } + } + + return queryBuilder; + } + + /** + * @param softConstraints + * @return + */ + @SuppressWarnings("unchecked") + private QueryBuilder getSoftConstraintQuery(Map softConstraints) { + BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery(); + for (String key : softConstraints.keySet()) { + List data = (List) softConstraints.get(key); + if (data.get(1) instanceof List) { + List dataList = (List) data.get(1); + for (Object value : dataList) { + queryBuilder + .should(QueryBuilders.matchQuery(key + SearchConstants.RAW_FIELD_EXTENSION, value) + .boost(Integer.valueOf((int) data.get(0)).floatValue()).fuzzyTranspositions(false)); + } + } else { + queryBuilder.should( + QueryBuilders.matchQuery(key + SearchConstants.RAW_FIELD_EXTENSION, data.get(1)) + .boost(Integer.valueOf((int) data.get(0)).floatValue()).fuzzyTranspositions(false)); + } + } + return queryBuilder; + } + + /** + * @param propertyName + * @param values + * @return + */ + private QueryBuilder getRangeQuery(String propertyName, List values, String operation) { + BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery(); + for (Object value : values) { + switch (operation) { + case SearchConstants.SEARCH_OPERATION_GREATER_THAN: { + queryBuilder.should(QueryBuilders + .rangeQuery(propertyName).gt(value)); + break; + } + case SearchConstants.SEARCH_OPERATION_GREATER_THAN_EQUALS: { + queryBuilder.should(QueryBuilders + .rangeQuery(propertyName).gte(value)); + break; + } + case SearchConstants.SEARCH_OPERATION_LESS_THAN: { + queryBuilder.should(QueryBuilders + .rangeQuery(propertyName).lt(value)); + break; + } + case SearchConstants.SEARCH_OPERATION_LESS_THAN_EQUALS: { + queryBuilder.should(QueryBuilders + .rangeQuery(propertyName).lte(value)); + break; + } + } + } + + return queryBuilder; + } + + /** + * @param propertyName + * @param values + * @return + */ + private QueryBuilder getExistsQuery(String propertyName, List values, boolean exists) { + BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery(); + for (Object value : values) { + if (exists) { + queryBuilder.should(QueryBuilders.existsQuery(String.valueOf(value))); + } else { + queryBuilder.mustNot(QueryBuilders.existsQuery(String.valueOf(value))); + } + } + return queryBuilder; + } + + /** + * @param propertyName + * @param values + * @return + */ + private QueryBuilder getNotInQuery(String propertyName, List values) { + BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery(); + queryBuilder + .mustNot(QueryBuilders.termsQuery(propertyName, values)); + return queryBuilder; + } + + /** + * @param propertyName + * @param values + * @return + */ + private QueryBuilder getMatchPhrasePrefixQuery(String propertyName, List values) { + BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery(); + for (Object value : values) { + queryBuilder.should(QueryBuilders.prefixQuery( + propertyName, ((String) value).toLowerCase())); + } + return queryBuilder; + } + + /** + * @param propertyName + * @param values + * @param match + * @return + */ + private QueryBuilder getMatchPhraseQuery(String propertyName, List values, boolean match) { + BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery(); + for (Object value : values) { + String stringValue = String.valueOf(value); + if (match) { + queryBuilder.should(QueryBuilders + .regexpQuery(propertyName, + ".*" + stringValue.toLowerCase() + ".*")); + } else { + queryBuilder.mustNot(QueryBuilders + .regexpQuery(propertyName, + ".*" + stringValue.toLowerCase() + ".*")); + } + } + return queryBuilder; + } + + /** + * @param propertyName + * @param values + * @return + */ + private QueryBuilder getRegexQuery(String propertyName, List values) { + BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery(); + for (Object value : values) { + String stringValue = String.valueOf(value); + queryBuilder.should(QueryBuilders.regexpQuery(propertyName, + ".*" + stringValue.toLowerCase())); + } + return queryBuilder; + } + + /** + * @param propertyName + * @param values + * @param match + * @return + */ + private QueryBuilder getMustTermQuery(String propertyName, List values, boolean match) { + BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery(); + for (Object value : values) { + if (match) { + queryBuilder.should( + QueryBuilders.matchQuery(propertyName, value).fuzzyTranspositions(false)); + } else { + queryBuilder.mustNot( + QueryBuilders.matchQuery(propertyName, value).fuzzyTranspositions(false)); + } + } + + return queryBuilder; + } + + /** + * @param propertyName + * @param values + * @return + */ + @SuppressWarnings("unchecked") + private QueryBuilder getRangeQuery(String propertyName, List values) { + RangeQueryBuilder queryBuilder = new RangeQueryBuilder(propertyName); + for (Object value : values) { + Map rangeMap = (Map) value; + if (!rangeMap.isEmpty()) { + for (String key : rangeMap.keySet()) { + switch (key) { + case SearchConstants.SEARCH_OPERATION_RANGE_GTE: { + queryBuilder.from(rangeMap.get(key)); + break; + } + case SearchConstants.SEARCH_OPERATION_RANGE_LTE: { + queryBuilder.to(rangeMap.get(key)); + break; + } + } + } + } + } + return queryBuilder; + } + + /** + * @param value + * @return + */ + private SortOrder getSortOrder(String value) { + return value.equalsIgnoreCase(ASC_ORDER) ? SortOrder.ASC : SortOrder.DESC; + } + + public Future> processSearchQuery(SearchDTO searchDTO, boolean includeResults, String index) + throws Exception { + return processSearchQuery(searchDTO, includeResults, index, true); + } + + public Future> processSearchQuery(SearchDTO searchDTO, boolean includeResults, String index, + boolean sort) + throws Exception { + List> groupByFinalList = new ArrayList>(); + if (searchDTO.getLimit() == 0) + searchDTO.setLimit(ElasticSearchUtil.defaultResultLimit); + SearchSourceBuilder query = processSearchQuery(searchDTO, groupByFinalList, sort); + TelemetryManager.log(" search query: " + query); + Future searchResponse = ElasticSearchUtil.search(index, query); + + return searchResponse.map(new Mapper>() { + public List apply(SearchResponse searchResult) { + List response = new ArrayList(); + TelemetryManager.log("search result from elastic search" + searchResult); + SearchHits resultMap = searchResult.getHits(); + SearchHit[] result = resultMap.getHits(); + for (SearchHit hit : result) { + response.add(hit.getSourceAsMap()); + } + TelemetryManager.log("search response size: " + response.size()); + return response; + } + }, ExecutionContext.Implicits$.MODULE$.global()); + + } + + public Future processSearchQueryWithSearchResult(SearchDTO searchDTO, boolean includeResults, + String index, + boolean sort) throws Exception { + List> groupByFinalList = new ArrayList>(); + if (searchDTO.getLimit() == 0) + searchDTO.setLimit(ElasticSearchUtil.defaultResultLimit); + SearchSourceBuilder query = processSearchQuery(searchDTO, groupByFinalList, sort); + TelemetryManager.log(" search query: " + query); + Future searchResult = ElasticSearchUtil.search(index, query); + TelemetryManager.log("search result from elastic search" + searchResult); + return searchResult; + } + + private void setAggregations(SearchSourceBuilder searchSourceBuilder, List> aggregations) { + if (CollectionUtils.isNotEmpty(aggregations)) { + for (Map aggregate : aggregations) { + TermsAggregationBuilder termBuilder = AggregationBuilders.terms((String) aggregate.get("l1")) + .field(aggregate.get("l1") + SearchConstants.RAW_FIELD_EXTENSION) + .size(ElasticSearchUtil.defaultResultLimit); + int level = 2; + termBuilder.subAggregation(getNextLevelAggregation(aggregate, level)); + searchSourceBuilder.aggregation(termBuilder); + } + } + } + + private AggregationBuilder getNextLevelAggregation(Map aggregate, int level) { + TermsAggregationBuilder termBuilder = AggregationBuilders.terms((String) aggregate.get("l" + level)) + .field(aggregate.get("l" + level) + SearchConstants.RAW_FIELD_EXTENSION) + .size(ElasticSearchUtil.defaultResultLimit); + + if (level == aggregate.keySet().size()) { + return termBuilder; + } else { + level += 1; + return termBuilder.subAggregation(getNextLevelAggregation(aggregate, level)); + } + } + + private List> aggregateResult(Aggregations aggregations) { + List> aggregationList = new ArrayList<>(); + if (null != aggregations) { + Map aggregationMap = aggregations.getAsMap(); + for (String key : aggregationMap.keySet()) { + Terms terms = (Terms) aggregationMap.get(key); + List buckets = (List) terms.getBuckets(); + List> values = new ArrayList<>(); + if (CollectionUtils.isNotEmpty(buckets)) { + for (Terms.Bucket bucket : buckets) { + Map termBucket = new HashMap() { + { + put("count", bucket.getDocCount()); + put("name", bucket.getKey()); + List> subAggregations = aggregateResult(bucket.getAggregations()); + if (CollectionUtils.isNotEmpty(subAggregations)) + put("aggregations", subAggregations); + } + }; + values.add(termBucket); + } + aggregationList.add(new HashMap() { + { + put("values", values); + put("name", key); + } + }); + } + } + + } + return aggregationList; + } + + private QueryBuilder getSearchQuery(SearchDTO searchDTO) { + BoolQueryBuilder boolQuery = new BoolQueryBuilder(); + QueryBuilder origFilterQry = getQuery(searchDTO); + QueryBuilder implFilterQuery = null; + + if (CollectionUtils.isNotEmpty(searchDTO.getImplicitFilterProperties())) { + List properties = searchDTO.getProperties(); + searchDTO.setProperties(searchDTO.getImplicitFilterProperties()); + implFilterQuery = getQuery(searchDTO); + searchDTO.setProperties(properties); + boolQuery.should(origFilterQry); + boolQuery.should(implFilterQuery); + return boolQuery; + } else { + return origFilterQry; + } + } + + private QueryBuilder getSearchQuery(SearchDTO searchDTO, boolean fuzzyEnabled) { + BoolQueryBuilder boolQuery = new BoolQueryBuilder(); + QueryBuilder origFilterQry = getQuery(searchDTO, fuzzyEnabled); + QueryBuilder implFilterQuery = null; + + if (CollectionUtils.isNotEmpty(searchDTO.getImplicitFilterProperties())) { + List properties = searchDTO.getProperties(); + searchDTO.setProperties(searchDTO.getImplicitFilterProperties()); + implFilterQuery = getQuery(searchDTO); + searchDTO.setProperties(properties); + boolQuery.should(origFilterQry); + boolQuery.should(implFilterQuery); + return boolQuery; + } else { + return origFilterQry; + } + } + + private QueryBuilder getQuery(SearchDTO searchDTO) { + return prepareSearchQuery(searchDTO); + } + + private QueryBuilder getQuery(SearchDTO searchDTO, boolean fuzzyEnabled) { + return prepareSearchQuery(searchDTO, fuzzyEnabled); + } } \ No newline at end of file diff --git a/search-api/search-service/app/controllers/SearchController.scala b/search-api/search-service/app/controllers/SearchController.scala index e76bcc5e3..55c803bf4 100644 --- a/search-api/search-service/app/controllers/SearchController.scala +++ b/search-api/search-service/app/controllers/SearchController.scala @@ -41,7 +41,7 @@ class SearchController @Inject()(@Named(ActorNames.SEARCH_ACTOR) searchActor: Ac val internalReq = getRequest(ApiId.APPLICATION_PRIVATE_SEARCH) setHeaderContext(internalReq) val channel = internalReq.getContext.getOrDefault("CHANNEL_ID", "").asInstanceOf[String] - if(channel.isBlank) { + if(channel.isEmpty) { getErrorResponse(ApiId.APPLICATION_PRIVATE_SEARCH, apiVersion, SearchConstants.ERR_INVALID_CHANNEL, "Please provide channel!") } else { val filters = internalReq.getRequest.getOrDefault(SearchConstants.filters,"").asInstanceOf[java.util.Map[String, Object]] @@ -57,4 +57,4 @@ class SearchController @Inject()(@Named(ActorNames.SEARCH_ACTOR) searchActor: Ac setHeaderContext(internalReq) getResult(mgr.count(internalReq, searchActor), ApiId.APPLICATION_COUNT) } -} +} \ No newline at end of file diff --git a/search-api/search-service/conf/application.conf b/search-api/search-service/conf/application.conf index 63dd77bd9..a2f9d3aca 100644 --- a/search-api/search-service/conf/application.conf +++ b/search-api/search-service/conf/application.conf @@ -293,7 +293,7 @@ search.config.path=/home/learning/platform/search cache.type="redis" search.es_conn_info="localhost:9200" -search.fields.query=["name^100.0","keywords^60.0","competencies_v3.name^60.0","systemTopics^60","purpose^60","source^50.0","description^5.0","language^5.0"] +search.fields.query=["name^100.0","keywords^60.0","competencies_v3.name^60.0","systemTopics^60","purpose^60", "source^50.0","description^5.0","language^5.0"] search.fields.date=["lastUpdatedOn","createdOn","versionDate","lastSubmittedOn","lastPublishedOn"] search.fields.mode_collection=["identifier","name","objectType","contentType","mimeType","size","childNodes","board","subject","medium","gradeLevel","appIcon", "origin", "originData"] search.batch.size=500 From eeba7de381b8920eb3c8d18fb21d53f70c113f23 Mon Sep 17 00:00:00 2001 From: "Radhesh.B.H" <44744695+Radheshhathwar@users.noreply.github.com> Date: Wed, 23 Nov 2022 17:42:49 +0530 Subject: [PATCH 4/5] Removed channel.isEmpty and added channel.isBlack --- .../search-service/app/controllers/SearchController.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/search-api/search-service/app/controllers/SearchController.scala b/search-api/search-service/app/controllers/SearchController.scala index 55c803bf4..e76bcc5e3 100644 --- a/search-api/search-service/app/controllers/SearchController.scala +++ b/search-api/search-service/app/controllers/SearchController.scala @@ -41,7 +41,7 @@ class SearchController @Inject()(@Named(ActorNames.SEARCH_ACTOR) searchActor: Ac val internalReq = getRequest(ApiId.APPLICATION_PRIVATE_SEARCH) setHeaderContext(internalReq) val channel = internalReq.getContext.getOrDefault("CHANNEL_ID", "").asInstanceOf[String] - if(channel.isEmpty) { + if(channel.isBlank) { getErrorResponse(ApiId.APPLICATION_PRIVATE_SEARCH, apiVersion, SearchConstants.ERR_INVALID_CHANNEL, "Please provide channel!") } else { val filters = internalReq.getRequest.getOrDefault(SearchConstants.filters,"").asInstanceOf[java.util.Map[String, Object]] @@ -57,4 +57,4 @@ class SearchController @Inject()(@Named(ActorNames.SEARCH_ACTOR) searchActor: Ac setHeaderContext(internalReq) getResult(mgr.count(internalReq, searchActor), ApiId.APPLICATION_COUNT) } -} \ No newline at end of file +} From 752cae5aa9dc4021ae22edd1953ee8c23f030775 Mon Sep 17 00:00:00 2001 From: "Radhesh.B.H" <44744695+Radheshhathwar@users.noreply.github.com> Date: Wed, 23 Nov 2022 17:52:23 +0530 Subject: [PATCH 5/5] Removed checkMatchExist() --- .../sunbird/search/client/ElasticSearchUtil.java | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/search-api/search-core/src/main/java/org/sunbird/search/client/ElasticSearchUtil.java b/search-api/search-core/src/main/java/org/sunbird/search/client/ElasticSearchUtil.java index 9f2d27037..7d18df88b 100644 --- a/search-api/search-core/src/main/java/org/sunbird/search/client/ElasticSearchUtil.java +++ b/search-api/search-core/src/main/java/org/sunbird/search/client/ElasticSearchUtil.java @@ -726,18 +726,6 @@ private static Map checkDocStringLength(Map doc) } return doc; } - public static boolean checkMatchExist(String indexName, SearchSourceBuilder searchSourceBuilder) throws IOException { - SearchResponse response = getClient(indexName) - .search(new SearchRequest().indices(indexName).source(searchSourceBuilder)); - try{ - SearchHit hitAt0 = response.getHits().getAt(0); - return true; - } - catch (ArrayIndexOutOfBoundsException e){ - return false; - } - - } -} \ No newline at end of file +}