From b4d0bce9cc9ec5715ed2357ba2be5b67a798c26d Mon Sep 17 00:00:00 2001 From: nivetha Date: Thu, 24 Mar 2022 20:37:02 +0530 Subject: [PATCH 001/126] Release-4.9 upgrade --- .../sunbird/content/util/RetireManager.scala | 2 +- .../app/controllers/BaseController.scala | 2 +- .../content-service/conf/application.conf | 3 ++ .../mgr/impl/HtmlMimeTypeMgrImpl.scala | 6 ++- schemas/asset/1.0/schema.json | 12 +++++ schemas/collection/1.0/schema.json | 12 +++++ schemas/content/1.0/schema.json | 29 ++++++++--- schemas/question/1.0/schema.json | 5 +- schemas/questionset/1.0/schema.json | 49 ++++++++++++++++++ .../java/org/sunbird/actors/SearchActor.java | 6 +++ .../search/client/ElasticSearchUtil.java | 12 ++++- .../org/sunbird/search/dto/SearchDTO.java | 10 +++- .../search/processor/SearchProcessor.java | 50 +++++++++++++++---- 13 files changed, 173 insertions(+), 25 deletions(-) diff --git a/content-api/content-actors/src/main/scala/org/sunbird/content/util/RetireManager.scala b/content-api/content-actors/src/main/scala/org/sunbird/content/util/RetireManager.scala index cbd8e4eba..360fcb567 100644 --- a/content-api/content-actors/src/main/scala/org/sunbird/content/util/RetireManager.scala +++ b/content-api/content-actors/src/main/scala/org/sunbird/content/util/RetireManager.scala @@ -57,7 +57,7 @@ object RetireManager { private def updateNodesToRetire(request: Request, updateMetadataMap: util.Map[String, AnyRef])(implicit ec: ExecutionContext, oec: OntologyEngineContext): Future[Response] = { RedisCache.delete(request.get(ContentConstants.IDENTIFIER).asInstanceOf[String]) val updateReq = new Request(request) - updateReq.put(ContentConstants.IDENTIFIERS, java.util.Arrays.asList(request.get(ContentConstants.IDENTIFIER).asInstanceOf[String], request.get(ContentConstants.IDENTIFIER).asInstanceOf[String] + HierarchyConstants.IMAGE_SUFFIX)) + updateReq.put(ContentConstants.IDENTIFIERS, java.util.Arrays.asList(request.get(ContentConstants.IDENTIFIER).asInstanceOf[String])) updateReq.put(ContentConstants.METADATA, updateMetadataMap) DataNode.bulkUpdate(updateReq).map(node => ResponseHandler.OK()) } diff --git a/content-api/content-service/app/controllers/BaseController.scala b/content-api/content-service/app/controllers/BaseController.scala index 4fc36e06e..54a94c0f2 100644 --- a/content-api/content-service/app/controllers/BaseController.scala +++ b/content-api/content-service/app/controllers/BaseController.scala @@ -83,7 +83,7 @@ abstract class BaseController(protected val cc: ControllerComponents)(implicit e } def getResult(apiId: String, actor: ActorRef, request: org.sunbird.common.dto.Request, categoryMapping: Boolean = false, version: String = "3.0") : Future[Result] = { - val future = Patterns.ask(actor, request, 30000) recoverWith {case e: Exception => Future(ResponseHandler.getErrorResponse(e))} + val future = Patterns.ask(actor, request, 120000) recoverWith {case e: Exception => Future(ResponseHandler.getErrorResponse(e))} future.map(f => { val result: Response = f.asInstanceOf[Response] result.setId(apiId) diff --git a/content-api/content-service/conf/application.conf b/content-api/content-service/conf/application.conf index 33ac60598..a38067351 100644 --- a/content-api/content-service/conf/application.conf +++ b/content-api/content-service/conf/application.conf @@ -736,3 +736,6 @@ collection { } } } + +#Index file validation +indexHtmlValidation.env=false \ No newline at end of file diff --git a/platform-modules/mimetype-manager/src/main/scala/org/sunbird/mimetype/mgr/impl/HtmlMimeTypeMgrImpl.scala b/platform-modules/mimetype-manager/src/main/scala/org/sunbird/mimetype/mgr/impl/HtmlMimeTypeMgrImpl.scala index b977d3683..8418d2180 100644 --- a/platform-modules/mimetype-manager/src/main/scala/org/sunbird/mimetype/mgr/impl/HtmlMimeTypeMgrImpl.scala +++ b/platform-modules/mimetype-manager/src/main/scala/org/sunbird/mimetype/mgr/impl/HtmlMimeTypeMgrImpl.scala @@ -9,6 +9,7 @@ import org.sunbird.graph.OntologyEngineContext import org.sunbird.graph.dac.model.Node import org.sunbird.mimetype.mgr.{BaseMimeTypeManager, MimeTypeManager} import org.sunbird.telemetry.logger.TelemetryManager +import org.sunbird.common.Platform import scala.concurrent.{ExecutionContext, Future} @@ -16,7 +17,10 @@ class HtmlMimeTypeMgrImpl(implicit ss: StorageService) extends BaseMimeTypeManag override def upload(objectId: String, node: Node, uploadFile: File, filePath: Option[String], params: UploadParams)(implicit ec: ExecutionContext): Future[Map[String, AnyRef]] = { validateUploadRequest(objectId, node, uploadFile) - if (isValidPackageStructure(uploadFile, List[String]("index.html"))) { + val indexHtmlValidation: Boolean = if (Platform.config.hasPath("indexHtmlValidation.env")) Platform.config.getBoolean("indexHtmlValidation.env") else true + TelemetryManager.error("Value of indexHtmlValidation: " + indexHtmlValidation) + val flag: Boolean = if (indexHtmlValidation) isValidPackageStructure(uploadFile, List[String]("index.html")) else true + if (flag) { val urls = uploadArtifactToCloud(uploadFile, objectId, filePath) node.getMetadata.put("s3Key", urls(IDX_S3_KEY)) node.getMetadata.put("artifactUrl", urls(IDX_S3_URL)) diff --git a/schemas/asset/1.0/schema.json b/schemas/asset/1.0/schema.json index 8aa9ad085..5a4dc078b 100644 --- a/schemas/asset/1.0/schema.json +++ b/schemas/asset/1.0/schema.json @@ -1249,6 +1249,18 @@ "items": { "type": "string" } + }, + "taxonomyPaths_v2": { + "type": "array", + "items": { + "type": "object" + } + }, + "competencies_v3": { + "type": "array", + "items": { + "type": "object" + } } } } \ No newline at end of file diff --git a/schemas/collection/1.0/schema.json b/schemas/collection/1.0/schema.json index 112e93841..473c36e7a 100644 --- a/schemas/collection/1.0/schema.json +++ b/schemas/collection/1.0/schema.json @@ -1286,6 +1286,18 @@ "items": { "type": "object" } + }, + "taxonomyPaths_v2": { + "type": "array", + "items": { + "type": "object" + } + }, + "competencies_v3": { + "type": "array", + "items": { + "type": "object" + } } } } diff --git a/schemas/content/1.0/schema.json b/schemas/content/1.0/schema.json index 73181a850..f9d8d172f 100644 --- a/schemas/content/1.0/schema.json +++ b/schemas/content/1.0/schema.json @@ -87,7 +87,8 @@ "audio/webm", "audio/x-wav", "audio/wav", - "application/json" + "application/json", + "application/quiz" ] }, "osId": { @@ -1387,14 +1388,26 @@ "transcripts": { "type": "array", "items": { - "type": "object" + "type": "string" + }, + "taxonomyPaths_v2": { + "type": "array", + "items": { + "type": "object" + } + }, + "accessibility": { + "type": "array", + "items": { + "type": "object" + } + }, + "competencies_v3": { + "type": "array", + "items": { + "type": "object" + } } - }, - "accessibility": { - "type": "array", - "items": { - "type": "object" - } } } } diff --git a/schemas/question/1.0/schema.json b/schemas/question/1.0/schema.json index 6054a39e3..c584ea51b 100644 --- a/schemas/question/1.0/schema.json +++ b/schemas/question/1.0/schema.json @@ -425,7 +425,10 @@ "enum": [ "MCQ", "FTB", - "SA" + "SA", + "MCQ-MCA", + "MCQ-SCA", + "MTF" ] }, "scoringMode": { diff --git a/schemas/questionset/1.0/schema.json b/schemas/questionset/1.0/schema.json index fcc691a25..bc1d34254 100644 --- a/schemas/questionset/1.0/schema.json +++ b/schemas/questionset/1.0/schema.json @@ -656,6 +656,55 @@ }, "originData": { "type": "object" + }, + "trackable": { + "type": "object", + "properties": { + "enabled": { + "type": "string", + "enum": ["Yes","No"], + "default": "No" + }, + "autoBatch": { + "type": "string", + "enum": ["Yes","No"], + "default": "No" + } + }, + "default": { + "enabled": "No", + "autoBatch": "No" + }, + "additionalProperties": false + }, + "purpose": { + "type": "string" + }, + "scoreCutoffType": { + "type": "string", + "enum": [ + "AssessmentLevel", + "SectionLevel" + ], + "default": "AssessmentLevel" + }, + "subTitle": { + "type": "string" + }, + "minimumPassPercentage": { + "type": "number" + }, + "additionalKeywords": { + "type": "array", + "items": { + "type": "string" + } + }, + "additionalInstructions": { + "type": "string" + }, + "reviewStatus": { + "type": "string" } }, "additionalProperties": false diff --git a/search-api/search-actors/src/main/java/org/sunbird/actors/SearchActor.java b/search-api/search-actors/src/main/java/org/sunbird/actors/SearchActor.java index 8dd9dcd92..68f175363 100644 --- a/search-api/search-actors/src/main/java/org/sunbird/actors/SearchActor.java +++ b/search-api/search-actors/src/main/java/org/sunbird/actors/SearchActor.java @@ -106,6 +106,7 @@ private SearchDTO getSearchDTO(Request request) throws Exception { wordChainsRequest = false; List properties = new ArrayList(); Map filters = (Map) req.get(SearchConstants.filters); + Map multiFilters = (Map) req.get("multiFilters"); if (null == filters) filters = new HashMap<>(); if (filters.containsKey("tags")) { @@ -239,6 +240,11 @@ private SearchDTO getSearchDTO(Request request) throws Exception { searchObj.setSortBy(sortBy); searchObj.setFacets(facets); searchObj.setProperties(properties); + if (multiFilters != null) { + List multiFilterProperties = new ArrayList(); + multiFilterProperties.addAll(getSearchFilterProperties(multiFilters, wordChainsRequest, request)); + searchObj.setMultiFilterProperties(multiFilterProperties); + } // Added Implicit Filter Properties To Support Collection content tagging to reuse by tenants. setImplicitFilters(filters, searchObj); searchObj.setLimit(limit); diff --git a/search-api/search-core/src/main/java/org/sunbird/search/client/ElasticSearchUtil.java b/search-api/search-core/src/main/java/org/sunbird/search/client/ElasticSearchUtil.java index cc031b378..a25be9575 100644 --- a/search-api/search-core/src/main/java/org/sunbird/search/client/ElasticSearchUtil.java +++ b/search-api/search-core/src/main/java/org/sunbird/search/client/ElasticSearchUtil.java @@ -49,6 +49,7 @@ import org.elasticsearch.search.aggregations.bucket.terms.Terms; import org.elasticsearch.search.aggregations.bucket.terms.Terms.Bucket; import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder; +import org.elasticsearch.search.aggregations.bucket.nested.Nested; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.sunbird.common.Platform; import org.sunbird.common.exception.ServerException; @@ -619,9 +620,16 @@ public static Object getCountFromAggregation(Aggregations aggregations, List aggregationsMap : groupByList) { Map parentCountMap = new HashMap(); String groupByParent = (String) aggregationsMap.get("groupByParent"); - Terms terms = aggregations.get(groupByParent); + Terms terms = null; + List buckets = null; + if (groupByParent.contains(".")) { + Nested nested = aggregations.get(groupByParent.split("\\.")[0]); + terms = nested.getAggregations().get(groupByParent.split("\\.")[1]); + } else { + terms = aggregations.get(groupByParent); + } + buckets = (List)terms.getBuckets(); List> parentGroupList = new ArrayList>(); - List buckets = (List) terms.getBuckets(); for (Bucket bucket : buckets) { Map parentCountObject = new HashMap(); parentCountObject.put("count", bucket.getDocCount()); diff --git a/search-api/search-core/src/main/java/org/sunbird/search/dto/SearchDTO.java b/search-api/search-core/src/main/java/org/sunbird/search/dto/SearchDTO.java index 71304a021..6fa3d15a9 100644 --- a/search-api/search-core/src/main/java/org/sunbird/search/dto/SearchDTO.java +++ b/search-api/search-core/src/main/java/org/sunbird/search/dto/SearchDTO.java @@ -20,7 +20,7 @@ public class SearchDTO { private Map softConstraints = new HashMap(); private List> aggregations = new ArrayList<>(); private List implicitFilterProperties; - + private List multiFilterProperties; public SearchDTO() { @@ -120,4 +120,12 @@ public List getImplicitFilterProperties() { public void setImplicitFilterProperties(List implicitFilterProperties) { this.implicitFilterProperties = implicitFilterProperties; } + + public List getMultiFilterProperties() { + return multiFilterProperties; + } + + public void setMultiFilterProperties(List multiFilterProperties) { + this.multiFilterProperties = multiFilterProperties; + } } diff --git a/search-api/search-core/src/main/java/org/sunbird/search/processor/SearchProcessor.java b/search-api/search-core/src/main/java/org/sunbird/search/processor/SearchProcessor.java index 5d163638d..489530e55 100644 --- a/search-api/search-core/src/main/java/org/sunbird/search/processor/SearchProcessor.java +++ b/search-api/search-core/src/main/java/org/sunbird/search/processor/SearchProcessor.java @@ -276,9 +276,11 @@ private void setAggregations(List> groupByList, SearchSourceBuilder searchSourceBuilder) { TermsAggregationBuilder termBuilder = null; if (groupByList != null && !groupByList.isEmpty()) { + HashMap> nestedAggregation = new HashMap<>(); for (Map groupByMap : groupByList) { String groupByParent = (String) groupByMap.get("groupByParent"); - termBuilder = AggregationBuilders.terms(groupByParent) + if (!groupByParent.contains(".")) { + termBuilder = AggregationBuilders.terms(groupByParent) .field(groupByParent + SearchConstants.RAW_FIELD_EXTENSION) .size(ElasticSearchUtil.defaultResultLimit); List groupByChildList = (List) groupByMap.get("groupByChildList"); @@ -290,6 +292,27 @@ private void setAggregations(List> groupByList, } } searchSourceBuilder.aggregation(termBuilder); + } else { + if (nestedAggregation.get(groupByParent.split("\\.")[0]) != null) { + nestedAggregation.get(groupByParent.split("\\.")[0]).add(groupByParent.split("\\.")[1]); + } else { + List nestedAggrList = new ArrayList<>(); + nestedAggrList.add(groupByParent.split("\\.")[1]); + nestedAggregation.put(groupByParent.split("\\.")[0], nestedAggrList); + } + } + } + if (!nestedAggregation.isEmpty()) { + for (Map.Entry> mapData : nestedAggregation.entrySet()) { + AggregationBuilder nestedAggregationBuilder = AggregationBuilders.nested(mapData.getKey(), mapData.getKey()); + for (String nestedValue : mapData.getValue()) { + termBuilder = AggregationBuilders.terms(nestedValue) + .field(mapData.getKey() + "." + nestedValue + SearchConstants.RAW_FIELD_EXTENSION) + .size(ElasticSearchUtil.defaultResultLimit); + nestedAggregationBuilder.subAggregation(termBuilder); + } + searchSourceBuilder.aggregation(nestedAggregationBuilder); + } } } } @@ -304,6 +327,21 @@ private QueryBuilder prepareSearchQuery(SearchDTO searchDTO) { QueryBuilder queryBuilder = null; String totalOperation = searchDTO.getOperation(); List properties = searchDTO.getProperties(); + formQuery(properties, queryBuilder, boolQuery, totalOperation); + if(searchDTO.getMultiFilterProperties() != null) { + formQuery(searchDTO.getMultiFilterProperties(), queryBuilder, boolQuery, SearchConstants.SEARCH_OPERATION_OR); + } + + Map softConstraints = searchDTO.getSoftConstraints(); + if (null != softConstraints && !softConstraints.isEmpty()) { + boolQuery.should(getSoftConstraintQuery(softConstraints)); + searchDTO.setSortBy(null); + // relevanceSort = true; + } + return boolQuery; + } + + private void formQuery(List properties, QueryBuilder queryBuilder, BoolQueryBuilder boolQuery, String operation) { for (Map property : properties) { String opertation = (String) property.get("operation"); @@ -408,21 +446,13 @@ private QueryBuilder prepareSearchQuery(SearchDTO searchDTO) { break; } } - if (totalOperation.equalsIgnoreCase(AND)) { + if (operation.equalsIgnoreCase(AND)) { boolQuery.must(queryBuilder); } else { boolQuery.should(queryBuilder); } } - - Map softConstraints = searchDTO.getSoftConstraints(); - if (null != softConstraints && !softConstraints.isEmpty()) { - boolQuery.should(getSoftConstraintQuery(softConstraints)); - searchDTO.setSortBy(null); - // relevanceSort = true; - } - return boolQuery; } private QueryBuilder checkNestedProperty(QueryBuilder queryBuilder, String propertyName) { From 85b46511c42d086fcb2ce4f07a6b04ac63777e0f Mon Sep 17 00:00:00 2001 From: nivetha Date: Tue, 5 Apr 2022 13:41:52 +0530 Subject: [PATCH 002/126] configuration param for timeout --- .../content-service/app/controllers/BaseController.scala | 3 ++- content-api/content-service/conf/application.conf | 5 ++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/content-api/content-service/app/controllers/BaseController.scala b/content-api/content-service/app/controllers/BaseController.scala index 54a94c0f2..3b7125e19 100644 --- a/content-api/content-service/app/controllers/BaseController.scala +++ b/content-api/content-service/app/controllers/BaseController.scala @@ -26,6 +26,7 @@ abstract class BaseController(protected val cc: ControllerComponents)(implicit e new util.HashMap[String, AnyRef]()).asInstanceOf[java.util.Map[String, AnyRef]] val mimeTypesToCheck = List("application/vnd.ekstep.h5p-archive", "application/vnd.ekstep.html-archive", "application/vnd.android.package-archive", "video/webm", "video/x-youtube", "video/mp4") + val actorTimeout: Long = Platform.getLong("actor.timeoutMillisec", 120000L) def requestBody()(implicit request: Request[AnyContent]) = { val body = request.body.asJson.getOrElse("{}").toString @@ -83,7 +84,7 @@ abstract class BaseController(protected val cc: ControllerComponents)(implicit e } def getResult(apiId: String, actor: ActorRef, request: org.sunbird.common.dto.Request, categoryMapping: Boolean = false, version: String = "3.0") : Future[Result] = { - val future = Patterns.ask(actor, request, 120000) recoverWith {case e: Exception => Future(ResponseHandler.getErrorResponse(e))} + val future = Patterns.ask(actor, request, actorTimeout) recoverWith {case e: Exception => Future(ResponseHandler.getErrorResponse(e))} future.map(f => { val result: Response = f.asInstanceOf[Response] result.setId(apiId) diff --git a/content-api/content-service/conf/application.conf b/content-api/content-service/conf/application.conf index a38067351..e0efe2541 100644 --- a/content-api/content-service/conf/application.conf +++ b/content-api/content-service/conf/application.conf @@ -738,4 +738,7 @@ collection { } #Index file validation -indexHtmlValidation.env=false \ No newline at end of file +indexHtmlValidation.env=false + +#timeout +actor.timeoutMillisec = 120000 \ No newline at end of file From 34e3d43d7f7b878bc77d407f1ff9a07076db20db Mon Sep 17 00:00:00 2001 From: nivetha Date: Thu, 5 May 2022 13:31:05 +0530 Subject: [PATCH 003/126] Hierarchy update fix --- .../content-service/conf/application.conf | 1 + .../managers/UpdateHierarchyManager.scala | 24 +++++++++++++++---- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/content-api/content-service/conf/application.conf b/content-api/content-service/conf/application.conf index e0efe2541..f942ee8fe 100644 --- a/content-api/content-service/conf/application.conf +++ b/content-api/content-service/conf/application.conf @@ -739,6 +739,7 @@ collection { #Index file validation indexHtmlValidation.env=false +root.resource.change=true #timeout actor.timeoutMillisec = 120000 \ No newline at end of file diff --git a/content-api/hierarchy-manager/src/main/scala/org/sunbird/managers/UpdateHierarchyManager.scala b/content-api/hierarchy-manager/src/main/scala/org/sunbird/managers/UpdateHierarchyManager.scala index 01c245c6a..457184247 100644 --- a/content-api/hierarchy-manager/src/main/scala/org/sunbird/managers/UpdateHierarchyManager.scala +++ b/content-api/hierarchy-manager/src/main/scala/org/sunbird/managers/UpdateHierarchyManager.scala @@ -345,7 +345,7 @@ object UpdateHierarchyManager { if (MapUtils.isNotEmpty(childrenIdentifiersMap)) { val updatedNodeList = getTempNode(nodeList, rootId) :: List() updateHierarchyRelatedData(childrenIdentifiersMap.getOrElse(rootId, Map[String, Int]()), 1, - rootId, nodeList, childrenIdentifiersMap, updatedNodeList, request).map(finalEnrichedNodeList => { + rootId, nodeList, childrenIdentifiersMap, updatedNodeList, request, rootId).map(finalEnrichedNodeList => { TelemetryManager.info("Final enriched list size: " + finalEnrichedNodeList.size) val childNodeIds = finalEnrichedNodeList.map(node => node.getIdentifier.replaceAll(".img", "")).filterNot(id => StringUtils.containsIgnoreCase(rootId, id)).distinct TelemetryManager.info("Final enriched ids (childNodes): " + childNodeIds + " :: size: " + childNodeIds.size) @@ -368,7 +368,8 @@ object UpdateHierarchyManager { } @throws[Exception] - private def updateHierarchyRelatedData(childrenIds: Map[String, Int], depth: Int, parent: String, nodeList: List[Node], hierarchyStructure: Map[String, Map[String, Int]], enrichedNodeList: scala.collection.immutable.List[Node], request: Request)(implicit oec: OntologyEngineContext, ec: ExecutionContext): Future[List[Node]] = { + private def updateHierarchyRelatedData(childrenIds: Map[String, Int], depth: Int, parent: String, nodeList: List[Node], hierarchyStructure: Map[String, Map[String, Int]], enrichedNodeList: scala.collection.immutable.List[Node], request: Request, rootId: String)(implicit oec: OntologyEngineContext, ec: ExecutionContext): Future[List[Node]] = { + val rootResourceChange: Boolean = if (Platform.config.hasPath("root.resource.change")) Platform.config.getBoolean("root.resource.change") else true val futures = childrenIds.map(child => { val id = child._1 val index = child._2 + 1 @@ -378,13 +379,19 @@ object UpdateHierarchyManager { val nxtEnrichedNodeList = tempNode :: enrichedNodeList if (MapUtils.isNotEmpty(hierarchyStructure.getOrDefault(child._1, Map[String, Int]()))) updateHierarchyRelatedData(hierarchyStructure.getOrDefault(child._1, Map[String, Int]()), - tempNode.getMetadata.get(HierarchyConstants.DEPTH).asInstanceOf[Int] + 1, id, nodeList, hierarchyStructure, nxtEnrichedNodeList, request) + tempNode.getMetadata.get(HierarchyConstants.DEPTH).asInstanceOf[Int] + 1, id, nodeList, hierarchyStructure, nxtEnrichedNodeList, request, rootId) else Future(nxtEnrichedNodeList) } else { // TelemetryManager.info("Get ContentNode as TempNode is null for ID: " + id) getContentNode(id, HierarchyConstants.TAXONOMY_ID).map(node => { - val parentNode: Node = nodeList.find(p => p.getIdentifier.equals(parent)).orNull + val parentNode: Node = if (rootResourceChange && nodeList.find(p => p.getIdentifier.equals(parent)).orNull == null) { + if (nodeList.find(p => p.getIdentifier.equals(rootId)).orNull == null) + nodeList.find(p => p.getIdentifier.equals(rootId + ".img")).orNull + else + nodeList.find(p => p.getIdentifier.equals(rootId)).orNull + } else + nodeList.find(p => p.getIdentifier.equals(parent)).orNull val nxtEnrichedNodeList = if (null != parentNode) { TelemetryManager.info(s"ObjectType for $parent is ${parentNode.getObjectType}...") val parentMetadata: java.util.Map[String, AnyRef] = NodeUtil.serialize(parentNode, new java.util.ArrayList[String](), parentNode.getObjectType.toLowerCase, "1.0") @@ -401,17 +408,24 @@ object UpdateHierarchyManager { enrichedNodeList } if (MapUtils.isNotEmpty(hierarchyStructure.getOrDefault(id, Map[String, Int]()))) { - updateHierarchyRelatedData(hierarchyStructure.getOrDefault(id, Map[String, Int]()), node.getMetadata.get(HierarchyConstants.DEPTH).asInstanceOf[Int] + 1, id, nodeList, hierarchyStructure, nxtEnrichedNodeList, request) + updateHierarchyRelatedData(hierarchyStructure.getOrDefault(id, Map[String, Int]()), node.getMetadata.get(HierarchyConstants.DEPTH).asInstanceOf[Int] + 1, id, nodeList, hierarchyStructure, nxtEnrichedNodeList, request, rootId) } else Future(nxtEnrichedNodeList) }).flatMap(f => f) recoverWith { case e: CompletionException => throw e.getCause } } }) + val outputNodesListFuture = if (rootResourceChange && CollectionUtils.isNotEmpty(futures)) { + val listOfFutures = Future.sequence(futures.toList) + listOfFutures.map(f => f.flatten.distinct) + } else + Future(enrichedNodeList) if (CollectionUtils.isNotEmpty(futures)) { val listOfFutures = Future.sequence(futures.toList) listOfFutures.map(f => f.flatten.distinct) } else Future(enrichedNodeList) + + outputNodesListFuture } private def populateHierarchyRelatedData(tempNode: Node, depth: Int, index: Int, parent: String) = { From fcbfe93ee54440b301b680854024413b8360f50e Mon Sep 17 00:00:00 2001 From: nivetha Date: Fri, 6 May 2022 10:48:21 +0530 Subject: [PATCH 004/126] hierarchy fix --- content-api/content-service/conf/application.conf | 3 ++- .../scala/org/sunbird/managers/UpdateHierarchyManager.scala | 4 +--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/content-api/content-service/conf/application.conf b/content-api/content-service/conf/application.conf index f942ee8fe..592809ab4 100644 --- a/content-api/content-service/conf/application.conf +++ b/content-api/content-service/conf/application.conf @@ -739,7 +739,8 @@ collection { #Index file validation indexHtmlValidation.env=false -root.resource.change=true +# Resource change for root Hierarchy level +root.resource.change=false #timeout actor.timeoutMillisec = 120000 \ No newline at end of file diff --git a/content-api/hierarchy-manager/src/main/scala/org/sunbird/managers/UpdateHierarchyManager.scala b/content-api/hierarchy-manager/src/main/scala/org/sunbird/managers/UpdateHierarchyManager.scala index 457184247..cd29bb30d 100644 --- a/content-api/hierarchy-manager/src/main/scala/org/sunbird/managers/UpdateHierarchyManager.scala +++ b/content-api/hierarchy-manager/src/main/scala/org/sunbird/managers/UpdateHierarchyManager.scala @@ -417,9 +417,7 @@ object UpdateHierarchyManager { val outputNodesListFuture = if (rootResourceChange && CollectionUtils.isNotEmpty(futures)) { val listOfFutures = Future.sequence(futures.toList) listOfFutures.map(f => f.flatten.distinct) - } else - Future(enrichedNodeList) - if (CollectionUtils.isNotEmpty(futures)) { + } else if (CollectionUtils.isNotEmpty(futures)) { val listOfFutures = Future.sequence(futures.toList) listOfFutures.map(f => f.flatten.distinct) } else From 41fb728785fb9f3e3a98be96f3fa6bf25ecb2169 Mon Sep 17 00:00:00 2001 From: nivetha Date: Fri, 6 May 2022 11:14:49 +0530 Subject: [PATCH 005/126] env variable --- content-api/content-service/conf/application.conf | 2 +- .../scala/org/sunbird/managers/UpdateHierarchyManager.scala | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/content-api/content-service/conf/application.conf b/content-api/content-service/conf/application.conf index 592809ab4..872c613af 100644 --- a/content-api/content-service/conf/application.conf +++ b/content-api/content-service/conf/application.conf @@ -740,7 +740,7 @@ collection { #Index file validation indexHtmlValidation.env=false # Resource change for root Hierarchy level -root.resource.change=false +hierarchyUpdate.use.rootNodeId.forNodeSearch=false #timeout actor.timeoutMillisec = 120000 \ No newline at end of file diff --git a/content-api/hierarchy-manager/src/main/scala/org/sunbird/managers/UpdateHierarchyManager.scala b/content-api/hierarchy-manager/src/main/scala/org/sunbird/managers/UpdateHierarchyManager.scala index cd29bb30d..dbc98be88 100644 --- a/content-api/hierarchy-manager/src/main/scala/org/sunbird/managers/UpdateHierarchyManager.scala +++ b/content-api/hierarchy-manager/src/main/scala/org/sunbird/managers/UpdateHierarchyManager.scala @@ -369,7 +369,7 @@ object UpdateHierarchyManager { @throws[Exception] private def updateHierarchyRelatedData(childrenIds: Map[String, Int], depth: Int, parent: String, nodeList: List[Node], hierarchyStructure: Map[String, Map[String, Int]], enrichedNodeList: scala.collection.immutable.List[Node], request: Request, rootId: String)(implicit oec: OntologyEngineContext, ec: ExecutionContext): Future[List[Node]] = { - val rootResourceChange: Boolean = if (Platform.config.hasPath("root.resource.change")) Platform.config.getBoolean("root.resource.change") else true + val rootResourceChange: Boolean = if (Platform.config.hasPath("hierarchyUpdate.use.rootNodeId.forNodeSearch")) Platform.config.getBoolean("hierarchyUpdate.use.rootNodeId.forNodeSearch") else true val futures = childrenIds.map(child => { val id = child._1 val index = child._2 + 1 From 433ee331410d8efed542bfb272993bfdded9a086 Mon Sep 17 00:00:00 2001 From: nivetha Date: Fri, 6 May 2022 12:43:33 +0530 Subject: [PATCH 006/126] Hierarchy update fix --- content-api/content-service/conf/application.conf | 2 +- .../org/sunbird/managers/UpdateHierarchyManager.scala | 9 ++------- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/content-api/content-service/conf/application.conf b/content-api/content-service/conf/application.conf index 872c613af..ed2c63d5e 100644 --- a/content-api/content-service/conf/application.conf +++ b/content-api/content-service/conf/application.conf @@ -740,7 +740,7 @@ collection { #Index file validation indexHtmlValidation.env=false # Resource change for root Hierarchy level -hierarchyUpdate.use.rootNodeId.forNodeSearch=false +hierarchyUpdate.allow.resource.at.root.level=false #timeout actor.timeoutMillisec = 120000 \ No newline at end of file diff --git a/content-api/hierarchy-manager/src/main/scala/org/sunbird/managers/UpdateHierarchyManager.scala b/content-api/hierarchy-manager/src/main/scala/org/sunbird/managers/UpdateHierarchyManager.scala index dbc98be88..525aba220 100644 --- a/content-api/hierarchy-manager/src/main/scala/org/sunbird/managers/UpdateHierarchyManager.scala +++ b/content-api/hierarchy-manager/src/main/scala/org/sunbird/managers/UpdateHierarchyManager.scala @@ -369,7 +369,7 @@ object UpdateHierarchyManager { @throws[Exception] private def updateHierarchyRelatedData(childrenIds: Map[String, Int], depth: Int, parent: String, nodeList: List[Node], hierarchyStructure: Map[String, Map[String, Int]], enrichedNodeList: scala.collection.immutable.List[Node], request: Request, rootId: String)(implicit oec: OntologyEngineContext, ec: ExecutionContext): Future[List[Node]] = { - val rootResourceChange: Boolean = if (Platform.config.hasPath("hierarchyUpdate.use.rootNodeId.forNodeSearch")) Platform.config.getBoolean("hierarchyUpdate.use.rootNodeId.forNodeSearch") else true + val rootResourceChange: Boolean = if (Platform.config.hasPath("hierarchyUpdate.allow.resource.at.root.level")) Platform.config.getBoolean("hierarchyUpdate.allow.resource.at.root.level") else true val futures = childrenIds.map(child => { val id = child._1 val index = child._2 + 1 @@ -414,16 +414,11 @@ object UpdateHierarchyManager { }).flatMap(f => f) recoverWith { case e: CompletionException => throw e.getCause } } }) - val outputNodesListFuture = if (rootResourceChange && CollectionUtils.isNotEmpty(futures)) { - val listOfFutures = Future.sequence(futures.toList) - listOfFutures.map(f => f.flatten.distinct) - } else if (CollectionUtils.isNotEmpty(futures)) { + if (CollectionUtils.isNotEmpty(futures)) { val listOfFutures = Future.sequence(futures.toList) listOfFutures.map(f => f.flatten.distinct) } else Future(enrichedNodeList) - - outputNodesListFuture } private def populateHierarchyRelatedData(tempNode: Node, depth: Int, index: Int, parent: String) = { From 18d81544a42ff8c36d42a24d389097c84117db27 Mon Sep 17 00:00:00 2001 From: Karthikeyan Rajendran <70887864+karthik-tarento@users.noreply.github.com> Date: Fri, 6 May 2022 14:12:49 +0530 Subject: [PATCH 007/126] Set default value as FALSE Setting default value as "FALSE" for config param "hierarchyUpdate.allow.resource.at.root.level" --- .../scala/org/sunbird/managers/UpdateHierarchyManager.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content-api/hierarchy-manager/src/main/scala/org/sunbird/managers/UpdateHierarchyManager.scala b/content-api/hierarchy-manager/src/main/scala/org/sunbird/managers/UpdateHierarchyManager.scala index 525aba220..fbb73934a 100644 --- a/content-api/hierarchy-manager/src/main/scala/org/sunbird/managers/UpdateHierarchyManager.scala +++ b/content-api/hierarchy-manager/src/main/scala/org/sunbird/managers/UpdateHierarchyManager.scala @@ -369,7 +369,7 @@ object UpdateHierarchyManager { @throws[Exception] private def updateHierarchyRelatedData(childrenIds: Map[String, Int], depth: Int, parent: String, nodeList: List[Node], hierarchyStructure: Map[String, Map[String, Int]], enrichedNodeList: scala.collection.immutable.List[Node], request: Request, rootId: String)(implicit oec: OntologyEngineContext, ec: ExecutionContext): Future[List[Node]] = { - val rootResourceChange: Boolean = if (Platform.config.hasPath("hierarchyUpdate.allow.resource.at.root.level")) Platform.config.getBoolean("hierarchyUpdate.allow.resource.at.root.level") else true + val rootResourceChange: Boolean = if (Platform.config.hasPath("hierarchyUpdate.allow.resource.at.root.level")) Platform.config.getBoolean("hierarchyUpdate.allow.resource.at.root.level") else false val futures = childrenIds.map(child => { val id = child._1 val index = child._2 + 1 From 8fd5fb09c63d1d8008e7534f19df45de6c8bd447 Mon Sep 17 00:00:00 2001 From: nivetha Date: Fri, 27 May 2022 14:16:04 +0530 Subject: [PATCH 008/126] updateNodesToRetire changes --- .../src/main/scala/org/sunbird/content/util/RetireManager.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content-api/content-actors/src/main/scala/org/sunbird/content/util/RetireManager.scala b/content-api/content-actors/src/main/scala/org/sunbird/content/util/RetireManager.scala index 360fcb567..cbd8e4eba 100644 --- a/content-api/content-actors/src/main/scala/org/sunbird/content/util/RetireManager.scala +++ b/content-api/content-actors/src/main/scala/org/sunbird/content/util/RetireManager.scala @@ -57,7 +57,7 @@ object RetireManager { private def updateNodesToRetire(request: Request, updateMetadataMap: util.Map[String, AnyRef])(implicit ec: ExecutionContext, oec: OntologyEngineContext): Future[Response] = { RedisCache.delete(request.get(ContentConstants.IDENTIFIER).asInstanceOf[String]) val updateReq = new Request(request) - updateReq.put(ContentConstants.IDENTIFIERS, java.util.Arrays.asList(request.get(ContentConstants.IDENTIFIER).asInstanceOf[String])) + updateReq.put(ContentConstants.IDENTIFIERS, java.util.Arrays.asList(request.get(ContentConstants.IDENTIFIER).asInstanceOf[String], request.get(ContentConstants.IDENTIFIER).asInstanceOf[String] + HierarchyConstants.IMAGE_SUFFIX)) updateReq.put(ContentConstants.METADATA, updateMetadataMap) DataNode.bulkUpdate(updateReq).map(node => ResponseHandler.OK()) } From c99425fda916651ca3c1a5af3e0dc3432c4cdf68 Mon Sep 17 00:00:00 2001 From: Karthikeyan Rajendran <70887864+karthik-tarento@users.noreply.github.com> Date: Tue, 12 Jul 2022 17:02:54 +0530 Subject: [PATCH 009/126] Added 'SentToPublish' state in 'reviewStatus' object --- schemas/content/1.0/schema.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/schemas/content/1.0/schema.json b/schemas/content/1.0/schema.json index f9d8d172f..bdad97d70 100644 --- a/schemas/content/1.0/schema.json +++ b/schemas/content/1.0/schema.json @@ -1247,7 +1247,8 @@ "type" : "string", "enum": [ "InReview", - "Reviewed" + "Reviewed", + "SentToPublish" ] }, "boardIds": { From 6300c00f658f4c520b3caddbb11678e38d105183 Mon Sep 17 00:00:00 2001 From: Karthikeyan Rajendran <70887864+karthik-tarento@users.noreply.github.com> Date: Tue, 12 Jul 2022 17:06:45 +0530 Subject: [PATCH 010/126] Added new type of attributes for Question --- schemas/question/1.0/schema.json | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/schemas/question/1.0/schema.json b/schemas/question/1.0/schema.json index c584ea51b..ae0c3e9d3 100644 --- a/schemas/question/1.0/schema.json +++ b/schemas/question/1.0/schema.json @@ -583,6 +583,17 @@ }, "originData": { "type": "object" + }, + "choices": { + "type": "object", + "description": "Choices which needs to be used in MCQ / MTF type question" + }, + "rhsChoices": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Choices which needs to be used in RHS of MTF type question" } }, "additionalProperties": false From d1170f54b12f5e79b51d41b263e5796d021632d7 Mon Sep 17 00:00:00 2001 From: Karthikeyan Rajendran <70887864+karthik-tarento@users.noreply.github.com> Date: Wed, 13 Jul 2022 11:34:49 +0530 Subject: [PATCH 011/126] Added "SentToPublish" in reviewStatus object --- schemas/collection/1.0/schema.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/schemas/collection/1.0/schema.json b/schemas/collection/1.0/schema.json index 473c36e7a..4d65aea24 100644 --- a/schemas/collection/1.0/schema.json +++ b/schemas/collection/1.0/schema.json @@ -1135,7 +1135,8 @@ "type" : "string", "enum": [ "InReview", - "Reviewed" + "Reviewed", + "SentToPublish" ] }, "boardIds": { From e01488a4c107745f121c3a16a658e03f96ccc66f Mon Sep 17 00:00:00 2001 From: Juhi Agarwal Date: Wed, 13 Jul 2022 15:49:38 +0530 Subject: [PATCH 012/126] implemented fuzzy logic to the search --- .../search/processor/SearchProcessor.java | 184 +++--------------- 1 file changed, 22 insertions(+), 162 deletions(-) diff --git a/search-api/search-core/src/main/java/org/sunbird/search/processor/SearchProcessor.java b/search-api/search-core/src/main/java/org/sunbird/search/processor/SearchProcessor.java index 489530e55..3a46c3ddb 100644 --- a/search-api/search-core/src/main/java/org/sunbird/search/processor/SearchProcessor.java +++ b/search-api/search-core/src/main/java/org/sunbird/search/processor/SearchProcessor.java @@ -6,17 +6,8 @@ import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang.StringUtils; import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.common.lucene.search.function.CombineFunction; -import org.elasticsearch.common.lucene.search.function.FunctionScoreQuery.ScoreMode; -import org.elasticsearch.index.query.BoolQueryBuilder; +import org.elasticsearch.index.query.*; import org.elasticsearch.index.query.MultiMatchQueryBuilder.Type; -import org.elasticsearch.index.query.Operator; -import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.index.query.RangeQueryBuilder; -import org.elasticsearch.index.query.functionscore.FunctionScoreQueryBuilder; -import org.elasticsearch.index.query.functionscore.FunctionScoreQueryBuilder.FilterFunctionBuilder; -import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHits; import org.elasticsearch.search.aggregations.Aggregation; @@ -38,11 +29,7 @@ import scala.concurrent.ExecutionContext; import scala.concurrent.Future; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.stream.Collectors; public class SearchProcessor { @@ -327,9 +314,9 @@ private QueryBuilder prepareSearchQuery(SearchDTO searchDTO) { QueryBuilder queryBuilder = null; String totalOperation = searchDTO.getOperation(); List properties = searchDTO.getProperties(); - formQuery(properties, queryBuilder, boolQuery, totalOperation); + formQuery(properties, queryBuilder, boolQuery, totalOperation, searchDTO.isFuzzySearch()); if(searchDTO.getMultiFilterProperties() != null) { - formQuery(searchDTO.getMultiFilterProperties(), queryBuilder, boolQuery, SearchConstants.SEARCH_OPERATION_OR); + formQuery(searchDTO.getMultiFilterProperties(), queryBuilder, boolQuery, SearchConstants.SEARCH_OPERATION_OR, searchDTO.isFuzzySearch()); } Map softConstraints = searchDTO.getSoftConstraints(); @@ -341,7 +328,7 @@ private QueryBuilder prepareSearchQuery(SearchDTO searchDTO) { return boolQuery; } - private void formQuery(List properties, QueryBuilder queryBuilder, BoolQueryBuilder boolQuery, String operation) { + private void formQuery(List properties, QueryBuilder queryBuilder, BoolQueryBuilder boolQuery, String operation, Boolean fuzzy) { for (Map property : properties) { String opertation = (String) property.get("operation"); @@ -358,7 +345,7 @@ private void formQuery(List properties, QueryBuilder queryBuilder, BoolQuer if (propertyName.equals("*")) { relevanceSort = true; propertyName = "all_fields"; - queryBuilder = getAllFieldsPropertyQuery(values); + queryBuilder = getAllFieldsPropertyQuery(values, fuzzy); boolQuery.must(queryBuilder); continue; } @@ -462,145 +449,12 @@ private QueryBuilder checkNestedProperty(QueryBuilder queryBuilder, String prope return queryBuilder; } - /** - * @param searchDTO - * @return - */ - @SuppressWarnings({ "unchecked", "rawtypes" }) - private QueryBuilder prepareFilteredSearchQuery(SearchDTO searchDTO) { - List filterFunctionBuilder = new ArrayList<>(); - - Map weightages = (Map) searchDTO.getAdditionalProperty("weightagesMap"); - if (weightages == null) { - weightages = new HashMap(); - weightages.put("default_weightage", 1.0f); - } - List querySearchFeilds = ElasticSearchUtil.getQuerySearchFields(); - List properties = searchDTO.getProperties(); - for (Map property : properties) { - String opertation = (String) property.get("operation"); - - List values; - try { - values = (List) property.get("values"); - } catch (Exception e) { - values = Arrays.asList(property.get("values")); - } - - values = values.stream().filter(value -> (null != value)).collect(Collectors.toList()); - String propertyName = (String) property.get("propertyName"); - if (propertyName.equals("*")) { - relevanceSort = true; - propertyName = "all_fields"; - filterFunctionBuilder - .add(new FilterFunctionBuilder(getAllFieldsPropertyQuery(values), - ScoreFunctionBuilders.weightFactorFunction(weightages.get("default_weightage")))); - continue; - } - - propertyName = propertyName + SearchConstants.RAW_FIELD_EXTENSION; - float weight = getweight(querySearchFeilds, propertyName); - switch (opertation) { - case SearchConstants.SEARCH_OPERATION_EQUAL: { - filterFunctionBuilder.add(new FilterFunctionBuilder( - getMustTermQuery(propertyName, values, true), - ScoreFunctionBuilders.weightFactorFunction(weight))); - break; - } - case SearchConstants.SEARCH_OPERATION_NOT_EQUAL: { - filterFunctionBuilder.add(new FilterFunctionBuilder( - getMustTermQuery(propertyName, values, true), - ScoreFunctionBuilders.weightFactorFunction(weight))); - break; - } - case SearchConstants.SEARCH_OPERATION_ENDS_WITH: { - filterFunctionBuilder.add(new FilterFunctionBuilder( - getRegexQuery(propertyName, values), ScoreFunctionBuilders.weightFactorFunction(weight))); - break; - } - case SearchConstants.SEARCH_OPERATION_LIKE: - case SearchConstants.SEARCH_OPERATION_CONTAINS: { - filterFunctionBuilder.add(new FilterFunctionBuilder( - getMatchPhraseQuery(propertyName, values, true), - ScoreFunctionBuilders.weightFactorFunction(weight))); - break; - } - case SearchConstants.SEARCH_OPERATION_NOT_LIKE: { - filterFunctionBuilder.add(new FilterFunctionBuilder( - getMatchPhraseQuery(propertyName, values, false), - ScoreFunctionBuilders.weightFactorFunction(weight))); - break; - } - case SearchConstants.SEARCH_OPERATION_STARTS_WITH: { - filterFunctionBuilder.add(new FilterFunctionBuilder( - getMatchPhrasePrefixQuery(propertyName, values), - ScoreFunctionBuilders.weightFactorFunction(weight))); - break; - } - case SearchConstants.SEARCH_OPERATION_EXISTS: { - filterFunctionBuilder.add( - new FilterFunctionBuilder(getExistsQuery(propertyName, values, true), - ScoreFunctionBuilders.weightFactorFunction(weight))); - break; - } - case SearchConstants.SEARCH_OPERATION_NOT_EXISTS: { - filterFunctionBuilder.add( - new FilterFunctionBuilder(getExistsQuery(propertyName, values, false), - ScoreFunctionBuilders.weightFactorFunction(weight))); - break; - } - case SearchConstants.SEARCH_OPERATION_NOT_IN: { - filterFunctionBuilder.add(new FilterFunctionBuilder( - getNotInQuery(propertyName, values), ScoreFunctionBuilders.weightFactorFunction(weight))); - break; - } - case SearchConstants.SEARCH_OPERATION_GREATER_THAN: { - filterFunctionBuilder.add(new FilterFunctionBuilder( - getRangeQuery(propertyName, values, SearchConstants.SEARCH_OPERATION_GREATER_THAN), - ScoreFunctionBuilders.weightFactorFunction(weight))); - break; - } - case SearchConstants.SEARCH_OPERATION_GREATER_THAN_EQUALS: { - filterFunctionBuilder.add(new FilterFunctionBuilder( - getRangeQuery(propertyName, values, - SearchConstants.SEARCH_OPERATION_GREATER_THAN_EQUALS), - ScoreFunctionBuilders.weightFactorFunction(weight))); - break; - } - case SearchConstants.SEARCH_OPERATION_LESS_THAN: { - filterFunctionBuilder.add(new FilterFunctionBuilder( - getRangeQuery(propertyName, values, SearchConstants.SEARCH_OPERATION_LESS_THAN), - ScoreFunctionBuilders.weightFactorFunction(weight))); - break; - } - case SearchConstants.SEARCH_OPERATION_LESS_THAN_EQUALS: { - filterFunctionBuilder.add(new FilterFunctionBuilder( - getRangeQuery(propertyName, values, SearchConstants.SEARCH_OPERATION_LESS_THAN_EQUALS), - ScoreFunctionBuilders.weightFactorFunction(weight))); - break; - } - case SearchConstants.SEARCH_OPERATION_AND: { - filterFunctionBuilder.add(new FilterFunctionBuilder( - getAndQuery(propertyName, values), - ScoreFunctionBuilders.weightFactorFunction(weight))); - break; - } - } - } - - FunctionScoreQueryBuilder queryBuilder = QueryBuilders - .functionScoreQuery( - filterFunctionBuilder.toArray(new FilterFunctionBuilder[filterFunctionBuilder.size()])) - .boostMode(CombineFunction.REPLACE).scoreMode(ScoreMode.SUM); - return queryBuilder; - - } private QueryBuilder getAndQuery(String propertyName, List values) { BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery(); for (Object value : values) { queryBuilder.must( - QueryBuilders.matchQuery(propertyName, value).operator(Operator.AND)); + QueryBuilders.matchQuery(propertyName, value).operator(Operator.AND).fuzzyTranspositions(false)); } return queryBuilder; } @@ -627,7 +481,7 @@ private float getweight(List querySearchFeilds, String propertyName) { * @param values * @return */ - private QueryBuilder getAllFieldsPropertyQuery(List values) { + private QueryBuilder getAllFieldsPropertyQuery(List values, Boolean fuzzy) { List queryFields = ElasticSearchUtil.getQuerySearchFields(); Map queryFieldsMap = new HashMap<>(); for (String field : queryFields) { @@ -638,9 +492,15 @@ private QueryBuilder getAllFieldsPropertyQuery(List values) { } BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery(); for (Object value : values) { - queryBuilder - .should(QueryBuilders.multiMatchQuery(value).fields(queryFieldsMap) - .operator(Operator.AND).type(Type.CROSS_FIELDS).lenient(true)); + if (fuzzy) { + queryBuilder + .should(QueryBuilders.multiMatchQuery(value).fields(queryFieldsMap) + .operator(Operator.AND).fuzziness("AUTO").lenient(true)); + } else { + queryBuilder + .should(QueryBuilders.multiMatchQuery(value).fields(queryFieldsMap) + .operator(Operator.AND).type(Type.CROSS_FIELDS).fuzzyTranspositions(false).lenient(true)); + } } return queryBuilder; @@ -660,13 +520,13 @@ private QueryBuilder getSoftConstraintQuery(Map softConstraints) for(Object value: dataList) { queryBuilder .should(QueryBuilders.matchQuery(key + SearchConstants.RAW_FIELD_EXTENSION, value) - .boost(Integer.valueOf((int) data.get(0)).floatValue())); + .boost(Integer.valueOf((int) data.get(0)).floatValue()).fuzzyTranspositions(false)); } } else { queryBuilder.should( QueryBuilders.matchQuery(key + SearchConstants.RAW_FIELD_EXTENSION, data.get(1)) - .boost(Integer.valueOf((int) data.get(0)).floatValue())); + .boost(Integer.valueOf((int) data.get(0)).floatValue()).fuzzyTranspositions(false)); } } return queryBuilder; @@ -799,10 +659,10 @@ private QueryBuilder getMustTermQuery(String propertyName, List values, for (Object value : values) { if (match) { queryBuilder.should( - QueryBuilders.matchQuery(propertyName, value)); + QueryBuilders.matchQuery(propertyName, value).fuzzyTranspositions(false)); } else { queryBuilder.mustNot( - QueryBuilders.matchQuery(propertyName, value)); + QueryBuilders.matchQuery(propertyName, value).fuzzyTranspositions(false)); } } @@ -965,7 +825,7 @@ private QueryBuilder getSearchQuery(SearchDTO searchDTO) { } private QueryBuilder getQuery(SearchDTO searchDTO) { - return searchDTO.isFuzzySearch() ? prepareFilteredSearchQuery(searchDTO) : prepareSearchQuery(searchDTO); + return prepareSearchQuery(searchDTO); } From 4e7d18a18c5b499a033ee4c5cc84b085c85812cf Mon Sep 17 00:00:00 2001 From: wilkysingh-tarento Date: Fri, 29 Jul 2022 13:59:04 +0530 Subject: [PATCH 013/126] Copy content artifacturl must not included bug ' --- .../main/scala/org/sunbird/content/util/CopyManager.scala | 8 ++++++-- content-api/content-service/conf/application.conf | 5 ++++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/content-api/content-actors/src/main/scala/org/sunbird/content/util/CopyManager.scala b/content-api/content-actors/src/main/scala/org/sunbird/content/util/CopyManager.scala index cbf52207d..15b3b266f 100644 --- a/content-api/content-actors/src/main/scala/org/sunbird/content/util/CopyManager.scala +++ b/content-api/content-actors/src/main/scala/org/sunbird/content/util/CopyManager.scala @@ -38,7 +38,7 @@ object CopyManager { private val originMetadataKeys: util.List[String] = Platform.getStringList("content.copy.origin_data", new util.ArrayList[String]()) private val internalHierarchyProps = List("identifier", "parent", "index", "depth") private val restrictedMimeTypesForUpload = List("application/vnd.ekstep.ecml-archive","application/vnd.ekstep.content-collection") - + private val copyArtifactUrl = Platform.config.getBoolean("content.copy.is_copy_artifacturl") private var copySchemeMap: util.Map[String, AnyRef] = new util.HashMap[String, AnyRef]() def copy(request: Request)(implicit ec: ExecutionContext, oec: OntologyEngineContext, ss: StorageService): Future[Response] = { @@ -73,7 +73,11 @@ object CopyManager { val copyCreateReq: Future[Request] = getCopyRequest(node, request) copyCreateReq.map(req => { DataNode.create(req).map(copiedNode => { - artifactUpload(node, copiedNode, request) + if(copyArtifactUrl){ + artifactUpload(node, copiedNode, request) + }else{ + Future(copiedNode) + } }).flatMap(f => f) }).flatMap(f => f) } diff --git a/content-api/content-service/conf/application.conf b/content-api/content-service/conf/application.conf index ed2c63d5e..fa3214e9f 100644 --- a/content-api/content-service/conf/application.conf +++ b/content-api/content-service/conf/application.conf @@ -743,4 +743,7 @@ indexHtmlValidation.env=false hierarchyUpdate.allow.resource.at.root.level=false #timeout -actor.timeoutMillisec = 120000 \ No newline at end of file +actor.timeoutMillisec = 120000 + +# Artifact Url allowed for copy resource +content.copy.is_copy_artifacturl= true \ No newline at end of file From 622dd4477765717416ffcac73f6f39ad874a36b7 Mon Sep 17 00:00:00 2001 From: Karthikeyan Rajendran <70887864+karthik-tarento@users.noreply.github.com> Date: Tue, 9 Aug 2022 11:23:15 +0530 Subject: [PATCH 014/126] Added MimeType for Survey --- schemas/content/1.0/schema.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/schemas/content/1.0/schema.json b/schemas/content/1.0/schema.json index bdad97d70..57c10e72e 100644 --- a/schemas/content/1.0/schema.json +++ b/schemas/content/1.0/schema.json @@ -88,7 +88,8 @@ "audio/x-wav", "audio/wav", "application/json", - "application/quiz" + "application/quiz", + "application/survey" ] }, "osId": { From e7ee48f9b70aaac9c791ccf4014b5555da65d230 Mon Sep 17 00:00:00 2001 From: Karthikeyan Rajendran <70887864+karthik-tarento@users.noreply.github.com> Date: Wed, 7 Sep 2022 18:40:25 +0530 Subject: [PATCH 015/126] Added competencies_v3 meta --- schemas/questionset/1.0/schema.json | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/schemas/questionset/1.0/schema.json b/schemas/questionset/1.0/schema.json index bc1d34254..0a9378a76 100644 --- a/schemas/questionset/1.0/schema.json +++ b/schemas/questionset/1.0/schema.json @@ -705,6 +705,12 @@ }, "reviewStatus": { "type": "string" + }, + "competencies_v3": { + "type": "array", + "items": { + "type": "object" + } } }, "additionalProperties": false From 40181c5fc86dd623c19b8490ac888a5d57ffc262 Mon Sep 17 00:00:00 2001 From: juhi agarwal Date: Fri, 18 Nov 2022 12:19:04 +0530 Subject: [PATCH 016/126] Update schema.json --- schemas/questionset/1.0/schema.json | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/schemas/questionset/1.0/schema.json b/schemas/questionset/1.0/schema.json index 0a9378a76..cbab7fc58 100644 --- a/schemas/questionset/1.0/schema.json +++ b/schemas/questionset/1.0/schema.json @@ -711,6 +711,10 @@ "items": { "type": "object" } + }, + "retakeAssessmentDuration": { + "type": "number", + "default": 5 } }, "additionalProperties": false From fb29f1208b5758ed1ba95b266854a01c7be1df13 Mon Sep 17 00:00:00 2001 From: Karthikeyan Rajendran <70887864+karthik-tarento@users.noreply.github.com> Date: Tue, 22 Nov 2022 17:42:35 +0530 Subject: [PATCH 017/126] Added meta to store content ratings --- schemas/collection/1.0/schema.json | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/schemas/collection/1.0/schema.json b/schemas/collection/1.0/schema.json index 4d65aea24..ed219dd85 100644 --- a/schemas/collection/1.0/schema.json +++ b/schemas/collection/1.0/schema.json @@ -1299,6 +1299,14 @@ "items": { "type": "object" } + }, + "sumOfTotalTatings": { + "type": "number", + "default": 0 + }, + "totalNumberOfRatings": { + "type": "number", + "default": 0 } } } From 7069cffd4bb514722705a3d722fae9e720b1847e Mon Sep 17 00:00:00 2001 From: karthik-tarento Date: Fri, 25 Nov 2022 17:39:39 +0530 Subject: [PATCH 018/126] Introduced new flag and changes to make fuzzy search mandatory when no results --- .../search/processor/SearchProcessor.java | 18 +++++++++++++++--- .../search-service/conf/application.conf | 5 ++++- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/search-api/search-core/src/main/java/org/sunbird/search/processor/SearchProcessor.java b/search-api/search-core/src/main/java/org/sunbird/search/processor/SearchProcessor.java index 3a46c3ddb..dc68b5fd7 100644 --- a/search-api/search-core/src/main/java/org/sunbird/search/processor/SearchProcessor.java +++ b/search-api/search-core/src/main/java/org/sunbird/search/processor/SearchProcessor.java @@ -52,9 +52,21 @@ public Future> processSearch(SearchDTO searchDTO, boolean in throws Exception { List> groupByFinalList = new ArrayList>(); SearchSourceBuilder query = processSearchQuery(searchDTO, groupByFinalList, true); - Future searchResponse = ElasticSearchUtil.search( - SearchConstants.COMPOSITE_SEARCH_INDEX, - query); + Future searchResponse = null; + boolean enableFuzzyWhenNoResults = Platform.config.hasPath("search.fields.enable.fuzzy.when.noresult") && + Platform.config.getBoolean("search.fields.enable.fuzzy.when.noresult"); + if (enableFuzzyWhenNoResults) { + //Let's call with Default fuzzy value given in request + int exactMatchCount = ElasticSearchUtil.count(SearchConstants.COMPOSITE_SEARCH_INDEX, query); + + //If no results and fuzzy was false then set fuzzy to true + //If no results when fuzzy was true, return the same. + if (exactMatchCount == 0 && !searchDTO.isFuzzySearch()) { + searchDTO.setFuzzySearch(true); + query = processSearchQuery(searchDTO, groupByFinalList, true); + } + } + searchResponse = ElasticSearchUtil.search(SearchConstants.COMPOSITE_SEARCH_INDEX, query); return searchResponse.map(new Mapper>() { public Map apply(SearchResponse searchResult) { diff --git a/search-api/search-service/conf/application.conf b/search-api/search-service/conf/application.conf index a803688db..0b038102c 100644 --- a/search-api/search-service/conf/application.conf +++ b/search-api/search-service/conf/application.conf @@ -312,4 +312,7 @@ ekstepPlatformApiUserId="search-service" content.tagging.property=["subject","medium"] -search.payload.log_enable=true \ No newline at end of file +search.payload.log_enable=true + +#Folling configuration would enable the fuzzy search when there are no matches found for given query. +search.fields.enable.fuzzy.when.noresult=false \ No newline at end of file From 7f14ae01e491a303873ef5e6dca06287ef31e1cf Mon Sep 17 00:00:00 2001 From: Karthikeyan Rajendran <70887864+karthik-tarento@users.noreply.github.com> Date: Wed, 30 Nov 2022 14:13:14 +0530 Subject: [PATCH 019/126] Updated meta param types to String to store float values --- schemas/collection/1.0/schema.json | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/schemas/collection/1.0/schema.json b/schemas/collection/1.0/schema.json index ed219dd85..1e0ee9cbf 100644 --- a/schemas/collection/1.0/schema.json +++ b/schemas/collection/1.0/schema.json @@ -1301,12 +1301,10 @@ } }, "sumOfTotalTatings": { - "type": "number", - "default": 0 + "type": "string" }, "totalNumberOfRatings": { - "type": "number", - "default": 0 + "type": "string" } } } From ab8ec79b7604ac8941201b3097acbb7a034553b0 Mon Sep 17 00:00:00 2001 From: Karthikeyan Rajendran <70887864+karthik-tarento@users.noreply.github.com> Date: Thu, 8 Dec 2022 17:20:39 +0530 Subject: [PATCH 020/126] 4.8.0 search fix v2 (#53) * Fixes for listing by facets in search --- .../main/java/org/sunbird/search/processor/SearchProcessor.java | 1 + 1 file changed, 1 insertion(+) diff --git a/search-api/search-core/src/main/java/org/sunbird/search/processor/SearchProcessor.java b/search-api/search-core/src/main/java/org/sunbird/search/processor/SearchProcessor.java index dc68b5fd7..c95fc4f24 100644 --- a/search-api/search-core/src/main/java/org/sunbird/search/processor/SearchProcessor.java +++ b/search-api/search-core/src/main/java/org/sunbird/search/processor/SearchProcessor.java @@ -63,6 +63,7 @@ public Future> processSearch(SearchDTO searchDTO, boolean in //If no results when fuzzy was true, return the same. if (exactMatchCount == 0 && !searchDTO.isFuzzySearch()) { searchDTO.setFuzzySearch(true); + groupByFinalList.clear(); query = processSearchQuery(searchDTO, groupByFinalList, true); } } From 2ff925f237527049e7ff71a1b7251b01fc775843 Mon Sep 17 00:00:00 2001 From: juhi agarwal Date: Tue, 20 Dec 2022 18:22:37 +0530 Subject: [PATCH 021/126] Update schema.json (#54) --- schemas/questionset/1.0/schema.json | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/schemas/questionset/1.0/schema.json b/schemas/questionset/1.0/schema.json index cbab7fc58..05674b5c1 100644 --- a/schemas/questionset/1.0/schema.json +++ b/schemas/questionset/1.0/schema.json @@ -715,6 +715,10 @@ "retakeAssessmentDuration": { "type": "number", "default": 5 + }, + "maxAssessmentRetakeAttempts": { + "type": "number", + "default": 5 } }, "additionalProperties": false From eecbb23034b72cc67256354f84e72c810994d717 Mon Sep 17 00:00:00 2001 From: gohilamariappan <41056032+gohilamariappan@users.noreply.github.com> Date: Wed, 11 Jan 2023 12:31:56 +0530 Subject: [PATCH 022/126] schema repo config added --- build/content-service/Jenkinsfile | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/build/content-service/Jenkinsfile b/build/content-service/Jenkinsfile index 6909c139c..8d9956323 100644 --- a/build/content-service/Jenkinsfile +++ b/build/content-service/Jenkinsfile @@ -28,7 +28,13 @@ node('build-slave') { sh 'mvn clean install -DskipTests=true ' } + stage('schema-pull') { + dir("${env.WORKSPACE}/schema") { + checkout scm: [$class: 'GitSCM', branches: [[name: assets_repo_branch]], extensions: [[$class: 'CloneOption', depth: 1, noTags: true, reference: '', shallow: true]], userRemoteConfigs: [[credentialsId: schema_repo_credentials, url: schema_repo_url]]] + + } + } stage('Package') { dir('content-api') { sh 'mvn play2:dist -pl content-service' @@ -47,4 +53,4 @@ node('build-slave') { currentBuild.result = "FAILURE" throw err } -} \ No newline at end of file +} From 5c162a957d1a3f147d867f50a0299d7c63996727 Mon Sep 17 00:00:00 2001 From: gohilamariappan <41056032+gohilamariappan@users.noreply.github.com> Date: Wed, 11 Jan 2023 12:32:23 +0530 Subject: [PATCH 023/126] schema path updated --- build/content-service/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/content-service/Dockerfile b/build/content-service/Dockerfile index 2e829d061..a22e80d32 100644 --- a/build/content-service/Dockerfile +++ b/build/content-service/Dockerfile @@ -9,6 +9,6 @@ USER sunbird COPY ./content-api/content-service/target/content-service-1.0-SNAPSHOT-dist.zip /home/sunbird/ RUN unzip /home/sunbird/content-service-1.0-SNAPSHOT-dist.zip -d /home/sunbird/ RUN rm /home/sunbird/content-service-1.0-SNAPSHOT-dist.zip -COPY --chown=sunbird ./schemas /home/sunbird/content-service-1.0-SNAPSHOT/schemas +COPY --chown=sunbird ./schema /home/sunbird/content-service-1.0-SNAPSHOT/schemas WORKDIR /home/sunbird/ CMD java -XX:+PrintFlagsFinal $JAVA_OPTIONS -cp '/home/sunbird/content-service-1.0-SNAPSHOT/lib/*' -Dconfig.file=/home/sunbird/content-service-1.0-SNAPSHOT/config/application.conf -Dlogger.file=/home/sunbird/content-service-1.0-SNAPSHOT/config/logback.xml play.core.server.ProdServerStart /home/sunbird/content-service-1.0-SNAPSHOT From 5824c02cfdd4b094e859fab7b44498e60327a817 Mon Sep 17 00:00:00 2001 From: gohilamariappan <41056032+gohilamariappan@users.noreply.github.com> Date: Wed, 11 Jan 2023 12:45:06 +0530 Subject: [PATCH 024/126] updated schema_repo_branch --- build/content-service/Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/content-service/Jenkinsfile b/build/content-service/Jenkinsfile index 8d9956323..fb4204f8a 100644 --- a/build/content-service/Jenkinsfile +++ b/build/content-service/Jenkinsfile @@ -31,7 +31,7 @@ node('build-slave') { stage('schema-pull') { dir("${env.WORKSPACE}/schema") { - checkout scm: [$class: 'GitSCM', branches: [[name: assets_repo_branch]], extensions: [[$class: 'CloneOption', depth: 1, noTags: true, reference: '', shallow: true]], userRemoteConfigs: [[credentialsId: schema_repo_credentials, url: schema_repo_url]]] + checkout scm: [$class: 'GitSCM', branches: [[name: schema_repo_branch]], extensions: [[$class: 'CloneOption', depth: 1, noTags: true, reference: '', shallow: true]], userRemoteConfigs: [[credentialsId: schema_repo_credentials, url: schema_repo_url]]] } } From 2b9b6c1edc991d4bcbc256f70a6e12cb3663c3f9 Mon Sep 17 00:00:00 2001 From: gohilamariappan <41056032+gohilamariappan@users.noreply.github.com> Date: Wed, 11 Jan 2023 12:53:59 +0530 Subject: [PATCH 025/126] path updated --- build/content-service/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/content-service/Dockerfile b/build/content-service/Dockerfile index a22e80d32..234423fdb 100644 --- a/build/content-service/Dockerfile +++ b/build/content-service/Dockerfile @@ -9,6 +9,6 @@ USER sunbird COPY ./content-api/content-service/target/content-service-1.0-SNAPSHOT-dist.zip /home/sunbird/ RUN unzip /home/sunbird/content-service-1.0-SNAPSHOT-dist.zip -d /home/sunbird/ RUN rm /home/sunbird/content-service-1.0-SNAPSHOT-dist.zip -COPY --chown=sunbird ./schema /home/sunbird/content-service-1.0-SNAPSHOT/schemas +COPY --chown=sunbird ./schema/schemas /home/sunbird/content-service-1.0-SNAPSHOT/schemas WORKDIR /home/sunbird/ CMD java -XX:+PrintFlagsFinal $JAVA_OPTIONS -cp '/home/sunbird/content-service-1.0-SNAPSHOT/lib/*' -Dconfig.file=/home/sunbird/content-service-1.0-SNAPSHOT/config/application.conf -Dlogger.file=/home/sunbird/content-service-1.0-SNAPSHOT/config/logback.xml play.core.server.ProdServerStart /home/sunbird/content-service-1.0-SNAPSHOT From c712b7d197a528c5e32731bdaa23bd03adbeeb44 Mon Sep 17 00:00:00 2001 From: gohilamariappan <41056032+gohilamariappan@users.noreply.github.com> Date: Tue, 17 Jan 2023 19:57:12 +0530 Subject: [PATCH 026/126] schema path updated --- build/search-service/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/search-service/Dockerfile b/build/search-service/Dockerfile index be9830e24..1ac8932b1 100644 --- a/build/search-service/Dockerfile +++ b/build/search-service/Dockerfile @@ -9,6 +9,6 @@ USER sunbird COPY ./search-api/search-service/target/search-service-1.0-SNAPSHOT-dist.zip /home/sunbird/ RUN unzip /home/sunbird/search-service-1.0-SNAPSHOT-dist.zip -d /home/sunbird/ RUN rm /home/sunbird/search-service-1.0-SNAPSHOT-dist.zip -COPY --chown=sunbird ./schemas /home/sunbird/search-service-1.0-SNAPSHOT/schemas +COPY --chown=sunbird ./schema/schemas /home/sunbird/search-service-1.0-SNAPSHOT/schemas WORKDIR /home/sunbird/ CMD java -XX:+PrintFlagsFinal $JAVA_OPTIONS -cp '/home/sunbird/search-service-1.0-SNAPSHOT/lib/*' -Dconfig.file=/home/sunbird/search-service-1.0-SNAPSHOT/config/application.conf -Dlogger.file=/home/sunbird/search-service-1.0-SNAPSHOT/config/logback.xml play.core.server.ProdServerStart /home/sunbird/search-service-1.0-SNAPSHOT From 0eefe684a243b0a84f0d5b1e41750aa1dea3a0ec Mon Sep 17 00:00:00 2001 From: gohilamariappan <41056032+gohilamariappan@users.noreply.github.com> Date: Tue, 17 Jan 2023 19:58:19 +0530 Subject: [PATCH 027/126] schema repo added --- build/search-service/Jenkinsfile | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/build/search-service/Jenkinsfile b/build/search-service/Jenkinsfile index a565e7da3..24909a49a 100644 --- a/build/search-service/Jenkinsfile +++ b/build/search-service/Jenkinsfile @@ -28,6 +28,13 @@ node('build-slave') { sh 'mvn clean install -DskipTests=true ' } + stage('schema-pull') { + + dir("${env.WORKSPACE}/schema") { + checkout scm: [$class: 'GitSCM', branches: [[name: schema_repo_branch]], extensions: [[$class: 'CloneOption', depth: 1, noTags: true, reference: '', shallow: true]], userRemoteConfigs: [[credentialsId: schema_repo_credentials, url: schema_repo_url]]] + + } + } stage('Package') { dir('search-api') { From da261805e116a96acf5fd534395f5cab795c82c8 Mon Sep 17 00:00:00 2001 From: gohilamariappan <41056032+gohilamariappan@users.noreply.github.com> Date: Tue, 17 Jan 2023 20:00:16 +0530 Subject: [PATCH 028/126] schema pull added --- build/assessment-service/Jenkinsfile | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/build/assessment-service/Jenkinsfile b/build/assessment-service/Jenkinsfile index ebc9f1a63..e848b98d1 100644 --- a/build/assessment-service/Jenkinsfile +++ b/build/assessment-service/Jenkinsfile @@ -27,6 +27,13 @@ node('build-slave') { print "Environment will be : ${env.NODE_ENV}" sh 'mvn clean install -DskipTests=true ' } + stage('schema-pull') { + + dir("${env.WORKSPACE}/schema") { + checkout scm: [$class: 'GitSCM', branches: [[name: schema_repo_branch]], extensions: [[$class: 'CloneOption', depth: 1, noTags: true, reference: '', shallow: true]], userRemoteConfigs: [[credentialsId: schema_repo_credentials, url: schema_repo_url]]] + + } + } stage('Package') { dir('assessment-api') { From 3fc4a6e432c06da39a37aec3f3517f10b0a7738d Mon Sep 17 00:00:00 2001 From: gohilamariappan <41056032+gohilamariappan@users.noreply.github.com> Date: Tue, 17 Jan 2023 20:00:47 +0530 Subject: [PATCH 029/126] schema pull added --- build/taxonomy-service/Jenkinsfile | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/build/taxonomy-service/Jenkinsfile b/build/taxonomy-service/Jenkinsfile index 453a603f6..96ae351d9 100644 --- a/build/taxonomy-service/Jenkinsfile +++ b/build/taxonomy-service/Jenkinsfile @@ -28,6 +28,13 @@ node('build-slave') { sh 'mvn clean install -DskipTests=true ' } + stage('schema-pull') { + + dir("${env.WORKSPACE}/schema") { + checkout scm: [$class: 'GitSCM', branches: [[name: schema_repo_branch]], extensions: [[$class: 'CloneOption', depth: 1, noTags: true, reference: '', shallow: true]], userRemoteConfigs: [[credentialsId: schema_repo_credentials, url: schema_repo_url]]] + + } + } stage('Package') { dir('taxonomy-api') { From 7aa516100ed50d05e4f0e7ae96bc1a2315fd7b01 Mon Sep 17 00:00:00 2001 From: gohilamariappan <41056032+gohilamariappan@users.noreply.github.com> Date: Tue, 17 Jan 2023 20:01:36 +0530 Subject: [PATCH 030/126] path updated --- build/taxonomy-service/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/taxonomy-service/Dockerfile b/build/taxonomy-service/Dockerfile index 89dae255c..ea9796631 100644 --- a/build/taxonomy-service/Dockerfile +++ b/build/taxonomy-service/Dockerfile @@ -9,6 +9,6 @@ USER sunbird COPY ./taxonomy-api/taxonomy-service/target/taxonomy-service-1.0-SNAPSHOT-dist.zip /home/sunbird/ RUN unzip /home/sunbird/taxonomy-service-1.0-SNAPSHOT-dist.zip -d /home/sunbird/ RUN rm /home/sunbird/taxonomy-service-1.0-SNAPSHOT-dist.zip -COPY --chown=sunbird ./schemas /home/sunbird/taxonomy-service-1.0-SNAPSHOT/schemas +COPY --chown=sunbird ./schema/schemas /home/sunbird/taxonomy-service-1.0-SNAPSHOT/schemas WORKDIR /home/sunbird/ CMD java -XX:+PrintFlagsFinal $JAVA_OPTIONS -cp '/home/sunbird/taxonomy-service-1.0-SNAPSHOT/lib/*' -Dconfig.file=/home/sunbird/taxonomy-service-1.0-SNAPSHOT/config/application.conf -Dlogger.file=/home/sunbird/taxonomy-service-1.0-SNAPSHOT/config/logback.xml play.core.server.ProdServerStart /home/sunbird/taxonomy-service-1.0-SNAPSHOT From 442a34ea00f31ebe5a28ee636731c6ba32bc1ea7 Mon Sep 17 00:00:00 2001 From: gohilamariappan <41056032+gohilamariappan@users.noreply.github.com> Date: Tue, 17 Jan 2023 20:02:08 +0530 Subject: [PATCH 031/126] path updated --- build/assessment-service/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/assessment-service/Dockerfile b/build/assessment-service/Dockerfile index d3b28b2cd..e9f6ebcc1 100644 --- a/build/assessment-service/Dockerfile +++ b/build/assessment-service/Dockerfile @@ -9,6 +9,6 @@ USER sunbird COPY ./assessment-api/assessment-service/target/assessment-service-1.0-SNAPSHOT-dist.zip /home/sunbird/ RUN unzip /home/sunbird/assessment-service-1.0-SNAPSHOT-dist.zip -d /home/sunbird/ RUN rm /home/sunbird/assessment-service-1.0-SNAPSHOT-dist.zip -COPY --chown=sunbird ./schemas /home/sunbird/assessment-service-1.0-SNAPSHOT/schemas +COPY --chown=sunbird ./schema/schemas /home/sunbird/assessment-service-1.0-SNAPSHOT/schemas WORKDIR /home/sunbird/ CMD java -XX:+PrintFlagsFinal $JAVA_OPTIONS -cp '/home/sunbird/assessment-service-1.0-SNAPSHOT/lib/*' -Dconfig.file=/home/sunbird/assessment-service-1.0-SNAPSHOT/config/application.conf -Dlogger.file=/home/sunbird/assessment-service-1.0-SNAPSHOT/config/logback.xml play.core.server.ProdServerStart /home/sunbird/assessment-service-1.0-SNAPSHOT From 1d38f65267a68e2978f805c31be2404603b35134 Mon Sep 17 00:00:00 2001 From: karthik-tarento Date: Fri, 10 Feb 2023 14:41:08 +0530 Subject: [PATCH 032/126] Added new DefinitionScript for AssessmentFeature --- .../Course_assessment_QuestionSet.sh | 39 +++++++++++++++++++ .../Match_the_Following_Question.sh | 15 +++++++ 2 files changed, 54 insertions(+) create mode 100644 scripts/definition-scripts/Course_assessment_QuestionSet.sh create mode 100644 scripts/definition-scripts/Match_the_Following_Question.sh diff --git a/scripts/definition-scripts/Course_assessment_QuestionSet.sh b/scripts/definition-scripts/Course_assessment_QuestionSet.sh new file mode 100644 index 000000000..170bd1d05 --- /dev/null +++ b/scripts/definition-scripts/Course_assessment_QuestionSet.sh @@ -0,0 +1,39 @@ +curl -L -X POST '{{host}}/object/category/definition/v4/create' \ +-H 'Content-Type: application/json' \ +--data-raw '{ + "request": { + "objectCategoryDefinition": { + "categoryId": "obj-cat:course-assessment", + "targetObjectType": "QuestionSet", + "objectMetadata": { + "config": {}, + "schema": { + "properties": { + "trackable": { + "type": "object", + "properties": { + "enabled": { + "type": "string", + "enum": [ + "Yes", + "No" + ], + "default": "No" + }, + "autoBatch": { + "type": "string", + "enum": [ + "Yes", + "No" + ], + "default": "No" + } + }, + "additionalProperties": false + } + } + } + } + } + } +}' \ No newline at end of file diff --git a/scripts/definition-scripts/Match_the_Following_Question.sh b/scripts/definition-scripts/Match_the_Following_Question.sh new file mode 100644 index 000000000..e76736227 --- /dev/null +++ b/scripts/definition-scripts/Match_the_Following_Question.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +curl -L -X POST '{{host}}/object/category/definition/v4/create' \ +-H 'Content-Type: application/json' \ +--data-raw '{ + "request": { + "objectCategoryDefinition": { + "categoryId": "obj-cat:mtf-question", + "targetObjectType": "Question", + "objectMetadata": { + "config": {}, + "schema": {} + } + } + } +}' \ No newline at end of file From 9f4959b55cf45e282e06bd5140079b838595f283 Mon Sep 17 00:00:00 2001 From: karthik-tarento Date: Mon, 13 Feb 2023 06:16:11 +0530 Subject: [PATCH 033/126] Added definition script for practice assessment questionset --- .../Practice_assessment_QuestionSet.sh | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 scripts/definition-scripts/Practice_assessment_QuestionSet.sh diff --git a/scripts/definition-scripts/Practice_assessment_QuestionSet.sh b/scripts/definition-scripts/Practice_assessment_QuestionSet.sh new file mode 100644 index 000000000..d55cb4318 --- /dev/null +++ b/scripts/definition-scripts/Practice_assessment_QuestionSet.sh @@ -0,0 +1,40 @@ +curl -L -X POST '{{host}}/object/category/definition/v4/create' \ +-H 'Content-Type: application/json' \ +--data-raw '{ + "request": { + "objectCategoryDefinition": { + "categoryId": "obj-cat:practice-question-set", + "targetObjectType": "QuestionSet", + "objectMetadata": { + "config": {}, + "schema": { + "properties": { + "trackable": { + "type": "object", + "properties": { + "enabled": { + "type": "string", + "enum": [ + "Yes", + "No" + ], + "default": "No" + }, + "autoBatch": { + "type": "string", + "enum": ["Yes","No"], + "default": "No" + } + }, + "default": { + "enabled": "No", + "autoBatch": "No" + }, + "additionalProperties": false + } + } + } + } + } + } +}' \ No newline at end of file From dedf2a4bc9ece9300e8b41f1456c484a3af0d10f Mon Sep 17 00:00:00 2001 From: Karthikeyan Rajendran <70887864+karthik-tarento@users.noreply.github.com> Date: Wed, 31 May 2023 18:16:45 +0530 Subject: [PATCH 034/126] 4.8.0 content security (#61) * Added validation for secureSettings in content READ API * Added hierarchy API changes * Adding prefix _rc for secure contents * Updated content security validation properly * Added csJwtToken in hierarchy response * default search is modified for contentSecurity (#55) * Updated the attribute name for adding token * Using proper header for reading user's orgId * Content security search service enhancement (#56) * Content security search service enhancement * Using user channel id header only for read * Added logic to generate jwt token * Content Security search API enhancement * Modifications * Modifications on default condition * method name change to formQueryImpl * Enhanced code to read private key from config * Using HS256 algorithm to generate token * Added version details in parent pom file --------- Co-authored-by: wilkysingh-tarento <97211740+wilkysingh-tarento@users.noreply.github.com> Co-authored-by: Sreerag K S <58926794+sreeragksgh@users.noreply.github.com> Co-authored-by: sreeragksgh Co-authored-by: Rangabashyam P K` <46988344+pkranga@users.noreply.github.com> --- .../sunbird/content/actors/ContentActor.scala | 42 +- .../app/controllers/BaseController.scala | 15 + .../controllers/v3/ContentController.scala | 4 +- content-api/hierarchy-manager/pom.xml | 5 + .../sunbird/managers/HierarchyManager.scala | 64 +- .../service/util/BaseQueryGenerationUtil.java | 3 + platform-core/auth-verifier/pom.xml | 93 +++ .../org/sunbird/auth/verifier/Base64Util.java | 741 ++++++++++++++++++ .../org/sunbird/auth/verifier/CryptoUtil.java | 59 ++ .../org/sunbird/auth/verifier/JWTUtil.java | 77 ++ .../sunbird/auth/verifier/JWTokenType.java | 22 + .../org/sunbird/auth/verifier/KeyData.java | 40 + .../org/sunbird/auth/verifier/KeyManager.java | 97 +++ .../auth/verifier/PropertiesCache.java | 97 +++ .../sunbird/auth/verifier/KeyManagerTest.java | 26 + platform-core/pom.xml | 1 + pom.xml | 3 + .../java/org/sunbird/actors/SearchActor.java | 6 + .../org/sunbird/search/dto/SearchDTO.java | 11 +- .../search/processor/SearchProcessor.java | 60 +- .../sunbird/search/util/SearchConstants.java | 1 + .../controllers/SearchBaseController.scala | 2 +- .../search-service/conf/application.conf | 5 +- 23 files changed, 1439 insertions(+), 35 deletions(-) create mode 100644 platform-core/auth-verifier/pom.xml create mode 100644 platform-core/auth-verifier/src/main/java/org/sunbird/auth/verifier/Base64Util.java create mode 100644 platform-core/auth-verifier/src/main/java/org/sunbird/auth/verifier/CryptoUtil.java create mode 100644 platform-core/auth-verifier/src/main/java/org/sunbird/auth/verifier/JWTUtil.java create mode 100644 platform-core/auth-verifier/src/main/java/org/sunbird/auth/verifier/JWTokenType.java create mode 100644 platform-core/auth-verifier/src/main/java/org/sunbird/auth/verifier/KeyData.java create mode 100644 platform-core/auth-verifier/src/main/java/org/sunbird/auth/verifier/KeyManager.java create mode 100644 platform-core/auth-verifier/src/main/java/org/sunbird/auth/verifier/PropertiesCache.java create mode 100644 platform-core/auth-verifier/src/test/java/org/sunbird/auth/verifier/KeyManagerTest.java diff --git a/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala b/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala index 16ebcc64d..f8d641180 100644 --- a/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala +++ b/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala @@ -5,13 +5,15 @@ import java.util.concurrent.CompletionException import java.io.File import org.apache.commons.io.FilenameUtils import javax.inject.Inject +import org.apache.commons.lang3.ObjectUtils import org.apache.commons.lang3.StringUtils +import org.apache.commons.collections4.{CollectionUtils, MapUtils} import org.sunbird.`object`.importer.{ImportConfig, ImportManager} import org.sunbird.actor.core.BaseActor import org.sunbird.cache.impl.RedisCache import org.sunbird.content.util.{AcceptFlagManager, ContentConstants, CopyManager, DiscardManager, FlagManager, RetireManager} import org.sunbird.cloudstore.StorageService -import org.sunbird.common.{ContentParams, Platform, Slug} +import org.sunbird.common.{ContentParams, JsonUtils, Platform, Slug} import org.sunbird.common.dto.{Request, Response, ResponseHandler} import org.sunbird.common.exception.ClientException import org.sunbird.content.dial.DIALManager @@ -73,19 +75,35 @@ class ContentActor @Inject() (implicit oec: OntologyEngineContext, ss: StorageSe DataNode.read(request).map(node => { val metadata: util.Map[String, AnyRef] = NodeUtil.serialize(node, fields, node.getObjectType.toLowerCase.replace("image", ""), request.getContext.get("version").asInstanceOf[String]) metadata.put("identifier", node.getIdentifier.replace(".img", "")) - val response: Response = ResponseHandler.OK - if (responseSchemaName.isEmpty) { - response.put("content", metadata) - } - else { - response.put(responseSchemaName, metadata) - } - if(!StringUtils.equalsIgnoreCase(metadata.get("visibility").asInstanceOf[String],"Private")) { - response - } - else { + if (StringUtils.equalsIgnoreCase(metadata.get("visibility").asInstanceOf[String],"Private")) { throw new ClientException("ERR_ACCESS_DENIED", "content visibility is private, hence access denied") } + var sa = metadata.get("secureSettings") + var securityAttribute : util.Map[String, AnyRef] = new util.HashMap[String, AnyRef] + if(sa.isInstanceOf[String]) { + securityAttribute = JsonUtils.deserialize(sa.asInstanceOf[String], classOf[java.util.Map[String, AnyRef]]) + metadata.put("secureSettings", securityAttribute) + } else if (sa.isInstanceOf[util.Map[String, AnyRef]]) { + securityAttribute = metadata.getOrDefault("secureSettings", new util.HashMap[String, AnyRef]).asInstanceOf[util.Map[String, AnyRef]] + } + //var securityAttribute : util.Map[String, AnyRef] = metadata.getOrDefault("secureSettings", new util.HashMap[String, AnyRef]).asInstanceOf[util.Map[String, AnyRef]] + if (MapUtils.isNotEmpty(securityAttribute)) { + var orgList : util.ArrayList[String] = securityAttribute.getOrDefault("organisation", new util.ArrayList[String]).asInstanceOf[util.ArrayList[String]] + if (!CollectionUtils.isEmpty(orgList)) { + //Content should be read by unique org users only. + var userChannelId : String = request.getRequest.getOrDefault("x-user-channel-id", "").asInstanceOf[String] + if (!orgList.contains(userChannelId)) { + throw new ClientException("ERR_ACCESS_DENIED", "User is not allowed to read this content.") + } + } + } + val response: Response = ResponseHandler.OK + if (responseSchemaName.isEmpty) { + response.put("content", metadata) + } else { + response.put(responseSchemaName, metadata) + } + response }) } diff --git a/content-api/content-service/app/controllers/BaseController.scala b/content-api/content-service/app/controllers/BaseController.scala index 3b7125e19..f915e9240 100644 --- a/content-api/content-service/app/controllers/BaseController.scala +++ b/content-api/content-service/app/controllers/BaseController.scala @@ -209,4 +209,19 @@ abstract class BaseController(protected val cc: ControllerComponents)(implicit e Future(BadRequest(JavaJsonUtils.serialize(result)).as("application/json")) } + def commonReadHeaders(ignoreHeaders: Option[List[String]] = Option(List()))(implicit request: Request[AnyContent]): java.util.Map[String, Object] = { + val customHeaders = Map("x-authenticated-user-orgid" -> "x-user-channel-id", "x-channel-id" -> "channel", "X-Consumer-ID" -> "consumerId", "X-App-Id" -> "appId").filterKeys(key => !ignoreHeaders.getOrElse(List()).contains(key)) + customHeaders.map(ch => { + val value = request.headers.get(ch._1) + if (value.isDefined && !value.isEmpty) { + collection.mutable.HashMap[String, Object](ch._2 -> value.get).asJava + } else { + collection.mutable.HashMap[String, Object]().asJava + } + }).reduce((a, b) => { + a.putAll(b) + return a + }) + } + } diff --git a/content-api/content-service/app/controllers/v3/ContentController.scala b/content-api/content-service/app/controllers/v3/ContentController.scala index 05c5b2470..a5c73c51a 100644 --- a/content-api/content-service/app/controllers/v3/ContentController.scala +++ b/content-api/content-service/app/controllers/v3/ContentController.scala @@ -43,7 +43,7 @@ class ContentController @Inject()(@Named(ActorNames.CONTENT_ACTOR) contentActor: * @return */ def read(identifier: String, mode: Option[String], fields: Option[String]) = Action.async { implicit request => - val headers = commonHeaders() + val headers = commonReadHeaders() val content = new java.util.HashMap().asInstanceOf[java.util.Map[String, Object]] content.putAll(headers) content.putAll(Map("identifier" -> identifier, "mode" -> mode.getOrElse("read"), "fields" -> fields.getOrElse("")).asJava) @@ -94,7 +94,7 @@ class ContentController @Inject()(@Named(ActorNames.CONTENT_ACTOR) contentActor: } def getHierarchy(identifier: String, mode: Option[String]) = Action.async { implicit request => - val headers = commonHeaders() + val headers = commonReadHeaders() val content = new java.util.HashMap().asInstanceOf[java.util.Map[String, Object]] content.putAll(headers) content.putAll(Map("rootId" -> identifier, "mode" -> mode.getOrElse("")).asJava) diff --git a/content-api/hierarchy-manager/pom.xml b/content-api/hierarchy-manager/pom.xml index 828dac9fc..3ef8d21c8 100644 --- a/content-api/hierarchy-manager/pom.xml +++ b/content-api/hierarchy-manager/pom.xml @@ -12,6 +12,11 @@ hierarchy-manager + + org.sunbird + auth-verifier + 1.0-SNAPSHOT + org.sunbird graph-engine_2.11 diff --git a/content-api/hierarchy-manager/src/main/scala/org/sunbird/managers/HierarchyManager.scala b/content-api/hierarchy-manager/src/main/scala/org/sunbird/managers/HierarchyManager.scala index 186232d39..1019d0fc4 100644 --- a/content-api/hierarchy-manager/src/main/scala/org/sunbird/managers/HierarchyManager.scala +++ b/content-api/hierarchy-manager/src/main/scala/org/sunbird/managers/HierarchyManager.scala @@ -4,6 +4,7 @@ import java.util import java.util.concurrent.CompletionException import com.fasterxml.jackson.databind.ObjectMapper import org.apache.commons.lang3.StringUtils +import org.sunbird.auth.verifier.JWTUtil import org.sunbird.cache.impl.RedisCache import org.sunbird.common.dto.{Request, Response, ResponseHandler, ResponseParams} import org.sunbird.common.exception.{ClientException, ErrorCodes, ResourceNotFoundException, ResponseCode, ServerException} @@ -132,7 +133,9 @@ object HierarchyManager { } val bookmarkId = request.get("bookmarkId").asInstanceOf[String] var metadata: util.Map[String, AnyRef] = NodeUtil.serialize(rootNode, new util.ArrayList[String](), request.getContext.get("schemaName").asInstanceOf[String], request.getContext.get("version").asInstanceOf[String]) - + if (!validateContentSecurity(request, metadata)) { + Future(ResponseHandler.ERROR(ResponseCode.RESOURCE_NOT_FOUND, ResponseCode.RESOURCE_NOT_FOUND.name(), "User can't read content with Id: " + request.get("rootId"))) + } fetchRelationalMetadata(request, rootNode.getIdentifier).map(collRelationalMetadata => { val hierarchy = fetchHierarchy(request, rootNode.getIdentifier) @@ -211,15 +214,24 @@ object HierarchyManager { if (!result.isEmpty) { val bookmarkId = request.get("bookmarkId").asInstanceOf[String] val rootHierarchy = result.get("content").asInstanceOf[util.Map[String, AnyRef]] - if (StringUtils.isEmpty(bookmarkId)) { - ResponseHandler.OK.put("content", rootHierarchy) + if (!validateContentSecurity(request, rootHierarchy)) { + ResponseHandler.ERROR(ResponseCode.RESOURCE_NOT_FOUND, ResponseCode.RESOURCE_NOT_FOUND.name(), "User can't read content with Id: " + request.get("rootId")) } else { - val children = rootHierarchy.getOrElse("children", new util.ArrayList[util.Map[String, AnyRef]]()).asInstanceOf[util.List[util.Map[String, AnyRef]]] - val bookmarkHierarchy = filterBookmarkHierarchy(children, bookmarkId) - if (MapUtils.isEmpty(bookmarkHierarchy)) { - ResponseHandler.ERROR(ResponseCode.RESOURCE_NOT_FOUND, ResponseCode.RESOURCE_NOT_FOUND.name(), "bookmarkId " + bookmarkId + " does not exist") + if (isSecureContent(rootHierarchy)) { + val csToken = generateCSToken(rootHierarchy.get("childNodes").asInstanceOf[util.List[String]]) + rootHierarchy.put("cstoken", csToken) + } + + if (StringUtils.isEmpty(bookmarkId)) { + ResponseHandler.OK.put("content", rootHierarchy) } else { - ResponseHandler.OK.put("content", bookmarkHierarchy) + val children = rootHierarchy.getOrElse("children", new util.ArrayList[util.Map[String, AnyRef]]()).asInstanceOf[util.List[util.Map[String, AnyRef]]] + val bookmarkHierarchy = filterBookmarkHierarchy(children, bookmarkId) + if (MapUtils.isEmpty(bookmarkHierarchy)) { + ResponseHandler.ERROR(ResponseCode.RESOURCE_NOT_FOUND, ResponseCode.RESOURCE_NOT_FOUND.name(), "bookmarkId " + bookmarkId + " does not exist") + } else { + ResponseHandler.OK.put("content", bookmarkHierarchy) + } } } } else @@ -713,4 +725,40 @@ object HierarchyManager { if(configObjTypes.nonEmpty && !configObjTypes.contains(childNode.getOrDefault("objectType", "").asInstanceOf[String])) throw new ClientException("ERR_INVALID_CHILDREN", "Invalid Children objectType "+childNode.get("objectType")+" found for : "+childNode.get("identifier") + "| Please provide children having one of the objectType from "+ configObjTypes.asJava) } + + def isSecureContent (metadata: util.Map[String, AnyRef])(implicit ec: ExecutionContext): Boolean = { + var securityAttribute : util.Map[String, AnyRef] = metadata.getOrDefault("secureSettings", new util.HashMap[String, AnyRef]).asInstanceOf[util.Map[String, AnyRef]] + var isSecureContent = false + if (MapUtils.isNotEmpty(securityAttribute)) { + var orgList : util.ArrayList[String] = securityAttribute.getOrDefault("organisation", new util.ArrayList[String]).asInstanceOf[util.ArrayList[String]] + if (!CollectionUtils.isEmpty(orgList)) { + isSecureContent = true + } + } + isSecureContent + } + + def validateContentSecurity(request: Request, metadata: util.Map[String, AnyRef])(implicit ec: ExecutionContext): Boolean = { + var securityAttribute : util.Map[String, AnyRef] = metadata.getOrDefault("secureSettings", new util.HashMap[String, AnyRef]).asInstanceOf[util.Map[String, AnyRef]] + var isUserAllowedToRead = true + if (MapUtils.isNotEmpty(securityAttribute)) { + var orgList : util.ArrayList[String] = securityAttribute.getOrDefault("organisation", new util.ArrayList[String]).asInstanceOf[util.ArrayList[String]] + if (!CollectionUtils.isEmpty(orgList)) { + //Content should be read by unique org users only. + var userChannelId : String = request.getRequest.getOrDefault("x-user-channel-id", "").asInstanceOf[String] + if (!orgList.contains(userChannelId)) { + isUserAllowedToRead = false + } + } + } + isUserAllowedToRead + } + + def generateCSToken(children: util.List[String])(implicit ec: ExecutionContext): String = { + var csToken = ""; + var claimsMap : util.Map[String, AnyRef] = new util.HashMap[String, AnyRef] + claimsMap.put("contentIdentifier", children) + csToken = JWTUtil.createHS256Token(claimsMap) + csToken + } } diff --git a/ontology-engine/graph-dac-api/src/main/java/org/sunbird/graph/service/util/BaseQueryGenerationUtil.java b/ontology-engine/graph-dac-api/src/main/java/org/sunbird/graph/service/util/BaseQueryGenerationUtil.java index df732461d..09fabece2 100644 --- a/ontology-engine/graph-dac-api/src/main/java/org/sunbird/graph/service/util/BaseQueryGenerationUtil.java +++ b/ontology-engine/graph-dac-api/src/main/java/org/sunbird/graph/service/util/BaseQueryGenerationUtil.java @@ -172,6 +172,9 @@ protected static Map getSystemPropertyQueryMap(Node node, String if (StringUtils.isBlank(node.getIdentifier())) node.setIdentifier(Identifier.getIdentifier(node.getGraphId(), Identifier.getUniqueIdFromTimestamp())); + if (node.getMetadata().containsKey("secureSettings")) { + node.setIdentifier(node.getIdentifier() + "_rc"); + } // Adding 'IL_UNIQUE_ID' Property query.append( SystemProperties.IL_UNIQUE_ID.name() + ": { SP_" + SystemProperties.IL_UNIQUE_ID.name() + " }, "); diff --git a/platform-core/auth-verifier/pom.xml b/platform-core/auth-verifier/pom.xml new file mode 100644 index 000000000..196b91bde --- /dev/null +++ b/platform-core/auth-verifier/pom.xml @@ -0,0 +1,93 @@ + + + + platform-core + org.sunbird + 1.0-SNAPSHOT + + 4.0.0 + + auth-verifier + + + + com.fasterxml.jackson.core + jackson-core + ${fasterxml.jackson.version} + + + org.sunbird + common-util + 0.0.1-SNAPSHOT + + + org.sunbird + platform-telemetry + 1.0-SNAPSHOT + jar + + + org.mockito + mockito-core + ${mockito.core.version} + test + + + org.powermock + powermock-api-mockito2 + ${powermock.api.mockito2.version} + test + + + org.powermock + powermock-module-junit4 + ${powermock.module.junit4.version} + test + + + ch.qos.logback + logback-classic + 1.2.3 + + + net.logstash.logback + logstash-logback-encoder + 6.3 + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.8.1 + + 11 + + + + org.jacoco + jacoco-maven-plugin + 0.8.5 + + + default-prepare-agent + + prepare-agent + + + + default-report + prepare-package + + report + + + + + + + \ No newline at end of file diff --git a/platform-core/auth-verifier/src/main/java/org/sunbird/auth/verifier/Base64Util.java b/platform-core/auth-verifier/src/main/java/org/sunbird/auth/verifier/Base64Util.java new file mode 100644 index 000000000..619330d24 --- /dev/null +++ b/platform-core/auth-verifier/src/main/java/org/sunbird/auth/verifier/Base64Util.java @@ -0,0 +1,741 @@ +package org.sunbird.auth.verifier; + +/* + * Copyright (C) 2010 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import java.io.UnsupportedEncodingException; + +/** + * Utilities for encoding and decoding the Base64 representation of + * binary data. See RFCs 2045 and 3548. + */ +public class Base64Util { + /** + * Default values for encoder/decoder flags. + */ + public static final int DEFAULT = 0; + + /** + * Encoder flag bit to omit the padding '=' characters at the end + * of the output (if any). + */ + public static final int NO_PADDING = 1; + + /** + * Encoder flag bit to omit all line terminators (i.e., the output + * will be on one long line). + */ + public static final int NO_WRAP = 2; + + /** + * Encoder flag bit to indicate lines should be terminated with a + * CRLF pair instead of just an LF. Has no effect if {@code + * NO_WRAP} is specified as well. + */ + public static final int CRLF = 4; + + /** + * Encoder/decoder flag bit to indicate using the "URL and + * filename safe" variant of Base64 (see RFC 3548 section 4) where + * {@code -} and {@code _} are used in place of {@code +} and + * {@code /}. + */ + public static final int URL_SAFE = 8; + + /** + * Flag to pass to {Base64OutputStream} to indicate that it + * should not close the output stream it is wrapping when it + * itself is closed. + */ + public static final int NO_CLOSE = 16; + + // -------------------------------------------------------- + // shared code + // -------------------------------------------------------- + + private Base64Util() { + } // don't instantiate + + // -------------------------------------------------------- + // decoding + // -------------------------------------------------------- + + /** + * Decode the Base64-encoded data in input and return the data in + * a new byte array. + *

+ *

The padding '=' characters at the end are considered optional, but + * if any are present, there must be the correct number of them. + * + * @param str the input String to decode, which is converted to + * bytes using the default charset + * @param flags controls certain features of the decoded output. + * Pass {@code DEFAULT} to decode standard Base64. + * @throws IllegalArgumentException if the input contains + * incorrect padding + */ + public static byte[] decode(String str, int flags) { + return decode(str.getBytes(), flags); + } + + /** + * Decode the Base64-encoded data in input and return the data in + * a new byte array. + *

+ *

The padding '=' characters at the end are considered optional, but + * if any are present, there must be the correct number of them. + * + * @param input the input array to decode + * @param flags controls certain features of the decoded output. + * Pass {@code DEFAULT} to decode standard Base64. + * @throws IllegalArgumentException if the input contains + * incorrect padding + */ + public static byte[] decode(byte[] input, int flags) { + return decode(input, 0, input.length, flags); + } + + /** + * Decode the Base64-encoded data in input and return the data in + * a new byte array. + *

+ *

The padding '=' characters at the end are considered optional, but + * if any are present, there must be the correct number of them. + * + * @param input the data to decode + * @param offset the position within the input array at which to start + * @param len the number of bytes of input to decode + * @param flags controls certain features of the decoded output. + * Pass {@code DEFAULT} to decode standard Base64. + * @throws IllegalArgumentException if the input contains + * incorrect padding + */ + public static byte[] decode(byte[] input, int offset, int len, int flags) { + // Allocate space for the most data the input could represent. + // (It could contain less if it contains whitespace, etc.) + Decoder decoder = new Decoder(flags, new byte[len * 3 / 4]); + + if (!decoder.process(input, offset, len, true)) { + throw new IllegalArgumentException("bad base-64"); + } + + // Maybe we got lucky and allocated exactly enough output space. + if (decoder.op == decoder.output.length) { + return decoder.output; + } + + // Need to shorten the array, so allocate a new one of the + // right size and copy. + byte[] temp = new byte[decoder.op]; + System.arraycopy(decoder.output, 0, temp, 0, decoder.op); + return temp; + } + + /** + * Base64-encode the given data and return a newly allocated + * String with the result. + * + * @param input the data to encode + * @param flags controls certain features of the encoded output. + * Passing {@code DEFAULT} results in output that + * adheres to RFC 2045. + */ + public static String encodeToString(byte[] input, int flags) { + try { + return new String(encode(input, flags), "US-ASCII"); + } catch (UnsupportedEncodingException e) { + // US-ASCII is guaranteed to be available. + throw new AssertionError(e); + } + } + + // -------------------------------------------------------- + // encoding + // -------------------------------------------------------- + + /** + * Base64-encode the given data and return a newly allocated + * String with the result. + * + * @param input the data to encode + * @param offset the position within the input array at which to + * start + * @param len the number of bytes of input to encode + * @param flags controls certain features of the encoded output. + * Passing {@code DEFAULT} results in output that + * adheres to RFC 2045. + */ + public static String encodeToString(byte[] input, int offset, int len, int flags) { + try { + return new String(encode(input, offset, len, flags), "US-ASCII"); + } catch (UnsupportedEncodingException e) { + // US-ASCII is guaranteed to be available. + throw new AssertionError(e); + } + } + + /** + * Base64-encode the given data and return a newly allocated + * byte[] with the result. + * + * @param input the data to encode + * @param flags controls certain features of the encoded output. + * Passing {@code DEFAULT} results in output that + * adheres to RFC 2045. + */ + public static byte[] encode(byte[] input, int flags) { + return encode(input, 0, input.length, flags); + } + + /** + * Base64-encode the given data and return a newly allocated + * byte[] with the result. + * + * @param input the data to encode + * @param offset the position within the input array at which to + * start + * @param len the number of bytes of input to encode + * @param flags controls certain features of the encoded output. + * Passing {@code DEFAULT} results in output that + * adheres to RFC 2045. + */ + public static byte[] encode(byte[] input, int offset, int len, int flags) { + Encoder encoder = new Encoder(flags, null); + + // Compute the exact length of the array we will produce. + int output_len = len / 3 * 4; + + // Account for the tail of the data and the padding bytes, if any. + if (encoder.do_padding) { + if (len % 3 > 0) { + output_len += 4; + } + } else { + switch (len % 3) { + case 0: + break; + case 1: + output_len += 2; + break; + case 2: + output_len += 3; + break; + } + } + + // Account for the newlines, if any. + if (encoder.do_newline && len > 0) { + output_len += (((len - 1) / (3 * Encoder.LINE_GROUPS)) + 1) * + (encoder.do_cr ? 2 : 1); + } + + encoder.output = new byte[output_len]; + encoder.process(input, offset, len, true); + + assert encoder.op == output_len; + + return encoder.output; + } + + /* package */ static abstract class Coder { + public byte[] output; + public int op; + + /** + * Encode/decode another block of input data. this.output is + * provided by the caller, and must be big enough to hold all + * the coded data. On exit, this.opwill be set to the length + * of the coded data. + * + * @param finish true if this is the final call to process for + * this object. Will finalize the coder state and + * include any final bytes in the output. + * @return true if the input so far is good; false if some + * error has been detected in the input stream.. + */ + public abstract boolean process(byte[] input, int offset, int len, boolean finish); + + /** + * @return the maximum number of bytes a call to process() + * could produce for the given number of input bytes. This may + * be an overestimate. + */ + public abstract int maxOutputSize(int len); + } + + /* package */ static class Decoder extends Coder { + /** + * Lookup table for turning bytes into their position in the + * Base64 alphabet. + */ + private static final int DECODE[] = { + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 62, -1, -1, -1, 63, + 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, -1, -1, -1, -2, -1, -1, + -1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, + 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, -1, -1, -1, -1, -1, + -1, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, + 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + }; + + /** + * Decode lookup table for the "web safe" variant (RFC 3548 + * sec. 4) where - and _ replace + and /. + */ + private static final int DECODE_WEBSAFE[] = { + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 62, -1, -1, + 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, -1, -1, -1, -2, -1, -1, + -1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, + 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, -1, -1, -1, -1, 63, + -1, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, + 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + }; + + /** + * Non-data values in the DECODE arrays. + */ + private static final int SKIP = -1; + private static final int EQUALS = -2; + final private int[] alphabet; + /** + * States 0-3 are reading through the next input tuple. + * State 4 is having read one '=' and expecting exactly + * one more. + * State 5 is expecting no more data or padding characters + * in the input. + * State 6 is the error state; an error has been detected + * in the input and no future input can "fix" it. + */ + private int state; // state number (0 to 6) + private int value; + + public Decoder(int flags, byte[] output) { + this.output = output; + + alphabet = ((flags & URL_SAFE) == 0) ? DECODE : DECODE_WEBSAFE; + state = 0; + value = 0; + } + + /** + * @return an overestimate for the number of bytes {@code + * len} bytes could decode to. + */ + public int maxOutputSize(int len) { + return len * 3 / 4 + 10; + } + + /** + * Decode another block of input data. + * + * @return true if the state machine is still healthy. false if + * bad base-64 data has been detected in the input stream. + */ + public boolean process(byte[] input, int offset, int len, boolean finish) { + if (this.state == 6) return false; + + int p = offset; + len += offset; + + // Using local variables makes the decoder about 12% + // faster than if we manipulate the member variables in + // the loop. (Even alphabet makes a measurable + // difference, which is somewhat surprising to me since + // the member variable is final.) + int state = this.state; + int value = this.value; + int op = 0; + final byte[] output = this.output; + final int[] alphabet = this.alphabet; + + while (p < len) { + // Try the fast path: we're starting a new tuple and the + // next four bytes of the input stream are all data + // bytes. This corresponds to going through states + // 0-1-2-3-0. We expect to use this method for most of + // the data. + // + // If any of the next four bytes of input are non-data + // (whitespace, etc.), value will end up negative. (All + // the non-data values in decode are small negative + // numbers, so shifting any of them up and or'ing them + // together will result in a value with its top bit set.) + // + // You can remove this whole block and the output should + // be the same, just slower. + if (state == 0) { + while (p + 4 <= len && + (value = ((alphabet[input[p] & 0xff] << 18) | + (alphabet[input[p + 1] & 0xff] << 12) | + (alphabet[input[p + 2] & 0xff] << 6) | + (alphabet[input[p + 3] & 0xff]))) >= 0) { + output[op + 2] = (byte) value; + output[op + 1] = (byte) (value >> 8); + output[op] = (byte) (value >> 16); + op += 3; + p += 4; + } + if (p >= len) break; + } + + // The fast path isn't available -- either we've read a + // partial tuple, or the next four input bytes aren't all + // data, or whatever. Fall back to the slower state + // machine implementation. + + int d = alphabet[input[p++] & 0xff]; + + switch (state) { + case 0: + if (d >= 0) { + value = d; + ++state; + } else if (d != SKIP) { + this.state = 6; + return false; + } + break; + + case 1: + if (d >= 0) { + value = (value << 6) | d; + ++state; + } else if (d != SKIP) { + this.state = 6; + return false; + } + break; + + case 2: + if (d >= 0) { + value = (value << 6) | d; + ++state; + } else if (d == EQUALS) { + // Emit the last (partial) output tuple; + // expect exactly one more padding character. + output[op++] = (byte) (value >> 4); + state = 4; + } else if (d != SKIP) { + this.state = 6; + return false; + } + break; + + case 3: + if (d >= 0) { + // Emit the output triple and return to state 0. + value = (value << 6) | d; + output[op + 2] = (byte) value; + output[op + 1] = (byte) (value >> 8); + output[op] = (byte) (value >> 16); + op += 3; + state = 0; + } else if (d == EQUALS) { + // Emit the last (partial) output tuple; + // expect no further data or padding characters. + output[op + 1] = (byte) (value >> 2); + output[op] = (byte) (value >> 10); + op += 2; + state = 5; + } else if (d != SKIP) { + this.state = 6; + return false; + } + break; + + case 4: + if (d == EQUALS) { + ++state; + } else if (d != SKIP) { + this.state = 6; + return false; + } + break; + + case 5: + if (d != SKIP) { + this.state = 6; + return false; + } + break; + } + } + + if (!finish) { + // We're out of input, but a future call could provide + // more. + this.state = state; + this.value = value; + this.op = op; + return true; + } + + // Done reading input. Now figure out where we are left in + // the state machine and finish up. + + switch (state) { + case 0: + // Output length is a multiple of three. Fine. + break; + case 1: + // Read one extra input byte, which isn't enough to + // make another output byte. Illegal. + this.state = 6; + return false; + case 2: + // Read two extra input bytes, enough to emit 1 more + // output byte. Fine. + output[op++] = (byte) (value >> 4); + break; + case 3: + // Read three extra input bytes, enough to emit 2 more + // output bytes. Fine. + output[op++] = (byte) (value >> 10); + output[op++] = (byte) (value >> 2); + break; + case 4: + // Read one padding '=' when we expected 2. Illegal. + this.state = 6; + return false; + case 5: + // Read all the padding '='s we expected and no more. + // Fine. + break; + } + + this.state = state; + this.op = op; + return true; + } + } + + /* package */ static class Encoder extends Coder { + /** + * Emit a new line every this many output tuples. Corresponds to + * a 76-character line length (the maximum allowable according to + * RFC 2045). + */ + public static final int LINE_GROUPS = 19; + + /** + * Lookup table for turning Base64 alphabet positions (6 bits) + * into output bytes. + */ + private static final byte ENCODE[] = { + 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', + 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', + 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', + 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '+', '/', + }; + + /** + * Lookup table for turning Base64 alphabet positions (6 bits) + * into output bytes. + */ + private static final byte ENCODE_WEBSAFE[] = { + 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', + 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', + 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', + 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '-', '_', + }; + final public boolean do_padding; + final public boolean do_newline; + final public boolean do_cr; + final private byte[] tail; + final private byte[] alphabet; + /* package */ int tailLen; + private int count; + + public Encoder(int flags, byte[] output) { + this.output = output; + + do_padding = (flags & NO_PADDING) == 0; + do_newline = (flags & NO_WRAP) == 0; + do_cr = (flags & CRLF) != 0; + alphabet = ((flags & URL_SAFE) == 0) ? ENCODE : ENCODE_WEBSAFE; + + tail = new byte[2]; + tailLen = 0; + + count = do_newline ? LINE_GROUPS : -1; + } + + /** + * @return an overestimate for the number of bytes {@code + * len} bytes could encode to. + */ + public int maxOutputSize(int len) { + return len * 8 / 5 + 10; + } + + public boolean process(byte[] input, int offset, int len, boolean finish) { + // Using local variables makes the encoder about 9% faster. + final byte[] alphabet = this.alphabet; + final byte[] output = this.output; + int op = 0; + int count = this.count; + + int p = offset; + len += offset; + int v = -1; + + // First we need to concatenate the tail of the previous call + // with any input bytes available now and see if we can empty + // the tail. + + switch (tailLen) { + case 0: + // There was no tail. + break; + + case 1: + if (p + 2 <= len) { + // A 1-byte tail with at least 2 bytes of + // input available now. + v = ((tail[0] & 0xff) << 16) | + ((input[p++] & 0xff) << 8) | + (input[p++] & 0xff); + tailLen = 0; + } + ; + break; + + case 2: + if (p + 1 <= len) { + // A 2-byte tail with at least 1 byte of input. + v = ((tail[0] & 0xff) << 16) | + ((tail[1] & 0xff) << 8) | + (input[p++] & 0xff); + tailLen = 0; + } + break; + } + + if (v != -1) { + output[op++] = alphabet[(v >> 18) & 0x3f]; + output[op++] = alphabet[(v >> 12) & 0x3f]; + output[op++] = alphabet[(v >> 6) & 0x3f]; + output[op++] = alphabet[v & 0x3f]; + if (--count == 0) { + if (do_cr) output[op++] = '\r'; + output[op++] = '\n'; + count = LINE_GROUPS; + } + } + + // At this point either there is no tail, or there are fewer + // than 3 bytes of input available. + + // The main loop, turning 3 input bytes into 4 output bytes on + // each iteration. + while (p + 3 <= len) { + v = ((input[p] & 0xff) << 16) | + ((input[p + 1] & 0xff) << 8) | + (input[p + 2] & 0xff); + output[op] = alphabet[(v >> 18) & 0x3f]; + output[op + 1] = alphabet[(v >> 12) & 0x3f]; + output[op + 2] = alphabet[(v >> 6) & 0x3f]; + output[op + 3] = alphabet[v & 0x3f]; + p += 3; + op += 4; + if (--count == 0) { + if (do_cr) output[op++] = '\r'; + output[op++] = '\n'; + count = LINE_GROUPS; + } + } + + if (finish) { + // Finish up the tail of the input. Note that we need to + // consume any bytes in tail before any bytes + // remaining in input; there should be at most two bytes + // total. + + if (p - tailLen == len - 1) { + int t = 0; + v = ((tailLen > 0 ? tail[t++] : input[p++]) & 0xff) << 4; + tailLen -= t; + output[op++] = alphabet[(v >> 6) & 0x3f]; + output[op++] = alphabet[v & 0x3f]; + if (do_padding) { + output[op++] = '='; + output[op++] = '='; + } + if (do_newline) { + if (do_cr) output[op++] = '\r'; + output[op++] = '\n'; + } + } else if (p - tailLen == len - 2) { + int t = 0; + v = (((tailLen > 1 ? tail[t++] : input[p++]) & 0xff) << 10) | + (((tailLen > 0 ? tail[t++] : input[p++]) & 0xff) << 2); + tailLen -= t; + output[op++] = alphabet[(v >> 12) & 0x3f]; + output[op++] = alphabet[(v >> 6) & 0x3f]; + output[op++] = alphabet[v & 0x3f]; + if (do_padding) { + output[op++] = '='; + } + if (do_newline) { + if (do_cr) output[op++] = '\r'; + output[op++] = '\n'; + } + } else if (do_newline && op > 0 && count != LINE_GROUPS) { + if (do_cr) output[op++] = '\r'; + output[op++] = '\n'; + } + + assert tailLen == 0; + assert p == len; + } else { + // Save the leftovers in tail to be consumed on the next + // call to encodeInternal. + + if (p == len - 1) { + tail[tailLen++] = input[p]; + } else if (p == len - 2) { + tail[tailLen++] = input[p]; + tail[tailLen++] = input[p + 1]; + } + } + + this.op = op; + this.count = count; + + return true; + } + } +} diff --git a/platform-core/auth-verifier/src/main/java/org/sunbird/auth/verifier/CryptoUtil.java b/platform-core/auth-verifier/src/main/java/org/sunbird/auth/verifier/CryptoUtil.java new file mode 100644 index 000000000..2f83d55b6 --- /dev/null +++ b/platform-core/auth-verifier/src/main/java/org/sunbird/auth/verifier/CryptoUtil.java @@ -0,0 +1,59 @@ +package org.sunbird.auth.verifier; + +import java.nio.charset.Charset; +import java.security.*; + +import javax.crypto.Mac; +import javax.crypto.spec.SecretKeySpec; + +import org.sunbird.telemetry.logger.TelemetryManager; + + +public class CryptoUtil { + private static final Charset US_ASCII = Charset.forName("US-ASCII"); + + public static boolean verifyRSASign(String payLoad, byte[] signature, PublicKey key, String algorithm) { + Signature sign; + try { + sign = Signature.getInstance(algorithm); + sign.initVerify(key); + sign.update(payLoad.getBytes(US_ASCII)); + return sign.verify(signature); + } catch (NoSuchAlgorithmException e) { + return false; + } catch (InvalidKeyException e){ + return false; + } catch (SignatureException e){ + return false; + } + } + + public static byte[] generateHMAC(String payLoad, String secretKey, String algorithm) { + Mac mac; + byte[] signature; + try { + mac = Mac.getInstance(algorithm); + mac.init(new SecretKeySpec(secretKey.getBytes(), algorithm)); + signature = mac.doFinal(payLoad.getBytes(US_ASCII)); + } catch (NoSuchAlgorithmException | InvalidKeyException e) { + TelemetryManager.error("CryptoUtil:generateHMAC :: failed to generate signature. Exception: ", e); + return null; + } + return signature; + } + + public static byte[] generateRSASign(String payLoad, PrivateKey key, String algorithm) { + Signature sign; + byte[] signature; + try { + sign = Signature.getInstance(algorithm); + sign.initSign(key); + sign.update(payLoad.getBytes(US_ASCII)); + signature = sign.sign(); + } catch (NoSuchAlgorithmException | InvalidKeyException | SignatureException e) { + TelemetryManager.error("CryptoUtil:generateRSASign :: failed to generate signature. Exception: ", e); + return null; + } + return signature; + } +} diff --git a/platform-core/auth-verifier/src/main/java/org/sunbird/auth/verifier/JWTUtil.java b/platform-core/auth-verifier/src/main/java/org/sunbird/auth/verifier/JWTUtil.java new file mode 100644 index 000000000..b382ae3d3 --- /dev/null +++ b/platform-core/auth-verifier/src/main/java/org/sunbird/auth/verifier/JWTUtil.java @@ -0,0 +1,77 @@ +package org.sunbird.auth.verifier; + +import org.sunbird.common.JsonUtils; +import org.sunbird.common.Platform; +import org.sunbird.telemetry.logger.TelemetryManager; + +import java.util.HashMap; +import java.util.Map; + +public class JWTUtil { + private static String SEPARATOR = "."; + + public static String JWT_SECRET_STRING = Platform.config.hasPath("content.security.jwt.secret") ? + Platform.config.getString("content.security.jwt.secret"): "sunbird"; + + public static String createHS256Token(Map claimsMap) { + String token = ""; + JWTokenType tokenType = JWTokenType.HS256; + try { + Map headerOptions = new HashMap(); + String payLoad = createHeader(tokenType, headerOptions) + SEPARATOR + createClaimsMap(claimsMap); + String signature = encodeToBase64Uri( + CryptoUtil.generateHMAC(payLoad, JWT_SECRET_STRING, tokenType.getAlgorithmName())); + token = payLoad + SEPARATOR + signature; + } catch (Exception e) { + TelemetryManager.error("JWTUtil.createHS256Token :: Failed to create RS256 token. Exception: ", e); + } + return token; + } + + public static String createRS256Token(Map claimsMap) { + String token = ""; + JWTokenType tokenType = JWTokenType.RS256; + try { + KeyData keyData = KeyManager.getRandomKey(); + if (keyData != null) { + Map headerOptions = createHeaderOptions(keyData.getKeyId()); + String payLoad = createHeader(tokenType, headerOptions) + SEPARATOR + createClaimsMap(claimsMap); + String signature = encodeToBase64Uri( + CryptoUtil.generateRSASign(payLoad, keyData.getPrivateKey(), tokenType.getAlgorithmName())); + token = payLoad + SEPARATOR + signature; + } else { + TelemetryManager.error("JWTUtil.createRS256Token :: KeyManager is not initialized properly."); + } + } catch (Exception e) { + TelemetryManager.error("JWTUtil.createRS256Token :: Failed to create RS256 token. Exception: ", e); + } + return token; + } + + private static String createHeader(JWTokenType tokenType, Map headerOptions) throws Exception { + Map headerData = new HashMap<>(); + if (headerOptions != null) + headerData.putAll(headerOptions); + headerData.put("alg", tokenType.getTokenType()); + headerData.put("typ", "JWT"); + return encodeToBase64Uri(JsonUtils.serialize(headerData).getBytes()); + } + + private static Map createHeaderOptions(String keyId) { + Map headers = new HashMap<>(); + headers.put("kid", keyId); + return headers; + } + + private static String createClaimsMap(Map claimsMap) throws Exception { + Map payloadData = new HashMap<>(); + if (claimsMap != null && claimsMap.size() > 0) { + payloadData.putAll(claimsMap); + } + return encodeToBase64Uri(JsonUtils.serialize(payloadData).getBytes()); + } + + private static String encodeToBase64Uri(byte[] data) { + return Base64Util.encodeToString(data, 11); + } +} diff --git a/platform-core/auth-verifier/src/main/java/org/sunbird/auth/verifier/JWTokenType.java b/platform-core/auth-verifier/src/main/java/org/sunbird/auth/verifier/JWTokenType.java new file mode 100644 index 000000000..0632e084d --- /dev/null +++ b/platform-core/auth-verifier/src/main/java/org/sunbird/auth/verifier/JWTokenType.java @@ -0,0 +1,22 @@ +package org.sunbird.auth.verifier; + +public enum JWTokenType { + HS256("HS256", "HmacSHA256"), + RS256("RS256", "SHA256withRSA"); + + private String algorithmName; + private String tokenType; + + JWTokenType(String tokenType, String algorithmName) { + this.algorithmName = algorithmName; + this.tokenType = tokenType; + } + + public String getAlgorithmName() { + return algorithmName; + } + + public String getTokenType() { + return tokenType; + } +} diff --git a/platform-core/auth-verifier/src/main/java/org/sunbird/auth/verifier/KeyData.java b/platform-core/auth-verifier/src/main/java/org/sunbird/auth/verifier/KeyData.java new file mode 100644 index 000000000..6028025f7 --- /dev/null +++ b/platform-core/auth-verifier/src/main/java/org/sunbird/auth/verifier/KeyData.java @@ -0,0 +1,40 @@ +package org.sunbird.auth.verifier; + +import java.security.PrivateKey; +import java.security.PublicKey; + +public class KeyData { + private String keyId; + private PrivateKey privateKey; + private PublicKey publicKey; + + public KeyData(String keyId, PublicKey publicKey, PrivateKey privateKey) { + this.keyId = keyId; + this.publicKey = publicKey; + this.privateKey = privateKey; + } + + public String getKeyId() { + return keyId; + } + + public void setKeyId(String keyId) { + this.keyId = keyId; + } + + public PublicKey getPublicKey() { + return publicKey; + } + + public void setPublicKey(PublicKey publicKey) { + this.publicKey = publicKey; + } + + public PrivateKey getPrivateKey() { + return privateKey; + } + + public void setPrivateKey(PrivateKey privateKey) { + this.privateKey = privateKey; + } +} diff --git a/platform-core/auth-verifier/src/main/java/org/sunbird/auth/verifier/KeyManager.java b/platform-core/auth-verifier/src/main/java/org/sunbird/auth/verifier/KeyManager.java new file mode 100644 index 000000000..a55ccb4ab --- /dev/null +++ b/platform-core/auth-verifier/src/main/java/org/sunbird/auth/verifier/KeyManager.java @@ -0,0 +1,97 @@ +package org.sunbird.auth.verifier; + +import java.util.Map; +import org.sunbird.telemetry.logger.TelemetryManager; + +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.security.KeyFactory; +import java.security.PublicKey; +import java.security.PrivateKey; +import java.security.spec.PKCS8EncodedKeySpec; +import java.security.spec.X509EncodedKeySpec; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Random; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +public class KeyManager { + public static final String ACCESS_TOKEN_PUBLICKEY_BASEPATH = "accesstoken_privatekey_basepath"; + + private static PropertiesCache propertiesCache = PropertiesCache.getInstance(); + private static Map keyMap = new HashMap(); + + public static void init() { + TelemetryManager.info("KeyManager:init :: Starting initialization..."); + String basePath = propertiesCache.getProperty(ACCESS_TOKEN_PUBLICKEY_BASEPATH); + try (Stream walk = Files.walk(Paths.get(basePath))) { + List result = + walk.filter(Files::isRegularFile).map(x -> x.toString()).collect(Collectors.toList()); + result.forEach( + file -> { + try { + StringBuilder contentBuilder = new StringBuilder(); + Path path = Paths.get(file); + Files.lines(path, StandardCharsets.UTF_8) + .forEach( + x -> { + contentBuilder.append(x); + }); + KeyData keyData = + new KeyData( + path.getFileName().toString(), null, loadPrivateKey(contentBuilder.toString())); + keyMap.put(path.getFileName().toString(), keyData); + } catch (Exception e) { + TelemetryManager.error("KeyManager:init: exception in reading public keys ", e); + } + }); + } catch (Exception e) { + TelemetryManager.error("KeyManager:init: exception in loading publickeys ", e); + } + } + + public static KeyData getRandomKey() { + if (keyMap.size() == 0) { + init(); + } + if (keyMap.size() > 0) { + Random random = new Random(); + List keys = new ArrayList(keyMap.keySet()); + String randomKey = keys.get(random.nextInt(keys.size())); + return keyMap.get(randomKey); + } + return null; + } + + public static PublicKey loadPublicKey(String key) throws Exception { + String publicKey = new String(key.getBytes(), StandardCharsets.UTF_8); + publicKey = publicKey.replaceAll("(-+BEGIN PUBLIC KEY-+)", ""); + publicKey = publicKey.replaceAll("(-+END PUBLIC KEY-+)", ""); + publicKey = publicKey.replaceAll("[\\r\\n]+", ""); + byte[] keyBytes = Base64Util.decode(publicKey.getBytes("UTF-8"), Base64Util.DEFAULT); + + X509EncodedKeySpec X509publicKey = new X509EncodedKeySpec(keyBytes); + KeyFactory kf = KeyFactory.getInstance("RSA"); + return kf.generatePublic(X509publicKey); + } + + public static PrivateKey loadPrivateKey(String key) throws Exception { + String privateKey = new String(key.getBytes(), StandardCharsets.UTF_8); + privateKey = privateKey.replaceAll("(-+BEGIN RSA PRIVATE KEY-+)", ""); + privateKey = privateKey.replaceAll("(-+END RSA PRIVATE KEY-+)", ""); + privateKey = privateKey.replaceAll("(-+BEGIN PRIVATE KEY-+)", ""); + privateKey = privateKey.replaceAll("(-+END PRIVATE KEY-+)", ""); + privateKey = privateKey.replaceAll("[\\r\\n]+", ""); + byte[] keyBytes = Base64Util.decode(privateKey.getBytes("UTF-8"), Base64Util.DEFAULT); + + // generate private key + PKCS8EncodedKeySpec spec = new PKCS8EncodedKeySpec(keyBytes); + KeyFactory keyFactory = KeyFactory.getInstance("RSA"); + return keyFactory.generatePrivate(spec); + } +} diff --git a/platform-core/auth-verifier/src/main/java/org/sunbird/auth/verifier/PropertiesCache.java b/platform-core/auth-verifier/src/main/java/org/sunbird/auth/verifier/PropertiesCache.java new file mode 100644 index 000000000..79d143f6a --- /dev/null +++ b/platform-core/auth-verifier/src/main/java/org/sunbird/auth/verifier/PropertiesCache.java @@ -0,0 +1,97 @@ +package org.sunbird.auth.verifier; + +import java.io.IOException; +import java.io.InputStream; +import java.util.Map; +import java.util.Properties; +import java.util.concurrent.ConcurrentHashMap; +import org.apache.commons.lang3.StringUtils; + +import org.sunbird.telemetry.logger.TelemetryManager; + + +/* + * @author Amit Kumar + * + * this class is used for reading properties file + */ +public class PropertiesCache { + + private final String[] fileName = { + "externalresource.properties" + }; + private final Properties configProp = new Properties(); + public final Map attributePercentageMap = new ConcurrentHashMap<>(); + private static PropertiesCache propertiesCache = null; + + /** private default constructor */ + private PropertiesCache() { + for (String file : fileName) { + InputStream in = this.getClass().getClassLoader().getResourceAsStream(file); + try { + configProp.load(in); + } catch (IOException e) { + TelemetryManager.error("Error in properties cache", e); + } + } + loadWeighted(); + } + + public static PropertiesCache getInstance() { + + // change the lazy holder implementation to simple singleton implementation ... + if (null == propertiesCache) { + synchronized (PropertiesCache.class) { + if (null == propertiesCache) { + propertiesCache = new PropertiesCache(); + } + } + } + + return propertiesCache; + } + + public void saveConfigProperty(String key, String value) { + configProp.setProperty(key, value); + } + + public String getProperty(String key) { + String value = System.getenv(key); + if (StringUtils.isNotBlank(value)) return value; + return configProp.getProperty(key) != null ? configProp.getProperty(key) : key; + } + + private void loadWeighted() { + String key = configProp.getProperty("user.profile.attribute"); + String value = configProp.getProperty("user.profile.weighted"); + if (StringUtils.isBlank(key)) { + TelemetryManager.info("Profile completeness value is not set"); + } else { + String keys[] = key.split(","); + String values[] = value.split(","); + if (keys.length == value.length()) { + // then take the value from user + TelemetryManager.log("weighted value is provided by user."); + for (int i = 0; i < keys.length; i++) + attributePercentageMap.put(keys[i], new Float(values[i])); + } else { + // equally divide all the provided field. + TelemetryManager.log("weighted value is not provided by user."); + float perc = (float) 100.0 / keys.length; + for (int i = 0; i < keys.length; i++) attributePercentageMap.put(keys[i], perc); + } + } + } + + /** + * Method to read value from resource file . + * + * @param key + * @return + */ + public String readProperty(String key) { + String value = System.getenv(key); + if (StringUtils.isNotBlank(value)) return value; + return configProp.getProperty(key); + } +} \ No newline at end of file diff --git a/platform-core/auth-verifier/src/test/java/org/sunbird/auth/verifier/KeyManagerTest.java b/platform-core/auth-verifier/src/test/java/org/sunbird/auth/verifier/KeyManagerTest.java new file mode 100644 index 000000000..245fa563c --- /dev/null +++ b/platform-core/auth-verifier/src/test/java/org/sunbird/auth/verifier/KeyManagerTest.java @@ -0,0 +1,26 @@ +package org.sunbird.auth.verifier; + +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; + +import java.security.PublicKey; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.core.classloader.annotations.PowerMockIgnore; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; +import org.sunbird.common.models.util.PropertiesCache; + +@RunWith(PowerMockRunner.class) +@PrepareForTest({PropertiesCache.class}) +@PowerMockIgnore({"javax.management.*"}) +public class KeyManagerTest { + + @Test + public void testLoadPublicKey() throws Exception { + PublicKey key = + KeyManager.loadPublicKey( + "MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAysH/wWtg0IjBL1JZZDYvUJC42JCxVobalckr2/3d3eEiWkk7Zh/4DAPYOs4UPjAevTs5VMUjq9EZu/u4H5hNzoVmYNvhtxbhWNY3n4mxpA4Lgt4sNGiGYNNGrN34ML+7+TR3Z1dlrhA271PiuanHI11YymskQRPhBfuwK923Kl/lgI4rS9OQ4GnkvwkUPvMUIRfNt8wL9uTbWm3V9p8VTcmQbW+pPw9QhO9v95NOgXQrLnT8xwnzQE6UCTY2al3B0fc3ULmcxvK+7P1R3/0w1qJLEKSiHl0xnv4WNEfS+2UmN+8jfdSCfoyVIglQl5/tb05j89nfZZp8k24AWLxIJQIDAQAB"); + assertNotNull(key); + } +} diff --git a/platform-core/pom.xml b/platform-core/pom.xml index 915ae188d..f8f4799e1 100755 --- a/platform-core/pom.xml +++ b/platform-core/pom.xml @@ -14,6 +14,7 @@ platform-common platform-telemetry actor-core + auth-verifier schema-validator platform-cache cassandra-connector diff --git a/pom.xml b/pom.xml index f3f14193a..f3b94a318 100644 --- a/pom.xml +++ b/pom.xml @@ -16,6 +16,9 @@ 2.11.12 3.0.8 2.9.8 + 2.22.0 + 2.0.7 + 2.0.0-beta.5 platform-core diff --git a/search-api/search-actors/src/main/java/org/sunbird/actors/SearchActor.java b/search-api/search-actors/src/main/java/org/sunbird/actors/SearchActor.java index 68f175363..136b52454 100644 --- a/search-api/search-actors/src/main/java/org/sunbird/actors/SearchActor.java +++ b/search-api/search-actors/src/main/java/org/sunbird/actors/SearchActor.java @@ -95,6 +95,12 @@ private SearchDTO getSearchDTO(Request request) throws Exception { SearchDTO searchObj = new SearchDTO(); try { Map req = request.getRequest(); + if (req.get("secureSettings") != null) { + searchObj.setSecureSettings((Boolean) req.get("secureSettings")); + } else { + searchObj.setSecureSettings(false); + } + searchObj.setUserOrgId((String) request.getContext().get("x-user-channel-id")); TelemetryManager.log("Search Request: ", req); String queryString = (String) req.get(SearchConstants.query); int limit = getIntValue(req.get(SearchConstants.limit)); diff --git a/search-api/search-core/src/main/java/org/sunbird/search/dto/SearchDTO.java b/search-api/search-core/src/main/java/org/sunbird/search/dto/SearchDTO.java index 6fa3d15a9..328d45212 100644 --- a/search-api/search-core/src/main/java/org/sunbird/search/dto/SearchDTO.java +++ b/search-api/search-core/src/main/java/org/sunbird/search/dto/SearchDTO.java @@ -16,6 +16,8 @@ public class SearchDTO { private int limit; private int offset; boolean fuzzySearch = false; + boolean secureSettings = false; + String userOrgId = ""; private Map additionalProperties = new HashMap(); private Map softConstraints = new HashMap(); private List> aggregations = new ArrayList<>(); @@ -65,13 +67,20 @@ public Map getSortBy() { public void setSortBy(Map sortBy) { this.sortBy = sortBy; } - public boolean isFuzzySearch() { return fuzzySearch; } public void setFuzzySearch(boolean fuzzySearch) { this.fuzzySearch = fuzzySearch; } + public boolean isSecureSettings() { + return secureSettings; + } + public void setSecureSettings(boolean secureSettings) { + this.secureSettings = secureSettings; + } + public String getUserOrgId() {return userOrgId;} + public void setUserOrgId(String userOrgId) {this.userOrgId = userOrgId;} public Map getAdditionalProperties() { return additionalProperties; } diff --git a/search-api/search-core/src/main/java/org/sunbird/search/processor/SearchProcessor.java b/search-api/search-core/src/main/java/org/sunbird/search/processor/SearchProcessor.java index c95fc4f24..58a734a5a 100644 --- a/search-api/search-core/src/main/java/org/sunbird/search/processor/SearchProcessor.java +++ b/search-api/search-core/src/main/java/org/sunbird/search/processor/SearchProcessor.java @@ -5,6 +5,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang.StringUtils; +import org.apache.lucene.search.join.ScoreMode; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.index.query.*; import org.elasticsearch.index.query.MultiMatchQueryBuilder.Type; @@ -43,7 +44,7 @@ public SearchProcessor() { ElasticSearchUtil.initialiseESClient(SearchConstants.COMPOSITE_SEARCH_INDEX, Platform.config.getString("search.es_conn_info")); } - + public SearchProcessor(String indexName) { } @@ -112,7 +113,7 @@ public Map processCount(SearchDTO searchDTO) throws Exception { /** * Returns the list of words which are synonyms of the synsetIds passed in the * request - * + * * @param synsetIds * @return * @throws Exception @@ -177,7 +178,7 @@ public Map multiWordDocSearch(List synsetIds) throws Exc /** * Returns list of synsetsIds which has valid documents in composite index - * + * * @param synsetIds * @return * @throws Exception @@ -327,11 +328,17 @@ private QueryBuilder prepareSearchQuery(SearchDTO searchDTO) { QueryBuilder queryBuilder = null; String totalOperation = searchDTO.getOperation(); List properties = searchDTO.getProperties(); - formQuery(properties, queryBuilder, boolQuery, totalOperation, searchDTO.isFuzzySearch()); - if(searchDTO.getMultiFilterProperties() != null) { - formQuery(searchDTO.getMultiFilterProperties(), queryBuilder, boolQuery, SearchConstants.SEARCH_OPERATION_OR, searchDTO.isFuzzySearch()); + if (searchDTO.isSecureSettings() == false) + formQuery(properties, queryBuilder, boolQuery, totalOperation, searchDTO.isFuzzySearch()); + else + formQueryImpl(properties, queryBuilder, boolQuery, totalOperation, searchDTO.isFuzzySearch(), searchDTO); + if (searchDTO.getMultiFilterProperties() != null) { + if (searchDTO.isSecureSettings() == false) + formQuery(searchDTO.getMultiFilterProperties(), queryBuilder, boolQuery, SearchConstants.SEARCH_OPERATION_OR, searchDTO.isFuzzySearch()); + else { + formQueryImpl(searchDTO.getMultiFilterProperties(), queryBuilder, boolQuery, totalOperation, searchDTO.isFuzzySearch(), searchDTO); + } } - Map softConstraints = searchDTO.getSoftConstraints(); if (null != softConstraints && !softConstraints.isEmpty()) { boolQuery.should(getSoftConstraintQuery(softConstraints)); @@ -342,6 +349,13 @@ private QueryBuilder prepareSearchQuery(SearchDTO searchDTO) { } private void formQuery(List properties, QueryBuilder queryBuilder, BoolQueryBuilder boolQuery, String operation, Boolean fuzzy) { + formQueryImpl(properties, queryBuilder, boolQuery, operation, fuzzy, null); + } + + private void formQueryImpl(List properties, QueryBuilder queryBuilder, BoolQueryBuilder boolQuery, String operation, Boolean fuzzy, SearchDTO searchDTO) { + boolean enableSecureSettings = false; + if (searchDTO != null) + enableSecureSettings = searchDTO.isSecureSettings(); for (Map property : properties) { String opertation = (String) property.get("operation"); @@ -359,6 +373,11 @@ private void formQuery(List properties, QueryBuilder queryBuilder, BoolQuer relevanceSort = true; propertyName = "all_fields"; queryBuilder = getAllFieldsPropertyQuery(values, fuzzy); + if (enableSecureSettings) { + boolQuery.must(getSecureSettingsSearchQuery(searchDTO.getUserOrgId())); + } else { + boolQuery.mustNot(getSecureSettingsSearchDefaultQuery()); + } boolQuery.must(queryBuilder); continue; } @@ -447,6 +466,11 @@ private void formQuery(List properties, QueryBuilder queryBuilder, BoolQuer } } if (operation.equalsIgnoreCase(AND)) { + if (enableSecureSettings) { + boolQuery.must(getSecureSettingsSearchQuery(searchDTO.getUserOrgId())); + } else { + boolQuery.mustNot(getSecureSettingsSearchDefaultQuery()); + } boolQuery.must(queryBuilder); } else { boolQuery.should(queryBuilder); @@ -515,10 +539,26 @@ private QueryBuilder getAllFieldsPropertyQuery(List values, Boolean fuzz .operator(Operator.AND).type(Type.CROSS_FIELDS).fuzzyTranspositions(false).lenient(true)); } } - return queryBuilder; } + private QueryBuilder getSecureSettingsSearchDefaultQuery() { + + QueryBuilder firstNestedQuery =new NestedQueryBuilder("secureSettings", + QueryBuilders.boolQuery() .mustNot(new ExistsQueryBuilder("organisation")), org.apache.lucene.search.join.ScoreMode.None); + QueryBuilder secondNestedQuery= new NestedQueryBuilder("secureSettings", QueryBuilders.boolQuery() + .filter(new RangeQueryBuilder("organisation" + ".length").lte(0)) , org.apache.lucene.search.join.ScoreMode.None); + QueryBuilder query = QueryBuilders.boolQuery() .should(firstNestedQuery).should (secondNestedQuery); + + return query; + } + + private QueryBuilder getSecureSettingsSearchQuery(String org_id) { + QueryBuilder query = new NestedQueryBuilder("secureSettings", + QueryBuilders.boolQuery().must(new ExistsQueryBuilder("secureSettings.organisation")).must(QueryBuilders.termQuery("secureSettings.organisation", org_id)), org.apache.lucene.search.join.ScoreMode.None); + + return query; + } /** * @param softConstraints * @return @@ -732,7 +772,7 @@ public Future> processSearchQuery(SearchDTO searchDTO, boolean incl SearchSourceBuilder query = processSearchQuery(searchDTO, groupByFinalList, sort); TelemetryManager.log(" search query: " + query); Future searchResponse = ElasticSearchUtil.search(index, query); - + return searchResponse.map(new Mapper>() { public List apply(SearchResponse searchResult) { List response = new ArrayList(); @@ -746,7 +786,7 @@ public List apply(SearchResponse searchResult) { return response; } }, ExecutionContext.Implicits$.MODULE$.global()); - + } public Future processSearchQueryWithSearchResult(SearchDTO searchDTO, boolean includeResults, diff --git a/search-api/search-core/src/main/java/org/sunbird/search/util/SearchConstants.java b/search-api/search-core/src/main/java/org/sunbird/search/util/SearchConstants.java index 27b68a8ba..66d276168 100644 --- a/search-api/search-core/src/main/java/org/sunbird/search/util/SearchConstants.java +++ b/search-api/search-core/src/main/java/org/sunbird/search/util/SearchConstants.java @@ -114,4 +114,5 @@ public class SearchConstants { public static final String softConstraints = "softConstraints"; public static final String setDefaultVisibility = "setDefaultVisibility"; public static String soft = "soft"; + public static String secureSettings = "secureSettings"; } diff --git a/search-api/search-service/app/controllers/SearchBaseController.scala b/search-api/search-service/app/controllers/SearchBaseController.scala index 9e1735b35..5938c93ac 100644 --- a/search-api/search-service/app/controllers/SearchBaseController.scala +++ b/search-api/search-service/app/controllers/SearchBaseController.scala @@ -25,7 +25,7 @@ abstract class SearchBaseController(protected val cc: ControllerComponents)(impl } def commonHeaders()(implicit request: Request[AnyContent]): java.util.Map[String, Object] = { - val customHeaders = Map("x-channel-id" -> "CHANNEL_ID", "x-consumer-id" -> "CONSUMER_ID", "x-app-id" -> "APP_ID", "x-session-id" -> "SESSION_ID", "x-device-id" -> "DEVICE_ID") + val customHeaders = Map("x-authenticated-user-orgid" -> "x-user-channel-id","x-channel-id" -> "CHANNEL_ID", "x-consumer-id" -> "CONSUMER_ID", "x-app-id" -> "APP_ID", "x-session-id" -> "SESSION_ID", "x-device-id" -> "DEVICE_ID") val headers = request.headers.headers.groupBy(_._1).mapValues(_.map(_._2)) val appHeaders = headers.filter(header => customHeaders.keySet.contains(header._1.toLowerCase)) .map(entry => (customHeaders.get(entry._1.toLowerCase()).get, entry._2.head)) diff --git a/search-api/search-service/conf/application.conf b/search-api/search-service/conf/application.conf index 0b038102c..b39c0e2b5 100644 --- a/search-api/search-service/conf/application.conf +++ b/search-api/search-service/conf/application.conf @@ -315,4 +315,7 @@ content.tagging.property=["subject","medium"] search.payload.log_enable=true #Folling configuration would enable the fuzzy search when there are no matches found for given query. -search.fields.enable.fuzzy.when.noresult=false \ No newline at end of file +search.fields.enable.fuzzy.when.noresult=false + +#Following configuration would enable the secureSettings search +search.fields.enable.secureSettings=false \ No newline at end of file From cecc43e76a612f098dc50f8ef892ce9702bf4d5d Mon Sep 17 00:00:00 2001 From: karthik-tarento Date: Thu, 1 Jun 2023 12:27:50 +0530 Subject: [PATCH 035/126] Using common project for JWT token instead of new module --- .../content-service/conf/application.conf | 4 +- content-api/hierarchy-manager/pom.xml | 5 - .../sunbird/managers/HierarchyManager.scala | 3 +- platform-core/auth-verifier/pom.xml | 93 ------------------ .../org/sunbird/auth/verifier/CryptoUtil.java | 59 ----------- .../org/sunbird/auth/verifier/KeyData.java | 40 -------- .../org/sunbird/auth/verifier/KeyManager.java | 97 ------------------- .../auth/verifier/PropertiesCache.java | 97 ------------------- .../sunbird/auth/verifier/KeyManagerTest.java | 26 ----- .../java/org/sunbird/common}/Base64Util.java | 2 +- .../java/org/sunbird/common/CryptoUtil.java | 25 +++++ .../java/org/sunbird/common}/JWTUtil.java | 43 ++------ .../java/org/sunbird/common}/JWTokenType.java | 2 +- platform-core/pom.xml | 1 - pom.xml | 3 - 15 files changed, 37 insertions(+), 463 deletions(-) delete mode 100644 platform-core/auth-verifier/pom.xml delete mode 100644 platform-core/auth-verifier/src/main/java/org/sunbird/auth/verifier/CryptoUtil.java delete mode 100644 platform-core/auth-verifier/src/main/java/org/sunbird/auth/verifier/KeyData.java delete mode 100644 platform-core/auth-verifier/src/main/java/org/sunbird/auth/verifier/KeyManager.java delete mode 100644 platform-core/auth-verifier/src/main/java/org/sunbird/auth/verifier/PropertiesCache.java delete mode 100644 platform-core/auth-verifier/src/test/java/org/sunbird/auth/verifier/KeyManagerTest.java rename platform-core/{auth-verifier/src/main/java/org/sunbird/auth/verifier => platform-common/src/main/java/org/sunbird/common}/Base64Util.java (99%) create mode 100644 platform-core/platform-common/src/main/java/org/sunbird/common/CryptoUtil.java rename platform-core/{auth-verifier/src/main/java/org/sunbird/auth/verifier => platform-common/src/main/java/org/sunbird/common}/JWTUtil.java (50%) rename platform-core/{auth-verifier/src/main/java/org/sunbird/auth/verifier => platform-common/src/main/java/org/sunbird/common}/JWTokenType.java (92%) diff --git a/content-api/content-service/conf/application.conf b/content-api/content-service/conf/application.conf index fa3214e9f..751f86283 100644 --- a/content-api/content-service/conf/application.conf +++ b/content-api/content-service/conf/application.conf @@ -545,10 +545,10 @@ learning.service_provider=["youtube"] stream.mime.type=video/mp4 compositesearch.index.name="compositesearch" -hierarchy.keyspace.name=hierarchy_store +hierarchy.keyspace.name=dev_hierarchy_store content.hierarchy.table=content_hierarchy framework.hierarchy.table=framework_hierarchy -objectcategorydefinition.keyspace=category_store +objectcategorydefinition.keyspace=dev_category_store # Kafka topic for definition update event. kafka.topic.system.command="dev.system.command" diff --git a/content-api/hierarchy-manager/pom.xml b/content-api/hierarchy-manager/pom.xml index 3ef8d21c8..828dac9fc 100644 --- a/content-api/hierarchy-manager/pom.xml +++ b/content-api/hierarchy-manager/pom.xml @@ -12,11 +12,6 @@ hierarchy-manager - - org.sunbird - auth-verifier - 1.0-SNAPSHOT - org.sunbird graph-engine_2.11 diff --git a/content-api/hierarchy-manager/src/main/scala/org/sunbird/managers/HierarchyManager.scala b/content-api/hierarchy-manager/src/main/scala/org/sunbird/managers/HierarchyManager.scala index 1019d0fc4..4d591308f 100644 --- a/content-api/hierarchy-manager/src/main/scala/org/sunbird/managers/HierarchyManager.scala +++ b/content-api/hierarchy-manager/src/main/scala/org/sunbird/managers/HierarchyManager.scala @@ -4,11 +4,10 @@ import java.util import java.util.concurrent.CompletionException import com.fasterxml.jackson.databind.ObjectMapper import org.apache.commons.lang3.StringUtils -import org.sunbird.auth.verifier.JWTUtil import org.sunbird.cache.impl.RedisCache import org.sunbird.common.dto.{Request, Response, ResponseHandler, ResponseParams} import org.sunbird.common.exception.{ClientException, ErrorCodes, ResourceNotFoundException, ResponseCode, ServerException} -import org.sunbird.common.{JsonUtils, Platform} +import org.sunbird.common.{JsonUtils, JWTUtil, Platform} import org.sunbird.graph.dac.model.Node import org.sunbird.graph.nodes.DataNode import org.sunbird.graph.utils.{NodeUtil, ScalaJsonUtils} diff --git a/platform-core/auth-verifier/pom.xml b/platform-core/auth-verifier/pom.xml deleted file mode 100644 index 196b91bde..000000000 --- a/platform-core/auth-verifier/pom.xml +++ /dev/null @@ -1,93 +0,0 @@ - - - - platform-core - org.sunbird - 1.0-SNAPSHOT - - 4.0.0 - - auth-verifier - - - - com.fasterxml.jackson.core - jackson-core - ${fasterxml.jackson.version} - - - org.sunbird - common-util - 0.0.1-SNAPSHOT - - - org.sunbird - platform-telemetry - 1.0-SNAPSHOT - jar - - - org.mockito - mockito-core - ${mockito.core.version} - test - - - org.powermock - powermock-api-mockito2 - ${powermock.api.mockito2.version} - test - - - org.powermock - powermock-module-junit4 - ${powermock.module.junit4.version} - test - - - ch.qos.logback - logback-classic - 1.2.3 - - - net.logstash.logback - logstash-logback-encoder - 6.3 - - - - - - - org.apache.maven.plugins - maven-compiler-plugin - 3.8.1 - - 11 - - - - org.jacoco - jacoco-maven-plugin - 0.8.5 - - - default-prepare-agent - - prepare-agent - - - - default-report - prepare-package - - report - - - - - - - \ No newline at end of file diff --git a/platform-core/auth-verifier/src/main/java/org/sunbird/auth/verifier/CryptoUtil.java b/platform-core/auth-verifier/src/main/java/org/sunbird/auth/verifier/CryptoUtil.java deleted file mode 100644 index 2f83d55b6..000000000 --- a/platform-core/auth-verifier/src/main/java/org/sunbird/auth/verifier/CryptoUtil.java +++ /dev/null @@ -1,59 +0,0 @@ -package org.sunbird.auth.verifier; - -import java.nio.charset.Charset; -import java.security.*; - -import javax.crypto.Mac; -import javax.crypto.spec.SecretKeySpec; - -import org.sunbird.telemetry.logger.TelemetryManager; - - -public class CryptoUtil { - private static final Charset US_ASCII = Charset.forName("US-ASCII"); - - public static boolean verifyRSASign(String payLoad, byte[] signature, PublicKey key, String algorithm) { - Signature sign; - try { - sign = Signature.getInstance(algorithm); - sign.initVerify(key); - sign.update(payLoad.getBytes(US_ASCII)); - return sign.verify(signature); - } catch (NoSuchAlgorithmException e) { - return false; - } catch (InvalidKeyException e){ - return false; - } catch (SignatureException e){ - return false; - } - } - - public static byte[] generateHMAC(String payLoad, String secretKey, String algorithm) { - Mac mac; - byte[] signature; - try { - mac = Mac.getInstance(algorithm); - mac.init(new SecretKeySpec(secretKey.getBytes(), algorithm)); - signature = mac.doFinal(payLoad.getBytes(US_ASCII)); - } catch (NoSuchAlgorithmException | InvalidKeyException e) { - TelemetryManager.error("CryptoUtil:generateHMAC :: failed to generate signature. Exception: ", e); - return null; - } - return signature; - } - - public static byte[] generateRSASign(String payLoad, PrivateKey key, String algorithm) { - Signature sign; - byte[] signature; - try { - sign = Signature.getInstance(algorithm); - sign.initSign(key); - sign.update(payLoad.getBytes(US_ASCII)); - signature = sign.sign(); - } catch (NoSuchAlgorithmException | InvalidKeyException | SignatureException e) { - TelemetryManager.error("CryptoUtil:generateRSASign :: failed to generate signature. Exception: ", e); - return null; - } - return signature; - } -} diff --git a/platform-core/auth-verifier/src/main/java/org/sunbird/auth/verifier/KeyData.java b/platform-core/auth-verifier/src/main/java/org/sunbird/auth/verifier/KeyData.java deleted file mode 100644 index 6028025f7..000000000 --- a/platform-core/auth-verifier/src/main/java/org/sunbird/auth/verifier/KeyData.java +++ /dev/null @@ -1,40 +0,0 @@ -package org.sunbird.auth.verifier; - -import java.security.PrivateKey; -import java.security.PublicKey; - -public class KeyData { - private String keyId; - private PrivateKey privateKey; - private PublicKey publicKey; - - public KeyData(String keyId, PublicKey publicKey, PrivateKey privateKey) { - this.keyId = keyId; - this.publicKey = publicKey; - this.privateKey = privateKey; - } - - public String getKeyId() { - return keyId; - } - - public void setKeyId(String keyId) { - this.keyId = keyId; - } - - public PublicKey getPublicKey() { - return publicKey; - } - - public void setPublicKey(PublicKey publicKey) { - this.publicKey = publicKey; - } - - public PrivateKey getPrivateKey() { - return privateKey; - } - - public void setPrivateKey(PrivateKey privateKey) { - this.privateKey = privateKey; - } -} diff --git a/platform-core/auth-verifier/src/main/java/org/sunbird/auth/verifier/KeyManager.java b/platform-core/auth-verifier/src/main/java/org/sunbird/auth/verifier/KeyManager.java deleted file mode 100644 index a55ccb4ab..000000000 --- a/platform-core/auth-verifier/src/main/java/org/sunbird/auth/verifier/KeyManager.java +++ /dev/null @@ -1,97 +0,0 @@ -package org.sunbird.auth.verifier; - -import java.util.Map; -import org.sunbird.telemetry.logger.TelemetryManager; - -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.security.KeyFactory; -import java.security.PublicKey; -import java.security.PrivateKey; -import java.security.spec.PKCS8EncodedKeySpec; -import java.security.spec.X509EncodedKeySpec; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Random; -import java.util.Map; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -public class KeyManager { - public static final String ACCESS_TOKEN_PUBLICKEY_BASEPATH = "accesstoken_privatekey_basepath"; - - private static PropertiesCache propertiesCache = PropertiesCache.getInstance(); - private static Map keyMap = new HashMap(); - - public static void init() { - TelemetryManager.info("KeyManager:init :: Starting initialization..."); - String basePath = propertiesCache.getProperty(ACCESS_TOKEN_PUBLICKEY_BASEPATH); - try (Stream walk = Files.walk(Paths.get(basePath))) { - List result = - walk.filter(Files::isRegularFile).map(x -> x.toString()).collect(Collectors.toList()); - result.forEach( - file -> { - try { - StringBuilder contentBuilder = new StringBuilder(); - Path path = Paths.get(file); - Files.lines(path, StandardCharsets.UTF_8) - .forEach( - x -> { - contentBuilder.append(x); - }); - KeyData keyData = - new KeyData( - path.getFileName().toString(), null, loadPrivateKey(contentBuilder.toString())); - keyMap.put(path.getFileName().toString(), keyData); - } catch (Exception e) { - TelemetryManager.error("KeyManager:init: exception in reading public keys ", e); - } - }); - } catch (Exception e) { - TelemetryManager.error("KeyManager:init: exception in loading publickeys ", e); - } - } - - public static KeyData getRandomKey() { - if (keyMap.size() == 0) { - init(); - } - if (keyMap.size() > 0) { - Random random = new Random(); - List keys = new ArrayList(keyMap.keySet()); - String randomKey = keys.get(random.nextInt(keys.size())); - return keyMap.get(randomKey); - } - return null; - } - - public static PublicKey loadPublicKey(String key) throws Exception { - String publicKey = new String(key.getBytes(), StandardCharsets.UTF_8); - publicKey = publicKey.replaceAll("(-+BEGIN PUBLIC KEY-+)", ""); - publicKey = publicKey.replaceAll("(-+END PUBLIC KEY-+)", ""); - publicKey = publicKey.replaceAll("[\\r\\n]+", ""); - byte[] keyBytes = Base64Util.decode(publicKey.getBytes("UTF-8"), Base64Util.DEFAULT); - - X509EncodedKeySpec X509publicKey = new X509EncodedKeySpec(keyBytes); - KeyFactory kf = KeyFactory.getInstance("RSA"); - return kf.generatePublic(X509publicKey); - } - - public static PrivateKey loadPrivateKey(String key) throws Exception { - String privateKey = new String(key.getBytes(), StandardCharsets.UTF_8); - privateKey = privateKey.replaceAll("(-+BEGIN RSA PRIVATE KEY-+)", ""); - privateKey = privateKey.replaceAll("(-+END RSA PRIVATE KEY-+)", ""); - privateKey = privateKey.replaceAll("(-+BEGIN PRIVATE KEY-+)", ""); - privateKey = privateKey.replaceAll("(-+END PRIVATE KEY-+)", ""); - privateKey = privateKey.replaceAll("[\\r\\n]+", ""); - byte[] keyBytes = Base64Util.decode(privateKey.getBytes("UTF-8"), Base64Util.DEFAULT); - - // generate private key - PKCS8EncodedKeySpec spec = new PKCS8EncodedKeySpec(keyBytes); - KeyFactory keyFactory = KeyFactory.getInstance("RSA"); - return keyFactory.generatePrivate(spec); - } -} diff --git a/platform-core/auth-verifier/src/main/java/org/sunbird/auth/verifier/PropertiesCache.java b/platform-core/auth-verifier/src/main/java/org/sunbird/auth/verifier/PropertiesCache.java deleted file mode 100644 index 79d143f6a..000000000 --- a/platform-core/auth-verifier/src/main/java/org/sunbird/auth/verifier/PropertiesCache.java +++ /dev/null @@ -1,97 +0,0 @@ -package org.sunbird.auth.verifier; - -import java.io.IOException; -import java.io.InputStream; -import java.util.Map; -import java.util.Properties; -import java.util.concurrent.ConcurrentHashMap; -import org.apache.commons.lang3.StringUtils; - -import org.sunbird.telemetry.logger.TelemetryManager; - - -/* - * @author Amit Kumar - * - * this class is used for reading properties file - */ -public class PropertiesCache { - - private final String[] fileName = { - "externalresource.properties" - }; - private final Properties configProp = new Properties(); - public final Map attributePercentageMap = new ConcurrentHashMap<>(); - private static PropertiesCache propertiesCache = null; - - /** private default constructor */ - private PropertiesCache() { - for (String file : fileName) { - InputStream in = this.getClass().getClassLoader().getResourceAsStream(file); - try { - configProp.load(in); - } catch (IOException e) { - TelemetryManager.error("Error in properties cache", e); - } - } - loadWeighted(); - } - - public static PropertiesCache getInstance() { - - // change the lazy holder implementation to simple singleton implementation ... - if (null == propertiesCache) { - synchronized (PropertiesCache.class) { - if (null == propertiesCache) { - propertiesCache = new PropertiesCache(); - } - } - } - - return propertiesCache; - } - - public void saveConfigProperty(String key, String value) { - configProp.setProperty(key, value); - } - - public String getProperty(String key) { - String value = System.getenv(key); - if (StringUtils.isNotBlank(value)) return value; - return configProp.getProperty(key) != null ? configProp.getProperty(key) : key; - } - - private void loadWeighted() { - String key = configProp.getProperty("user.profile.attribute"); - String value = configProp.getProperty("user.profile.weighted"); - if (StringUtils.isBlank(key)) { - TelemetryManager.info("Profile completeness value is not set"); - } else { - String keys[] = key.split(","); - String values[] = value.split(","); - if (keys.length == value.length()) { - // then take the value from user - TelemetryManager.log("weighted value is provided by user."); - for (int i = 0; i < keys.length; i++) - attributePercentageMap.put(keys[i], new Float(values[i])); - } else { - // equally divide all the provided field. - TelemetryManager.log("weighted value is not provided by user."); - float perc = (float) 100.0 / keys.length; - for (int i = 0; i < keys.length; i++) attributePercentageMap.put(keys[i], perc); - } - } - } - - /** - * Method to read value from resource file . - * - * @param key - * @return - */ - public String readProperty(String key) { - String value = System.getenv(key); - if (StringUtils.isNotBlank(value)) return value; - return configProp.getProperty(key); - } -} \ No newline at end of file diff --git a/platform-core/auth-verifier/src/test/java/org/sunbird/auth/verifier/KeyManagerTest.java b/platform-core/auth-verifier/src/test/java/org/sunbird/auth/verifier/KeyManagerTest.java deleted file mode 100644 index 245fa563c..000000000 --- a/platform-core/auth-verifier/src/test/java/org/sunbird/auth/verifier/KeyManagerTest.java +++ /dev/null @@ -1,26 +0,0 @@ -package org.sunbird.auth.verifier; - -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; - -import java.security.PublicKey; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.powermock.core.classloader.annotations.PowerMockIgnore; -import org.powermock.core.classloader.annotations.PrepareForTest; -import org.powermock.modules.junit4.PowerMockRunner; -import org.sunbird.common.models.util.PropertiesCache; - -@RunWith(PowerMockRunner.class) -@PrepareForTest({PropertiesCache.class}) -@PowerMockIgnore({"javax.management.*"}) -public class KeyManagerTest { - - @Test - public void testLoadPublicKey() throws Exception { - PublicKey key = - KeyManager.loadPublicKey( - "MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAysH/wWtg0IjBL1JZZDYvUJC42JCxVobalckr2/3d3eEiWkk7Zh/4DAPYOs4UPjAevTs5VMUjq9EZu/u4H5hNzoVmYNvhtxbhWNY3n4mxpA4Lgt4sNGiGYNNGrN34ML+7+TR3Z1dlrhA271PiuanHI11YymskQRPhBfuwK923Kl/lgI4rS9OQ4GnkvwkUPvMUIRfNt8wL9uTbWm3V9p8VTcmQbW+pPw9QhO9v95NOgXQrLnT8xwnzQE6UCTY2al3B0fc3ULmcxvK+7P1R3/0w1qJLEKSiHl0xnv4WNEfS+2UmN+8jfdSCfoyVIglQl5/tb05j89nfZZp8k24AWLxIJQIDAQAB"); - assertNotNull(key); - } -} diff --git a/platform-core/auth-verifier/src/main/java/org/sunbird/auth/verifier/Base64Util.java b/platform-core/platform-common/src/main/java/org/sunbird/common/Base64Util.java similarity index 99% rename from platform-core/auth-verifier/src/main/java/org/sunbird/auth/verifier/Base64Util.java rename to platform-core/platform-common/src/main/java/org/sunbird/common/Base64Util.java index 619330d24..73fb8e50a 100644 --- a/platform-core/auth-verifier/src/main/java/org/sunbird/auth/verifier/Base64Util.java +++ b/platform-core/platform-common/src/main/java/org/sunbird/common/Base64Util.java @@ -1,4 +1,4 @@ -package org.sunbird.auth.verifier; +package org.sunbird.common; /* * Copyright (C) 2010 The Android Open Source Project diff --git a/platform-core/platform-common/src/main/java/org/sunbird/common/CryptoUtil.java b/platform-core/platform-common/src/main/java/org/sunbird/common/CryptoUtil.java new file mode 100644 index 000000000..6a794063d --- /dev/null +++ b/platform-core/platform-common/src/main/java/org/sunbird/common/CryptoUtil.java @@ -0,0 +1,25 @@ +package org.sunbird.common; + +import java.nio.charset.Charset; +import java.security.InvalidKeyException; +import java.security.NoSuchAlgorithmException; + +import javax.crypto.Mac; +import javax.crypto.spec.SecretKeySpec; + +public class CryptoUtil { + private static final Charset US_ASCII = Charset.forName("US-ASCII"); + + public static byte[] generateHMAC(String payLoad, String secretKey, String algorithm) { + Mac mac; + byte[] signature; + try { + mac = Mac.getInstance(algorithm); + mac.init(new SecretKeySpec(secretKey.getBytes(), algorithm)); + signature = mac.doFinal(payLoad.getBytes(US_ASCII)); + } catch (NoSuchAlgorithmException | InvalidKeyException e) { + return null; + } + return signature; + } +} diff --git a/platform-core/auth-verifier/src/main/java/org/sunbird/auth/verifier/JWTUtil.java b/platform-core/platform-common/src/main/java/org/sunbird/common/JWTUtil.java similarity index 50% rename from platform-core/auth-verifier/src/main/java/org/sunbird/auth/verifier/JWTUtil.java rename to platform-core/platform-common/src/main/java/org/sunbird/common/JWTUtil.java index b382ae3d3..9f948967f 100644 --- a/platform-core/auth-verifier/src/main/java/org/sunbird/auth/verifier/JWTUtil.java +++ b/platform-core/platform-common/src/main/java/org/sunbird/common/JWTUtil.java @@ -1,18 +1,15 @@ -package org.sunbird.auth.verifier; - -import org.sunbird.common.JsonUtils; -import org.sunbird.common.Platform; -import org.sunbird.telemetry.logger.TelemetryManager; +package org.sunbird.common; import java.util.HashMap; import java.util.Map; +import org.sunbird.common.exception.ServerException; + public class JWTUtil { private static String SEPARATOR = "."; - - public static String JWT_SECRET_STRING = Platform.config.hasPath("content.security.jwt.secret") ? - Platform.config.getString("content.security.jwt.secret"): "sunbird"; - + private static String JWT_SECRET_STRING = Platform.config.hasPath("content_security_jwt_secret") ? + Platform.config.getString("content_security_jwt_secret"): "sunbird"; + public static String createHS256Token(Map claimsMap) { String token = ""; JWTokenType tokenType = JWTokenType.HS256; @@ -23,27 +20,7 @@ public static String createHS256Token(Map claimsMap) { CryptoUtil.generateHMAC(payLoad, JWT_SECRET_STRING, tokenType.getAlgorithmName())); token = payLoad + SEPARATOR + signature; } catch (Exception e) { - TelemetryManager.error("JWTUtil.createHS256Token :: Failed to create RS256 token. Exception: ", e); - } - return token; - } - - public static String createRS256Token(Map claimsMap) { - String token = ""; - JWTokenType tokenType = JWTokenType.RS256; - try { - KeyData keyData = KeyManager.getRandomKey(); - if (keyData != null) { - Map headerOptions = createHeaderOptions(keyData.getKeyId()); - String payLoad = createHeader(tokenType, headerOptions) + SEPARATOR + createClaimsMap(claimsMap); - String signature = encodeToBase64Uri( - CryptoUtil.generateRSASign(payLoad, keyData.getPrivateKey(), tokenType.getAlgorithmName())); - token = payLoad + SEPARATOR + signature; - } else { - TelemetryManager.error("JWTUtil.createRS256Token :: KeyManager is not initialized properly."); - } - } catch (Exception e) { - TelemetryManager.error("JWTUtil.createRS256Token :: Failed to create RS256 token. Exception: ", e); + throw new ServerException("ERR_INVALID_HEADER_PARAM", "JWTUtil.createHS256Token :: Failed to create RS256 token. Err is : " + e.getMessage()); } return token; } @@ -57,12 +34,6 @@ private static String createHeader(JWTokenType tokenType, Map he return encodeToBase64Uri(JsonUtils.serialize(headerData).getBytes()); } - private static Map createHeaderOptions(String keyId) { - Map headers = new HashMap<>(); - headers.put("kid", keyId); - return headers; - } - private static String createClaimsMap(Map claimsMap) throws Exception { Map payloadData = new HashMap<>(); if (claimsMap != null && claimsMap.size() > 0) { diff --git a/platform-core/auth-verifier/src/main/java/org/sunbird/auth/verifier/JWTokenType.java b/platform-core/platform-common/src/main/java/org/sunbird/common/JWTokenType.java similarity index 92% rename from platform-core/auth-verifier/src/main/java/org/sunbird/auth/verifier/JWTokenType.java rename to platform-core/platform-common/src/main/java/org/sunbird/common/JWTokenType.java index 0632e084d..1d3adbf39 100644 --- a/platform-core/auth-verifier/src/main/java/org/sunbird/auth/verifier/JWTokenType.java +++ b/platform-core/platform-common/src/main/java/org/sunbird/common/JWTokenType.java @@ -1,4 +1,4 @@ -package org.sunbird.auth.verifier; +package org.sunbird.common; public enum JWTokenType { HS256("HS256", "HmacSHA256"), diff --git a/platform-core/pom.xml b/platform-core/pom.xml index f8f4799e1..915ae188d 100755 --- a/platform-core/pom.xml +++ b/platform-core/pom.xml @@ -14,7 +14,6 @@ platform-common platform-telemetry actor-core - auth-verifier schema-validator platform-cache cassandra-connector diff --git a/pom.xml b/pom.xml index f3b94a318..f3f14193a 100644 --- a/pom.xml +++ b/pom.xml @@ -16,9 +16,6 @@ 2.11.12 3.0.8 2.9.8 - 2.22.0 - 2.0.7 - 2.0.0-beta.5 platform-core From 77ede59420a412faafae3f4958c20ffd1c65fc08 Mon Sep 17 00:00:00 2001 From: karthik-tarento Date: Thu, 1 Jun 2023 12:42:43 +0530 Subject: [PATCH 036/126] reverted the changes for conf file --- content-api/content-service/conf/application.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/content-api/content-service/conf/application.conf b/content-api/content-service/conf/application.conf index 751f86283..fa3214e9f 100644 --- a/content-api/content-service/conf/application.conf +++ b/content-api/content-service/conf/application.conf @@ -545,10 +545,10 @@ learning.service_provider=["youtube"] stream.mime.type=video/mp4 compositesearch.index.name="compositesearch" -hierarchy.keyspace.name=dev_hierarchy_store +hierarchy.keyspace.name=hierarchy_store content.hierarchy.table=content_hierarchy framework.hierarchy.table=framework_hierarchy -objectcategorydefinition.keyspace=dev_category_store +objectcategorydefinition.keyspace=category_store # Kafka topic for definition update event. kafka.topic.system.command="dev.system.command" From 0546b1f7bca53055deddc5243a9ef7a024dcd8df Mon Sep 17 00:00:00 2001 From: Karthikeyan Rajendran <70887864+karthik-tarento@users.noreply.github.com> Date: Tue, 5 Sep 2023 19:23:01 +0530 Subject: [PATCH 037/126] 4.8.3 upload fix (#63) * Using Async process for uploading zip file content --- .../sunbird/mimetype/mgr/BaseMimeTypeManager.scala | 14 ++++++++++++++ .../mimetype/mgr/impl/HtmlMimeTypeMgrImpl.scala | 4 ++-- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/platform-modules/mimetype-manager/src/main/scala/org/sunbird/mimetype/mgr/BaseMimeTypeManager.scala b/platform-modules/mimetype-manager/src/main/scala/org/sunbird/mimetype/mgr/BaseMimeTypeManager.scala index c357443c5..416d75925 100644 --- a/platform-modules/mimetype-manager/src/main/scala/org/sunbird/mimetype/mgr/BaseMimeTypeManager.scala +++ b/platform-modules/mimetype-manager/src/main/scala/org/sunbird/mimetype/mgr/BaseMimeTypeManager.scala @@ -214,6 +214,20 @@ class BaseMimeTypeManager(implicit ss: StorageService) { } } + def extractPackageInCloudAsync(objectId: String, uploadFile: File, node: Node, extractionType: String, slugFile: Boolean)(implicit ec: ExecutionContext): Future[List[String]] = { + val file = Slug.createSlugFile(uploadFile) + val mimeType = node.getMetadata.get("mimeType").asInstanceOf[String] + validationForCloudExtraction(file, extractionType, mimeType) + if(extractableMimeTypes.contains(mimeType)){ + val extractionBasePath = getBasePath(objectId) + extractPackage(file, extractionBasePath) + ss.uploadDirectoryAsync(getExtractionPath(objectId, node, extractionType, mimeType), new File(extractionBasePath), Option(slugFile)) + } else { + val emptyFuture: Future[List[String]] = Future.successful(List.empty[String]) + emptyFuture + } + } + def extractH5PPackageInCloud(objectId: String, extractionBasePath: String, node: Node, extractionType: String, slugFile: Boolean)(implicit ec: ExecutionContext): Future[List[String]] = { val mimeType = node.getMetadata.get("mimeType").asInstanceOf[String] if(null == extractionType) diff --git a/platform-modules/mimetype-manager/src/main/scala/org/sunbird/mimetype/mgr/impl/HtmlMimeTypeMgrImpl.scala b/platform-modules/mimetype-manager/src/main/scala/org/sunbird/mimetype/mgr/impl/HtmlMimeTypeMgrImpl.scala index 8418d2180..1ba192dec 100644 --- a/platform-modules/mimetype-manager/src/main/scala/org/sunbird/mimetype/mgr/impl/HtmlMimeTypeMgrImpl.scala +++ b/platform-modules/mimetype-manager/src/main/scala/org/sunbird/mimetype/mgr/impl/HtmlMimeTypeMgrImpl.scala @@ -18,13 +18,13 @@ class HtmlMimeTypeMgrImpl(implicit ss: StorageService) extends BaseMimeTypeManag override def upload(objectId: String, node: Node, uploadFile: File, filePath: Option[String], params: UploadParams)(implicit ec: ExecutionContext): Future[Map[String, AnyRef]] = { validateUploadRequest(objectId, node, uploadFile) val indexHtmlValidation: Boolean = if (Platform.config.hasPath("indexHtmlValidation.env")) Platform.config.getBoolean("indexHtmlValidation.env") else true - TelemetryManager.error("Value of indexHtmlValidation: " + indexHtmlValidation) + TelemetryManager.log("Value of indexHtmlValidation: " + indexHtmlValidation) val flag: Boolean = if (indexHtmlValidation) isValidPackageStructure(uploadFile, List[String]("index.html")) else true if (flag) { val urls = uploadArtifactToCloud(uploadFile, objectId, filePath) node.getMetadata.put("s3Key", urls(IDX_S3_KEY)) node.getMetadata.put("artifactUrl", urls(IDX_S3_URL)) - extractPackageInCloud(objectId, uploadFile, node, "snapshot", false) + Future { extractPackageInCloudAsync(objectId, uploadFile, node, "snapshot", false) } Future(Map[String, AnyRef]("identifier" -> objectId, "artifactUrl" -> urls(IDX_S3_URL), "s3Key" -> urls(IDX_S3_KEY), "size" -> getFileSize(uploadFile).asInstanceOf[AnyRef])) } else { TelemetryManager.error("ERR_INVALID_FILE" + "Please Provide Valid File! with file name: " + uploadFile.getName) From 71c0e00262924b7ed57f9987ac19ff73b99f4ebf Mon Sep 17 00:00:00 2001 From: Karthikeyan Rajendran <70887864+karthik-tarento@users.noreply.github.com> Date: Mon, 20 Nov 2023 19:30:27 +0530 Subject: [PATCH 038/126] Event Update API changes for update, retire (#64) (#66) Co-authored-by: Sahil-tarento <140611066+Sahil-tarento@users.noreply.github.com> --- .../org/sunbird/content/actors/EventActor.scala | 15 ++------------- .../app/controllers/v4/EventController.scala | 13 ++++++++++++- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/content-api/content-actors/src/main/scala/org/sunbird/content/actors/EventActor.scala b/content-api/content-actors/src/main/scala/org/sunbird/content/actors/EventActor.scala index cd42072d8..2be44d103 100644 --- a/content-api/content-actors/src/main/scala/org/sunbird/content/actors/EventActor.scala +++ b/content-api/content-actors/src/main/scala/org/sunbird/content/actors/EventActor.scala @@ -29,22 +29,11 @@ class EventActor @Inject()(implicit oec: OntologyEngineContext, ss: StorageServi } override def update(request: Request): Future[Response] = { - verifyStandaloneEventAndApply(super.update, request, Some(node => { - if (!"Draft".equalsIgnoreCase(node.getMetadata.getOrDefault("status", "").toString)) { - throw new ClientException(ContentConstants.ERR_CONTENT_NOT_DRAFT, "Update not allowed! Event status isn't draft") - } - })) + verifyStandaloneEventAndApply(super.update, request) } def publish(request: Request): Future[Response] = { - verifyStandaloneEventAndApply(super.update, request, Some(node => { - if (!"Draft".equalsIgnoreCase(node.getMetadata.getOrDefault("status", "").toString)) { - throw new ClientException(ContentConstants.ERR_CONTENT_NOT_DRAFT, "Publish not allowed! Event status isn't draft") - } - val versionKey = node.getMetadata.getOrDefault("versionKey", "").toString - if (StringUtils.isNotBlank(versionKey)) - request.put("versionKey", versionKey) - })) + verifyStandaloneEventAndApply(super.update, request) } override def discard(request: Request): Future[Response] = { diff --git a/content-api/content-service/app/controllers/v4/EventController.scala b/content-api/content-service/app/controllers/v4/EventController.scala index 3b609d32b..0f2589cc7 100644 --- a/content-api/content-service/app/controllers/v4/EventController.scala +++ b/content-api/content-service/app/controllers/v4/EventController.scala @@ -57,7 +57,8 @@ class EventController @Inject()(@Named(ActorNames.EVENT_ACTOR) eventActor: Actor def publish(identifier: String): Action[AnyContent] = Action.async { implicit request => val headers = commonHeaders() - val content = new java.util.HashMap[String, Object]() + val body = requestBody() + val content = body.getOrDefault(schemaName, new java.util.HashMap()).asInstanceOf[java.util.Map[String, Object]]; content.put("status", "Live") content.put("identifier", identifier) content.putAll(headers) @@ -67,4 +68,14 @@ class EventController @Inject()(@Named(ActorNames.EVENT_ACTOR) eventActor: Actor getResult(ApiId.PUBLISH_EVENT, eventActor, contentRequest, version = apiVersion) } + override def retire(identifier: String) = Action.async { implicit request => + val headers = commonHeaders() + val body = requestBody() + val content = body.getOrDefault(schemaName, new java.util.HashMap()).asInstanceOf[java.util.Map[String, Object]] + content.put("identifier", identifier) + content.putAll(headers) + val contentRequest = getRequest(content, headers, "retireContent") + setRequestContext(contentRequest, version, objectType, schemaName) + getResult(ApiId.RETIRE_CONTENT, eventActor, contentRequest, version = apiVersion) + } } \ No newline at end of file From 80d348fa259ffe0e8e883bff332a82353611e332 Mon Sep 17 00:00:00 2001 From: Karthikeyan Rajendran <70887864+karthik-tarento@users.noreply.github.com> Date: Wed, 22 Nov 2023 08:45:37 +0530 Subject: [PATCH 039/126] Added API to read the content by admin (#67) --- .../sunbird/content/actors/ContentActor.scala | 18 ++++++++++++++++++ .../app/controllers/v4/ContentController.scala | 10 ++++++++++ content-api/content-service/conf/routes | 1 + 3 files changed, 29 insertions(+) diff --git a/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala b/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala index f8d641180..2f41ca5f0 100644 --- a/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala +++ b/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala @@ -55,6 +55,7 @@ class ContentActor @Inject() (implicit oec: OntologyEngineContext, ss: StorageSe case "systemUpdate" => systemUpdate(request) case "reviewContent" => reviewContent(request) case "rejectContent" => rejectContent(request) + case "adminReadContent" => adminRead(request) case _ => ERROR(request.getOperation) } } @@ -323,4 +324,21 @@ class ContentActor @Inject() (implicit oec: OntologyEngineContext, ss: StorageSe }).flatMap(f => f) } + def adminRead(request: Request): Future[Response] = { + val responseSchemaName: String = request.getContext.getOrDefault(ContentConstants.RESPONSE_SCHEMA_NAME, "").asInstanceOf[String] + val fields: util.List[String] = JavaConverters.seqAsJavaListConverter(request.get("fields").asInstanceOf[String].split(",").filter(field => StringUtils.isNotBlank(field) && !StringUtils.equalsIgnoreCase(field, "null"))).asJava + request.getRequest.put("fields", fields) + DataNode.read(request).map(node => { + val metadata: util.Map[String, AnyRef] = NodeUtil.serialize(node, fields, node.getObjectType.toLowerCase.replace("image", ""), request.getContext.get("version").asInstanceOf[String]) + metadata.put("identifier", node.getIdentifier.replace(".img", "")) + val response: Response = ResponseHandler.OK + if (responseSchemaName.isEmpty) { + response.put("content", metadata) + } else { + response.put(responseSchemaName, metadata) + } + response + }) + } + } diff --git a/content-api/content-service/app/controllers/v4/ContentController.scala b/content-api/content-service/app/controllers/v4/ContentController.scala index ddf2dc45c..3657b2fc5 100644 --- a/content-api/content-service/app/controllers/v4/ContentController.scala +++ b/content-api/content-service/app/controllers/v4/ContentController.scala @@ -208,4 +208,14 @@ class ContentController @Inject()(@Named(ActorNames.CONTENT_ACTOR) contentActor: getResult(ApiId.REJECT_CONTENT, contentActor, contentRequest, version = apiVersion) } + def adminRead(identifier: String, mode: Option[String], fields: Option[String]) = Action.async { implicit request => + val headers = commonHeaders() + val content = new java.util.HashMap().asInstanceOf[java.util.Map[String, Object]] + content.putAll(headers) + content.putAll(Map("identifier" -> identifier, "mode" -> mode.getOrElse("read"), "fields" -> fields.getOrElse("")).asJava) + val readRequest = getRequest(content, headers, "adminReadContent") + setRequestContext(readRequest, version, objectType, schemaName) + getResult(ApiId.READ_PRIVATE_CONTENT, contentActor, readRequest, version = apiVersion) + } + } diff --git a/content-api/content-service/conf/routes b/content-api/content-service/conf/routes index fc4c8ad83..3cc5616cd 100644 --- a/content-api/content-service/conf/routes +++ b/content-api/content-service/conf/routes @@ -87,6 +87,7 @@ POST /content/v4/create controllers.v4.ContentControl PATCH /content/v4/update/:identifier controllers.v4.ContentController.update(identifier:String) GET /content/v4/read/:identifier controllers.v4.ContentController.read(identifier:String, mode:Option[String], fields:Option[String]) GET /content/v4/private/read/:identifier controllers.v4.ContentController.privateRead(identifier:String, mode:Option[String], fields:Option[String]) +GET /content/v4/admin/read/:identifier controllers.v4.ContentController.adminRead(identifier:String, mode:Option[String], fields:Option[String]) POST /content/v4/upload/url/:identifier controllers.v4.ContentController.uploadPreSigned(identifier:String, type: Option[String]) POST /content/v4/upload/:identifier controllers.v4.ContentController.upload(identifier:String, fileFormat: Option[String], validation: Option[String]) POST /content/v4/copy/:identifier controllers.v4.ContentController.copy(identifier:String, mode:Option[String], type:String ?= "deep") From 5310ceeee5bf68b8076cd47bafdcf3ce32d84e37 Mon Sep 17 00:00:00 2001 From: vikrantsingh Date: Tue, 16 Jan 2024 18:05:50 +0530 Subject: [PATCH 040/126] KAR-262:Live Moderated Course Editing Fix --- .../org/sunbird/graph/service/util/BaseQueryGenerationUtil.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ontology-engine/graph-dac-api/src/main/java/org/sunbird/graph/service/util/BaseQueryGenerationUtil.java b/ontology-engine/graph-dac-api/src/main/java/org/sunbird/graph/service/util/BaseQueryGenerationUtil.java index 09fabece2..57e538da8 100644 --- a/ontology-engine/graph-dac-api/src/main/java/org/sunbird/graph/service/util/BaseQueryGenerationUtil.java +++ b/ontology-engine/graph-dac-api/src/main/java/org/sunbird/graph/service/util/BaseQueryGenerationUtil.java @@ -172,7 +172,7 @@ protected static Map getSystemPropertyQueryMap(Node node, String if (StringUtils.isBlank(node.getIdentifier())) node.setIdentifier(Identifier.getIdentifier(node.getGraphId(), Identifier.getUniqueIdFromTimestamp())); - if (node.getMetadata().containsKey("secureSettings")) { + if (node.getMetadata().containsKey("secureSettings") && !node.getIdentifier().contains("_rc")) { node.setIdentifier(node.getIdentifier() + "_rc"); } // Adding 'IL_UNIQUE_ID' Property From 3eeefb3f053b6d3001b53e8da95a7ec2505c00bc Mon Sep 17 00:00:00 2001 From: sreeragksgh Date: Tue, 16 Jan 2024 18:12:23 +0530 Subject: [PATCH 041/126] Validations to ensure boolean input for MCQ answers --- .../managers/UpdateHierarchyManager.scala | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/assessment-api/qs-hierarchy-manager/src/main/scala/org/sunbird/managers/UpdateHierarchyManager.scala b/assessment-api/qs-hierarchy-manager/src/main/scala/org/sunbird/managers/UpdateHierarchyManager.scala index 59cdd1810..78284b687 100644 --- a/assessment-api/qs-hierarchy-manager/src/main/scala/org/sunbird/managers/UpdateHierarchyManager.scala +++ b/assessment-api/qs-hierarchy-manager/src/main/scala/org/sunbird/managers/UpdateHierarchyManager.scala @@ -71,6 +71,22 @@ object UpdateHierarchyManager { .getOrDefault(HierarchyConstants.OBJECT_TYPE, "").asInstanceOf[String], "Question")) throw new ClientException("ERR_QS_UPDATE_HIERARCHY", "Question cannot have children in hierarchy") }) + val iterator = nodesModified.entrySet().iterator() + while (iterator.hasNext) { + val entry = iterator.next() + val questionData = entry.getValue().asInstanceOf[java.util.HashMap[String, AnyRef]] + val metadata = questionData.get(HierarchyConstants.METADATA).asInstanceOf[java.util.HashMap[String, AnyRef]] + val choices = metadata.get("choices").asInstanceOf[java.util.HashMap[String, AnyRef]].get("options").asInstanceOf[java.util.List[java.util.HashMap[String, AnyRef]]].asScala.toList + for (option <- choices) { + val answerType = option.get("answer") + if (!answerType.isInstanceOf[java.lang.Boolean]) { + val questionBody = metadata.get("body").asInstanceOf[String] + val optionBody = option.get("value").asInstanceOf[java.util.HashMap[String, AnyRef]].get("body") + println(s"Error: Question '$questionBody', Option '$optionBody' has a non-boolean answer.") + throw new ClientException("ERR_QS_UPDATE_HIERARCHY", s"Error: Question : '$questionBody', Option : '$optionBody' answer value should be either true or false") + } + } + } (nodesModified, hierarchy) } From 93cde23b99bacd654407b0e368790a7ff511746c Mon Sep 17 00:00:00 2001 From: sreeragksgh Date: Wed, 17 Jan 2024 09:23:27 +0530 Subject: [PATCH 042/126] Additional check on MCQ questions --- .../managers/UpdateHierarchyManager.scala | 17 +++++++++-------- .../org/sunbird/utils/HierarchyConstants.scala | 7 +++++++ 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/assessment-api/qs-hierarchy-manager/src/main/scala/org/sunbird/managers/UpdateHierarchyManager.scala b/assessment-api/qs-hierarchy-manager/src/main/scala/org/sunbird/managers/UpdateHierarchyManager.scala index 78284b687..cfa90e6d8 100644 --- a/assessment-api/qs-hierarchy-manager/src/main/scala/org/sunbird/managers/UpdateHierarchyManager.scala +++ b/assessment-api/qs-hierarchy-manager/src/main/scala/org/sunbird/managers/UpdateHierarchyManager.scala @@ -76,14 +76,15 @@ object UpdateHierarchyManager { val entry = iterator.next() val questionData = entry.getValue().asInstanceOf[java.util.HashMap[String, AnyRef]] val metadata = questionData.get(HierarchyConstants.METADATA).asInstanceOf[java.util.HashMap[String, AnyRef]] - val choices = metadata.get("choices").asInstanceOf[java.util.HashMap[String, AnyRef]].get("options").asInstanceOf[java.util.List[java.util.HashMap[String, AnyRef]]].asScala.toList - for (option <- choices) { - val answerType = option.get("answer") - if (!answerType.isInstanceOf[java.lang.Boolean]) { - val questionBody = metadata.get("body").asInstanceOf[String] - val optionBody = option.get("value").asInstanceOf[java.util.HashMap[String, AnyRef]].get("body") - println(s"Error: Question '$questionBody', Option '$optionBody' has a non-boolean answer.") - throw new ClientException("ERR_QS_UPDATE_HIERARCHY", s"Error: Question : '$questionBody', Option : '$optionBody' answer value should be either true or false") + if (metadata.get(HierarchyConstants.PRIMARY_CATEGORY) == "Multiple Choice Question") { + val choices = metadata.get(HierarchyConstants.CHOICES).asInstanceOf[java.util.HashMap[String, AnyRef]].get(HierarchyConstants.OPTIONS).asInstanceOf[java.util.List[java.util.HashMap[String, AnyRef]]].asScala.toList + for (option <- choices) { + val answerType = option.get(HierarchyConstants.ANSWER) + if (!answerType.isInstanceOf[java.lang.Boolean]) { + val questionBody = metadata.get(HierarchyConstants.BODY).asInstanceOf[String] + val optionBody = option.get(HierarchyConstants.VALUE).asInstanceOf[java.util.HashMap[String, AnyRef]].get(HierarchyConstants.BODY) + throw new ClientException("ERR_QS_UPDATE_HIERARCHY", s"Error: Question : '$questionBody', Option : '$optionBody' has a non-boolean answer. The answer value should be either true or false.") + } } } } diff --git a/assessment-api/qs-hierarchy-manager/src/main/scala/org/sunbird/utils/HierarchyConstants.scala b/assessment-api/qs-hierarchy-manager/src/main/scala/org/sunbird/utils/HierarchyConstants.scala index 25408886c..3141661ae 100644 --- a/assessment-api/qs-hierarchy-manager/src/main/scala/org/sunbird/utils/HierarchyConstants.scala +++ b/assessment-api/qs-hierarchy-manager/src/main/scala/org/sunbird/utils/HierarchyConstants.scala @@ -57,4 +57,11 @@ object HierarchyConstants { val SOURCE: String = "source" val PRE_CONDITION: String = "preCondition" val QUESTION_VISIBILITY: List[String] = List("Default", "Parent") + val OPTIONS = "options" + val CHOICES = "choices" + val ANSWER = "answer" + val BODY = "body" + val VALUE = "value" + val PRIMARY_CATEGORY = "primaryCategory" + } From f39c77837a92782d2e8008921986b98e4de790f4 Mon Sep 17 00:00:00 2001 From: sreeragksgh Date: Tue, 23 Jan 2024 10:20:51 +0530 Subject: [PATCH 043/126] Review changes --- .../managers/UpdateHierarchyManager.scala | 57 ++++++++++++++++--- 1 file changed, 48 insertions(+), 9 deletions(-) diff --git a/assessment-api/qs-hierarchy-manager/src/main/scala/org/sunbird/managers/UpdateHierarchyManager.scala b/assessment-api/qs-hierarchy-manager/src/main/scala/org/sunbird/managers/UpdateHierarchyManager.scala index cfa90e6d8..41dac361c 100644 --- a/assessment-api/qs-hierarchy-manager/src/main/scala/org/sunbird/managers/UpdateHierarchyManager.scala +++ b/assessment-api/qs-hierarchy-manager/src/main/scala/org/sunbird/managers/UpdateHierarchyManager.scala @@ -74,16 +74,55 @@ object UpdateHierarchyManager { val iterator = nodesModified.entrySet().iterator() while (iterator.hasNext) { val entry = iterator.next() - val questionData = entry.getValue().asInstanceOf[java.util.HashMap[String, AnyRef]] - val metadata = questionData.get(HierarchyConstants.METADATA).asInstanceOf[java.util.HashMap[String, AnyRef]] + val questionData = entry.getValue() match { + case map: java.util.HashMap[_, _] => map.asInstanceOf[java.util.HashMap[String, AnyRef]] + case _ => throw new ClientException("ERR_QS_UPDATE_HIERARCHY", "Error: Invalid question data.") + } + val metadata = Option(questionData.get(HierarchyConstants.METADATA)) match { + case Some(map: java.util.HashMap[_, _]) => map.asInstanceOf[java.util.HashMap[String, AnyRef]] + case _ => throw new ClientException("ERR_QS_UPDATE_HIERARCHY", "Error: Missing metadata.") + } if (metadata.get(HierarchyConstants.PRIMARY_CATEGORY) == "Multiple Choice Question") { - val choices = metadata.get(HierarchyConstants.CHOICES).asInstanceOf[java.util.HashMap[String, AnyRef]].get(HierarchyConstants.OPTIONS).asInstanceOf[java.util.List[java.util.HashMap[String, AnyRef]]].asScala.toList - for (option <- choices) { - val answerType = option.get(HierarchyConstants.ANSWER) - if (!answerType.isInstanceOf[java.lang.Boolean]) { - val questionBody = metadata.get(HierarchyConstants.BODY).asInstanceOf[String] - val optionBody = option.get(HierarchyConstants.VALUE).asInstanceOf[java.util.HashMap[String, AnyRef]].get(HierarchyConstants.BODY) - throw new ClientException("ERR_QS_UPDATE_HIERARCHY", s"Error: Question : '$questionBody', Option : '$optionBody' has a non-boolean answer. The answer value should be either true or false.") + val primaryCategory = Option(metadata.get(HierarchyConstants.PRIMARY_CATEGORY)) match { + case Some(category: String) => category + case _ => throw new ClientException("ERR_QS_UPDATE_HIERARCHY", "Error: Missing primary category.") + } + + if (primaryCategory == "Multiple Choice Question") { + val choices = Option(metadata.get(HierarchyConstants.CHOICES)) match { + case Some(choiceMap: java.util.HashMap[_, _]) => + val options = Option(choiceMap.get(HierarchyConstants.OPTIONS)) match { + case Some(optionsList: java.util.List[_]) => + optionsList.asInstanceOf[java.util.List[java.util.HashMap[String, AnyRef]]].asScala.toList + case _ => throw new ClientException("ERR_QS_UPDATE_HIERARCHY", "Error: Missing options.") + } + options + case _ => throw new ClientException("ERR_QS_UPDATE_HIERARCHY", "Error: Missing choices.") + } + + for (option <- choices) { + val answerType = Option(option.get(HierarchyConstants.ANSWER)) match { + case Some(answer) => answer + case _ => throw new ClientException("ERR_QS_UPDATE_HIERARCHY", "Error: Missing answer type.") + } + + if (!answerType.isInstanceOf[java.lang.Boolean]) { + val questionBody = Option(metadata.get(HierarchyConstants.BODY)) match { + case Some(body: String) => body + case _ => throw new ClientException("ERR_QS_UPDATE_HIERARCHY", "Error: Missing question body.") + } + + val optionBody = Option(option.get(HierarchyConstants.VALUE)) match { + case Some(valueMap: java.util.HashMap[_, _]) => + Option(valueMap.get(HierarchyConstants.BODY)) match { + case Some(body) => body + case _ => throw new ClientException("ERR_QS_UPDATE_HIERARCHY", "Error: Missing option body.") + } + case _ => throw new ClientException("ERR_QS_UPDATE_HIERARCHY", "Error: Missing option value map.") + } + + throw new ClientException("ERR_QS_UPDATE_HIERARCHY", s"Error: Question : '$questionBody', Option : '$optionBody' has a non-boolean answer. The answer value should be either true or false.") + } } } } From 6c587bcf5cfd20dc666d01bb3307d3d4d71adb7f Mon Sep 17 00:00:00 2001 From: sreeragksgh Date: Tue, 23 Jan 2024 10:42:05 +0530 Subject: [PATCH 044/126] Adding constant value for MCQ - review changes --- .../scala/org/sunbird/managers/UpdateHierarchyManager.scala | 2 +- .../src/main/scala/org/sunbird/utils/HierarchyConstants.scala | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/assessment-api/qs-hierarchy-manager/src/main/scala/org/sunbird/managers/UpdateHierarchyManager.scala b/assessment-api/qs-hierarchy-manager/src/main/scala/org/sunbird/managers/UpdateHierarchyManager.scala index 41dac361c..729f6bfa7 100644 --- a/assessment-api/qs-hierarchy-manager/src/main/scala/org/sunbird/managers/UpdateHierarchyManager.scala +++ b/assessment-api/qs-hierarchy-manager/src/main/scala/org/sunbird/managers/UpdateHierarchyManager.scala @@ -82,7 +82,7 @@ object UpdateHierarchyManager { case Some(map: java.util.HashMap[_, _]) => map.asInstanceOf[java.util.HashMap[String, AnyRef]] case _ => throw new ClientException("ERR_QS_UPDATE_HIERARCHY", "Error: Missing metadata.") } - if (metadata.get(HierarchyConstants.PRIMARY_CATEGORY) == "Multiple Choice Question") { + if (metadata.get(HierarchyConstants.PRIMARY_CATEGORY) == HierarchyConstants.MCQ) { val primaryCategory = Option(metadata.get(HierarchyConstants.PRIMARY_CATEGORY)) match { case Some(category: String) => category case _ => throw new ClientException("ERR_QS_UPDATE_HIERARCHY", "Error: Missing primary category.") diff --git a/assessment-api/qs-hierarchy-manager/src/main/scala/org/sunbird/utils/HierarchyConstants.scala b/assessment-api/qs-hierarchy-manager/src/main/scala/org/sunbird/utils/HierarchyConstants.scala index 3141661ae..16114a3fc 100644 --- a/assessment-api/qs-hierarchy-manager/src/main/scala/org/sunbird/utils/HierarchyConstants.scala +++ b/assessment-api/qs-hierarchy-manager/src/main/scala/org/sunbird/utils/HierarchyConstants.scala @@ -63,5 +63,7 @@ object HierarchyConstants { val BODY = "body" val VALUE = "value" val PRIMARY_CATEGORY = "primaryCategory" + val MCQ = "Multiple Choice Question" + } From ddd6136ee037c835d1b2a945074391a5e1f30e8f Mon Sep 17 00:00:00 2001 From: sreeragksgh Date: Tue, 23 Jan 2024 11:14:23 +0530 Subject: [PATCH 045/126] Adding constant value for MCQ - review changes --- .../scala/org/sunbird/managers/UpdateHierarchyManager.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/assessment-api/qs-hierarchy-manager/src/main/scala/org/sunbird/managers/UpdateHierarchyManager.scala b/assessment-api/qs-hierarchy-manager/src/main/scala/org/sunbird/managers/UpdateHierarchyManager.scala index 729f6bfa7..6143a8f71 100644 --- a/assessment-api/qs-hierarchy-manager/src/main/scala/org/sunbird/managers/UpdateHierarchyManager.scala +++ b/assessment-api/qs-hierarchy-manager/src/main/scala/org/sunbird/managers/UpdateHierarchyManager.scala @@ -88,7 +88,7 @@ object UpdateHierarchyManager { case _ => throw new ClientException("ERR_QS_UPDATE_HIERARCHY", "Error: Missing primary category.") } - if (primaryCategory == "Multiple Choice Question") { + if (primaryCategory == HierarchyConstants.MCQ) { val choices = Option(metadata.get(HierarchyConstants.CHOICES)) match { case Some(choiceMap: java.util.HashMap[_, _]) => val options = Option(choiceMap.get(HierarchyConstants.OPTIONS)) match { From 3322b1d5a0856a878f81e5a8ce56a13cf2deb733 Mon Sep 17 00:00:00 2001 From: Karthikeyan Rajendran <70887864+karthik-tarento@users.noreply.github.com> Date: Tue, 6 Feb 2024 18:40:45 +0530 Subject: [PATCH 046/126] Adding the Moderated courses Logic in normal courses (#68) (#71) * Adding the Moderated courses Logic in normal courses (#68) * Adding the Moderated courses Logic in normal courses * Adding search V4 for moderated courses * Added the additional filter for securedSettings (#72) * Added the additional filter for securedSettings * Updated the review comments --------- Co-authored-by: Sahil-tarento <140611066+Sahil-tarento@users.noreply.github.com> --- .../java/org/sunbird/actors/SearchActor.java | 20 ++++++ .../org/sunbird/search/dto/SearchDTO.java | 19 ++++- .../search/processor/SearchProcessor.java | 72 ++++++++++++++++--- .../sunbird/search/util/SearchConstants.java | 2 + .../app/controllers/SearchController.scala | 21 ++++++ search-api/search-service/conf/routes | 3 +- 6 files changed, 126 insertions(+), 11 deletions(-) diff --git a/search-api/search-actors/src/main/java/org/sunbird/actors/SearchActor.java b/search-api/search-actors/src/main/java/org/sunbird/actors/SearchActor.java index 136b52454..e6f45f37a 100644 --- a/search-api/search-actors/src/main/java/org/sunbird/actors/SearchActor.java +++ b/search-api/search-actors/src/main/java/org/sunbird/actors/SearchActor.java @@ -100,6 +100,11 @@ private SearchDTO getSearchDTO(Request request) throws Exception { } else { searchObj.setSecureSettings(false); } + if (req.get(SearchConstants.isSecureSettingsDisabled) != null) { + searchObj.setSecureSettingsDisabled((Boolean) req.get(SearchConstants.isSecureSettingsDisabled)); + } else { + searchObj.setSecureSettingsDisabled(false); + } searchObj.setUserOrgId((String) request.getContext().get("x-user-channel-id")); TelemetryManager.log("Search Request: ", req); String queryString = (String) req.get(SearchConstants.query); @@ -125,6 +130,21 @@ private SearchDTO getSearchDTO(Request request) throws Exception { if (filters.containsKey("relatedBoards")) filters.remove("relatedBoards"); + Map secureSettingsFilter = new HashMap<>(); + for (String key : filters.keySet()) { + if (key.startsWith(SearchConstants.secureSettings)) { + secureSettingsFilter.put(key, filters.get(key)); + } + } + searchObj.setPostFilter(secureSettingsFilter); + if (MapUtils.isEmpty(searchObj.getPostFilter())) { + secureSettingsFilter.put(SearchConstants.secureSettingsOrganisation, searchObj.getUserOrgId()); + searchObj.setPostFilter(secureSettingsFilter); + } else { + for(String key: searchObj.getPostFilter().keySet()) { + filters.remove(key); + } + } Object objectTypeFromFilter = filters.get(SearchConstants.objectType); String objectType = null; if (objectTypeFromFilter != null) { diff --git a/search-api/search-core/src/main/java/org/sunbird/search/dto/SearchDTO.java b/search-api/search-core/src/main/java/org/sunbird/search/dto/SearchDTO.java index 328d45212..1d1392572 100644 --- a/search-api/search-core/src/main/java/org/sunbird/search/dto/SearchDTO.java +++ b/search-api/search-core/src/main/java/org/sunbird/search/dto/SearchDTO.java @@ -23,7 +23,8 @@ public class SearchDTO { private List> aggregations = new ArrayList<>(); private List implicitFilterProperties; private List multiFilterProperties; - + boolean isSecureSettingsDisabled = false; + private Map postFilter = new HashMap<>(); public SearchDTO() { super(); @@ -137,4 +138,20 @@ public List getMultiFilterProperties() { public void setMultiFilterProperties(List multiFilterProperties) { this.multiFilterProperties = multiFilterProperties; } + + public boolean isSecureSettingsDisabled() { + return isSecureSettingsDisabled; + } + + public void setSecureSettingsDisabled(boolean secureSettingsDisabled) { + isSecureSettingsDisabled = secureSettingsDisabled; + } + + public Map getPostFilter() { + return postFilter; + } + + public void setPostFilter(Map postFilter) { + this.postFilter = postFilter; + } } diff --git a/search-api/search-core/src/main/java/org/sunbird/search/processor/SearchProcessor.java b/search-api/search-core/src/main/java/org/sunbird/search/processor/SearchProcessor.java index 58a734a5a..9926bcb59 100644 --- a/search-api/search-core/src/main/java/org/sunbird/search/processor/SearchProcessor.java +++ b/search-api/search-core/src/main/java/org/sunbird/search/processor/SearchProcessor.java @@ -53,6 +53,11 @@ public Future> processSearch(SearchDTO searchDTO, boolean in throws Exception { List> groupByFinalList = new ArrayList>(); SearchSourceBuilder query = processSearchQuery(searchDTO, groupByFinalList, true); + + if (searchDTO.isSecureSettingsDisabled()) { + query.postFilter(getPostFilterQuery(searchDTO.getPostFilter())); + } + Future searchResponse = null; boolean enableFuzzyWhenNoResults = Platform.config.hasPath("search.fields.enable.fuzzy.when.noresult") && Platform.config.getBoolean("search.fields.enable.fuzzy.when.noresult"); @@ -328,15 +333,24 @@ private QueryBuilder prepareSearchQuery(SearchDTO searchDTO) { QueryBuilder queryBuilder = null; String totalOperation = searchDTO.getOperation(); List properties = searchDTO.getProperties(); - if (searchDTO.isSecureSettings() == false) - formQuery(properties, queryBuilder, boolQuery, totalOperation, searchDTO.isFuzzySearch()); - else + if (searchDTO.isSecureSettingsDisabled()) { formQueryImpl(properties, queryBuilder, boolQuery, totalOperation, searchDTO.isFuzzySearch(), searchDTO); - if (searchDTO.getMultiFilterProperties() != null) { + } else { if (searchDTO.isSecureSettings() == false) - formQuery(searchDTO.getMultiFilterProperties(), queryBuilder, boolQuery, SearchConstants.SEARCH_OPERATION_OR, searchDTO.isFuzzySearch()); - else { + formQuery(properties, queryBuilder, boolQuery, totalOperation, searchDTO.isFuzzySearch()); + else + formQueryImpl(properties, queryBuilder, boolQuery, totalOperation, searchDTO.isFuzzySearch(), searchDTO); + } + + if (searchDTO.getMultiFilterProperties() != null) { + if (searchDTO.isSecureSettingsDisabled()) { formQueryImpl(searchDTO.getMultiFilterProperties(), queryBuilder, boolQuery, totalOperation, searchDTO.isFuzzySearch(), searchDTO); + } else { + if (searchDTO.isSecureSettings() == false) + formQuery(searchDTO.getMultiFilterProperties(), queryBuilder, boolQuery, SearchConstants.SEARCH_OPERATION_OR, searchDTO.isFuzzySearch()); + else { + formQueryImpl(searchDTO.getMultiFilterProperties(), queryBuilder, boolQuery, totalOperation, searchDTO.isFuzzySearch(), searchDTO); + } } } Map softConstraints = searchDTO.getSoftConstraints(); @@ -354,8 +368,11 @@ private void formQuery(List properties, QueryBuilder queryBuilder, BoolQuer private void formQueryImpl(List properties, QueryBuilder queryBuilder, BoolQueryBuilder boolQuery, String operation, Boolean fuzzy, SearchDTO searchDTO) { boolean enableSecureSettings = false; - if (searchDTO != null) + boolean disableSecureSettings = false; + if (searchDTO != null) { enableSecureSettings = searchDTO.isSecureSettings(); + disableSecureSettings = searchDTO.isSecureSettingsDisabled(); + } for (Map property : properties) { String opertation = (String) property.get("operation"); @@ -376,7 +393,9 @@ private void formQueryImpl(List properties, QueryBuilder queryBuilder, Bool if (enableSecureSettings) { boolQuery.must(getSecureSettingsSearchQuery(searchDTO.getUserOrgId())); } else { - boolQuery.mustNot(getSecureSettingsSearchDefaultQuery()); + if (!disableSecureSettings) { + boolQuery.mustNot(getSecureSettingsSearchDefaultQuery()); + } } boolQuery.must(queryBuilder); continue; @@ -469,7 +488,9 @@ private void formQueryImpl(List properties, QueryBuilder queryBuilder, Bool if (enableSecureSettings) { boolQuery.must(getSecureSettingsSearchQuery(searchDTO.getUserOrgId())); } else { - boolQuery.mustNot(getSecureSettingsSearchDefaultQuery()); + if (!disableSecureSettings) { + boolQuery.mustNot(getSecureSettingsSearchDefaultQuery()); + } } boolQuery.must(queryBuilder); } else { @@ -881,5 +902,38 @@ private QueryBuilder getQuery(SearchDTO searchDTO) { return prepareSearchQuery(searchDTO); } + private static QueryBuilder getPostFilterQuery(Map postFilter) { + // Creating the post_filter bool query + BoolQueryBuilder postFilterBoolQuery = QueryBuilders.boolQuery(); + BoolQueryBuilder nestedBoolQuery = QueryBuilders.boolQuery(); + for (Map.Entry filters : postFilter.entrySet()) { + if (filters.getValue() instanceof List) { + for (String value : (List) filters.getValue()) { + nestedBoolQuery.should(QueryBuilders.termQuery(filters.getKey(), value)); + } + } else if (filters.getValue() instanceof String) { + nestedBoolQuery.should(QueryBuilders.termQuery(filters.getKey(), ((String) filters.getValue()).toLowerCase())); + } + } + + NestedQueryBuilder nestedQuery = QueryBuilders.nestedQuery( + "secureSettings", + nestedBoolQuery, + org.apache.lucene.search.join.ScoreMode.None + ); + postFilterBoolQuery.should(nestedQuery); + // Nested query for "secureSettings" with must_not exists + BoolQueryBuilder mustNotBoolQuery = QueryBuilders.boolQuery(); + NestedQueryBuilder nestedMustNotQuery = QueryBuilders.nestedQuery( + "secureSettings", + new ExistsQueryBuilder("secureSettings.organisation") + .boost(1.0f), + org.apache.lucene.search.join.ScoreMode.None + ); + mustNotBoolQuery.mustNot(nestedMustNotQuery); + postFilterBoolQuery.should(mustNotBoolQuery); + + return postFilterBoolQuery; + } } \ No newline at end of file diff --git a/search-api/search-core/src/main/java/org/sunbird/search/util/SearchConstants.java b/search-api/search-core/src/main/java/org/sunbird/search/util/SearchConstants.java index 66d276168..bbd41f9ed 100644 --- a/search-api/search-core/src/main/java/org/sunbird/search/util/SearchConstants.java +++ b/search-api/search-core/src/main/java/org/sunbird/search/util/SearchConstants.java @@ -115,4 +115,6 @@ public class SearchConstants { public static final String setDefaultVisibility = "setDefaultVisibility"; public static String soft = "soft"; public static String secureSettings = "secureSettings"; + public static String isSecureSettingsDisabled = "isSecureSettingsDisabled"; + public static String secureSettingsOrganisation = "secureSettings.organisation"; } diff --git a/search-api/search-service/app/controllers/SearchController.scala b/search-api/search-service/app/controllers/SearchController.scala index e76bcc5e3..451e7c33c 100644 --- a/search-api/search-service/app/controllers/SearchController.scala +++ b/search-api/search-service/app/controllers/SearchController.scala @@ -57,4 +57,25 @@ class SearchController @Inject()(@Named(ActorNames.SEARCH_ACTOR) searchActor: Ac setHeaderContext(internalReq) getResult(mgr.count(internalReq, searchActor), ApiId.APPLICATION_COUNT) } + + def searchV4() = loggingAction.async { implicit request => + val internalReq = getRequest(ApiId.APPLICATION_SEARCH) + val requestMap: java.util.Map[String, Any] = internalReq.getRequest.asInstanceOf[util.Map[String, Any]] + requestMap.put(SearchConstants.isSecureSettingsDisabled, true) + setHeaderContext(internalReq) + val filters = internalReq.getRequest.getOrDefault(SearchConstants.filters, new java.util.HashMap()).asInstanceOf[java.util.Map[String, Object]] + val visibilityObject = filters.getOrDefault("visibility","") + var visibility:util.List[String] = null + if (visibilityObject != null) { + if (visibilityObject.isInstanceOf[util.ArrayList[_]]) visibility = visibilityObject.asInstanceOf[util.ArrayList[String]] + else if (visibilityObject.isInstanceOf[String]) visibility = util.Arrays.asList(visibilityObject.asInstanceOf[String]) + } + if (visibility.contains("Private")) { + getErrorResponse(ApiId.APPLICATION_SEARCH, apiVersion, SearchConstants.ERR_ACCESS_DENIED, "Cannot access private content through public search api") + } + else { + internalReq.getContext.put(SearchConstants.setDefaultVisibility, "true") + getResult(mgr.search(internalReq, searchActor), ApiId.APPLICATION_SEARCH) + } + } } diff --git a/search-api/search-service/conf/routes b/search-api/search-service/conf/routes index 44e6c9ad5..5a39e14a6 100644 --- a/search-api/search-service/conf/routes +++ b/search-api/search-service/conf/routes @@ -10,4 +10,5 @@ POST /v3/private/search controllers.SearchController.privateSearch() POST /v2/search/count controllers.SearchController.count() POST /v3/count controllers.SearchController.count() #POST /v2/metrics controllers.MetricsController.search() -#POST /v3/metrics controllers.MetricsController.search() \ No newline at end of file +#POST /v3/metrics controllers.MetricsController.search() +POST /v4/search controllers.SearchController.searchV4() \ No newline at end of file From 5f634682f0e5a61635351280105b32f0cece9493 Mon Sep 17 00:00:00 2001 From: Sahil-tarento <140611066+Sahil-tarento@users.noreply.github.com> Date: Tue, 13 Feb 2024 17:52:37 +0530 Subject: [PATCH 047/126] Fix the issue for checking the SecureSetting variables (#73) --- .../main/java/org/sunbird/search/processor/SearchProcessor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/search-api/search-core/src/main/java/org/sunbird/search/processor/SearchProcessor.java b/search-api/search-core/src/main/java/org/sunbird/search/processor/SearchProcessor.java index 9926bcb59..f8f40268b 100644 --- a/search-api/search-core/src/main/java/org/sunbird/search/processor/SearchProcessor.java +++ b/search-api/search-core/src/main/java/org/sunbird/search/processor/SearchProcessor.java @@ -910,7 +910,7 @@ private static QueryBuilder getPostFilterQuery(Map postFilter) { for (Map.Entry filters : postFilter.entrySet()) { if (filters.getValue() instanceof List) { for (String value : (List) filters.getValue()) { - nestedBoolQuery.should(QueryBuilders.termQuery(filters.getKey(), value)); + nestedBoolQuery.must(QueryBuilders.termQuery(filters.getKey(), value)); } } else if (filters.getValue() instanceof String) { nestedBoolQuery.should(QueryBuilders.termQuery(filters.getKey(), ((String) filters.getValue()).toLowerCase())); From c0be98fe561b81cc971fef9c2c001d86b491271a Mon Sep 17 00:00:00 2001 From: Sahil-tarento <140611066+Sahil-tarento@users.noreply.github.com> Date: Mon, 19 Feb 2024 12:13:40 +0530 Subject: [PATCH 048/126] Adding the mustCondition for all subFilter (#74) --- .../main/java/org/sunbird/search/processor/SearchProcessor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/search-api/search-core/src/main/java/org/sunbird/search/processor/SearchProcessor.java b/search-api/search-core/src/main/java/org/sunbird/search/processor/SearchProcessor.java index f8f40268b..d51a749f3 100644 --- a/search-api/search-core/src/main/java/org/sunbird/search/processor/SearchProcessor.java +++ b/search-api/search-core/src/main/java/org/sunbird/search/processor/SearchProcessor.java @@ -913,7 +913,7 @@ private static QueryBuilder getPostFilterQuery(Map postFilter) { nestedBoolQuery.must(QueryBuilders.termQuery(filters.getKey(), value)); } } else if (filters.getValue() instanceof String) { - nestedBoolQuery.should(QueryBuilders.termQuery(filters.getKey(), ((String) filters.getValue()).toLowerCase())); + nestedBoolQuery.must(QueryBuilders.termQuery(filters.getKey(), ((String) filters.getValue()).toLowerCase())); } } From 825049b81b79e76e454c44b8c22c08168106a3da Mon Sep 17 00:00:00 2001 From: Sahil-tarento <140611066+Sahil-tarento@users.noreply.github.com> Date: Mon, 19 Feb 2024 13:39:25 +0530 Subject: [PATCH 049/126] Adding the query implementation changes (#75) --- .../org/sunbird/search/processor/SearchProcessor.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/search-api/search-core/src/main/java/org/sunbird/search/processor/SearchProcessor.java b/search-api/search-core/src/main/java/org/sunbird/search/processor/SearchProcessor.java index d51a749f3..41099f584 100644 --- a/search-api/search-core/src/main/java/org/sunbird/search/processor/SearchProcessor.java +++ b/search-api/search-core/src/main/java/org/sunbird/search/processor/SearchProcessor.java @@ -54,10 +54,6 @@ public Future> processSearch(SearchDTO searchDTO, boolean in List> groupByFinalList = new ArrayList>(); SearchSourceBuilder query = processSearchQuery(searchDTO, groupByFinalList, true); - if (searchDTO.isSecureSettingsDisabled()) { - query.postFilter(getPostFilterQuery(searchDTO.getPostFilter())); - } - Future searchResponse = null; boolean enableFuzzyWhenNoResults = Platform.config.hasPath("search.fields.enable.fuzzy.when.noresult") && Platform.config.getBoolean("search.fields.enable.fuzzy.when.noresult"); @@ -73,6 +69,11 @@ public Future> processSearch(SearchDTO searchDTO, boolean in query = processSearchQuery(searchDTO, groupByFinalList, true); } } + + if (searchDTO.isSecureSettingsDisabled()) { + query.postFilter(getPostFilterQuery(searchDTO.getPostFilter())); + } + searchResponse = ElasticSearchUtil.search(SearchConstants.COMPOSITE_SEARCH_INDEX, query); return searchResponse.map(new Mapper>() { From ed535ff7e385f91f9bbad09e1c8be1299e95a6d6 Mon Sep 17 00:00:00 2001 From: Karthikeyan Rajendran <70887864+karthik-tarento@users.noreply.github.com> Date: Tue, 20 Feb 2024 16:26:38 +0530 Subject: [PATCH 050/126] Revert MCQ question commit (#76) (#78) (#79) * Revert "Adding constant value for MCQ - review changes" This reverts commit ddd6136ee037c835d1b2a945074391a5e1f30e8f. * Revert "Adding constant value for MCQ - review changes" This reverts commit 6c587bcf5cfd20dc666d01bb3307d3d4d71adb7f. * Revert "Review changes" This reverts commit f39c77837a92782d2e8008921986b98e4de790f4. * Revert "Additional check on MCQ questions" This reverts commit 93cde23b99bacd654407b0e368790a7ff511746c. * Revert "Validations to ensure boolean input for MCQ answers" This reverts commit 3eeefb3f053b6d3001b53e8da95a7ec2505c00bc. Co-authored-by: Sahil-tarento <140611066+Sahil-tarento@users.noreply.github.com> --- .../managers/UpdateHierarchyManager.scala | 56 ------------------- .../sunbird/utils/HierarchyConstants.scala | 9 --- 2 files changed, 65 deletions(-) diff --git a/assessment-api/qs-hierarchy-manager/src/main/scala/org/sunbird/managers/UpdateHierarchyManager.scala b/assessment-api/qs-hierarchy-manager/src/main/scala/org/sunbird/managers/UpdateHierarchyManager.scala index 6143a8f71..59cdd1810 100644 --- a/assessment-api/qs-hierarchy-manager/src/main/scala/org/sunbird/managers/UpdateHierarchyManager.scala +++ b/assessment-api/qs-hierarchy-manager/src/main/scala/org/sunbird/managers/UpdateHierarchyManager.scala @@ -71,62 +71,6 @@ object UpdateHierarchyManager { .getOrDefault(HierarchyConstants.OBJECT_TYPE, "").asInstanceOf[String], "Question")) throw new ClientException("ERR_QS_UPDATE_HIERARCHY", "Question cannot have children in hierarchy") }) - val iterator = nodesModified.entrySet().iterator() - while (iterator.hasNext) { - val entry = iterator.next() - val questionData = entry.getValue() match { - case map: java.util.HashMap[_, _] => map.asInstanceOf[java.util.HashMap[String, AnyRef]] - case _ => throw new ClientException("ERR_QS_UPDATE_HIERARCHY", "Error: Invalid question data.") - } - val metadata = Option(questionData.get(HierarchyConstants.METADATA)) match { - case Some(map: java.util.HashMap[_, _]) => map.asInstanceOf[java.util.HashMap[String, AnyRef]] - case _ => throw new ClientException("ERR_QS_UPDATE_HIERARCHY", "Error: Missing metadata.") - } - if (metadata.get(HierarchyConstants.PRIMARY_CATEGORY) == HierarchyConstants.MCQ) { - val primaryCategory = Option(metadata.get(HierarchyConstants.PRIMARY_CATEGORY)) match { - case Some(category: String) => category - case _ => throw new ClientException("ERR_QS_UPDATE_HIERARCHY", "Error: Missing primary category.") - } - - if (primaryCategory == HierarchyConstants.MCQ) { - val choices = Option(metadata.get(HierarchyConstants.CHOICES)) match { - case Some(choiceMap: java.util.HashMap[_, _]) => - val options = Option(choiceMap.get(HierarchyConstants.OPTIONS)) match { - case Some(optionsList: java.util.List[_]) => - optionsList.asInstanceOf[java.util.List[java.util.HashMap[String, AnyRef]]].asScala.toList - case _ => throw new ClientException("ERR_QS_UPDATE_HIERARCHY", "Error: Missing options.") - } - options - case _ => throw new ClientException("ERR_QS_UPDATE_HIERARCHY", "Error: Missing choices.") - } - - for (option <- choices) { - val answerType = Option(option.get(HierarchyConstants.ANSWER)) match { - case Some(answer) => answer - case _ => throw new ClientException("ERR_QS_UPDATE_HIERARCHY", "Error: Missing answer type.") - } - - if (!answerType.isInstanceOf[java.lang.Boolean]) { - val questionBody = Option(metadata.get(HierarchyConstants.BODY)) match { - case Some(body: String) => body - case _ => throw new ClientException("ERR_QS_UPDATE_HIERARCHY", "Error: Missing question body.") - } - - val optionBody = Option(option.get(HierarchyConstants.VALUE)) match { - case Some(valueMap: java.util.HashMap[_, _]) => - Option(valueMap.get(HierarchyConstants.BODY)) match { - case Some(body) => body - case _ => throw new ClientException("ERR_QS_UPDATE_HIERARCHY", "Error: Missing option body.") - } - case _ => throw new ClientException("ERR_QS_UPDATE_HIERARCHY", "Error: Missing option value map.") - } - - throw new ClientException("ERR_QS_UPDATE_HIERARCHY", s"Error: Question : '$questionBody', Option : '$optionBody' has a non-boolean answer. The answer value should be either true or false.") - } - } - } - } - } (nodesModified, hierarchy) } diff --git a/assessment-api/qs-hierarchy-manager/src/main/scala/org/sunbird/utils/HierarchyConstants.scala b/assessment-api/qs-hierarchy-manager/src/main/scala/org/sunbird/utils/HierarchyConstants.scala index 16114a3fc..25408886c 100644 --- a/assessment-api/qs-hierarchy-manager/src/main/scala/org/sunbird/utils/HierarchyConstants.scala +++ b/assessment-api/qs-hierarchy-manager/src/main/scala/org/sunbird/utils/HierarchyConstants.scala @@ -57,13 +57,4 @@ object HierarchyConstants { val SOURCE: String = "source" val PRE_CONDITION: String = "preCondition" val QUESTION_VISIBILITY: List[String] = List("Default", "Parent") - val OPTIONS = "options" - val CHOICES = "choices" - val ANSWER = "answer" - val BODY = "body" - val VALUE = "value" - val PRIMARY_CATEGORY = "primaryCategory" - val MCQ = "Multiple Choice Question" - - } From 797324b2eaf67258eb80a12eb8a49f9065bd98f0 Mon Sep 17 00:00:00 2001 From: Karthikeyan Rajendran <70887864+karthik-tarento@users.noreply.github.com> Date: Tue, 26 Mar 2024 16:27:15 +0530 Subject: [PATCH 051/126] Retire Fix Impl (#81) Co-authored-by: sahilchaudhary Co-authored-by: Sahil-tarento <140611066+Sahil-tarento@users.noreply.github.com> --- .../src/main/scala/org/sunbird/content/util/RetireManager.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content-api/content-actors/src/main/scala/org/sunbird/content/util/RetireManager.scala b/content-api/content-actors/src/main/scala/org/sunbird/content/util/RetireManager.scala index cbd8e4eba..2be7ca06c 100644 --- a/content-api/content-actors/src/main/scala/org/sunbird/content/util/RetireManager.scala +++ b/content-api/content-actors/src/main/scala/org/sunbird/content/util/RetireManager.scala @@ -50,7 +50,7 @@ object RetireManager { private def validateRequest(request: Request) = { val contentId: String = request.get(ContentConstants.IDENTIFIER).asInstanceOf[String] - if (StringUtils.isBlank(contentId) || StringUtils.endsWithIgnoreCase(contentId, HierarchyConstants.IMAGE_SUFFIX)) + if (StringUtils.isBlank(contentId)) throw new ClientException(ContentConstants.ERR_INVALID_CONTENT_ID, "Please Provide Valid Content Identifier.") } From 5d63b84ee36d5c13fc484044387b9ca16c1ec768 Mon Sep 17 00:00:00 2001 From: Sahil-tarento <140611066+Sahil-tarento@users.noreply.github.com> Date: Tue, 2 Apr 2024 17:32:47 +0530 Subject: [PATCH 052/126] Retire Fix Impl (#83) --- .../main/scala/org/sunbird/content/util/DiscardManager.scala | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/content-api/content-actors/src/main/scala/org/sunbird/content/util/DiscardManager.scala b/content-api/content-actors/src/main/scala/org/sunbird/content/util/DiscardManager.scala index 6a2aa5ed6..13f17d240 100644 --- a/content-api/content-actors/src/main/scala/org/sunbird/content/util/DiscardManager.scala +++ b/content-api/content-actors/src/main/scala/org/sunbird/content/util/DiscardManager.scala @@ -48,8 +48,7 @@ object DiscardManager { } def validateRequest(request: Request): Unit = { - if (StringUtils.isBlank(request.getRequest.getOrDefault(ContentConstants.IDENTIFIER, "").asInstanceOf[String]) - || StringUtils.endsWith(request.getRequest.getOrDefault(ContentConstants.IDENTIFIER, "").asInstanceOf[String], ContentConstants.IMAGE_SUFFIX)) + if (StringUtils.isBlank(request.getRequest.getOrDefault(ContentConstants.IDENTIFIER, "").asInstanceOf[String])) throw new ClientException(ContentConstants.ERR_INVALID_CONTENT_ID, "Please provide valid content identifier") } From ffa1525994e43f44227d20fec9649f0226657b8c Mon Sep 17 00:00:00 2001 From: Karthikeyan Rajendran <70887864+karthik-tarento@users.noreply.github.com> Date: Thu, 25 Apr 2024 17:24:41 +0530 Subject: [PATCH 053/126] Added options to search multiple keywords inside filter using OR (#84) --- .../java/org/sunbird/actors/SearchActor.java | 9 ++++- .../search/processor/SearchProcessor.java | 37 +++++++++++++++++-- .../sunbird/search/util/SearchConstants.java | 1 + 3 files changed, 43 insertions(+), 4 deletions(-) diff --git a/search-api/search-actors/src/main/java/org/sunbird/actors/SearchActor.java b/search-api/search-actors/src/main/java/org/sunbird/actors/SearchActor.java index e6f45f37a..ef6c76002 100644 --- a/search-api/search-actors/src/main/java/org/sunbird/actors/SearchActor.java +++ b/search-api/search-actors/src/main/java/org/sunbird/actors/SearchActor.java @@ -448,7 +448,14 @@ private List> getSearchFilterProperties(Map Object filterObject = entry.getValue(); if (filterObject instanceof Map) { Map filterMap = (Map) filterObject; - if (!filterMap.containsKey(SearchConstants.SEARCH_OPERATION_RANGE_MIN) + if (SearchConstants.must.equalsIgnoreCase(entry.getKey())) { + Map property = new HashMap(); + property.put(SearchConstants.values, entry.getValue()); + property.put(SearchConstants.propertyName, entry.getKey()); + property.put(SearchConstants.operation, + SearchConstants.SEARCH_OPERATION_EQUAL); + properties.add(property); + } else if (!filterMap.containsKey(SearchConstants.SEARCH_OPERATION_RANGE_MIN) && !filterMap.containsKey(SearchConstants.SEARCH_OPERATION_RANGE_MAX)) { for (Map.Entry filterEntry : filterMap.entrySet()) { Map property = new HashMap(); diff --git a/search-api/search-core/src/main/java/org/sunbird/search/processor/SearchProcessor.java b/search-api/search-core/src/main/java/org/sunbird/search/processor/SearchProcessor.java index 41099f584..1a8ccd8b9 100644 --- a/search-api/search-core/src/main/java/org/sunbird/search/processor/SearchProcessor.java +++ b/search-api/search-core/src/main/java/org/sunbird/search/processor/SearchProcessor.java @@ -4,8 +4,8 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.MapUtils; import org.apache.commons.lang.StringUtils; -import org.apache.lucene.search.join.ScoreMode; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.index.query.*; import org.elasticsearch.index.query.MultiMatchQueryBuilder.Type; @@ -377,6 +377,12 @@ private void formQueryImpl(List properties, QueryBuilder queryBuilder, Bool for (Map property : properties) { String opertation = (String) property.get("operation"); + Object objValues = property.get("values"); + Map valuesMap = new HashMap<>(); + if (objValues instanceof Map) { + valuesMap = (Map) property.get("values"); + } + List values; try { values = (List) property.get("values"); @@ -406,8 +412,12 @@ private void formQueryImpl(List properties, QueryBuilder queryBuilder, Bool switch (opertation) { case SearchConstants.SEARCH_OPERATION_EQUAL: { - queryBuilder = getMustTermQuery(propertyName, values, true); - queryBuilder = checkNestedProperty(queryBuilder, propertyName); + if (MapUtils.isNotEmpty(valuesMap)) { + queryBuilder = getMustTermQuery(valuesMap, true); + } else { + queryBuilder = getMustTermQuery(propertyName, values, true); + queryBuilder = checkNestedProperty(queryBuilder, propertyName); + } break; } case SearchConstants.SEARCH_OPERATION_NOT_EQUAL: { @@ -937,4 +947,25 @@ private static QueryBuilder getPostFilterQuery(Map postFilter) { return postFilterBoolQuery; } + + private QueryBuilder getMustTermQuery(Map propertyMap, boolean match) { + BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery(); + Set> entrySet = propertyMap.entrySet(); + for (Map.Entry entry : entrySet) { + String propertyName = (String) entry.getKey(); + propertyName = propertyName + SearchConstants.RAW_FIELD_EXTENSION; + List valueList = (List) entry.getValue(); + for (Object value : valueList) { + if (match) { + queryBuilder.should( + QueryBuilders.matchQuery(propertyName, value).fuzzyTranspositions(false)); + } else { + queryBuilder.mustNot( + QueryBuilders.matchQuery(propertyName, value).fuzzyTranspositions(false)); + } + } + } + + return queryBuilder; + } } \ No newline at end of file diff --git a/search-api/search-core/src/main/java/org/sunbird/search/util/SearchConstants.java b/search-api/search-core/src/main/java/org/sunbird/search/util/SearchConstants.java index bbd41f9ed..72599abea 100644 --- a/search-api/search-core/src/main/java/org/sunbird/search/util/SearchConstants.java +++ b/search-api/search-core/src/main/java/org/sunbird/search/util/SearchConstants.java @@ -117,4 +117,5 @@ public class SearchConstants { public static String secureSettings = "secureSettings"; public static String isSecureSettingsDisabled = "isSecureSettingsDisabled"; public static String secureSettingsOrganisation = "secureSettings.organisation"; + public static final String must = "must"; } From 833ebde99a5b52adc5f7476ddde463b62a78ab47 Mon Sep 17 00:00:00 2001 From: mathewjpallan Date: Wed, 29 May 2024 10:42:32 +0530 Subject: [PATCH 054/126] Updated for handling google storage account. --- platform-modules/mimetype-manager/pom.xml | 2 +- .../sunbird/cloudstore/StorageService.scala | 34 ++++++------------- 2 files changed, 11 insertions(+), 25 deletions(-) diff --git a/platform-modules/mimetype-manager/pom.xml b/platform-modules/mimetype-manager/pom.xml index eb25c58f1..3a988277b 100644 --- a/platform-modules/mimetype-manager/pom.xml +++ b/platform-modules/mimetype-manager/pom.xml @@ -30,7 +30,7 @@ org.sunbird cloud-store-sdk - 1.3.0 + 1.4.7 org.scala-lang diff --git a/platform-modules/mimetype-manager/src/main/scala/org/sunbird/cloudstore/StorageService.scala b/platform-modules/mimetype-manager/src/main/scala/org/sunbird/cloudstore/StorageService.scala index 83fabffc8..942492479 100644 --- a/platform-modules/mimetype-manager/src/main/scala/org/sunbird/cloudstore/StorageService.scala +++ b/platform-modules/mimetype-manager/src/main/scala/org/sunbird/cloudstore/StorageService.scala @@ -20,36 +20,22 @@ class StorageService { @throws[Exception] def getService(): BaseStorageService = { if (null == storageService) { - if (StringUtils.equalsIgnoreCase(storageType, "azure")) { - val storageKey = Platform.config.getString("azure_storage_key") - val storageSecret = Platform.config.getString("azure_storage_secret") - storageService = StorageServiceFactory.getStorageService(new StorageConfig(storageType, storageKey, storageSecret)) - } else if (StringUtils.equalsIgnoreCase(storageType, "aws")) { - val storageKey = Platform.config.getString("aws_storage_key") - val storageSecret = Platform.config.getString("aws_storage_secret") - storageService = StorageServiceFactory.getStorageService(new StorageConfig(storageType, storageKey, storageSecret)) - } - else if (StringUtils.equalsIgnoreCase(storageType, "cephs3")) { - val storageKey = Platform.config.getString("cephs3_storage_key") - val storageSecret = Platform.config.getString("cephs3_storage_secret") - val endpoint = Platform.config.getString("cephs3_storage_endpoint") - storageService = StorageServiceFactory.getStorageService(new StorageConfig(storageType, storageKey, storageSecret, Option(endpoint))) - } - else throw new ServerException("ERR_INVALID_CLOUD_STORAGE", "Error while initialising cloud storage") + val storageKey = Platform.config.getString("cloud_storage_key") + val storageSecret = Platform.config.getString("cloud_storage_secret") + // TODO: endPoint defined to support "cephs3". Make code changes after cloud-store-sdk 2.11 support it. + val endPoint = if (Platform.config.hasPath("cloud_storage_endpoint")) Option(Platform.config.getString("cloud_storage_endpoint")) else None + println("StorageService --> params: " + storageType + "," + storageKey) + storageService = StorageServiceFactory.getStorageService(new StorageConfig(storageType, storageKey, storageSecret, endPoint)) } storageService } def getContainerName(): String = { - if (StringUtils.equalsIgnoreCase(storageType, "azure")) - Platform.config.getString("azure_storage_container") - else if (StringUtils.equalsIgnoreCase(storageType, "aws")) - Platform.config.getString("aws_storage_container") - else if (StringUtils.equalsIgnoreCase(storageType, "cephs3")) - Platform.config.getString("cephs3_storage_container") - else - throw new ServerException("ERR_INVALID_CLOUD_STORAGE", "Container name not configured.") + if(Platform.config.hasPath("cloud_storage_container")) + Platform.config.getString("cloud_storage_container") + else + throw new ServerException("ERR_INVALID_CLOUD_STORAGE", "Cloud Storage Container name not configured.") } def uploadFile(folderName: String, file: File, slug: Option[Boolean] = Option(true)): Array[String] = { From 05d4bfe1d2f7d5f595066710c49ad45e40722d19 Mon Sep 17 00:00:00 2001 From: karthik-tarento Date: Tue, 11 Jun 2024 16:51:15 +0530 Subject: [PATCH 055/126] Using updated cloud-store-sdk --- platform-modules/mimetype-manager/pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/platform-modules/mimetype-manager/pom.xml b/platform-modules/mimetype-manager/pom.xml index 3a988277b..3dabb8f30 100644 --- a/platform-modules/mimetype-manager/pom.xml +++ b/platform-modules/mimetype-manager/pom.xml @@ -28,9 +28,9 @@ jar - org.sunbird + io.github.karthik-tarento cloud-store-sdk - 1.4.7 + 1.4.5 org.scala-lang From a44fd907e78aad39f16e20cf633a4956e1b36358 Mon Sep 17 00:00:00 2001 From: mathewjpallan Date: Tue, 25 Jun 2024 15:41:25 +0530 Subject: [PATCH 056/126] Moving to our own build of cloud-storage-sdk. This is necessary as the sunbird cloud storage sdk has a createContainer call during upload and this is unncessary and rate limited by google --- platform-modules/mimetype-manager/pom.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/platform-modules/mimetype-manager/pom.xml b/platform-modules/mimetype-manager/pom.xml index 3dabb8f30..a7a80d411 100644 --- a/platform-modules/mimetype-manager/pom.xml +++ b/platform-modules/mimetype-manager/pom.xml @@ -28,9 +28,9 @@ jar - io.github.karthik-tarento - cloud-store-sdk - 1.4.5 + net.karmayogibharat + cloud-store-sdk_2.11 + 1.4.6 org.scala-lang From 67aaab32080c2db4e97c17f52a73c5a1f5e1a81f Mon Sep 17 00:00:00 2001 From: sahilchaudhary Date: Thu, 11 Jul 2024 10:46:08 +0530 Subject: [PATCH 057/126] Adding Redis cache on systemUpdate API --- .../src/main/scala/org/sunbird/actors/QuestionSetActor.scala | 4 +++- .../main/scala/org/sunbird/content/actors/ContentActor.scala | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/assessment-api/assessment-actors/src/main/scala/org/sunbird/actors/QuestionSetActor.scala b/assessment-api/assessment-actors/src/main/scala/org/sunbird/actors/QuestionSetActor.scala index 7ce683869..150e10a91 100644 --- a/assessment-api/assessment-actors/src/main/scala/org/sunbird/actors/QuestionSetActor.scala +++ b/assessment-api/assessment-actors/src/main/scala/org/sunbird/actors/QuestionSetActor.scala @@ -142,8 +142,10 @@ class QuestionSetActor @Inject()(implicit oec: OntologyEngineContext) extends Ba def systemUpdate(request: Request): Future[Response] = { val identifier = request.getContext.get("identifier").asInstanceOf[String] RequestUtil.validateRequest(request) - if(Platform.getBoolean("questionset.cache.enable", false)) + if(Platform.getBoolean("questionset.cache.enable", false)) { RedisCache.delete(hierarchyPrefix + identifier) + RedisCache.delete(identifier) + } val readReq = new Request(request) val identifiers = new util.ArrayList[String](){{ diff --git a/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala b/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala index 2f41ca5f0..b2cc6b70a 100644 --- a/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala +++ b/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala @@ -279,6 +279,7 @@ class ContentActor @Inject() (implicit oec: OntologyEngineContext, ss: StorageSe val identifier = request.getContext.get("identifier").asInstanceOf[String] RequestUtil.validateRequest(request) RedisCache.delete(hierarchyPrefix + request.get("rootId")) + RedisCache.delete(identifier) val readReq = new Request(request) val identifiers = new util.ArrayList[String](){{ From 08ab7bb654d1b06bb34ba182fcb70dacf44275aa Mon Sep 17 00:00:00 2001 From: sahilchaudhary Date: Thu, 11 Jul 2024 14:51:43 +0530 Subject: [PATCH 058/126] Adding Redis cache on systemUpdate API For QuestionSet --- .../src/main/scala/org/sunbird/actors/QuestionSetActor.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/assessment-api/assessment-actors/src/main/scala/org/sunbird/actors/QuestionSetActor.scala b/assessment-api/assessment-actors/src/main/scala/org/sunbird/actors/QuestionSetActor.scala index 150e10a91..fd77db8ab 100644 --- a/assessment-api/assessment-actors/src/main/scala/org/sunbird/actors/QuestionSetActor.scala +++ b/assessment-api/assessment-actors/src/main/scala/org/sunbird/actors/QuestionSetActor.scala @@ -144,9 +144,9 @@ class QuestionSetActor @Inject()(implicit oec: OntologyEngineContext) extends Ba RequestUtil.validateRequest(request) if(Platform.getBoolean("questionset.cache.enable", false)) { RedisCache.delete(hierarchyPrefix + identifier) - RedisCache.delete(identifier) } + RedisCache.delete(identifier) val readReq = new Request(request) val identifiers = new util.ArrayList[String](){{ add(identifier) From 4641e1a1667a9da49926e65906e11f68dc707718 Mon Sep 17 00:00:00 2001 From: sahilchaudhary Date: Fri, 12 Jul 2024 11:06:28 +0530 Subject: [PATCH 059/126] Adding Redis cache on systemUpdate API For Question --- .../src/main/scala/org/sunbird/actors/QuestionActor.scala | 2 ++ 1 file changed, 2 insertions(+) diff --git a/assessment-api/assessment-actors/src/main/scala/org/sunbird/actors/QuestionActor.scala b/assessment-api/assessment-actors/src/main/scala/org/sunbird/actors/QuestionActor.scala index 3f0ed87fc..ffefa8ad6 100644 --- a/assessment-api/assessment-actors/src/main/scala/org/sunbird/actors/QuestionActor.scala +++ b/assessment-api/assessment-actors/src/main/scala/org/sunbird/actors/QuestionActor.scala @@ -3,6 +3,7 @@ package org.sunbird.actors import org.apache.commons.lang3.StringUtils import org.sunbird.`object`.importer.{ImportConfig, ImportManager} import org.sunbird.actor.core.BaseActor +import org.sunbird.cache.impl.RedisCache import org.sunbird.common.dto.{Request, Response, ResponseHandler} import org.sunbird.common.{DateUtils, Platform} import org.sunbird.graph.OntologyEngineContext @@ -92,6 +93,7 @@ class QuestionActor @Inject()(implicit oec: OntologyEngineContext) extends BaseA def systemUpdate(request: Request): Future[Response] = { val identifier = request.getContext.get("identifier").asInstanceOf[String] RequestUtil.validateRequest(request) + RedisCache.delete(identifier) val readReq = new Request(request) val identifiers = new util.ArrayList[String](){{ add(identifier) From 075f664ddb1e35b349213cac1819ec5b62473344 Mon Sep 17 00:00:00 2001 From: saipradeep_ravipati Date: Thu, 5 Sep 2024 12:38:54 +0530 Subject: [PATCH 060/126] Added List for QuestionSet --- .../org/sunbird/actors/QuestionSetActor.scala | 16 +++++++++++- .../v4/QuestionSetController.scala | 26 ++++++++++++++++--- .../assessment-service/app/utils/ApiId.scala | 1 + .../app/utils/QuestionOperations.scala | 2 +- assessment-api/assessment-service/conf/routes | 3 ++- 5 files changed, 42 insertions(+), 6 deletions(-) diff --git a/assessment-api/assessment-actors/src/main/scala/org/sunbird/actors/QuestionSetActor.scala b/assessment-api/assessment-actors/src/main/scala/org/sunbird/actors/QuestionSetActor.scala index fd77db8ab..721271348 100644 --- a/assessment-api/assessment-actors/src/main/scala/org/sunbird/actors/QuestionSetActor.scala +++ b/assessment-api/assessment-actors/src/main/scala/org/sunbird/actors/QuestionSetActor.scala @@ -1,9 +1,9 @@ package org.sunbird.actors import java.util - import javax.inject.Inject import org.apache.commons.collections4.CollectionUtils +import org.apache.commons.lang3.StringUtils import org.sunbird.`object`.importer.{ImportConfig, ImportManager} import org.sunbird.actor.core.BaseActor import org.sunbird.cache.impl.RedisCache @@ -12,10 +12,12 @@ import org.sunbird.common.dto.{Request, Response, ResponseHandler} import org.sunbird.graph.OntologyEngineContext import org.sunbird.graph.nodes.DataNode import org.sunbird.graph.dac.model.Node +import org.sunbird.graph.utils.NodeUtil import org.sunbird.managers.HierarchyManager.hierarchyPrefix import org.sunbird.managers.{AssessmentManager, HierarchyManager, UpdateHierarchyManager} import org.sunbird.utils.RequestUtil +import scala.collection.JavaConverters import scala.collection.JavaConverters._ import scala.concurrent.{ExecutionContext, Future} @@ -40,9 +42,21 @@ class QuestionSetActor @Inject()(implicit oec: OntologyEngineContext) extends Ba case "rejectQuestionSet" => reject(request) case "importQuestionSet" => importQuestionSet(request) case "systemUpdateQuestionSet" => systemUpdate(request) + case "listQuestionSet" => listQuestionSet(request) case _ => ERROR(request.getOperation) } + def listQuestionSet(request: Request): Future[Response] = { + RequestUtil.validateListRequest(request) + val fields: util.List[String] = JavaConverters.seqAsJavaListConverter(request.get("fields").asInstanceOf[String].split(",").filter(field => StringUtils.isNotBlank(field) && !StringUtils.equalsIgnoreCase(field, "null"))).asJava + request.getRequest.put("fields", fields) + DataNode.search(request).map(nodeList => { + val questionList = nodeList.map(node => { + NodeUtil.serialize(node, fields, node.getObjectType.toLowerCase.replace("Image", ""), request.getContext.get("version").asInstanceOf[String]) + }).asJava + ResponseHandler.OK.put("questionSets", questionList).put("count", questionList.size) + }) + } def update(request: Request): Future[Response] = { RequestUtil.restrictProperties(request) request.getRequest.put("identifier", request.getContext.get("identifier")) diff --git a/assessment-api/assessment-service/app/controllers/v4/QuestionSetController.scala b/assessment-api/assessment-service/app/controllers/v4/QuestionSetController.scala index a82f320ae..8bc0bdcea 100644 --- a/assessment-api/assessment-service/app/controllers/v4/QuestionSetController.scala +++ b/assessment-api/assessment-service/app/controllers/v4/QuestionSetController.scala @@ -2,12 +2,20 @@ package controllers.v4 import akka.actor.{ActorRef, ActorSystem} import controllers.BaseController +import org.apache.commons.lang3.StringUtils +import org.sunbird.common.dto.{Request, Response, ResponseHandler} +import org.sunbird.graph.nodes.DataNode +import org.sunbird.graph.utils.NodeUtil +import org.sunbird.utils.RequestUtil + import javax.inject.{Inject, Named} import play.api.mvc.ControllerComponents -import utils.{ActorNames, ApiId, QuestionSetOperations} +import utils.{ActorNames, ApiId, QuestionOperations, QuestionSetOperations} +import java.util +import scala.collection.JavaConverters import scala.collection.JavaConverters._ -import scala.concurrent.ExecutionContext +import scala.concurrent.{ExecutionContext, Future} class QuestionSetController @Inject()(@Named(ActorNames.QUESTION_SET_ACTOR) questionSetActor: ActorRef, cc: ControllerComponents, actorSystem: ActorSystem)(implicit exec: ExecutionContext) extends BaseController(cc) { @@ -15,7 +23,7 @@ class QuestionSetController @Inject()(@Named(ActorNames.QUESTION_SET_ACTOR) ques val schemaName: String = "questionset" val version = "1.0" - def create() = Action.async { implicit request => + def create() = Action.async { implicit request =>list val headers = commonHeaders() val body = requestBody() val questionSet = body.getOrDefault("questionset", new java.util.HashMap()).asInstanceOf[java.util.Map[String, AnyRef]] @@ -25,6 +33,18 @@ class QuestionSetController @Inject()(@Named(ActorNames.QUESTION_SET_ACTOR) ques getResult(ApiId.CREATE_QUESTION_SET, questionSetActor, questionSetRequest) } + def list(fields: Option[String]) = Action.async { implicit request => + val headers = commonHeaders() + val body = requestBody() + val question = body.getOrDefault("search", new java.util.HashMap()).asInstanceOf[java.util.Map[String, Object]]; + question.putAll(headers) + question.put("fields", fields.getOrElse("")) + val questionSetRequest = getRequest(question, headers, QuestionOperations.listQuestionSet.toString) + questionSetRequest.put("identifiers", questionSetRequest.get("identifier")) + setRequestContext(questionSetRequest, version, objectType, schemaName) + getResult(ApiId.LIST_QUESTIONSET, questionSetActor, questionSetRequest) + } + def read(identifier: String, mode: Option[String], fields: Option[String]) = Action.async { implicit request => val headers = commonHeaders() val questionSet = new java.util.HashMap().asInstanceOf[java.util.Map[String, Object]] diff --git a/assessment-api/assessment-service/app/utils/ApiId.scala b/assessment-api/assessment-service/app/utils/ApiId.scala index d45e57597..a5479f271 100644 --- a/assessment-api/assessment-service/app/utils/ApiId.scala +++ b/assessment-api/assessment-service/app/utils/ApiId.scala @@ -24,6 +24,7 @@ object ApiId { val SYSTEM_UPDATE_QUESTION = "api.question.system.update" val LIST_QUESTIONS = "api.questions.list" val REJECT_QUESTION = "api.question.reject" + val LIST_QUESTIONSET = "api.questionset.list" //QuestionSet APIs val CREATE_QUESTION_SET = "api.questionset.create" diff --git a/assessment-api/assessment-service/app/utils/QuestionOperations.scala b/assessment-api/assessment-service/app/utils/QuestionOperations.scala index 57e9e3815..37865fc44 100644 --- a/assessment-api/assessment-service/app/utils/QuestionOperations.scala +++ b/assessment-api/assessment-service/app/utils/QuestionOperations.scala @@ -1,5 +1,5 @@ package utils object QuestionOperations extends Enumeration { - val createQuestion, readQuestion, readPrivateQuestion, updateQuestion, reviewQuestion, publishQuestion, retireQuestion, importQuestion, systemUpdateQuestion, listQuestions, rejectQuestion = Value + val createQuestion, readQuestion, readPrivateQuestion, updateQuestion, reviewQuestion, publishQuestion, retireQuestion, importQuestion, systemUpdateQuestion, listQuestions, listQuestionSet,rejectQuestion = Value } diff --git a/assessment-api/assessment-service/conf/routes b/assessment-api/assessment-service/conf/routes index e33c9c1da..716a12b76 100644 --- a/assessment-api/assessment-service/conf/routes +++ b/assessment-api/assessment-service/conf/routes @@ -38,4 +38,5 @@ PATCH /questionset/v4/hierarchy/update controllers.v4.QuestionSetC GET /questionset/v4/hierarchy/:identifier controllers.v4.QuestionSetController.getHierarchy(identifier:String, mode:Option[String]) POST /questionset/v4/reject/:identifier controllers.v4.QuestionSetController.reject(identifier:String) POST /questionset/v4/import controllers.v4.QuestionSetController.importQuestionSet() -PATCH /questionset/v4/system/update/:identifier controllers.v4.QuestionSetController.systemUpdate(identifier:String) \ No newline at end of file +PATCH /questionset/v4/system/update/:identifier controllers.v4.QuestionSetController.systemUpdate(identifier:String) +POST /questionset/v4/list controllers.v4.QuestionSetController.list(fields:Option[String]) From 9eb8cfbd5f013abb30c8d78efdfeedb13ab94952 Mon Sep 17 00:00:00 2001 From: saipradeep_ravipati Date: Thu, 5 Sep 2024 12:53:56 +0530 Subject: [PATCH 061/126] Compilation Error Fix --- .../app/controllers/v4/QuestionSetController.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/assessment-api/assessment-service/app/controllers/v4/QuestionSetController.scala b/assessment-api/assessment-service/app/controllers/v4/QuestionSetController.scala index 8bc0bdcea..727306dcc 100644 --- a/assessment-api/assessment-service/app/controllers/v4/QuestionSetController.scala +++ b/assessment-api/assessment-service/app/controllers/v4/QuestionSetController.scala @@ -23,7 +23,7 @@ class QuestionSetController @Inject()(@Named(ActorNames.QUESTION_SET_ACTOR) ques val schemaName: String = "questionset" val version = "1.0" - def create() = Action.async { implicit request =>list + def create() = Action.async { implicit request => val headers = commonHeaders() val body = requestBody() val questionSet = body.getOrDefault("questionset", new java.util.HashMap()).asInstanceOf[java.util.Map[String, AnyRef]] From af5f70bfee010445255cadd60cd1838264d81547 Mon Sep 17 00:00:00 2001 From: saipradeep_ravipati Date: Thu, 5 Sep 2024 15:57:59 +0530 Subject: [PATCH 062/126] updated as per code conventional --- .../app/controllers/v4/QuestionSetController.scala | 2 +- .../assessment-service/app/utils/QuestionOperations.scala | 2 +- .../assessment-service/app/utils/QuestionSetOperations.scala | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/assessment-api/assessment-service/app/controllers/v4/QuestionSetController.scala b/assessment-api/assessment-service/app/controllers/v4/QuestionSetController.scala index 727306dcc..2eb127c77 100644 --- a/assessment-api/assessment-service/app/controllers/v4/QuestionSetController.scala +++ b/assessment-api/assessment-service/app/controllers/v4/QuestionSetController.scala @@ -39,7 +39,7 @@ class QuestionSetController @Inject()(@Named(ActorNames.QUESTION_SET_ACTOR) ques val question = body.getOrDefault("search", new java.util.HashMap()).asInstanceOf[java.util.Map[String, Object]]; question.putAll(headers) question.put("fields", fields.getOrElse("")) - val questionSetRequest = getRequest(question, headers, QuestionOperations.listQuestionSet.toString) + val questionSetRequest = getRequest(question, headers, QuestionSetOperations.listQuestionSet.toString) questionSetRequest.put("identifiers", questionSetRequest.get("identifier")) setRequestContext(questionSetRequest, version, objectType, schemaName) getResult(ApiId.LIST_QUESTIONSET, questionSetActor, questionSetRequest) diff --git a/assessment-api/assessment-service/app/utils/QuestionOperations.scala b/assessment-api/assessment-service/app/utils/QuestionOperations.scala index 37865fc44..57e9e3815 100644 --- a/assessment-api/assessment-service/app/utils/QuestionOperations.scala +++ b/assessment-api/assessment-service/app/utils/QuestionOperations.scala @@ -1,5 +1,5 @@ package utils object QuestionOperations extends Enumeration { - val createQuestion, readQuestion, readPrivateQuestion, updateQuestion, reviewQuestion, publishQuestion, retireQuestion, importQuestion, systemUpdateQuestion, listQuestions, listQuestionSet,rejectQuestion = Value + val createQuestion, readQuestion, readPrivateQuestion, updateQuestion, reviewQuestion, publishQuestion, retireQuestion, importQuestion, systemUpdateQuestion, listQuestions, rejectQuestion = Value } diff --git a/assessment-api/assessment-service/app/utils/QuestionSetOperations.scala b/assessment-api/assessment-service/app/utils/QuestionSetOperations.scala index afcd22e2b..643205710 100644 --- a/assessment-api/assessment-service/app/utils/QuestionSetOperations.scala +++ b/assessment-api/assessment-service/app/utils/QuestionSetOperations.scala @@ -3,5 +3,5 @@ package utils object QuestionSetOperations extends Enumeration { val createQuestionSet, readQuestionSet, readPrivateQuestionSet, updateQuestionSet, reviewQuestionSet, publishQuestionSet, retireQuestionSet, addQuestion, removeQuestion, updateHierarchyQuestion, readHierarchyQuestion, - rejectQuestionSet, importQuestionSet, systemUpdateQuestionSet = Value + rejectQuestionSet, importQuestionSet, listQuestionSet,systemUpdateQuestionSet = Value } From 586549f3798a025f63cd67c640cfe8bc38b221e9 Mon Sep 17 00:00:00 2001 From: tarentomaheshvakkund <139739142+tarentomaheshvakkund@users.noreply.github.com> Date: Thu, 19 Sep 2024 12:22:02 +0530 Subject: [PATCH 063/126] KB-6563 | DEV| Assessment | BE | Consumption Logic for the CQF Assessment 1. Added cqfVersion to the content meta. --- .../src/main/scala/org/sunbird/actors/QuestionSetActor.scala | 5 +++++ .../main/scala/org/sunbird/managers/AssessmentManager.scala | 4 ++++ 2 files changed, 9 insertions(+) diff --git a/assessment-api/assessment-actors/src/main/scala/org/sunbird/actors/QuestionSetActor.scala b/assessment-api/assessment-actors/src/main/scala/org/sunbird/actors/QuestionSetActor.scala index 721271348..0dcd27728 100644 --- a/assessment-api/assessment-actors/src/main/scala/org/sunbird/actors/QuestionSetActor.scala +++ b/assessment-api/assessment-actors/src/main/scala/org/sunbird/actors/QuestionSetActor.scala @@ -59,6 +59,11 @@ class QuestionSetActor @Inject()(implicit oec: OntologyEngineContext) extends Ba } def update(request: Request): Future[Response] = { RequestUtil.restrictProperties(request) + val primaryCategory: String = request.getContext.getOrDefault("primaryCategory", "").asInstanceOf[String] + val status: String = request.getContext.getOrDefault("status", "").asInstanceOf[String] + if (primaryCategory.equalsIgnoreCase("CQF Assessment") && status.equalsIgnoreCase("Draft")) { + request.getRequest.put("cqfVersion", System.currentTimeMillis().toString) + } request.getRequest.put("identifier", request.getContext.get("identifier")) AssessmentManager.getValidatedNodeForUpdate(request, "ERR_QUESTION_SET_UPDATE").flatMap(_ => AssessmentManager.updateNode(request)) } diff --git a/assessment-api/assessment-actors/src/main/scala/org/sunbird/managers/AssessmentManager.scala b/assessment-api/assessment-actors/src/main/scala/org/sunbird/managers/AssessmentManager.scala index fc3d99331..649714f05 100644 --- a/assessment-api/assessment-actors/src/main/scala/org/sunbird/managers/AssessmentManager.scala +++ b/assessment-api/assessment-actors/src/main/scala/org/sunbird/managers/AssessmentManager.scala @@ -25,6 +25,10 @@ object AssessmentManager { def create(request: Request, errCode: String)(implicit oec: OntologyEngineContext, ec: ExecutionContext): Future[Response] = { val visibility: String = request.getRequest.getOrDefault("visibility", "").asInstanceOf[String] + val primaryCategory: String = request.getContext.getOrDefault("primaryCategory", "").asInstanceOf[String] + if (primaryCategory.equalsIgnoreCase("CQF Assessment")) { + request.getRequest.put("cqfVersion", System.currentTimeMillis().toString) + } if (StringUtils.isNotBlank(visibility) && StringUtils.equalsIgnoreCase(visibility, "Parent")) throw new ClientException(errCode, "Visibility cannot be Parent!") DataNode.create(request).map(node => { From 1f5f2a8d41f4d21523c7885d06d54b849ad68d31 Mon Sep 17 00:00:00 2001 From: tarentomaheshvakkund <139739142+tarentomaheshvakkund@users.noreply.github.com> Date: Thu, 19 Sep 2024 15:59:42 +0530 Subject: [PATCH 064/126] KB-6563 | DEV| Assessment | BE | Consumption Logic for the CQF Assessment 1. Added cqfVersion to the content meta. --- .../src/main/scala/org/sunbird/actors/QuestionSetActor.scala | 5 ----- .../main/scala/org/sunbird/managers/AssessmentManager.scala | 4 ---- .../main/scala/org/sunbird/content/actors/ContentActor.scala | 5 +++++ 3 files changed, 5 insertions(+), 9 deletions(-) diff --git a/assessment-api/assessment-actors/src/main/scala/org/sunbird/actors/QuestionSetActor.scala b/assessment-api/assessment-actors/src/main/scala/org/sunbird/actors/QuestionSetActor.scala index 0dcd27728..721271348 100644 --- a/assessment-api/assessment-actors/src/main/scala/org/sunbird/actors/QuestionSetActor.scala +++ b/assessment-api/assessment-actors/src/main/scala/org/sunbird/actors/QuestionSetActor.scala @@ -59,11 +59,6 @@ class QuestionSetActor @Inject()(implicit oec: OntologyEngineContext) extends Ba } def update(request: Request): Future[Response] = { RequestUtil.restrictProperties(request) - val primaryCategory: String = request.getContext.getOrDefault("primaryCategory", "").asInstanceOf[String] - val status: String = request.getContext.getOrDefault("status", "").asInstanceOf[String] - if (primaryCategory.equalsIgnoreCase("CQF Assessment") && status.equalsIgnoreCase("Draft")) { - request.getRequest.put("cqfVersion", System.currentTimeMillis().toString) - } request.getRequest.put("identifier", request.getContext.get("identifier")) AssessmentManager.getValidatedNodeForUpdate(request, "ERR_QUESTION_SET_UPDATE").flatMap(_ => AssessmentManager.updateNode(request)) } diff --git a/assessment-api/assessment-actors/src/main/scala/org/sunbird/managers/AssessmentManager.scala b/assessment-api/assessment-actors/src/main/scala/org/sunbird/managers/AssessmentManager.scala index 649714f05..fc3d99331 100644 --- a/assessment-api/assessment-actors/src/main/scala/org/sunbird/managers/AssessmentManager.scala +++ b/assessment-api/assessment-actors/src/main/scala/org/sunbird/managers/AssessmentManager.scala @@ -25,10 +25,6 @@ object AssessmentManager { def create(request: Request, errCode: String)(implicit oec: OntologyEngineContext, ec: ExecutionContext): Future[Response] = { val visibility: String = request.getRequest.getOrDefault("visibility", "").asInstanceOf[String] - val primaryCategory: String = request.getContext.getOrDefault("primaryCategory", "").asInstanceOf[String] - if (primaryCategory.equalsIgnoreCase("CQF Assessment")) { - request.getRequest.put("cqfVersion", System.currentTimeMillis().toString) - } if (StringUtils.isNotBlank(visibility) && StringUtils.equalsIgnoreCase(visibility, "Parent")) throw new ClientException(errCode, "Visibility cannot be Parent!") DataNode.create(request).map(node => { diff --git a/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala b/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala index b2cc6b70a..efabf1385 100644 --- a/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala +++ b/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala @@ -63,6 +63,7 @@ class ContentActor @Inject() (implicit oec: OntologyEngineContext, ss: StorageSe def create(request: Request): Future[Response] = { populateDefaultersForCreation(request) RequestUtil.restrictProperties(request) + request.getRequest.put("cqfVersion", System.currentTimeMillis().toString) DataNode.create(request, dataModifier).map(node => { ResponseHandler.OK.put("identifier", node.getIdentifier).put("node_id", node.getIdentifier) .put("versionKey", node.getMetadata.get("versionKey")) @@ -136,6 +137,10 @@ class ContentActor @Inject() (implicit oec: OntologyEngineContext, ss: StorageSe populateDefaultersForUpdation(request) if (StringUtils.isBlank(request.getRequest.getOrDefault("versionKey", "").asInstanceOf[String])) throw new ClientException("ERR_INVALID_REQUEST", "Please Provide Version Key!") RequestUtil.restrictProperties(request) + val status: String = request.getContext.getOrDefault("status", "").asInstanceOf[String] + if (status.equalsIgnoreCase("Draft")) { + request.getRequest.put("cqfVersion", System.currentTimeMillis().toString) + } DataNode.update(request, dataModifier).map(node => { val identifier: String = node.getIdentifier.replace(".img", "") ResponseHandler.OK.put("node_id", identifier).put("identifier", identifier) From e8ddd9024cbaf3de4e4576c3e7fe7e2aa9383ba6 Mon Sep 17 00:00:00 2001 From: tarentomaheshvakkund <139739142+tarentomaheshvakkund@users.noreply.github.com> Date: Thu, 19 Sep 2024 21:47:42 +0530 Subject: [PATCH 065/126] KB-6563 | DEV| Assessment | BE | Consumption Logic for the CQF Assessment 1. Added logs. --- .../scala/org/sunbird/content/actors/ContentActor.scala | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala b/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala index efabf1385..c7a9725ae 100644 --- a/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala +++ b/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala @@ -4,10 +4,12 @@ import java.util import java.util.concurrent.CompletionException import java.io.File import org.apache.commons.io.FilenameUtils + import javax.inject.Inject import org.apache.commons.lang3.ObjectUtils import org.apache.commons.lang3.StringUtils import org.apache.commons.collections4.{CollectionUtils, MapUtils} +import org.slf4j.{Logger, LoggerFactory} import org.sunbird.`object`.importer.{ImportConfig, ImportManager} import org.sunbird.actor.core.BaseActor import org.sunbird.cache.impl.RedisCache @@ -36,6 +38,7 @@ class ContentActor @Inject() (implicit oec: OntologyEngineContext, ss: StorageSe implicit val ec: ExecutionContext = getContext().dispatcher private lazy val importConfig = getImportConfig() private lazy val importMgr = new ImportManager(importConfig) + private val logger: Logger = LoggerFactory.getLogger("ContentActor") override def onReceive(request: Request): Future[Response] = { request.getOperation match { @@ -137,10 +140,8 @@ class ContentActor @Inject() (implicit oec: OntologyEngineContext, ss: StorageSe populateDefaultersForUpdation(request) if (StringUtils.isBlank(request.getRequest.getOrDefault("versionKey", "").asInstanceOf[String])) throw new ClientException("ERR_INVALID_REQUEST", "Please Provide Version Key!") RequestUtil.restrictProperties(request) - val status: String = request.getContext.getOrDefault("status", "").asInstanceOf[String] - if (status.equalsIgnoreCase("Draft")) { - request.getRequest.put("cqfVersion", System.currentTimeMillis().toString) - } + request.getRequest.put("cqfVersion", System.currentTimeMillis().toString) + logger.info("Inside the update method of content actor : " + request.toString) DataNode.update(request, dataModifier).map(node => { val identifier: String = node.getIdentifier.replace(".img", "") ResponseHandler.OK.put("node_id", identifier).put("identifier", identifier) From 479feeae6aacb4a18a758ad2d69696bef109134f Mon Sep 17 00:00:00 2001 From: tarentomaheshvakkund <139739142+tarentomaheshvakkund@users.noreply.github.com> Date: Thu, 19 Sep 2024 22:20:37 +0530 Subject: [PATCH 066/126] KB-6563 | DEV| Assessment | BE | Consumption Logic for the CQF Assessment 1. Added check for updating cqfVersion --- .../scala/org/sunbird/content/actors/ContentActor.scala | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala b/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala index c7a9725ae..121002f44 100644 --- a/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala +++ b/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala @@ -140,8 +140,12 @@ class ContentActor @Inject() (implicit oec: OntologyEngineContext, ss: StorageSe populateDefaultersForUpdation(request) if (StringUtils.isBlank(request.getRequest.getOrDefault("versionKey", "").asInstanceOf[String])) throw new ClientException("ERR_INVALID_REQUEST", "Please Provide Version Key!") RequestUtil.restrictProperties(request) - request.getRequest.put("cqfVersion", System.currentTimeMillis().toString) + val reviewStatus: String = request.getContext.getOrDefault("reviewStatus", "").asInstanceOf[String] logger.info("Inside the update method of content actor : " + request.toString) + if(reviewStatus == null || reviewStatus.isEmpty ) { + logger.info("i am inside if condition") + request.getRequest.put("cqfVersion", System.currentTimeMillis().toString) + } DataNode.update(request, dataModifier).map(node => { val identifier: String = node.getIdentifier.replace(".img", "") ResponseHandler.OK.put("node_id", identifier).put("identifier", identifier) From 3caf0d9ac4763cd6483b6f81bccd65609000681c Mon Sep 17 00:00:00 2001 From: tarentomaheshvakkund <139739142+tarentomaheshvakkund@users.noreply.github.com> Date: Thu, 19 Sep 2024 22:42:36 +0530 Subject: [PATCH 067/126] KB-6563 | DEV| Assessment | BE | Consumption Logic for the CQF Assessment 1. Fetch value from request body. --- .../main/scala/org/sunbird/content/actors/ContentActor.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala b/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala index 121002f44..343d1f048 100644 --- a/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala +++ b/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala @@ -140,7 +140,7 @@ class ContentActor @Inject() (implicit oec: OntologyEngineContext, ss: StorageSe populateDefaultersForUpdation(request) if (StringUtils.isBlank(request.getRequest.getOrDefault("versionKey", "").asInstanceOf[String])) throw new ClientException("ERR_INVALID_REQUEST", "Please Provide Version Key!") RequestUtil.restrictProperties(request) - val reviewStatus: String = request.getContext.getOrDefault("reviewStatus", "").asInstanceOf[String] + val reviewStatus: String = request.getRequest.getOrDefault("reviewStatus", "").asInstanceOf[String] logger.info("Inside the update method of content actor : " + request.toString) if(reviewStatus == null || reviewStatus.isEmpty ) { logger.info("i am inside if condition") From d81e9f8ef1e9b649a294b8d8f38c4815b9b1dc7f Mon Sep 17 00:00:00 2001 From: tarentomaheshvakkund <139739142+tarentomaheshvakkund@users.noreply.github.com> Date: Thu, 19 Sep 2024 23:03:05 +0530 Subject: [PATCH 068/126] KB-6563 | DEV| Assessment | BE | Consumption Logic for the CQF Assessment 1. Removed the logs. --- .../main/scala/org/sunbird/content/actors/ContentActor.scala | 2 -- 1 file changed, 2 deletions(-) diff --git a/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala b/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala index 343d1f048..be17caece 100644 --- a/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala +++ b/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala @@ -141,9 +141,7 @@ class ContentActor @Inject() (implicit oec: OntologyEngineContext, ss: StorageSe if (StringUtils.isBlank(request.getRequest.getOrDefault("versionKey", "").asInstanceOf[String])) throw new ClientException("ERR_INVALID_REQUEST", "Please Provide Version Key!") RequestUtil.restrictProperties(request) val reviewStatus: String = request.getRequest.getOrDefault("reviewStatus", "").asInstanceOf[String] - logger.info("Inside the update method of content actor : " + request.toString) if(reviewStatus == null || reviewStatus.isEmpty ) { - logger.info("i am inside if condition") request.getRequest.put("cqfVersion", System.currentTimeMillis().toString) } DataNode.update(request, dataModifier).map(node => { From e907437b566e81c5d8454b74a2d41c37c8f9f5dc Mon Sep 17 00:00:00 2001 From: karthik-tarento Date: Tue, 15 Oct 2024 04:46:13 +0530 Subject: [PATCH 069/126] Added kakfa message generation for event publish --- .../sunbird/content/actors/EventActor.scala | 54 +++++++++++++++++-- 1 file changed, 51 insertions(+), 3 deletions(-) diff --git a/content-api/content-actors/src/main/scala/org/sunbird/content/actors/EventActor.scala b/content-api/content-actors/src/main/scala/org/sunbird/content/actors/EventActor.scala index 2be44d103..b9d49124c 100644 --- a/content-api/content-actors/src/main/scala/org/sunbird/content/actors/EventActor.scala +++ b/content-api/content-actors/src/main/scala/org/sunbird/content/actors/EventActor.scala @@ -1,6 +1,7 @@ package org.sunbird.content.actors import org.apache.commons.lang.StringUtils +import org.sunbird.common.Platform import org.sunbird.cloudstore.StorageService import org.sunbird.common.dto.{Request, Response, ResponseHandler} import org.sunbird.common.exception.{ClientException, ResponseCode} @@ -8,12 +9,18 @@ import org.sunbird.content.util.ContentConstants import org.sunbird.graph.OntologyEngineContext import org.sunbird.graph.dac.model.{Node, Relation} import org.sunbird.graph.nodes.DataNode +import org.sunbird.telemetry.logger.TelemetryManager +import org.sunbird.telemetry.util.LogTelemetryEventUtil import java.util import javax.inject.Inject import scala.collection.JavaConverters.asScalaBufferConverter import scala.concurrent.Future +import scala.collection.JavaConversions._ +import scala.collection.JavaConverters +import scala.collection.JavaConverters._ + class EventActor @Inject()(implicit oec: OntologyEngineContext, ss: StorageService) extends ContentActor { override def onReceive(request: Request): Future[Response] = { @@ -33,7 +40,8 @@ class EventActor @Inject()(implicit oec: OntologyEngineContext, ss: StorageServi } def publish(request: Request): Future[Response] = { - verifyStandaloneEventAndApply(super.update, request) + TelemetryManager.log("EventActor::publish Identifier: " + request.getRequest.getOrDefault("identifier", "")) + verifyStandaloneEventAndApply(super.update, request, true) } override def discard(request: Request): Future[Response] = { @@ -44,7 +52,7 @@ class EventActor @Inject()(implicit oec: OntologyEngineContext, ss: StorageServi verifyStandaloneEventAndApply(super.retire, request) } - private def verifyStandaloneEventAndApply(f: Request => Future[Response], request: Request, dataUpdater: Option[Node => Unit] = None): Future[Response] = { + private def verifyStandaloneEventAndApply(f: Request => Future[Response], request: Request, isPublish: Boolean = false, dataUpdater: Option[Node => Unit] = None): Future[Response] = { DataNode.read(request).flatMap(node => { val inRelations = if (node.getInRelations == null) new util.ArrayList[Relation]() else node.getInRelations; val hasEventSetParent = inRelations.asScala.exists(rel => "EventSet".equalsIgnoreCase(rel.getStartNodeObjectType)) @@ -54,18 +62,58 @@ class EventActor @Inject()(implicit oec: OntologyEngineContext, ss: StorageServi if (dataUpdater.isDefined) { dataUpdater.get.apply(node) } - f.apply(request) + f.apply(request).flatMap(response => { + // Check if the response is OK + if (response.getResponseCode == ResponseCode.OK) { + if (isPublish) { + TelemetryManager.log("EventActor::verifyStandaloneEventAndApply publish request for Identifier: " + request.getRequest.getOrDefault("identifier", "")) + pushInstructionEvent(node.getIdentifier, node) + } else { + TelemetryManager.log("EventActor::verifyStandaloneEventAndApply Identifier: " + request.getRequest.getOrDefault("identifier", "")) + } + Future.successful(response) + } else { + // Return the response if it's not OK as it is + Future.successful(response) + } + }) } }) } override def dataModifier(node: Node): Node = { + TelemetryManager.log("EventActor::dataModifier Identifier: " + node.getIdentifier) if (node.getMetadata.containsKey("trackable") && node.getMetadata.getOrDefault("trackable", new java.util.HashMap[String, AnyRef]).asInstanceOf[java.util.Map[String, AnyRef]].containsKey("enabled") && "Yes".equalsIgnoreCase(node.getMetadata.getOrDefault("trackable", new java.util.HashMap[String, AnyRef]).asInstanceOf[java.util.Map[String, AnyRef]].getOrDefault("enabled", "").asInstanceOf[String])) { node.getMetadata.put("contentType", "Event") + node.getMetadata.put("objectType", "Event") } node } + @throws[Exception] + def pushInstructionEvent(identifier: String, node: Node)(implicit oec: OntologyEngineContext): Unit = { + val (actor, context, objData, eData) = generateInstructionEventMetadata(identifier.replace(".img", ""), node) + val beJobRequestEvent: String = LogTelemetryEventUtil.logInstructionEvent(actor.asJava, context.asJava, objData.asJava, eData) + val topic: String = Platform.getString("kafka.topics.event.publish", "dev.publish.job.request") + if (StringUtils.isBlank(beJobRequestEvent)) throw new ClientException("BE_JOB_REQUEST_EXCEPTION", "Event is not generated properly.") + oec.kafkaClient.send(beJobRequestEvent, topic) + } + + def generateInstructionEventMetadata(identifier: String, node: Node): (Map[String, AnyRef], Map[String, AnyRef], Map[String, AnyRef], util.Map[String, AnyRef]) = { + val metadata: util.Map[String, AnyRef] = node.getMetadata + val publishType = if (StringUtils.equalsIgnoreCase(metadata.getOrDefault("status", "").asInstanceOf[String], "Unlisted")) "unlisted" else "public" + val eventMetadata = Map("identifier" -> identifier, "mimeType" -> metadata.getOrDefault("mimeType", ""), "objectType" -> node.getObjectType.replace("Image", ""), "pkgVersion" -> metadata.getOrDefault("pkgVersion", 0.asInstanceOf[AnyRef]), "lastPublishedBy" -> metadata.getOrDefault("lastPublishedBy", "")) + val actor = Map("id" -> s"${node.getObjectType.toLowerCase().replace("image", "")}-publish", "type" -> "System".asInstanceOf[AnyRef]) + val context = Map("channel" -> metadata.getOrDefault("channel", ""), "pdata" -> Map("id" -> "org.sunbird.platform", "ver" -> "1.0").asJava, "env" -> Platform.getString("cloud_storage.env", "dev")) + val objData = Map("id" -> identifier, "ver" -> metadata.getOrDefault("versionKey", "")) + val eData: util.Map[String, AnyRef] = new util.HashMap[String, AnyRef] {{ + put("action", "publish") + put("publish_type", publishType) + put("metadata", eventMetadata.asJava) + }} + (actor, context, objData, eData) + } + } \ No newline at end of file From a959c0d545273cf4fab5ce35d826d7c4622821a8 Mon Sep 17 00:00:00 2001 From: tarentomaheshvakkund <139739142+tarentomaheshvakkund@users.noreply.github.com> Date: Tue, 11 Feb 2025 10:57:50 +0530 Subject: [PATCH 070/126] KB-8297 | [Backend] : Include a review step in the Events Flow and add meta fields to support competencies and associated files. 1. Event Rejection API added for rejecting the events by SPV publisher --- .../sunbird/content/actors/EventActor.scala | 24 +++++++++++++++++++ .../app/controllers/v4/EventController.scala | 13 ++++++++++ .../content-service/app/utils/ApiId.scala | 1 + content-api/content-service/conf/routes | 1 + schemas/event/1.0/schema.json | 5 +++- 5 files changed, 43 insertions(+), 1 deletion(-) diff --git a/content-api/content-actors/src/main/scala/org/sunbird/content/actors/EventActor.scala b/content-api/content-actors/src/main/scala/org/sunbird/content/actors/EventActor.scala index b9d49124c..64891b0ae 100644 --- a/content-api/content-actors/src/main/scala/org/sunbird/content/actors/EventActor.scala +++ b/content-api/content-actors/src/main/scala/org/sunbird/content/actors/EventActor.scala @@ -11,6 +11,7 @@ import org.sunbird.graph.dac.model.{Node, Relation} import org.sunbird.graph.nodes.DataNode import org.sunbird.telemetry.logger.TelemetryManager import org.sunbird.telemetry.util.LogTelemetryEventUtil +import org.sunbird.util.RequestUtil import java.util import javax.inject.Inject @@ -31,6 +32,7 @@ class EventActor @Inject()(implicit oec: OntologyEngineContext, ss: StorageServi case "retireContent" => retire(request) case "discardContent" => discard(request) case "publishContent" => publish(request) + case "rejectEvent" => rejectEvent(request) case _ => ERROR(request.getOperation) } } @@ -116,4 +118,26 @@ class EventActor @Inject()(implicit oec: OntologyEngineContext, ss: StorageServi (actor, context, objData, eData) } + def rejectEvent(request: Request): Future[Response] = { + RequestUtil.validateRequest(request) + DataNode.read(request).map(node => { + val status = node.getMetadata.get("status").asInstanceOf[String] + if (StringUtils.isBlank(status)) { + throw new ClientException("ERR_METADATA_ISSUE", "Event metadata error, status is blank for identifier:" + node.getIdentifier) + } + if (StringUtils.equals("sentToPublish", status) || StringUtils.equalsIgnoreCase("sentToPublish",status)) { + request.getRequest.put("status", "Rejected") + request.getRequest.put("prevStatus", "sentToPublish") + } + + else new ClientException("ERR_INVALID_REQUEST", "Content not in Review status.") + request.getRequest.put("versionKey", node.getMetadata.get("versionKey")) + request.putIn("publishChecklist", null).putIn("publishComment", null) + RequestUtil.restrictProperties(request) + DataNode.update(request).map(node => { + val identifier: String = node.getIdentifier.replace(".img", "") + ResponseHandler.OK.put("node_id", identifier).put("identifier", identifier) + }) + }).flatMap(f => f) + } } \ No newline at end of file diff --git a/content-api/content-service/app/controllers/v4/EventController.scala b/content-api/content-service/app/controllers/v4/EventController.scala index 0f2589cc7..18c539936 100644 --- a/content-api/content-service/app/controllers/v4/EventController.scala +++ b/content-api/content-service/app/controllers/v4/EventController.scala @@ -78,4 +78,17 @@ class EventController @Inject()(@Named(ActorNames.EVENT_ACTOR) eventActor: Actor setRequestContext(contentRequest, version, objectType, schemaName) getResult(ApiId.RETIRE_CONTENT, eventActor, contentRequest, version = apiVersion) } + + override def reviewReject(identifier: String) = Action.async { implicit request => + val headers = commonHeaders() + val body = requestBody() + val content = body.getOrDefault(schemaName, new java.util.HashMap()).asInstanceOf[java.util.Map[String, Object]]; + content.putAll(headers) + content.putAll(Map("identifier" -> identifier).asJava) + val contentRequest = getRequest(content, headers, "rejectEvent") + contentRequest.put("mode", "edit") + setRequestContext(contentRequest, version, objectType, schemaName) + contentRequest.getContext.put("identifier", identifier); + getResult(ApiId.REJECT_EVENT, eventActor, contentRequest, version = apiVersion) + } } \ No newline at end of file diff --git a/content-api/content-service/app/utils/ApiId.scala b/content-api/content-service/app/utils/ApiId.scala index 7aa95ac57..035ffc7cf 100644 --- a/content-api/content-service/app/utils/ApiId.scala +++ b/content-api/content-service/app/utils/ApiId.scala @@ -100,4 +100,5 @@ object ApiId { val IMPORT_CSV = "api.collection.import" val EXPORT_CSV = "api.collection.export" + val REJECT_EVENT = "api.event.review.reject" } diff --git a/content-api/content-service/conf/routes b/content-api/content-service/conf/routes index 3cc5616cd..93f378b69 100644 --- a/content-api/content-service/conf/routes +++ b/content-api/content-service/conf/routes @@ -113,6 +113,7 @@ POST /event/v4/publish/:identifier controllers.v4.EventControll GET /event/v4/read/:identifier controllers.v4.EventController.read(identifier:String, mode:Option[String], fields:Option[String]) DELETE /event/v4/discard/:identifier controllers.v4.EventController.discard(identifier:String) DELETE /private/event/v4/retire/:identifier controllers.v4.EventController.retire(identifier:String) +POST /event/v4/reject/:identifier controllers.v4.EventController.reviewReject(identifier:String) # EventSet v4 Api's POST /eventset/v4/create controllers.v4.EventSetController.create diff --git a/schemas/event/1.0/schema.json b/schemas/event/1.0/schema.json index 69556357c..cfec01e57 100644 --- a/schemas/event/1.0/schema.json +++ b/schemas/event/1.0/schema.json @@ -36,7 +36,10 @@ "enum": [ "Draft", "Live", - "Retired" + "Retired", + "SentToPublish", + "Rejected", + "Cancelled" ], "default": "Draft" }, From 0d0b1eb24fa3d7283cff94df382ba72dcdbb8d04 Mon Sep 17 00:00:00 2001 From: tarentomaheshvakkund <139739142+tarentomaheshvakkund@users.noreply.github.com> Date: Tue, 11 Feb 2025 14:45:57 +0530 Subject: [PATCH 071/126] KB-8297 | [Backend] : Include a review step in the Events Flow and add meta fields to support competencies and associated files. 1. Allow to update the status untill its not Live. --- .../content-service/app/controllers/v4/EventController.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content-api/content-service/app/controllers/v4/EventController.scala b/content-api/content-service/app/controllers/v4/EventController.scala index 18c539936..e8bac6078 100644 --- a/content-api/content-service/app/controllers/v4/EventController.scala +++ b/content-api/content-service/app/controllers/v4/EventController.scala @@ -44,7 +44,7 @@ class EventController @Inject()(@Named(ActorNames.EVENT_ACTOR) eventActor: Actor val headers = commonHeaders() val body = requestBody() val content = body.getOrDefault(schemaName, new java.util.HashMap()).asInstanceOf[java.util.Map[String, Object]]; - if (content.containsKey("status")) { + if (content.containsKey("status") && content.get("status").equals("Live")) { getErrorResponse(ApiId.UPDATE_EVENT, apiVersion, "VALIDATION_ERROR", "status update is restricted, use status APIs.") } else { content.putAll(headers) From 2ce79e3ae93c5da7126d4afc8517ff64ac400355 Mon Sep 17 00:00:00 2001 From: tarentomaheshvakkund <139739142+tarentomaheshvakkund@users.noreply.github.com> Date: Tue, 11 Feb 2025 14:56:24 +0530 Subject: [PATCH 072/126] KB-8297 | [Backend] : Include a review step in the Events Flow and add meta fields to support competencies and associated files. 1. Fixed casting issue. --- .../content-service/app/controllers/v4/EventController.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content-api/content-service/app/controllers/v4/EventController.scala b/content-api/content-service/app/controllers/v4/EventController.scala index e8bac6078..dca939de3 100644 --- a/content-api/content-service/app/controllers/v4/EventController.scala +++ b/content-api/content-service/app/controllers/v4/EventController.scala @@ -44,7 +44,7 @@ class EventController @Inject()(@Named(ActorNames.EVENT_ACTOR) eventActor: Actor val headers = commonHeaders() val body = requestBody() val content = body.getOrDefault(schemaName, new java.util.HashMap()).asInstanceOf[java.util.Map[String, Object]]; - if (content.containsKey("status") && content.get("status").equals("Live")) { + if (content.containsKey("status") && "Live".equals(content.get("status").toString)) { getErrorResponse(ApiId.UPDATE_EVENT, apiVersion, "VALIDATION_ERROR", "status update is restricted, use status APIs.") } else { content.putAll(headers) From 22ddf61aa8edca164b83fa034f7750adf318bd06 Mon Sep 17 00:00:00 2001 From: tarentomaheshvakkund <139739142+tarentomaheshvakkund@users.noreply.github.com> Date: Fri, 14 Feb 2025 12:04:58 +0530 Subject: [PATCH 073/126] KB-8297 | [Backend] : Include a review step in the Events Flow and add meta fields to support competencies and associated files. 1. Update should be allowed when the event is live. --- .../app/controllers/v4/EventController.scala | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/content-api/content-service/app/controllers/v4/EventController.scala b/content-api/content-service/app/controllers/v4/EventController.scala index dca939de3..6bf421414 100644 --- a/content-api/content-service/app/controllers/v4/EventController.scala +++ b/content-api/content-service/app/controllers/v4/EventController.scala @@ -44,15 +44,12 @@ class EventController @Inject()(@Named(ActorNames.EVENT_ACTOR) eventActor: Actor val headers = commonHeaders() val body = requestBody() val content = body.getOrDefault(schemaName, new java.util.HashMap()).asInstanceOf[java.util.Map[String, Object]]; - if (content.containsKey("status") && "Live".equals(content.get("status").toString)) { - getErrorResponse(ApiId.UPDATE_EVENT, apiVersion, "VALIDATION_ERROR", "status update is restricted, use status APIs.") - } else { - content.putAll(headers) - val contentRequest = getRequest(content, headers, "updateContent") - setRequestContext(contentRequest, version, objectType, schemaName) - contentRequest.getContext.put("identifier", identifier); - getResult(ApiId.UPDATE_EVENT, eventActor, contentRequest, version = apiVersion) - } + content.putAll(headers) + val contentRequest = getRequest(content, headers, "updateContent") + setRequestContext(contentRequest, version, objectType, schemaName) + contentRequest.getContext.put("identifier", identifier); + getResult(ApiId.UPDATE_EVENT, eventActor, contentRequest, version = apiVersion) + } def publish(identifier: String): Action[AnyContent] = Action.async { implicit request => From 3208942c2f6043bb0ab8eac91e5fa0a466bb39ac Mon Sep 17 00:00:00 2001 From: tarentomaheshvakkund <139739142+tarentomaheshvakkund@users.noreply.github.com> Date: Wed, 5 Mar 2025 09:42:45 +0530 Subject: [PATCH 074/126] Added new Mimetype for assets and content. --- schemas/asset/1.0/schema.json | 3 ++- schemas/content/1.0/schema.json | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/schemas/asset/1.0/schema.json b/schemas/asset/1.0/schema.json index 5a4dc078b..1b3ea5f0d 100644 --- a/schemas/asset/1.0/schema.json +++ b/schemas/asset/1.0/schema.json @@ -86,7 +86,8 @@ "audio/webm", "audio/x-wav", "audio/wav", - "application/json" + "application/json", + "application/vnd.openxmlformats-officedocument.presentationml.presentation" ] }, "osId": { diff --git a/schemas/content/1.0/schema.json b/schemas/content/1.0/schema.json index 57c10e72e..c9a9b4b02 100644 --- a/schemas/content/1.0/schema.json +++ b/schemas/content/1.0/schema.json @@ -89,7 +89,8 @@ "audio/wav", "application/json", "application/quiz", - "application/survey" + "application/survey", + "application/vnd.openxmlformats-officedocument.presentationml.presentation" ] }, "osId": { From 367e5914db5abf77158c2e35309ba85697d31289 Mon Sep 17 00:00:00 2001 From: tarentomaheshvakkund <139739142+tarentomaheshvakkund@users.noreply.github.com> Date: Mon, 10 Mar 2025 15:07:02 +0530 Subject: [PATCH 075/126] KB-8855 | DEV | BE | Q7 | Events Hub | MDO Admin | Event Editing 1. Allowing the edit of events even when the content is live. --- .../sunbird/content/actors/EventActor.scala | 18 +++++++++++++++++- schemas/event/1.0/config.json | 2 +- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/content-api/content-actors/src/main/scala/org/sunbird/content/actors/EventActor.scala b/content-api/content-actors/src/main/scala/org/sunbird/content/actors/EventActor.scala index 64891b0ae..54eabaaed 100644 --- a/content-api/content-actors/src/main/scala/org/sunbird/content/actors/EventActor.scala +++ b/content-api/content-actors/src/main/scala/org/sunbird/content/actors/EventActor.scala @@ -38,7 +38,23 @@ class EventActor @Inject()(implicit oec: OntologyEngineContext, ss: StorageServi } override def update(request: Request): Future[Response] = { - verifyStandaloneEventAndApply(super.update, request) + populateDefaultersForUpdation(request) + val versionKey = request.getRequest.getOrDefault("versionKey", "").asInstanceOf[String] + if (StringUtils.isBlank(versionKey)) { + throw new ClientException("ERR_INVALID_REQUEST", "Please Provide Version Key!") + } + RequestUtil.restrictProperties(request) + val reviewStatus: String = request.getRequest.getOrDefault("reviewStatus", "").asInstanceOf[String] + if (reviewStatus == null || reviewStatus.isEmpty) { + + request.getRequest.put("cqfVersion", System.currentTimeMillis().toString) + } + DataNode.update(request, dataModifier).map(node => { + val identifier: String = node.getIdentifier.replace(".img", "") + ResponseHandler.OK.put("node_id", identifier) + .put("identifier", identifier) + .put("versionKey", node.getMetadata.get("versionKey")) + }) } def publish(request: Request): Future[Response] = { diff --git a/schemas/event/1.0/config.json b/schemas/event/1.0/config.json index 52bdf3ba8..04b1fc8c3 100644 --- a/schemas/event/1.0/config.json +++ b/schemas/event/1.0/config.json @@ -55,7 +55,7 @@ "targetMediumIds", "targetTopicIds" ], - "version": "disable", + "version": "enable", "versionCheckMode": "OFF", "cacheEnabled": false, "schema_restrict_api": false From 576c7c4c1024699e1ec02aec72cf268d80b2590b Mon Sep 17 00:00:00 2001 From: tarentomaheshvakkund <139739142+tarentomaheshvakkund@users.noreply.github.com> Date: Mon, 17 Mar 2025 10:29:57 +0530 Subject: [PATCH 076/126] Revert "KB-8855 | DEV | BE | Q7 | Events Hub | MDO Admin | Event Editing" This reverts commit 367e5914db5abf77158c2e35309ba85697d31289. --- .../sunbird/content/actors/EventActor.scala | 18 +----------------- schemas/event/1.0/config.json | 2 +- 2 files changed, 2 insertions(+), 18 deletions(-) diff --git a/content-api/content-actors/src/main/scala/org/sunbird/content/actors/EventActor.scala b/content-api/content-actors/src/main/scala/org/sunbird/content/actors/EventActor.scala index 54eabaaed..64891b0ae 100644 --- a/content-api/content-actors/src/main/scala/org/sunbird/content/actors/EventActor.scala +++ b/content-api/content-actors/src/main/scala/org/sunbird/content/actors/EventActor.scala @@ -38,23 +38,7 @@ class EventActor @Inject()(implicit oec: OntologyEngineContext, ss: StorageServi } override def update(request: Request): Future[Response] = { - populateDefaultersForUpdation(request) - val versionKey = request.getRequest.getOrDefault("versionKey", "").asInstanceOf[String] - if (StringUtils.isBlank(versionKey)) { - throw new ClientException("ERR_INVALID_REQUEST", "Please Provide Version Key!") - } - RequestUtil.restrictProperties(request) - val reviewStatus: String = request.getRequest.getOrDefault("reviewStatus", "").asInstanceOf[String] - if (reviewStatus == null || reviewStatus.isEmpty) { - - request.getRequest.put("cqfVersion", System.currentTimeMillis().toString) - } - DataNode.update(request, dataModifier).map(node => { - val identifier: String = node.getIdentifier.replace(".img", "") - ResponseHandler.OK.put("node_id", identifier) - .put("identifier", identifier) - .put("versionKey", node.getMetadata.get("versionKey")) - }) + verifyStandaloneEventAndApply(super.update, request) } def publish(request: Request): Future[Response] = { diff --git a/schemas/event/1.0/config.json b/schemas/event/1.0/config.json index 04b1fc8c3..52bdf3ba8 100644 --- a/schemas/event/1.0/config.json +++ b/schemas/event/1.0/config.json @@ -55,7 +55,7 @@ "targetMediumIds", "targetTopicIds" ], - "version": "enable", + "version": "disable", "versionCheckMode": "OFF", "cacheEnabled": false, "schema_restrict_api": false From 33e283c0a2570c830049f288ffc66293d81c3bf2 Mon Sep 17 00:00:00 2001 From: arpithasureshappa Date: Wed, 19 Mar 2025 11:16:36 +0530 Subject: [PATCH 077/126] Query Search with Sorting enabled --- .../java/org/sunbird/search/processor/SearchProcessor.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/search-api/search-core/src/main/java/org/sunbird/search/processor/SearchProcessor.java b/search-api/search-core/src/main/java/org/sunbird/search/processor/SearchProcessor.java index 1a8ccd8b9..5ef2b7821 100644 --- a/search-api/search-core/src/main/java/org/sunbird/search/processor/SearchProcessor.java +++ b/search-api/search-core/src/main/java/org/sunbird/search/processor/SearchProcessor.java @@ -247,6 +247,9 @@ private SearchSourceBuilder processSearchQuery(SearchDTO searchDTO, List Date: Wed, 19 Mar 2025 11:40:20 +0530 Subject: [PATCH 078/126] updated with null check --- .../main/java/org/sunbird/search/processor/SearchProcessor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/search-api/search-core/src/main/java/org/sunbird/search/processor/SearchProcessor.java b/search-api/search-core/src/main/java/org/sunbird/search/processor/SearchProcessor.java index 5ef2b7821..9ed710dd4 100644 --- a/search-api/search-core/src/main/java/org/sunbird/search/processor/SearchProcessor.java +++ b/search-api/search-core/src/main/java/org/sunbird/search/processor/SearchProcessor.java @@ -247,7 +247,7 @@ private SearchSourceBuilder processSearchQuery(SearchDTO searchDTO, List Date: Mon, 10 Mar 2025 15:07:02 +0530 Subject: [PATCH 079/126] KB-8855 | DEV | BE | Q7 | Events Hub | MDO Admin | Event Editing 1. Allowing the edit of events even when the content is live. --- .../sunbird/content/actors/EventActor.scala | 18 +++++++++++++++++- schemas/event/1.0/config.json | 2 +- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/content-api/content-actors/src/main/scala/org/sunbird/content/actors/EventActor.scala b/content-api/content-actors/src/main/scala/org/sunbird/content/actors/EventActor.scala index 64891b0ae..54eabaaed 100644 --- a/content-api/content-actors/src/main/scala/org/sunbird/content/actors/EventActor.scala +++ b/content-api/content-actors/src/main/scala/org/sunbird/content/actors/EventActor.scala @@ -38,7 +38,23 @@ class EventActor @Inject()(implicit oec: OntologyEngineContext, ss: StorageServi } override def update(request: Request): Future[Response] = { - verifyStandaloneEventAndApply(super.update, request) + populateDefaultersForUpdation(request) + val versionKey = request.getRequest.getOrDefault("versionKey", "").asInstanceOf[String] + if (StringUtils.isBlank(versionKey)) { + throw new ClientException("ERR_INVALID_REQUEST", "Please Provide Version Key!") + } + RequestUtil.restrictProperties(request) + val reviewStatus: String = request.getRequest.getOrDefault("reviewStatus", "").asInstanceOf[String] + if (reviewStatus == null || reviewStatus.isEmpty) { + + request.getRequest.put("cqfVersion", System.currentTimeMillis().toString) + } + DataNode.update(request, dataModifier).map(node => { + val identifier: String = node.getIdentifier.replace(".img", "") + ResponseHandler.OK.put("node_id", identifier) + .put("identifier", identifier) + .put("versionKey", node.getMetadata.get("versionKey")) + }) } def publish(request: Request): Future[Response] = { diff --git a/schemas/event/1.0/config.json b/schemas/event/1.0/config.json index 52bdf3ba8..04b1fc8c3 100644 --- a/schemas/event/1.0/config.json +++ b/schemas/event/1.0/config.json @@ -55,7 +55,7 @@ "targetMediumIds", "targetTopicIds" ], - "version": "disable", + "version": "enable", "versionCheckMode": "OFF", "cacheEnabled": false, "schema_restrict_api": false From 25a2d5e50f58a26764258c8b62275e441a413cc1 Mon Sep 17 00:00:00 2001 From: sahilchaudhary Date: Thu, 20 Mar 2025 13:18:27 +0530 Subject: [PATCH 080/126] Revert "KB-8855 | DEV | BE | Q7 | Events Hub | MDO Admin | Event Editing" This reverts commit 3f84fbc7a0fc31fc5482085c19b8072bbc05097c. --- .../sunbird/content/actors/EventActor.scala | 18 +----------------- schemas/event/1.0/config.json | 2 +- 2 files changed, 2 insertions(+), 18 deletions(-) diff --git a/content-api/content-actors/src/main/scala/org/sunbird/content/actors/EventActor.scala b/content-api/content-actors/src/main/scala/org/sunbird/content/actors/EventActor.scala index 54eabaaed..64891b0ae 100644 --- a/content-api/content-actors/src/main/scala/org/sunbird/content/actors/EventActor.scala +++ b/content-api/content-actors/src/main/scala/org/sunbird/content/actors/EventActor.scala @@ -38,23 +38,7 @@ class EventActor @Inject()(implicit oec: OntologyEngineContext, ss: StorageServi } override def update(request: Request): Future[Response] = { - populateDefaultersForUpdation(request) - val versionKey = request.getRequest.getOrDefault("versionKey", "").asInstanceOf[String] - if (StringUtils.isBlank(versionKey)) { - throw new ClientException("ERR_INVALID_REQUEST", "Please Provide Version Key!") - } - RequestUtil.restrictProperties(request) - val reviewStatus: String = request.getRequest.getOrDefault("reviewStatus", "").asInstanceOf[String] - if (reviewStatus == null || reviewStatus.isEmpty) { - - request.getRequest.put("cqfVersion", System.currentTimeMillis().toString) - } - DataNode.update(request, dataModifier).map(node => { - val identifier: String = node.getIdentifier.replace(".img", "") - ResponseHandler.OK.put("node_id", identifier) - .put("identifier", identifier) - .put("versionKey", node.getMetadata.get("versionKey")) - }) + verifyStandaloneEventAndApply(super.update, request) } def publish(request: Request): Future[Response] = { diff --git a/schemas/event/1.0/config.json b/schemas/event/1.0/config.json index 04b1fc8c3..52bdf3ba8 100644 --- a/schemas/event/1.0/config.json +++ b/schemas/event/1.0/config.json @@ -55,7 +55,7 @@ "targetMediumIds", "targetTopicIds" ], - "version": "enable", + "version": "disable", "versionCheckMode": "OFF", "cacheEnabled": false, "schema_restrict_api": false From 44de48141c02f698d69c9651e30cbcfb5d266e87 Mon Sep 17 00:00:00 2001 From: vikrantsingh Date: Tue, 25 Mar 2025 16:47:14 +0530 Subject: [PATCH 081/126] getHierarchy?mode=edit: handled null and "null" --- .../main/scala/org/sunbird/managers/HierarchyManager.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/content-api/hierarchy-manager/src/main/scala/org/sunbird/managers/HierarchyManager.scala b/content-api/hierarchy-manager/src/main/scala/org/sunbird/managers/HierarchyManager.scala index 4d591308f..966e2a470 100644 --- a/content-api/hierarchy-manager/src/main/scala/org/sunbird/managers/HierarchyManager.scala +++ b/content-api/hierarchy-manager/src/main/scala/org/sunbird/managers/HierarchyManager.scala @@ -464,7 +464,7 @@ object HierarchyManager { responseFuture.map(response => { if (!ResponseHandler.checkError(response)) { val relationalMetadataString = response.getResult.toMap.getOrDefault("relational_metadata", "").asInstanceOf[String] - if (StringUtils.isNotEmpty(relationalMetadataString)) { + if (StringUtils.isNotEmpty(relationalMetadataString) && !relationalMetadataString.equalsIgnoreCase("null")) { Future(JsonUtils.deserialize(relationalMetadataString, classOf[java.util.Map[String, AnyRef]]).toMap) } else Future(Map[String, AnyRef]()) @@ -475,7 +475,7 @@ object HierarchyManager { responseFuture.map(response => { if (!ResponseHandler.checkError(response)) { val relationalMetadataString = response.getResult.toMap.getOrDefault("relational_metadata", "").asInstanceOf[String] - if (StringUtils.isNotEmpty(relationalMetadataString)) { + if (StringUtils.isNotEmpty(relationalMetadataString) && !relationalMetadataString.equalsIgnoreCase("null")) { Future(JsonUtils.deserialize(relationalMetadataString, classOf[java.util.Map[String, AnyRef]]).toMap) } else Future(Map[String, AnyRef]()) From 059b29db27a6f634bab3f242087f7913c3a77782 Mon Sep 17 00:00:00 2001 From: arpithasureshappa Date: Wed, 2 Apr 2025 11:33:35 +0530 Subject: [PATCH 082/126] added secureSettings key map in request --- .../search-service/app/controllers/SearchController.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/search-api/search-service/app/controllers/SearchController.scala b/search-api/search-service/app/controllers/SearchController.scala index 451e7c33c..191e4bc6e 100644 --- a/search-api/search-service/app/controllers/SearchController.scala +++ b/search-api/search-service/app/controllers/SearchController.scala @@ -61,7 +61,8 @@ class SearchController @Inject()(@Named(ActorNames.SEARCH_ACTOR) searchActor: Ac def searchV4() = loggingAction.async { implicit request => val internalReq = getRequest(ApiId.APPLICATION_SEARCH) val requestMap: java.util.Map[String, Any] = internalReq.getRequest.asInstanceOf[util.Map[String, Any]] - requestMap.put(SearchConstants.isSecureSettingsDisabled, true) + val isSecureSettingsDisabled = requestMap.getOrDefault(SearchConstants.isSecureSettingsDisabled, true).asInstanceOf[Boolean] + requestMap.put(SearchConstants.isSecureSettingsDisabled, isSecureSettingsDisabled) setHeaderContext(internalReq) val filters = internalReq.getRequest.getOrDefault(SearchConstants.filters, new java.util.HashMap()).asInstanceOf[java.util.Map[String, Object]] val visibilityObject = filters.getOrDefault("visibility","") From ad4d58f4bac172368ae7d0c085c24bc577e2fb54 Mon Sep 17 00:00:00 2001 From: arpithasureshappa Date: Wed, 2 Apr 2025 19:04:42 +0530 Subject: [PATCH 083/126] KB-8946 issue fix --- .../org/sunbird/search/processor/SearchProcessor.java | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/search-api/search-core/src/main/java/org/sunbird/search/processor/SearchProcessor.java b/search-api/search-core/src/main/java/org/sunbird/search/processor/SearchProcessor.java index 9ed710dd4..682674d34 100644 --- a/search-api/search-core/src/main/java/org/sunbird/search/processor/SearchProcessor.java +++ b/search-api/search-core/src/main/java/org/sunbird/search/processor/SearchProcessor.java @@ -11,10 +11,8 @@ import org.elasticsearch.index.query.MultiMatchQueryBuilder.Type; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHits; -import org.elasticsearch.search.aggregations.Aggregation; -import org.elasticsearch.search.aggregations.AggregationBuilder; -import org.elasticsearch.search.aggregations.AggregationBuilders; -import org.elasticsearch.search.aggregations.Aggregations; +import org.elasticsearch.search.aggregations.*; +import org.elasticsearch.search.aggregations.bucket.filter.FilterAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.terms.Terms; import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder; import org.elasticsearch.search.builder.SearchSourceBuilder; @@ -71,7 +69,10 @@ public Future> processSearch(SearchDTO searchDTO, boolean in } if (searchDTO.isSecureSettingsDisabled()) { - query.postFilter(getPostFilterQuery(searchDTO.getPostFilter())); + BoolQueryBuilder mainQuery = QueryBuilders.boolQuery(); + mainQuery.must(query.query()); // Preserve the existing query + mainQuery.filter(getPostFilterQuery(searchDTO.getPostFilter())); // Apply filtering for aggregations + query.query(mainQuery); } searchResponse = ElasticSearchUtil.search(SearchConstants.COMPOSITE_SEARCH_INDEX, query); From a07f0fc2d1ff6542c314f42943713a07f0badb1f Mon Sep 17 00:00:00 2001 From: arpithasureshappa Date: Wed, 2 Apr 2025 19:07:21 +0530 Subject: [PATCH 084/126] KB-8946 issue fix --- .../search-service/app/controllers/SearchController.scala | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/search-api/search-service/app/controllers/SearchController.scala b/search-api/search-service/app/controllers/SearchController.scala index 191e4bc6e..451e7c33c 100644 --- a/search-api/search-service/app/controllers/SearchController.scala +++ b/search-api/search-service/app/controllers/SearchController.scala @@ -61,8 +61,7 @@ class SearchController @Inject()(@Named(ActorNames.SEARCH_ACTOR) searchActor: Ac def searchV4() = loggingAction.async { implicit request => val internalReq = getRequest(ApiId.APPLICATION_SEARCH) val requestMap: java.util.Map[String, Any] = internalReq.getRequest.asInstanceOf[util.Map[String, Any]] - val isSecureSettingsDisabled = requestMap.getOrDefault(SearchConstants.isSecureSettingsDisabled, true).asInstanceOf[Boolean] - requestMap.put(SearchConstants.isSecureSettingsDisabled, isSecureSettingsDisabled) + requestMap.put(SearchConstants.isSecureSettingsDisabled, true) setHeaderContext(internalReq) val filters = internalReq.getRequest.getOrDefault(SearchConstants.filters, new java.util.HashMap()).asInstanceOf[java.util.Map[String, Object]] val visibilityObject = filters.getOrDefault("visibility","") From 408c7eca866751bb3ca640f6d58ef663a0aa4941 Mon Sep 17 00:00:00 2001 From: arpithasureshappa Date: Wed, 2 Apr 2025 19:08:37 +0530 Subject: [PATCH 085/126] KB-8946 issue fix --- .../main/java/org/sunbird/search/processor/SearchProcessor.java | 1 - 1 file changed, 1 deletion(-) diff --git a/search-api/search-core/src/main/java/org/sunbird/search/processor/SearchProcessor.java b/search-api/search-core/src/main/java/org/sunbird/search/processor/SearchProcessor.java index 682674d34..69019e4fe 100644 --- a/search-api/search-core/src/main/java/org/sunbird/search/processor/SearchProcessor.java +++ b/search-api/search-core/src/main/java/org/sunbird/search/processor/SearchProcessor.java @@ -12,7 +12,6 @@ import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHits; import org.elasticsearch.search.aggregations.*; -import org.elasticsearch.search.aggregations.bucket.filter.FilterAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.terms.Terms; import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder; import org.elasticsearch.search.builder.SearchSourceBuilder; From a41bed8f33e92da8610b50962fe9a2518993f5bf Mon Sep 17 00:00:00 2001 From: tarentomaheshvakkund <139739142+tarentomaheshvakkund@users.noreply.github.com> Date: Fri, 4 Apr 2025 11:04:07 +0530 Subject: [PATCH 086/126] Deleting the cache before every update to reflect the latest data in content read. --- .../src/main/scala/org/sunbird/content/actors/EventActor.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/content-api/content-actors/src/main/scala/org/sunbird/content/actors/EventActor.scala b/content-api/content-actors/src/main/scala/org/sunbird/content/actors/EventActor.scala index 64891b0ae..82ba1fc63 100644 --- a/content-api/content-actors/src/main/scala/org/sunbird/content/actors/EventActor.scala +++ b/content-api/content-actors/src/main/scala/org/sunbird/content/actors/EventActor.scala @@ -1,6 +1,7 @@ package org.sunbird.content.actors import org.apache.commons.lang.StringUtils +import org.sunbird.cache.impl.RedisCache import org.sunbird.common.Platform import org.sunbird.cloudstore.StorageService import org.sunbird.common.dto.{Request, Response, ResponseHandler} @@ -17,7 +18,6 @@ import java.util import javax.inject.Inject import scala.collection.JavaConverters.asScalaBufferConverter import scala.concurrent.Future - import scala.collection.JavaConversions._ import scala.collection.JavaConverters import scala.collection.JavaConverters._ @@ -38,6 +38,7 @@ class EventActor @Inject()(implicit oec: OntologyEngineContext, ss: StorageServi } override def update(request: Request): Future[Response] = { + RedisCache.delete(request.get("identifier").asInstanceOf[String]) verifyStandaloneEventAndApply(super.update, request) } From 5c777e3b3f3661593fa40b6787b6f0be66d73c4b Mon Sep 17 00:00:00 2001 From: arpithasureshappa Date: Mon, 7 Apr 2025 10:35:57 +0530 Subject: [PATCH 087/126] event startDateTime endDateTime converting to timestamp during create and update event --- .../sunbird/content/actors/ContentActor.scala | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala b/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala index be17caece..d11d00655 100644 --- a/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala +++ b/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala @@ -29,6 +29,8 @@ import org.sunbird.graph.utils.NodeUtil import org.sunbird.managers.HierarchyManager import org.sunbird.managers.HierarchyManager.hierarchyPrefix +import java.time.ZonedDateTime +import java.time.format.DateTimeFormatter import scala.collection.JavaConverters import scala.collection.JavaConverters._ import scala.concurrent.{ExecutionContext, Future} @@ -66,6 +68,21 @@ class ContentActor @Inject() (implicit oec: OntologyEngineContext, ss: StorageSe def create(request: Request): Future[Response] = { populateDefaultersForCreation(request) RequestUtil.restrictProperties(request) + val startDateTimeStr = request.getRequest.getOrDefault("startDateTime", "").asInstanceOf[String] + val endDateTimeStr = request.getRequest.getOrDefault("endDateTime", "").asInstanceOf[String] + if (StringUtils.isNotBlank(startDateTimeStr) && StringUtils.isNotBlank(endDateTimeStr)) { + // Convert startDateTime string to epoch milliseconds + val formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSSXX") + val startDateTime = ZonedDateTime.parse(startDateTimeStr, formatter) + val startDateTimeEpochMillis = startDateTime.toInstant.toEpochMilli + request.getRequest.put("startDateTime", startDateTimeStr) + request.getRequest.put("startDateTimeInEpoch", startDateTimeEpochMillis.asInstanceOf[java.lang.Long]) + + val endDateTime = ZonedDateTime.parse(endDateTimeStr, formatter) + val endDateTimeEpochMillis = endDateTime.toInstant.toEpochMilli + request.getRequest.put("endDateTime", endDateTimeStr) + request.getRequest.put("endDateTimeInEpoch", endDateTimeEpochMillis.asInstanceOf[java.lang.Long]) + } request.getRequest.put("cqfVersion", System.currentTimeMillis().toString) DataNode.create(request, dataModifier).map(node => { ResponseHandler.OK.put("identifier", node.getIdentifier).put("node_id", node.getIdentifier) @@ -137,6 +154,21 @@ class ContentActor @Inject() (implicit oec: OntologyEngineContext, ss: StorageSe } def update(request: Request): Future[Response] = { + val startDateTimeStr = request.getRequest.getOrDefault("startDateTime", "").asInstanceOf[String] + val endDateTimeStr = request.getRequest.getOrDefault("endDateTime", "").asInstanceOf[String] + if (StringUtils.isNotBlank(startDateTimeStr) && StringUtils.isNotBlank(endDateTimeStr)) { + // Convert startDateTime string to epoch milliseconds + val formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSSXX") + val startDateTime = ZonedDateTime.parse(startDateTimeStr, formatter) + val startDateTimeEpochMillis = startDateTime.toInstant.toEpochMilli + request.getRequest.put("startDateTime", startDateTimeStr) + request.getRequest.put("startDateTimeInEpoch", startDateTimeEpochMillis.asInstanceOf[java.lang.Long]) + + val endDateTime = ZonedDateTime.parse(endDateTimeStr, formatter) + val endDateTimeEpochMillis = endDateTime.toInstant.toEpochMilli + request.getRequest.put("endDateTime", endDateTimeStr) + request.getRequest.put("endDateTimeInEpoch", endDateTimeEpochMillis.asInstanceOf[java.lang.Long]) + } populateDefaultersForUpdation(request) if (StringUtils.isBlank(request.getRequest.getOrDefault("versionKey", "").asInstanceOf[String])) throw new ClientException("ERR_INVALID_REQUEST", "Please Provide Version Key!") RequestUtil.restrictProperties(request) From be2913fcada56f3cec702aeeb96fd161063e1183 Mon Sep 17 00:00:00 2001 From: arpithasureshappa Date: Mon, 7 Apr 2025 17:13:21 +0530 Subject: [PATCH 088/126] try catch block added --- .../sunbird/content/actors/ContentActor.scala | 61 ++++++++++++------- 1 file changed, 38 insertions(+), 23 deletions(-) diff --git a/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala b/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala index d11d00655..4aaedff43 100644 --- a/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala +++ b/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala @@ -17,7 +17,7 @@ import org.sunbird.content.util.{AcceptFlagManager, ContentConstants, CopyManage import org.sunbird.cloudstore.StorageService import org.sunbird.common.{ContentParams, JsonUtils, Platform, Slug} import org.sunbird.common.dto.{Request, Response, ResponseHandler} -import org.sunbird.common.exception.ClientException +import org.sunbird.common.exception.{ClientException, ResponseCode} import org.sunbird.content.dial.DIALManager import org.sunbird.content.review.mgr.ReviewManager import org.sunbird.util.RequestUtil @@ -71,17 +71,25 @@ class ContentActor @Inject() (implicit oec: OntologyEngineContext, ss: StorageSe val startDateTimeStr = request.getRequest.getOrDefault("startDateTime", "").asInstanceOf[String] val endDateTimeStr = request.getRequest.getOrDefault("endDateTime", "").asInstanceOf[String] if (StringUtils.isNotBlank(startDateTimeStr) && StringUtils.isNotBlank(endDateTimeStr)) { - // Convert startDateTime string to epoch milliseconds - val formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSSXX") - val startDateTime = ZonedDateTime.parse(startDateTimeStr, formatter) - val startDateTimeEpochMillis = startDateTime.toInstant.toEpochMilli - request.getRequest.put("startDateTime", startDateTimeStr) - request.getRequest.put("startDateTimeInEpoch", startDateTimeEpochMillis.asInstanceOf[java.lang.Long]) - - val endDateTime = ZonedDateTime.parse(endDateTimeStr, formatter) - val endDateTimeEpochMillis = endDateTime.toInstant.toEpochMilli - request.getRequest.put("endDateTime", endDateTimeStr) - request.getRequest.put("endDateTimeInEpoch", endDateTimeEpochMillis.asInstanceOf[java.lang.Long]) + try { + // Convert startDateTime string to epoch milliseconds + val formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSSXX") + val startDateTime = ZonedDateTime.parse(startDateTimeStr, formatter) + val startDateTimeEpochMillis = startDateTime.toInstant.toEpochMilli + request.getRequest.put("startDateTime", startDateTimeStr) + request.getRequest.put("startDateTimeInEpoch", startDateTimeEpochMillis.asInstanceOf[java.lang.Long]) + + val endDateTime = ZonedDateTime.parse(endDateTimeStr, formatter) + val endDateTimeEpochMillis = endDateTime.toInstant.toEpochMilli + request.getRequest.put("endDateTime", endDateTimeStr) + request.getRequest.put("endDateTimeInEpoch", endDateTimeEpochMillis.asInstanceOf[java.lang.Long]) + } catch { + case ex: Exception => + return Future.successful(ResponseHandler. + ERROR(ResponseCode.CLIENT_ERROR, + "ERR_INVALID_DATE_FORMAT", + "startDateTime or endDateTime is not in the expected format yyyy-MM-dd'T'HH:mm:ss.SSSXX")) + } } request.getRequest.put("cqfVersion", System.currentTimeMillis().toString) DataNode.create(request, dataModifier).map(node => { @@ -157,17 +165,24 @@ class ContentActor @Inject() (implicit oec: OntologyEngineContext, ss: StorageSe val startDateTimeStr = request.getRequest.getOrDefault("startDateTime", "").asInstanceOf[String] val endDateTimeStr = request.getRequest.getOrDefault("endDateTime", "").asInstanceOf[String] if (StringUtils.isNotBlank(startDateTimeStr) && StringUtils.isNotBlank(endDateTimeStr)) { - // Convert startDateTime string to epoch milliseconds - val formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSSXX") - val startDateTime = ZonedDateTime.parse(startDateTimeStr, formatter) - val startDateTimeEpochMillis = startDateTime.toInstant.toEpochMilli - request.getRequest.put("startDateTime", startDateTimeStr) - request.getRequest.put("startDateTimeInEpoch", startDateTimeEpochMillis.asInstanceOf[java.lang.Long]) - - val endDateTime = ZonedDateTime.parse(endDateTimeStr, formatter) - val endDateTimeEpochMillis = endDateTime.toInstant.toEpochMilli - request.getRequest.put("endDateTime", endDateTimeStr) - request.getRequest.put("endDateTimeInEpoch", endDateTimeEpochMillis.asInstanceOf[java.lang.Long]) + try { + // Convert startDateTime string to epoch milliseconds + val formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSSXX") + val startDateTime = ZonedDateTime.parse(startDateTimeStr, formatter) + val startDateTimeEpochMillis = startDateTime.toInstant.toEpochMilli + request.getRequest.put("startDateTime", startDateTimeStr) + request.getRequest.put("startDateTimeInEpoch", startDateTimeEpochMillis.asInstanceOf[java.lang.Long]) + val endDateTime = ZonedDateTime.parse(endDateTimeStr, formatter) + val endDateTimeEpochMillis = endDateTime.toInstant.toEpochMilli + request.getRequest.put("endDateTime", endDateTimeStr) + request.getRequest.put("endDateTimeInEpoch", endDateTimeEpochMillis.asInstanceOf[java.lang.Long]) + } catch { + case ex: Exception => + return Future.successful(ResponseHandler. + ERROR(ResponseCode.CLIENT_ERROR, + "ERR_INVALID_DATE_FORMAT", + "startDateTime or endDateTime is not in the expected format yyyy-MM-dd'T'HH:mm:ss.SSSXX")) + } } populateDefaultersForUpdation(request) if (StringUtils.isBlank(request.getRequest.getOrDefault("versionKey", "").asInstanceOf[String])) throw new ClientException("ERR_INVALID_REQUEST", "Please Provide Version Key!") From 1f5720d15c264eb332a8c819b2096372541852fe Mon Sep 17 00:00:00 2001 From: arpithasureshappa Date: Tue, 8 Apr 2025 09:28:24 +0530 Subject: [PATCH 089/126] create and update API updated to save time in IST format --- .../sunbird/content/actors/ContentActor.scala | 51 ++++++++++++------- 1 file changed, 34 insertions(+), 17 deletions(-) diff --git a/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala b/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala index 4aaedff43..2ba64edfb 100644 --- a/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala +++ b/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala @@ -29,7 +29,7 @@ import org.sunbird.graph.utils.NodeUtil import org.sunbird.managers.HierarchyManager import org.sunbird.managers.HierarchyManager.hierarchyPrefix -import java.time.ZonedDateTime +import java.time.{ZoneId, ZonedDateTime} import java.time.format.DateTimeFormatter import scala.collection.JavaConverters import scala.collection.JavaConverters._ @@ -72,16 +72,24 @@ class ContentActor @Inject() (implicit oec: OntologyEngineContext, ss: StorageSe val endDateTimeStr = request.getRequest.getOrDefault("endDateTime", "").asInstanceOf[String] if (StringUtils.isNotBlank(startDateTimeStr) && StringUtils.isNotBlank(endDateTimeStr)) { try { - // Convert startDateTime string to epoch milliseconds - val formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSSXX") - val startDateTime = ZonedDateTime.parse(startDateTimeStr, formatter) - val startDateTimeEpochMillis = startDateTime.toInstant.toEpochMilli - request.getRequest.put("startDateTime", startDateTimeStr) + val inputUtcFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSSXX") + val outputIstFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSSZ") + val istZoneId = ZoneId.of("Asia/Kolkata") + + val startDateTimeUtc = ZonedDateTime.parse(startDateTimeStr, inputUtcFormatter) + val startDateTimeIst = startDateTimeUtc.withZoneSameInstant(istZoneId) + val formattedStartDateTimeIst = startDateTimeIst.format(outputIstFormatter) + val startDateTimeEpochMillis = startDateTimeIst.toInstant.toEpochMilli + + request.getRequest.put("startDateTime", formattedStartDateTimeIst) request.getRequest.put("startDateTimeInEpoch", startDateTimeEpochMillis.asInstanceOf[java.lang.Long]) - val endDateTime = ZonedDateTime.parse(endDateTimeStr, formatter) - val endDateTimeEpochMillis = endDateTime.toInstant.toEpochMilli - request.getRequest.put("endDateTime", endDateTimeStr) + val endDateTimeUtc = ZonedDateTime.parse(startDateTimeStr, inputUtcFormatter) + val endDateTimeIst = endDateTimeUtc.withZoneSameInstant(istZoneId) + val formattedEndDateTimeIst = endDateTimeIst.format(outputIstFormatter) + val endDateTimeEpochMillis = startDateTimeIst.toInstant.toEpochMilli + + request.getRequest.put("endDateTime", formattedEndDateTimeIst) request.getRequest.put("endDateTimeInEpoch", endDateTimeEpochMillis.asInstanceOf[java.lang.Long]) } catch { case ex: Exception => @@ -166,15 +174,24 @@ class ContentActor @Inject() (implicit oec: OntologyEngineContext, ss: StorageSe val endDateTimeStr = request.getRequest.getOrDefault("endDateTime", "").asInstanceOf[String] if (StringUtils.isNotBlank(startDateTimeStr) && StringUtils.isNotBlank(endDateTimeStr)) { try { - // Convert startDateTime string to epoch milliseconds - val formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSSXX") - val startDateTime = ZonedDateTime.parse(startDateTimeStr, formatter) - val startDateTimeEpochMillis = startDateTime.toInstant.toEpochMilli - request.getRequest.put("startDateTime", startDateTimeStr) + val inputUtcFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSSXX") + val outputIstFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSSZ") + val istZoneId = ZoneId.of("Asia/Kolkata") + + val startDateTimeUtc = ZonedDateTime.parse(startDateTimeStr, inputUtcFormatter) + val startDateTimeIst = startDateTimeUtc.withZoneSameInstant(istZoneId) + val formattedStartDateTimeIst = startDateTimeIst.format(outputIstFormatter) + val startDateTimeEpochMillis = startDateTimeIst.toInstant.toEpochMilli + + request.getRequest.put("startDateTime", formattedStartDateTimeIst) request.getRequest.put("startDateTimeInEpoch", startDateTimeEpochMillis.asInstanceOf[java.lang.Long]) - val endDateTime = ZonedDateTime.parse(endDateTimeStr, formatter) - val endDateTimeEpochMillis = endDateTime.toInstant.toEpochMilli - request.getRequest.put("endDateTime", endDateTimeStr) + + val endDateTimeUtc = ZonedDateTime.parse(startDateTimeStr, inputUtcFormatter) + val endDateTimeIst = endDateTimeUtc.withZoneSameInstant(istZoneId) + val formattedEndDateTimeIst = endDateTimeIst.format(outputIstFormatter) + val endDateTimeEpochMillis = startDateTimeIst.toInstant.toEpochMilli + + request.getRequest.put("endDateTime", formattedEndDateTimeIst) request.getRequest.put("endDateTimeInEpoch", endDateTimeEpochMillis.asInstanceOf[java.lang.Long]) } catch { case ex: Exception => From 8515d09cfcffd671bbda8a110d36ae2bddf9a14d Mon Sep 17 00:00:00 2001 From: arpithasureshappa Date: Tue, 8 Apr 2025 22:35:26 +0530 Subject: [PATCH 090/126] range filter issue fixed --- .../search/client/ElasticSearchUtil.java | 5 ++ .../search/processor/SearchProcessor.java | 48 +++++++++---------- .../search-service/conf/application.conf | 4 +- 3 files changed, 30 insertions(+), 27 deletions(-) diff --git a/search-api/search-core/src/main/java/org/sunbird/search/client/ElasticSearchUtil.java b/search-api/search-core/src/main/java/org/sunbird/search/client/ElasticSearchUtil.java index a25be9575..6b3283d75 100644 --- a/search-api/search-core/src/main/java/org/sunbird/search/client/ElasticSearchUtil.java +++ b/search-api/search-core/src/main/java/org/sunbird/search/client/ElasticSearchUtil.java @@ -136,6 +136,11 @@ public static List getQuerySearchFields() { return querySearchFields; } + public static List getNonTextFields() { + List nonTextFields = Platform.config.getStringList("non.text.fields"); + return nonTextFields; + } + public List getDateFields() { List dateFields = Platform.config.getStringList("search.fields.date"); return dateFields; diff --git a/search-api/search-core/src/main/java/org/sunbird/search/processor/SearchProcessor.java b/search-api/search-core/src/main/java/org/sunbird/search/processor/SearchProcessor.java index 69019e4fe..d68f4f46f 100644 --- a/search-api/search-core/src/main/java/org/sunbird/search/processor/SearchProcessor.java +++ b/search-api/search-core/src/main/java/org/sunbird/search/processor/SearchProcessor.java @@ -410,9 +410,10 @@ private void formQueryImpl(List properties, QueryBuilder queryBuilder, Bool boolQuery.must(queryBuilder); continue; } - - propertyName = propertyName + SearchConstants.RAW_FIELD_EXTENSION; - + List nonTextFields = ElasticSearchUtil.getNonTextFields(); + if (!nonTextFields.contains(propertyName)) { + propertyName = propertyName + SearchConstants.RAW_FIELD_EXTENSION; + } switch (opertation) { case SearchConstants.SEARCH_OPERATION_EQUAL: { if (MapUtils.isNotEmpty(valuesMap)) { @@ -626,33 +627,28 @@ private QueryBuilder getSoftConstraintQuery(Map softConstraints) * @return */ private QueryBuilder getRangeQuery(String propertyName, List values, String operation) { - BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery(); + RangeQueryBuilder rangeQuery = QueryBuilders.rangeQuery(propertyName); for (Object value : values) { switch (operation) { - case SearchConstants.SEARCH_OPERATION_GREATER_THAN: { - queryBuilder.should(QueryBuilders - .rangeQuery(propertyName).gt(value)); - break; - } - case SearchConstants.SEARCH_OPERATION_GREATER_THAN_EQUALS: { - queryBuilder.should(QueryBuilders - .rangeQuery(propertyName).gte(value)); - break; - } - case SearchConstants.SEARCH_OPERATION_LESS_THAN: { - queryBuilder.should(QueryBuilders - .rangeQuery(propertyName).lt(value)); - break; - } - case SearchConstants.SEARCH_OPERATION_LESS_THAN_EQUALS: { - queryBuilder.should(QueryBuilders - .rangeQuery(propertyName).lte(value)); - break; - } + case SearchConstants.SEARCH_OPERATION_GREATER_THAN: { + rangeQuery.gt(value); + break; + } + case SearchConstants.SEARCH_OPERATION_GREATER_THAN_EQUALS: { + rangeQuery.gte(value); + break; + } + case SearchConstants.SEARCH_OPERATION_LESS_THAN: { + rangeQuery.lt(value); + break; + } + case SearchConstants.SEARCH_OPERATION_LESS_THAN_EQUALS: { + rangeQuery.lte(value); + break; + } } } - - return queryBuilder; + return rangeQuery; } /** diff --git a/search-api/search-service/conf/application.conf b/search-api/search-service/conf/application.conf index b39c0e2b5..3dd0e8627 100644 --- a/search-api/search-service/conf/application.conf +++ b/search-api/search-service/conf/application.conf @@ -318,4 +318,6 @@ search.payload.log_enable=true search.fields.enable.fuzzy.when.noresult=false #Following configuration would enable the secureSettings search -search.fields.enable.secureSettings=false \ No newline at end of file +search.fields.enable.secureSettings=false + +non.text.fields=["startDateTimeInEpoch","endDateTimeInEpoch"] \ No newline at end of file From cf6474831b6d1a8e1c23f501be1a44ce4ca4f4fc Mon Sep 17 00:00:00 2001 From: arpithasureshappa Date: Wed, 9 Apr 2025 15:12:19 +0530 Subject: [PATCH 091/126] range filter issue fixed --- .../scala/org/sunbird/content/actors/ContentActor.scala | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala b/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala index 2ba64edfb..e5cb762b7 100644 --- a/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala +++ b/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala @@ -84,10 +84,10 @@ class ContentActor @Inject() (implicit oec: OntologyEngineContext, ss: StorageSe request.getRequest.put("startDateTime", formattedStartDateTimeIst) request.getRequest.put("startDateTimeInEpoch", startDateTimeEpochMillis.asInstanceOf[java.lang.Long]) - val endDateTimeUtc = ZonedDateTime.parse(startDateTimeStr, inputUtcFormatter) + val endDateTimeUtc = ZonedDateTime.parse(endDateTimeStr, inputUtcFormatter) val endDateTimeIst = endDateTimeUtc.withZoneSameInstant(istZoneId) val formattedEndDateTimeIst = endDateTimeIst.format(outputIstFormatter) - val endDateTimeEpochMillis = startDateTimeIst.toInstant.toEpochMilli + val endDateTimeEpochMillis = endDateTimeIst.toInstant.toEpochMilli request.getRequest.put("endDateTime", formattedEndDateTimeIst) request.getRequest.put("endDateTimeInEpoch", endDateTimeEpochMillis.asInstanceOf[java.lang.Long]) @@ -186,10 +186,10 @@ class ContentActor @Inject() (implicit oec: OntologyEngineContext, ss: StorageSe request.getRequest.put("startDateTime", formattedStartDateTimeIst) request.getRequest.put("startDateTimeInEpoch", startDateTimeEpochMillis.asInstanceOf[java.lang.Long]) - val endDateTimeUtc = ZonedDateTime.parse(startDateTimeStr, inputUtcFormatter) + val endDateTimeUtc = ZonedDateTime.parse(endDateTimeStr, inputUtcFormatter) val endDateTimeIst = endDateTimeUtc.withZoneSameInstant(istZoneId) val formattedEndDateTimeIst = endDateTimeIst.format(outputIstFormatter) - val endDateTimeEpochMillis = startDateTimeIst.toInstant.toEpochMilli + val endDateTimeEpochMillis = endDateTimeIst.toInstant.toEpochMilli request.getRequest.put("endDateTime", formattedEndDateTimeIst) request.getRequest.put("endDateTimeInEpoch", endDateTimeEpochMillis.asInstanceOf[java.lang.Long]) From 2137eb9a7675b40b7d9fa0e5a538103e9259d660 Mon Sep 17 00:00:00 2001 From: arpithasureshappa Date: Wed, 9 Apr 2025 15:39:36 +0530 Subject: [PATCH 092/126] fixed facet issue for non text fields --- .../sunbird/search/processor/SearchProcessor.java | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/search-api/search-core/src/main/java/org/sunbird/search/processor/SearchProcessor.java b/search-api/search-core/src/main/java/org/sunbird/search/processor/SearchProcessor.java index d68f4f46f..1c1eece78 100644 --- a/search-api/search-core/src/main/java/org/sunbird/search/processor/SearchProcessor.java +++ b/search-api/search-core/src/main/java/org/sunbird/search/processor/SearchProcessor.java @@ -285,14 +285,21 @@ private SearchSourceBuilder processSearchQuery(SearchDTO searchDTO, List> groupByList, SearchSourceBuilder searchSourceBuilder) { TermsAggregationBuilder termBuilder = null; + List nonTextFields = ElasticSearchUtil.getNonTextFields(); if (groupByList != null && !groupByList.isEmpty()) { HashMap> nestedAggregation = new HashMap<>(); for (Map groupByMap : groupByList) { String groupByParent = (String) groupByMap.get("groupByParent"); if (!groupByParent.contains(".")) { - termBuilder = AggregationBuilders.terms(groupByParent) - .field(groupByParent + SearchConstants.RAW_FIELD_EXTENSION) - .size(ElasticSearchUtil.defaultResultLimit); + if (nonTextFields.contains(groupByParent)) { + termBuilder = AggregationBuilders.terms(groupByParent) + .field(groupByParent) + .size(ElasticSearchUtil.defaultResultLimit); + }else { + termBuilder = AggregationBuilders.terms(groupByParent) + .field(groupByParent + SearchConstants.RAW_FIELD_EXTENSION) + .size(ElasticSearchUtil.defaultResultLimit); + } List groupByChildList = (List) groupByMap.get("groupByChildList"); if (groupByChildList != null && !groupByChildList.isEmpty()) { for (String childGroupBy : groupByChildList) { From d6178bff756ae081454bd1d792e143db306fcfe5 Mon Sep 17 00:00:00 2001 From: tarentomaheshvakkund <139739142+tarentomaheshvakkund@users.noreply.github.com> Date: Fri, 11 Apr 2025 11:57:16 +0530 Subject: [PATCH 093/126] Event Update and Publish API enhanced 1. After the event is published we need to allow the user to edit and publish , so enhanced the API for that. --- .../sunbird/content/actors/EventActor.scala | 36 +++++++++++++++-- .../org/sunbird/graph/nodes/DataNode.scala | 39 +++++++++++++++++++ .../schema/validator/VersioningNode.scala | 25 +++++++----- 3 files changed, 87 insertions(+), 13 deletions(-) diff --git a/content-api/content-actors/src/main/scala/org/sunbird/content/actors/EventActor.scala b/content-api/content-actors/src/main/scala/org/sunbird/content/actors/EventActor.scala index 82ba1fc63..7b17d11ed 100644 --- a/content-api/content-actors/src/main/scala/org/sunbird/content/actors/EventActor.scala +++ b/content-api/content-actors/src/main/scala/org/sunbird/content/actors/EventActor.scala @@ -38,13 +38,43 @@ class EventActor @Inject()(implicit oec: OntologyEngineContext, ss: StorageServi } override def update(request: Request): Future[Response] = { - RedisCache.delete(request.get("identifier").asInstanceOf[String]) - verifyStandaloneEventAndApply(super.update, request) + populateDefaultersForUpdation(request) + val versionKey = request.getRequest.getOrDefault("versionKey", "").asInstanceOf[String] + if (StringUtils.isBlank(versionKey)) { + throw new ClientException("ERR_INVALID_REQUEST", "Please Provide Version Key!") + } + RequestUtil.restrictProperties(request) + val reviewStatus: String = request.getRequest.getOrDefault("reviewStatus", "").asInstanceOf[String] + if (reviewStatus == null || reviewStatus.isEmpty) { + request.getRequest.put("cqfVersion", System.currentTimeMillis().toString) + } + DataNode.update(request, dataModifier).map(node => { + val identifier: String = node.getIdentifier.replace(".img", "") + ResponseHandler.OK.put("node_id", identifier) + .put("identifier", identifier) + .put("versionKey", node.getMetadata.get("versionKey")) + }) } def publish(request: Request): Future[Response] = { TelemetryManager.log("EventActor::publish Identifier: " + request.getRequest.getOrDefault("identifier", "")) - verifyStandaloneEventAndApply(super.update, request, true) + val identifier = request.get("identifier").asInstanceOf[String] + val updatedIdentifier = if (!identifier.endsWith(".img")) s"$identifier.img" else identifier + request.put("identifier", updatedIdentifier) + // Check if the node exists + DataNode.read(request).flatMap { node => + // If the node exists, proceed with update and delete + DataNode.updatev2(request).flatMap { _ => + DataNode.delete(request) + request.put("identifier", updatedIdentifier.replace(".img", "")) + verifyStandaloneEventAndApply(super.update, request, true) + } + }.recoverWith { + case ex: Exception => + // If the node does not exist, directly call verifyStandaloneEventAndApply + request.put("identifier", updatedIdentifier.replace(".img", "")) + verifyStandaloneEventAndApply(super.update, request, true) + } } override def discard(request: Request): Future[Response] = { diff --git a/ontology-engine/graph-engine_2.11/src/main/scala/org/sunbird/graph/nodes/DataNode.scala b/ontology-engine/graph-engine_2.11/src/main/scala/org/sunbird/graph/nodes/DataNode.scala index 1e7b83178..03c4d7deb 100644 --- a/ontology-engine/graph-engine_2.11/src/main/scala/org/sunbird/graph/nodes/DataNode.scala +++ b/ontology-engine/graph-engine_2.11/src/main/scala/org/sunbird/graph/nodes/DataNode.scala @@ -323,4 +323,43 @@ object DataNode { }) } + def delete(request: Request)(implicit oec: OntologyEngineContext, ec: ExecutionContext): Future[Response] = { + val identifier = request.get("identifier").asInstanceOf[String] + val updatedIdentifier = if (!identifier.endsWith(".img")) s"$identifier.img" else identifier + request.put("identifier", updatedIdentifier) + if (StringUtils.isBlank(identifier)) { + throw new ClientException("ERR_INVALID_REQUEST", "Identifier is required for deletion") + } + // Perform the deletion operation + oec.graphService.deleteNode(request.graphId, identifier, request).map { _ => + val response = new Response + response.put("message", "Node deleted successfully") + response + }.recover { + case ex: Exception => + throw new ClientException("ERR_NODE_DELETION_FAILED", s"Failed to delete node with identifier: $identifier", ex) + } + } + + def updatev2(request: Request, dataModifier: (Node) => Node = defaultDataModifier)(implicit oec: OntologyEngineContext, ec: ExecutionContext): Future[Response] = { + val identifier = request.get("identifier").asInstanceOf[String] + if (StringUtils.isBlank(identifier)) { + throw new ClientException("ERR_INVALID_REQUEST", "Identifier is required for update") + } + // Read the node to be updated + DataNode.read(request).flatMap { node => + val updatedNode = dataModifier(node) // Apply the data modifier to update the node + updatedNode.setIdentifier(identifier.replace(".img","")) + oec.graphService.upsertNode(request.graphId, updatedNode, request).map { updatedNode => + val response = new Response + response.put("message", "Node updated successfully") + response.put("identifier", updatedNode.getIdentifier) + response + } + }.recover { + case ex: Exception => + throw new ClientException("ERR_NODE_UPDATE_FAILED", s"Failed to update node with identifier: $identifier", ex) + } + } + } diff --git a/ontology-engine/graph-engine_2.11/src/main/scala/org/sunbird/graph/schema/validator/VersioningNode.scala b/ontology-engine/graph-engine_2.11/src/main/scala/org/sunbird/graph/schema/validator/VersioningNode.scala index 36190dca2..f35f4e0ae 100644 --- a/ontology-engine/graph-engine_2.11/src/main/scala/org/sunbird/graph/schema/validator/VersioningNode.scala +++ b/ontology-engine/graph-engine_2.11/src/main/scala/org/sunbird/graph/schema/validator/VersioningNode.scala @@ -85,17 +85,22 @@ trait VersioningNode extends IDefinition { node.getMetadata.put("status", "Draft") node.getMetadata.put("prevStatus", status) node.getMetadata.put(AuditProperties.lastStatusChangedOn.name, DateUtils.formatCurrentDate()) - oec.graphService.addNode(node.getGraphId, node).map(imgNode => { - imgNode.getMetadata.put("isImageNodeCreated", "yes"); - copyExternalProps(identifier, node.getGraphId, imgNode.getObjectType.toLowerCase().replace("image", "")).map(response => { - if(!ResponseHandler.checkError(response)) { - if(null != response.getResult && !response.getResult.isEmpty) - imgNode.setExternalData(response.getResult) + oec.graphService.addNode(node.getGraphId, node).map { imgNode => + imgNode.getMetadata.put("isImageNodeCreated", "yes") + val category = node.getMetadata.get("category").asInstanceOf[String] + if (!"event".equalsIgnoreCase(category)) { + copyExternalProps(identifier, node.getGraphId, imgNode.getObjectType.toLowerCase().replace("image", "")).map { response => + if (!ResponseHandler.checkError(response)) { + if (null != response.getResult && !response.getResult.isEmpty) + imgNode.setExternalData(response.getResult) + } + imgNode } - imgNode - }) - }).flatMap(f=>f) - } else + } else { + Future.successful(imgNode) + } + }.flatMap(f => f) + }else throw e.getCause } } From e51f6634034e60b75d6fd9ee58dce59de3e580c2 Mon Sep 17 00:00:00 2001 From: tarentomaheshvakkund <139739142+tarentomaheshvakkund@users.noreply.github.com> Date: Mon, 14 Apr 2025 18:51:00 +0530 Subject: [PATCH 094/126] Event Publish API enhanced --- .../src/main/scala/org/sunbird/content/actors/EventActor.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content-api/content-actors/src/main/scala/org/sunbird/content/actors/EventActor.scala b/content-api/content-actors/src/main/scala/org/sunbird/content/actors/EventActor.scala index 7b17d11ed..0b3e69ca3 100644 --- a/content-api/content-actors/src/main/scala/org/sunbird/content/actors/EventActor.scala +++ b/content-api/content-actors/src/main/scala/org/sunbird/content/actors/EventActor.scala @@ -65,7 +65,6 @@ class EventActor @Inject()(implicit oec: OntologyEngineContext, ss: StorageServi DataNode.read(request).flatMap { node => // If the node exists, proceed with update and delete DataNode.updatev2(request).flatMap { _ => - DataNode.delete(request) request.put("identifier", updatedIdentifier.replace(".img", "")) verifyStandaloneEventAndApply(super.update, request, true) } @@ -75,6 +74,7 @@ class EventActor @Inject()(implicit oec: OntologyEngineContext, ss: StorageServi request.put("identifier", updatedIdentifier.replace(".img", "")) verifyStandaloneEventAndApply(super.update, request, true) } + DataNode.delete(request) } override def discard(request: Request): Future[Response] = { From 0ce001902fbae128f5f78155cada49ade5a216cd Mon Sep 17 00:00:00 2001 From: tarentomaheshvakkund <139739142+tarentomaheshvakkund@users.noreply.github.com> Date: Mon, 14 Apr 2025 19:28:14 +0530 Subject: [PATCH 095/126] Handled deleting the event --- .../main/scala/org/sunbird/content/actors/EventActor.scala | 5 ++++- schemas/event/1.0/config.json | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/content-api/content-actors/src/main/scala/org/sunbird/content/actors/EventActor.scala b/content-api/content-actors/src/main/scala/org/sunbird/content/actors/EventActor.scala index 0b3e69ca3..6ace9c534 100644 --- a/content-api/content-actors/src/main/scala/org/sunbird/content/actors/EventActor.scala +++ b/content-api/content-actors/src/main/scala/org/sunbird/content/actors/EventActor.scala @@ -74,7 +74,10 @@ class EventActor @Inject()(implicit oec: OntologyEngineContext, ss: StorageServi request.put("identifier", updatedIdentifier.replace(".img", "")) verifyStandaloneEventAndApply(super.update, request, true) } - DataNode.delete(request) + DataNode.delete(request).recoverWith { + case ex: Exception => + Future.successful(ResponseHandler.OK) + } } override def discard(request: Request): Future[Response] = { diff --git a/schemas/event/1.0/config.json b/schemas/event/1.0/config.json index 52bdf3ba8..04b1fc8c3 100644 --- a/schemas/event/1.0/config.json +++ b/schemas/event/1.0/config.json @@ -55,7 +55,7 @@ "targetMediumIds", "targetTopicIds" ], - "version": "disable", + "version": "enable", "versionCheckMode": "OFF", "cacheEnabled": false, "schema_restrict_api": false From d3fb757280d9fb25928eb2544c948cdacf2c0166 Mon Sep 17 00:00:00 2001 From: tarentomaheshvakkund <139739142+tarentomaheshvakkund@users.noreply.github.com> Date: Tue, 15 Apr 2025 17:23:28 +0530 Subject: [PATCH 096/126] System Update to update the data to the main node when the event is live --- .../sunbird/content/actors/EventActor.scala | 30 +++++++++++++++++++ .../app/controllers/v4/EventController.scala | 11 +++++++ content-api/content-service/conf/routes | 1 + 3 files changed, 42 insertions(+) diff --git a/content-api/content-actors/src/main/scala/org/sunbird/content/actors/EventActor.scala b/content-api/content-actors/src/main/scala/org/sunbird/content/actors/EventActor.scala index 6ace9c534..ee054ab55 100644 --- a/content-api/content-actors/src/main/scala/org/sunbird/content/actors/EventActor.scala +++ b/content-api/content-actors/src/main/scala/org/sunbird/content/actors/EventActor.scala @@ -33,6 +33,7 @@ class EventActor @Inject()(implicit oec: OntologyEngineContext, ss: StorageServi case "discardContent" => discard(request) case "publishContent" => publish(request) case "rejectEvent" => rejectEvent(request) + case "systemUpdate" => systemUpdate(request) case _ => ERROR(request.getOperation) } } @@ -174,4 +175,33 @@ class EventActor @Inject()(implicit oec: OntologyEngineContext, ss: StorageServi }) }).flatMap(f => f) } + + override def systemUpdate(request: Request): Future[Response] = { + RedisCache.delete(request.get("identifier").asInstanceOf[String]) + val identifier = request.get("identifier").asInstanceOf[String] + val updatedIdentifier = if (!identifier.endsWith(".img")) s"$identifier.img" else identifier + DataNode.read(request).flatMap { node => + // Extract attributes to be updated from the request body + val attributesToUpdate = request.getRequest.asInstanceOf[java.util.Map[String, AnyRef]] + + // Append attributes from the request to the node's metadata + attributesToUpdate.forEach(new java.util.function.BiConsumer[String, AnyRef] { + override def accept(key: String, value: AnyRef): Unit = { + node.getMetadata.put(key, value) + } + }) + // Save the updated node + DataNode.updatev2(request, _ => node).recover { + case ex: Exception => + TelemetryManager.error("Error occurred during updatev2 operation", ex) + ResponseHandler.ERROR(ResponseCode.SERVER_ERROR, "ERR_UPDATE_FAILED", ex.getMessage) + }.map(response => { + if (response.getResponseCode == ResponseCode.OK) { + ResponseHandler.OK + } else { + response + } + }) + } + } } \ No newline at end of file diff --git a/content-api/content-service/app/controllers/v4/EventController.scala b/content-api/content-service/app/controllers/v4/EventController.scala index 6bf421414..93d215cd2 100644 --- a/content-api/content-service/app/controllers/v4/EventController.scala +++ b/content-api/content-service/app/controllers/v4/EventController.scala @@ -88,4 +88,15 @@ class EventController @Inject()(@Named(ActorNames.EVENT_ACTOR) eventActor: Actor contentRequest.getContext.put("identifier", identifier); getResult(ApiId.REJECT_EVENT, eventActor, contentRequest, version = apiVersion) } + + override def systemUpdate(identifier: String) = Action.async { implicit request => + val headers = commonHeaders() + val body = requestBody() + val content = body.getOrDefault(schemaName, new java.util.HashMap()).asInstanceOf[java.util.Map[String, Object]]; + content.putAll(headers) + val contentRequest = getRequest(content, headers, "systemUpdate") + setRequestContext(contentRequest, version, objectType, schemaName) + contentRequest.getContext.put("identifier", identifier); + getResult(ApiId.SYSTEM_UPDATE_CONTENT, eventActor, contentRequest, version = apiVersion) + } } \ No newline at end of file diff --git a/content-api/content-service/conf/routes b/content-api/content-service/conf/routes index 93f378b69..cce91ce12 100644 --- a/content-api/content-service/conf/routes +++ b/content-api/content-service/conf/routes @@ -114,6 +114,7 @@ GET /event/v4/read/:identifier controllers.v4.EventControll DELETE /event/v4/discard/:identifier controllers.v4.EventController.discard(identifier:String) DELETE /private/event/v4/retire/:identifier controllers.v4.EventController.retire(identifier:String) POST /event/v4/reject/:identifier controllers.v4.EventController.reviewReject(identifier:String) +PATCH /event/v4/system/update/:identifier controllers.v4.EventController.systemUpdate(identifier:String) # EventSet v4 Api's POST /eventset/v4/create controllers.v4.EventSetController.create From 4fde30ee4c168f3f9d95a8d24677ffbf1b5d9fec Mon Sep 17 00:00:00 2001 From: tarentomaheshvakkund <139739142+tarentomaheshvakkund@users.noreply.github.com> Date: Wed, 16 Apr 2025 12:12:33 +0530 Subject: [PATCH 097/126] Flag added to execute for replacing the identifier with image extension. --- .../main/scala/org/sunbird/content/actors/EventActor.scala | 4 ++-- .../src/main/scala/org/sunbird/graph/nodes/DataNode.scala | 6 ++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/content-api/content-actors/src/main/scala/org/sunbird/content/actors/EventActor.scala b/content-api/content-actors/src/main/scala/org/sunbird/content/actors/EventActor.scala index ee054ab55..816e145a2 100644 --- a/content-api/content-actors/src/main/scala/org/sunbird/content/actors/EventActor.scala +++ b/content-api/content-actors/src/main/scala/org/sunbird/content/actors/EventActor.scala @@ -65,7 +65,7 @@ class EventActor @Inject()(implicit oec: OntologyEngineContext, ss: StorageServi // Check if the node exists DataNode.read(request).flatMap { node => // If the node exists, proceed with update and delete - DataNode.updatev2(request).flatMap { _ => + DataNode.updatev2(request, flag = true).flatMap { _ => request.put("identifier", updatedIdentifier.replace(".img", "")) verifyStandaloneEventAndApply(super.update, request, true) } @@ -191,7 +191,7 @@ class EventActor @Inject()(implicit oec: OntologyEngineContext, ss: StorageServi } }) // Save the updated node - DataNode.updatev2(request, _ => node).recover { + DataNode.updatev2(request, _ => node, flag = false).recover { case ex: Exception => TelemetryManager.error("Error occurred during updatev2 operation", ex) ResponseHandler.ERROR(ResponseCode.SERVER_ERROR, "ERR_UPDATE_FAILED", ex.getMessage) diff --git a/ontology-engine/graph-engine_2.11/src/main/scala/org/sunbird/graph/nodes/DataNode.scala b/ontology-engine/graph-engine_2.11/src/main/scala/org/sunbird/graph/nodes/DataNode.scala index 03c4d7deb..61ca93352 100644 --- a/ontology-engine/graph-engine_2.11/src/main/scala/org/sunbird/graph/nodes/DataNode.scala +++ b/ontology-engine/graph-engine_2.11/src/main/scala/org/sunbird/graph/nodes/DataNode.scala @@ -341,7 +341,7 @@ object DataNode { } } - def updatev2(request: Request, dataModifier: (Node) => Node = defaultDataModifier)(implicit oec: OntologyEngineContext, ec: ExecutionContext): Future[Response] = { + def updatev2(request: Request, dataModifier: (Node) => Node = defaultDataModifier, flag: Boolean)(implicit oec: OntologyEngineContext, ec: ExecutionContext): Future[Response] = { val identifier = request.get("identifier").asInstanceOf[String] if (StringUtils.isBlank(identifier)) { throw new ClientException("ERR_INVALID_REQUEST", "Identifier is required for update") @@ -349,7 +349,9 @@ object DataNode { // Read the node to be updated DataNode.read(request).flatMap { node => val updatedNode = dataModifier(node) // Apply the data modifier to update the node - updatedNode.setIdentifier(identifier.replace(".img","")) + if(flag) { + updatedNode.setIdentifier(identifier.replace(".img","")) + } oec.graphService.upsertNode(request.graphId, updatedNode, request).map { updatedNode => val response = new Response response.put("message", "Node updated successfully") From ea815087f50859719249b3700f2cfc812e9dbac9 Mon Sep 17 00:00:00 2001 From: tarentomaheshvakkund <139739142+tarentomaheshvakkund@users.noreply.github.com> Date: Tue, 22 Apr 2025 09:19:09 +0530 Subject: [PATCH 098/126] Handled Deleting the node.img 1. If we are deleting the datanode outside it is considering the response structrue provided by the DateNode.delete "Node deleted successfully" due to which getting error in the getResult->setResponseEnvelope method saying null pointer exception. 2. So added the delete method inside the updateV2 and overriden the response of delete with verifyStandaloneEventAndApply method response. --- .../main/scala/org/sunbird/content/actors/EventActor.scala | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/content-api/content-actors/src/main/scala/org/sunbird/content/actors/EventActor.scala b/content-api/content-actors/src/main/scala/org/sunbird/content/actors/EventActor.scala index 816e145a2..183c301cd 100644 --- a/content-api/content-actors/src/main/scala/org/sunbird/content/actors/EventActor.scala +++ b/content-api/content-actors/src/main/scala/org/sunbird/content/actors/EventActor.scala @@ -66,6 +66,7 @@ class EventActor @Inject()(implicit oec: OntologyEngineContext, ss: StorageServi DataNode.read(request).flatMap { node => // If the node exists, proceed with update and delete DataNode.updatev2(request, flag = true).flatMap { _ => + DataNode.delete(request) request.put("identifier", updatedIdentifier.replace(".img", "")) verifyStandaloneEventAndApply(super.update, request, true) } @@ -75,10 +76,6 @@ class EventActor @Inject()(implicit oec: OntologyEngineContext, ss: StorageServi request.put("identifier", updatedIdentifier.replace(".img", "")) verifyStandaloneEventAndApply(super.update, request, true) } - DataNode.delete(request).recoverWith { - case ex: Exception => - Future.successful(ResponseHandler.OK) - } } override def discard(request: Request): Future[Response] = { From 67f7b61761f62083fc71963703c42f8a6d878943 Mon Sep 17 00:00:00 2001 From: tarentomaheshvakkund <139739142+tarentomaheshvakkund@users.noreply.github.com> Date: Fri, 9 May 2025 16:20:35 +0530 Subject: [PATCH 099/126] KB-7793 | Event details edited by SPV admin are not being reflected in SPV portal 1. The change for the updated event is not reflecting after editing and publishing the event the cache is not having the updated details . 2. So have introduced the delete cache here. 3. While publishing the event it will update the cache with latest value. --- .../scala/org/sunbird/content/actors/EventActor.scala | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/content-api/content-actors/src/main/scala/org/sunbird/content/actors/EventActor.scala b/content-api/content-actors/src/main/scala/org/sunbird/content/actors/EventActor.scala index 183c301cd..040d45f57 100644 --- a/content-api/content-actors/src/main/scala/org/sunbird/content/actors/EventActor.scala +++ b/content-api/content-actors/src/main/scala/org/sunbird/content/actors/EventActor.scala @@ -1,6 +1,7 @@ package org.sunbird.content.actors import org.apache.commons.lang.StringUtils +import org.slf4j.{Logger, LoggerFactory} import org.sunbird.cache.impl.RedisCache import org.sunbird.common.Platform import org.sunbird.cloudstore.StorageService @@ -23,7 +24,7 @@ import scala.collection.JavaConverters import scala.collection.JavaConverters._ class EventActor @Inject()(implicit oec: OntologyEngineContext, ss: StorageService) extends ContentActor { - + private val logger: Logger = LoggerFactory.getLogger("EventActor") override def onReceive(request: Request): Future[Response] = { request.getOperation match { case "createContent" => create(request) @@ -67,6 +68,12 @@ class EventActor @Inject()(implicit oec: OntologyEngineContext, ss: StorageServi // If the node exists, proceed with update and delete DataNode.updatev2(request, flag = true).flatMap { _ => DataNode.delete(request) + try { + RedisCache.delete(identifier) + } catch { + case e: Exception => + logger.error(s"Error deleting Redis cache entry for identifier: $identifier", e) + } request.put("identifier", updatedIdentifier.replace(".img", "")) verifyStandaloneEventAndApply(super.update, request, true) } From ec743f0b5de9471ece7becd72ff31bab5bb90dd6 Mon Sep 17 00:00:00 2001 From: DeepikaMalav28 Date: Thu, 12 Jun 2025 11:33:19 +0530 Subject: [PATCH 100/126] initial change for notification --- .../sunbird/content/actors/ContentActor.scala | 14 ++++-- .../content/util/NotificationManager.scala | 43 +++++++++++++++++++ .../content-service/conf/application.conf | 1 + 3 files changed, 55 insertions(+), 3 deletions(-) create mode 100644 content-api/content-actors/src/main/scala/org/sunbird/content/util/NotificationManager.scala diff --git a/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala b/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala index e5cb762b7..f6a8ee988 100644 --- a/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala +++ b/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala @@ -13,7 +13,7 @@ import org.slf4j.{Logger, LoggerFactory} import org.sunbird.`object`.importer.{ImportConfig, ImportManager} import org.sunbird.actor.core.BaseActor import org.sunbird.cache.impl.RedisCache -import org.sunbird.content.util.{AcceptFlagManager, ContentConstants, CopyManager, DiscardManager, FlagManager, RetireManager} +import org.sunbird.content.util.{AcceptFlagManager, ContentConstants, CopyManager, DiscardManager, FlagManager, NotificationManager, RetireManager} import org.sunbird.cloudstore.StorageService import org.sunbird.common.{ContentParams, JsonUtils, Platform, Slug} import org.sunbird.common.dto.{Request, Response, ResponseHandler} @@ -31,7 +31,7 @@ import org.sunbird.managers.HierarchyManager.hierarchyPrefix import java.time.{ZoneId, ZonedDateTime} import java.time.format.DateTimeFormatter -import scala.collection.JavaConverters +import scala.collection.{JavaConverters, Map} import scala.collection.JavaConverters._ import scala.concurrent.{ExecutionContext, Future} @@ -279,7 +279,15 @@ class ContentActor @Inject() (implicit oec: OntologyEngineContext, ss: StorageSe request.getContext.put("schemaName", node.getObjectType.toLowerCase()) if (StringUtils.equalsAnyIgnoreCase("Processing", node.getMetadata.getOrDefault("status", "").asInstanceOf[String])) throw new ClientException("ERR_NODE_ACCESS_DENIED", "Review Operation Can't Be Applied On Node Under Processing State") - else ReviewManager.review(request, node) + else { + val response = ReviewManager.review(request, node) + try { + NotificationManager.sendNotification("CONTENT_REVIEW_REQUEST", "ALERT", List(""), "", Map.empty) + } catch { + case e: Exception => println("Error sending notification :" + e) + } + response + } }).flatMap(f => f) } diff --git a/content-api/content-actors/src/main/scala/org/sunbird/content/util/NotificationManager.scala b/content-api/content-actors/src/main/scala/org/sunbird/content/util/NotificationManager.scala new file mode 100644 index 000000000..dc08a771b --- /dev/null +++ b/content-api/content-actors/src/main/scala/org/sunbird/content/util/NotificationManager.scala @@ -0,0 +1,43 @@ +package org.sunbird.content.util + +import com.mashape.unirest.http.Unirest +import org.sunbird.common.Platform +import org.sunbird.util.{HTTPResponse, HttpUtil} +import scala.collection.JavaConverters._ + +object NotificationManager { + + def sendNotification(subCategory: String, subType: String, userIds: List[String], title: String, data: Map[String, Any]): Unit = { + + val userIdsJson = userIds.map(id => s""""$id"""").mkString("[", ",", "]") + val dataJson = toJsonString(data) + + val body = + s""" + { + "request": { + "subCategory": "$subCategory", + "subType": "$subType", + "userIds": $userIdsJson, + "title": "$title", + "data": $dataJson + } + } + """ + + val url: String = Platform.getString("notification.api.url", "http://cb-notification-wrapper-service:8081/notifications/create") + val response = Unirest.post(url).headers(Map[String, String]("Content-Type"->"application/json").asJava).body(body).asString() + HTTPResponse(response.getStatus, response.getBody) + } + + private def toJsonString(map: Map[String, Any]): String = { + map.map { + case (k, v: String) => s""""$k":"$v"""" + case (k, v: Int) => s""""$k":$v""" + case (k, v: Boolean) => s""""$k":$v""" + case (k, v: Double) => s""""$k":$v""" + case (k, v: List[_]) => s""""$k":[${v.map(x => s""""$x"""").mkString(",")}]""" + case (k, v) => s""""$k":"${v.toString}"""" + }.mkString("{", ",", "}") + } +} \ No newline at end of file diff --git a/content-api/content-service/conf/application.conf b/content-api/content-service/conf/application.conf index fa3214e9f..2cfa3342c 100644 --- a/content-api/content-service/conf/application.conf +++ b/content-api/content-service/conf/application.conf @@ -588,6 +588,7 @@ collection.image.migration.enabled=true cloud_storage.upload.url.ttl=600 composite.search.url="https://dev.sunbirded.org/action/composite/v3/search" +notification.api.url="http://cb-notification-wrapper-service:8081/notifications/create" # Enable Suggested Framework in Get Channel API. channel.fetch.suggested_frameworks=true From 7c2c989335b89d9193fed28ed7fa9bb17086ef9c Mon Sep 17 00:00:00 2001 From: DeepikaMalav28 Date: Thu, 12 Jun 2025 13:48:34 +0530 Subject: [PATCH 101/126] contenct review notification --- .../main/scala/org/sunbird/content/actors/ContentActor.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala b/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala index f6a8ee988..8ef8fe6b5 100644 --- a/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala +++ b/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala @@ -282,7 +282,7 @@ class ContentActor @Inject() (implicit oec: OntologyEngineContext, ss: StorageSe else { val response = ReviewManager.review(request, node) try { - NotificationManager.sendNotification("CONTENT_REVIEW_REQUEST", "ALERT", List(""), "", Map.empty) + NotificationManager.sendNotification("CONTENT_REVIEW_REQUEST", "ALERT", List(node.getMetadata.get("createdBy").toString), node.getMetadata.get("name").toString, Map.empty) } catch { case e: Exception => println("Error sending notification :" + e) } From e7fc62d3139f3d3d1dd19c39098bb16e879d8ac5 Mon Sep 17 00:00:00 2001 From: DeepikaMalav28 Date: Thu, 12 Jun 2025 13:53:41 +0530 Subject: [PATCH 102/126] initial change for notification (#54) * initial change for notification * contenct review notification --- .../sunbird/content/actors/ContentActor.scala | 14 ++++-- .../content/util/NotificationManager.scala | 43 +++++++++++++++++++ .../content-service/conf/application.conf | 1 + 3 files changed, 55 insertions(+), 3 deletions(-) create mode 100644 content-api/content-actors/src/main/scala/org/sunbird/content/util/NotificationManager.scala diff --git a/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala b/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala index e5cb762b7..8ef8fe6b5 100644 --- a/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala +++ b/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala @@ -13,7 +13,7 @@ import org.slf4j.{Logger, LoggerFactory} import org.sunbird.`object`.importer.{ImportConfig, ImportManager} import org.sunbird.actor.core.BaseActor import org.sunbird.cache.impl.RedisCache -import org.sunbird.content.util.{AcceptFlagManager, ContentConstants, CopyManager, DiscardManager, FlagManager, RetireManager} +import org.sunbird.content.util.{AcceptFlagManager, ContentConstants, CopyManager, DiscardManager, FlagManager, NotificationManager, RetireManager} import org.sunbird.cloudstore.StorageService import org.sunbird.common.{ContentParams, JsonUtils, Platform, Slug} import org.sunbird.common.dto.{Request, Response, ResponseHandler} @@ -31,7 +31,7 @@ import org.sunbird.managers.HierarchyManager.hierarchyPrefix import java.time.{ZoneId, ZonedDateTime} import java.time.format.DateTimeFormatter -import scala.collection.JavaConverters +import scala.collection.{JavaConverters, Map} import scala.collection.JavaConverters._ import scala.concurrent.{ExecutionContext, Future} @@ -279,7 +279,15 @@ class ContentActor @Inject() (implicit oec: OntologyEngineContext, ss: StorageSe request.getContext.put("schemaName", node.getObjectType.toLowerCase()) if (StringUtils.equalsAnyIgnoreCase("Processing", node.getMetadata.getOrDefault("status", "").asInstanceOf[String])) throw new ClientException("ERR_NODE_ACCESS_DENIED", "Review Operation Can't Be Applied On Node Under Processing State") - else ReviewManager.review(request, node) + else { + val response = ReviewManager.review(request, node) + try { + NotificationManager.sendNotification("CONTENT_REVIEW_REQUEST", "ALERT", List(node.getMetadata.get("createdBy").toString), node.getMetadata.get("name").toString, Map.empty) + } catch { + case e: Exception => println("Error sending notification :" + e) + } + response + } }).flatMap(f => f) } diff --git a/content-api/content-actors/src/main/scala/org/sunbird/content/util/NotificationManager.scala b/content-api/content-actors/src/main/scala/org/sunbird/content/util/NotificationManager.scala new file mode 100644 index 000000000..dc08a771b --- /dev/null +++ b/content-api/content-actors/src/main/scala/org/sunbird/content/util/NotificationManager.scala @@ -0,0 +1,43 @@ +package org.sunbird.content.util + +import com.mashape.unirest.http.Unirest +import org.sunbird.common.Platform +import org.sunbird.util.{HTTPResponse, HttpUtil} +import scala.collection.JavaConverters._ + +object NotificationManager { + + def sendNotification(subCategory: String, subType: String, userIds: List[String], title: String, data: Map[String, Any]): Unit = { + + val userIdsJson = userIds.map(id => s""""$id"""").mkString("[", ",", "]") + val dataJson = toJsonString(data) + + val body = + s""" + { + "request": { + "subCategory": "$subCategory", + "subType": "$subType", + "userIds": $userIdsJson, + "title": "$title", + "data": $dataJson + } + } + """ + + val url: String = Platform.getString("notification.api.url", "http://cb-notification-wrapper-service:8081/notifications/create") + val response = Unirest.post(url).headers(Map[String, String]("Content-Type"->"application/json").asJava).body(body).asString() + HTTPResponse(response.getStatus, response.getBody) + } + + private def toJsonString(map: Map[String, Any]): String = { + map.map { + case (k, v: String) => s""""$k":"$v"""" + case (k, v: Int) => s""""$k":$v""" + case (k, v: Boolean) => s""""$k":$v""" + case (k, v: Double) => s""""$k":$v""" + case (k, v: List[_]) => s""""$k":[${v.map(x => s""""$x"""").mkString(",")}]""" + case (k, v) => s""""$k":"${v.toString}"""" + }.mkString("{", ",", "}") + } +} \ No newline at end of file diff --git a/content-api/content-service/conf/application.conf b/content-api/content-service/conf/application.conf index fa3214e9f..2cfa3342c 100644 --- a/content-api/content-service/conf/application.conf +++ b/content-api/content-service/conf/application.conf @@ -588,6 +588,7 @@ collection.image.migration.enabled=true cloud_storage.upload.url.ttl=600 composite.search.url="https://dev.sunbirded.org/action/composite/v3/search" +notification.api.url="http://cb-notification-wrapper-service:8081/notifications/create" # Enable Suggested Framework in Get Channel API. channel.fetch.suggested_frameworks=true From 847d0bb8998f0f4fc34ccf09f49af91b3280d7b0 Mon Sep 17 00:00:00 2001 From: DeepikaMalav28 Date: Thu, 12 Jun 2025 15:29:29 +0530 Subject: [PATCH 103/126] small fix for body change --- .../sunbird/content/util/NotificationManager.scala | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/content-api/content-actors/src/main/scala/org/sunbird/content/util/NotificationManager.scala b/content-api/content-actors/src/main/scala/org/sunbird/content/util/NotificationManager.scala index dc08a771b..82d0ff038 100644 --- a/content-api/content-actors/src/main/scala/org/sunbird/content/util/NotificationManager.scala +++ b/content-api/content-actors/src/main/scala/org/sunbird/content/util/NotificationManager.scala @@ -15,13 +15,11 @@ object NotificationManager { val body = s""" { - "request": { - "subCategory": "$subCategory", - "subType": "$subType", - "userIds": $userIdsJson, - "title": "$title", - "data": $dataJson - } + "subCategory": "$subCategory", + "subType": "$subType", + "userIds": $userIdsJson, + "title": "$title", + "data": $dataJson } """ From e51f31cffbb9a8f7458f877b07d171af7c7c9a6e Mon Sep 17 00:00:00 2001 From: DeepikaMalav28 Date: Thu, 12 Jun 2025 15:31:42 +0530 Subject: [PATCH 104/126] Dev 4.8.26 notification (#55) * initial change for notification * contenct review notification * small fix for body change --- .../sunbird/content/util/NotificationManager.scala | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/content-api/content-actors/src/main/scala/org/sunbird/content/util/NotificationManager.scala b/content-api/content-actors/src/main/scala/org/sunbird/content/util/NotificationManager.scala index dc08a771b..82d0ff038 100644 --- a/content-api/content-actors/src/main/scala/org/sunbird/content/util/NotificationManager.scala +++ b/content-api/content-actors/src/main/scala/org/sunbird/content/util/NotificationManager.scala @@ -15,13 +15,11 @@ object NotificationManager { val body = s""" { - "request": { - "subCategory": "$subCategory", - "subType": "$subType", - "userIds": $userIdsJson, - "title": "$title", - "data": $dataJson - } + "subCategory": "$subCategory", + "subType": "$subType", + "userIds": $userIdsJson, + "title": "$title", + "data": $dataJson } """ From aa04c3b08ec1aaa147d3338e3db96c421739e286 Mon Sep 17 00:00:00 2001 From: DeepikaMalav28 Date: Thu, 12 Jun 2025 15:55:29 +0530 Subject: [PATCH 105/126] more fields added to body --- .../org/sunbird/content/util/NotificationManager.scala | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/content-api/content-actors/src/main/scala/org/sunbird/content/util/NotificationManager.scala b/content-api/content-actors/src/main/scala/org/sunbird/content/util/NotificationManager.scala index 82d0ff038..2d7522c35 100644 --- a/content-api/content-actors/src/main/scala/org/sunbird/content/util/NotificationManager.scala +++ b/content-api/content-actors/src/main/scala/org/sunbird/content/util/NotificationManager.scala @@ -10,7 +10,10 @@ object NotificationManager { def sendNotification(subCategory: String, subType: String, userIds: List[String], title: String, data: Map[String, Any]): Unit = { val userIdsJson = userIds.map(id => s""""$id"""").mkString("[", ",", "]") - val dataJson = toJsonString(data) + + val placeholders = Map[String, Any]("title" -> title) + + val message = Map[String, Any]("placeholders" -> toJsonString(placeholders), "data" -> toJsonString(data)) val body = s""" @@ -18,13 +21,12 @@ object NotificationManager { "subCategory": "$subCategory", "subType": "$subType", "userIds": $userIdsJson, - "title": "$title", - "data": $dataJson + "message": ${toJsonString(message)} } """ val url: String = Platform.getString("notification.api.url", "http://cb-notification-wrapper-service:8081/notifications/create") - val response = Unirest.post(url).headers(Map[String, String]("Content-Type"->"application/json").asJava).body(body).asString() + val response = Unirest.post(url).headers(Map[String, String]("Content-Type" -> "application/json").asJava).body(body).asString() HTTPResponse(response.getStatus, response.getBody) } From f48e9c05530318215ba676d61d69688fdebb7dd3 Mon Sep 17 00:00:00 2001 From: DeepikaMalav28 Date: Thu, 12 Jun 2025 15:58:23 +0530 Subject: [PATCH 106/126] Dev 4.8.26 notification (#56) * initial change for notification * contenct review notification * small fix for body change * more fields added to body --- .../org/sunbird/content/util/NotificationManager.scala | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/content-api/content-actors/src/main/scala/org/sunbird/content/util/NotificationManager.scala b/content-api/content-actors/src/main/scala/org/sunbird/content/util/NotificationManager.scala index 82d0ff038..2d7522c35 100644 --- a/content-api/content-actors/src/main/scala/org/sunbird/content/util/NotificationManager.scala +++ b/content-api/content-actors/src/main/scala/org/sunbird/content/util/NotificationManager.scala @@ -10,7 +10,10 @@ object NotificationManager { def sendNotification(subCategory: String, subType: String, userIds: List[String], title: String, data: Map[String, Any]): Unit = { val userIdsJson = userIds.map(id => s""""$id"""").mkString("[", ",", "]") - val dataJson = toJsonString(data) + + val placeholders = Map[String, Any]("title" -> title) + + val message = Map[String, Any]("placeholders" -> toJsonString(placeholders), "data" -> toJsonString(data)) val body = s""" @@ -18,13 +21,12 @@ object NotificationManager { "subCategory": "$subCategory", "subType": "$subType", "userIds": $userIdsJson, - "title": "$title", - "data": $dataJson + "message": ${toJsonString(message)} } """ val url: String = Platform.getString("notification.api.url", "http://cb-notification-wrapper-service:8081/notifications/create") - val response = Unirest.post(url).headers(Map[String, String]("Content-Type"->"application/json").asJava).body(body).asString() + val response = Unirest.post(url).headers(Map[String, String]("Content-Type" -> "application/json").asJava).body(body).asString() HTTPResponse(response.getStatus, response.getBody) } From 74758a95fea5c9a8810001e08c00e47966a00b42 Mon Sep 17 00:00:00 2001 From: DeepikaMalav28 Date: Thu, 12 Jun 2025 16:20:34 +0530 Subject: [PATCH 107/126] json creation made better --- .../sunbird/content/actors/ContentActor.scala | 2 +- .../content/util/NotificationManager.scala | 49 +++++++++---------- 2 files changed, 24 insertions(+), 27 deletions(-) diff --git a/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala b/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala index 8ef8fe6b5..5a396742a 100644 --- a/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala +++ b/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala @@ -284,7 +284,7 @@ class ContentActor @Inject() (implicit oec: OntologyEngineContext, ss: StorageSe try { NotificationManager.sendNotification("CONTENT_REVIEW_REQUEST", "ALERT", List(node.getMetadata.get("createdBy").toString), node.getMetadata.get("name").toString, Map.empty) } catch { - case e: Exception => println("Error sending notification :" + e) + case e: Exception => logger.info("Error while sending notification ", e) } response } diff --git a/content-api/content-actors/src/main/scala/org/sunbird/content/util/NotificationManager.scala b/content-api/content-actors/src/main/scala/org/sunbird/content/util/NotificationManager.scala index 2d7522c35..51878f3a6 100644 --- a/content-api/content-actors/src/main/scala/org/sunbird/content/util/NotificationManager.scala +++ b/content-api/content-actors/src/main/scala/org/sunbird/content/util/NotificationManager.scala @@ -1,43 +1,40 @@ package org.sunbird.content.util import com.mashape.unirest.http.Unirest -import org.sunbird.common.Platform +import org.slf4j.{Logger, LoggerFactory} +import org.sunbird.common.{JsonUtils, Platform} import org.sunbird.util.{HTTPResponse, HttpUtil} -import scala.collection.JavaConverters._ object NotificationManager { + private val logger: Logger = LoggerFactory.getLogger("NotificationManager") + def sendNotification(subCategory: String, subType: String, userIds: List[String], title: String, data: Map[String, Any]): Unit = { - val userIdsJson = userIds.map(id => s""""$id"""").mkString("[", ",", "]") + logger.info("Notification construction started") + + val placeholders = Map("title" -> title) + val message = Map("placeholders" -> placeholders, "data" -> data) - val placeholders = Map[String, Any]("title" -> title) + val bodyMap = Map( + "subCategory" -> subCategory, + "subType" -> subType, + "userIds" -> userIds, + "message" -> message + ) - val message = Map[String, Any]("placeholders" -> toJsonString(placeholders), "data" -> toJsonString(data)) + val body = JsonUtils.serialize(bodyMap) - val body = - s""" - { - "subCategory": "$subCategory", - "subType": "$subType", - "userIds": $userIdsJson, - "message": ${toJsonString(message)} - } - """ + val url = Platform.getString("notification.api.url", "http://cb-notification-wrapper-service:8081/notifications/create") + logger.info("Started sending notification with body {}", body) + val response = Unirest.post(url) + .header("Content-Type", "application/json") + .body(body) + .asString() + + logger.info("Successfully sent notification {}", response) - val url: String = Platform.getString("notification.api.url", "http://cb-notification-wrapper-service:8081/notifications/create") - val response = Unirest.post(url).headers(Map[String, String]("Content-Type" -> "application/json").asJava).body(body).asString() HTTPResponse(response.getStatus, response.getBody) } - private def toJsonString(map: Map[String, Any]): String = { - map.map { - case (k, v: String) => s""""$k":"$v"""" - case (k, v: Int) => s""""$k":$v""" - case (k, v: Boolean) => s""""$k":$v""" - case (k, v: Double) => s""""$k":$v""" - case (k, v: List[_]) => s""""$k":[${v.map(x => s""""$x"""").mkString(",")}]""" - case (k, v) => s""""$k":"${v.toString}"""" - }.mkString("{", ",", "}") - } } \ No newline at end of file From 804a79b1759da23f62e5bd8e39fec9b4f0c4bb77 Mon Sep 17 00:00:00 2001 From: DeepikaMalav28 Date: Thu, 12 Jun 2025 16:36:22 +0530 Subject: [PATCH 108/126] Dev 4.8.26 notification (#57) * initial change for notification * contenct review notification * small fix for body change * more fields added to body * json creation made better --- .../sunbird/content/actors/ContentActor.scala | 2 +- .../content/util/NotificationManager.scala | 49 +++++++++---------- 2 files changed, 24 insertions(+), 27 deletions(-) diff --git a/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala b/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala index 8ef8fe6b5..5a396742a 100644 --- a/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala +++ b/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala @@ -284,7 +284,7 @@ class ContentActor @Inject() (implicit oec: OntologyEngineContext, ss: StorageSe try { NotificationManager.sendNotification("CONTENT_REVIEW_REQUEST", "ALERT", List(node.getMetadata.get("createdBy").toString), node.getMetadata.get("name").toString, Map.empty) } catch { - case e: Exception => println("Error sending notification :" + e) + case e: Exception => logger.info("Error while sending notification ", e) } response } diff --git a/content-api/content-actors/src/main/scala/org/sunbird/content/util/NotificationManager.scala b/content-api/content-actors/src/main/scala/org/sunbird/content/util/NotificationManager.scala index 2d7522c35..51878f3a6 100644 --- a/content-api/content-actors/src/main/scala/org/sunbird/content/util/NotificationManager.scala +++ b/content-api/content-actors/src/main/scala/org/sunbird/content/util/NotificationManager.scala @@ -1,43 +1,40 @@ package org.sunbird.content.util import com.mashape.unirest.http.Unirest -import org.sunbird.common.Platform +import org.slf4j.{Logger, LoggerFactory} +import org.sunbird.common.{JsonUtils, Platform} import org.sunbird.util.{HTTPResponse, HttpUtil} -import scala.collection.JavaConverters._ object NotificationManager { + private val logger: Logger = LoggerFactory.getLogger("NotificationManager") + def sendNotification(subCategory: String, subType: String, userIds: List[String], title: String, data: Map[String, Any]): Unit = { - val userIdsJson = userIds.map(id => s""""$id"""").mkString("[", ",", "]") + logger.info("Notification construction started") + + val placeholders = Map("title" -> title) + val message = Map("placeholders" -> placeholders, "data" -> data) - val placeholders = Map[String, Any]("title" -> title) + val bodyMap = Map( + "subCategory" -> subCategory, + "subType" -> subType, + "userIds" -> userIds, + "message" -> message + ) - val message = Map[String, Any]("placeholders" -> toJsonString(placeholders), "data" -> toJsonString(data)) + val body = JsonUtils.serialize(bodyMap) - val body = - s""" - { - "subCategory": "$subCategory", - "subType": "$subType", - "userIds": $userIdsJson, - "message": ${toJsonString(message)} - } - """ + val url = Platform.getString("notification.api.url", "http://cb-notification-wrapper-service:8081/notifications/create") + logger.info("Started sending notification with body {}", body) + val response = Unirest.post(url) + .header("Content-Type", "application/json") + .body(body) + .asString() + + logger.info("Successfully sent notification {}", response) - val url: String = Platform.getString("notification.api.url", "http://cb-notification-wrapper-service:8081/notifications/create") - val response = Unirest.post(url).headers(Map[String, String]("Content-Type" -> "application/json").asJava).body(body).asString() HTTPResponse(response.getStatus, response.getBody) } - private def toJsonString(map: Map[String, Any]): String = { - map.map { - case (k, v: String) => s""""$k":"$v"""" - case (k, v: Int) => s""""$k":$v""" - case (k, v: Boolean) => s""""$k":$v""" - case (k, v: Double) => s""""$k":$v""" - case (k, v: List[_]) => s""""$k":[${v.map(x => s""""$x"""").mkString(",")}]""" - case (k, v) => s""""$k":"${v.toString}"""" - }.mkString("{", ",", "}") - } } \ No newline at end of file From cfbc24aa5ffa76a8a29c0e96431be6f3d17379b1 Mon Sep 17 00:00:00 2001 From: DeepikaMalav28 Date: Thu, 12 Jun 2025 17:10:33 +0530 Subject: [PATCH 109/126] fix for message body --- .../sunbird/content/util/NotificationManager.scala | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/content-api/content-actors/src/main/scala/org/sunbird/content/util/NotificationManager.scala b/content-api/content-actors/src/main/scala/org/sunbird/content/util/NotificationManager.scala index 51878f3a6..fe6538f8d 100644 --- a/content-api/content-actors/src/main/scala/org/sunbird/content/util/NotificationManager.scala +++ b/content-api/content-actors/src/main/scala/org/sunbird/content/util/NotificationManager.scala @@ -3,7 +3,8 @@ package org.sunbird.content.util import com.mashape.unirest.http.Unirest import org.slf4j.{Logger, LoggerFactory} import org.sunbird.common.{JsonUtils, Platform} -import org.sunbird.util.{HTTPResponse, HttpUtil} +import org.sunbird.util.HTTPResponse +import scala.collection.JavaConverters._ object NotificationManager { @@ -13,15 +14,12 @@ object NotificationManager { logger.info("Notification construction started") - val placeholders = Map("title" -> title) - val message = Map("placeholders" -> placeholders, "data" -> data) - val bodyMap = Map( "subCategory" -> subCategory, "subType" -> subType, - "userIds" -> userIds, - "message" -> message - ) + "userIds" -> userIds.asJava, + "message" -> Map("placeholders" -> Map("title" -> title).asJava, "data" -> data.asJava).asJava + ).asJava val body = JsonUtils.serialize(bodyMap) @@ -32,7 +30,7 @@ object NotificationManager { .body(body) .asString() - logger.info("Successfully sent notification {}", response) + logger.info("Successfully sent notification status: {}, body: {}", response.getStatus, response.getBody) HTTPResponse(response.getStatus, response.getBody) } From e55a623777546667065c9067a134a24d7d3ff090 Mon Sep 17 00:00:00 2001 From: DeepikaMalav28 Date: Thu, 12 Jun 2025 17:26:16 +0530 Subject: [PATCH 110/126] Dev 4.8.26 notification (#58) * initial change for notification * contenct review notification * small fix for body change * more fields added to body * json creation made better * fix for message body --- .../sunbird/content/util/NotificationManager.scala | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/content-api/content-actors/src/main/scala/org/sunbird/content/util/NotificationManager.scala b/content-api/content-actors/src/main/scala/org/sunbird/content/util/NotificationManager.scala index 51878f3a6..fe6538f8d 100644 --- a/content-api/content-actors/src/main/scala/org/sunbird/content/util/NotificationManager.scala +++ b/content-api/content-actors/src/main/scala/org/sunbird/content/util/NotificationManager.scala @@ -3,7 +3,8 @@ package org.sunbird.content.util import com.mashape.unirest.http.Unirest import org.slf4j.{Logger, LoggerFactory} import org.sunbird.common.{JsonUtils, Platform} -import org.sunbird.util.{HTTPResponse, HttpUtil} +import org.sunbird.util.HTTPResponse +import scala.collection.JavaConverters._ object NotificationManager { @@ -13,15 +14,12 @@ object NotificationManager { logger.info("Notification construction started") - val placeholders = Map("title" -> title) - val message = Map("placeholders" -> placeholders, "data" -> data) - val bodyMap = Map( "subCategory" -> subCategory, "subType" -> subType, - "userIds" -> userIds, - "message" -> message - ) + "userIds" -> userIds.asJava, + "message" -> Map("placeholders" -> Map("title" -> title).asJava, "data" -> data.asJava).asJava + ).asJava val body = JsonUtils.serialize(bodyMap) @@ -32,7 +30,7 @@ object NotificationManager { .body(body) .asString() - logger.info("Successfully sent notification {}", response) + logger.info("Successfully sent notification status: {}, body: {}", response.getStatus, response.getBody) HTTPResponse(response.getStatus, response.getBody) } From 56d7be823645df8229673b62e86d56a839fd2005 Mon Sep 17 00:00:00 2001 From: DeepikaMalav28 Date: Thu, 12 Jun 2025 19:35:51 +0530 Subject: [PATCH 111/126] add more notification --- .../org/sunbird/content/actors/ContentActor.scala | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala b/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala index 5a396742a..e07bb4bc3 100644 --- a/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala +++ b/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala @@ -282,7 +282,7 @@ class ContentActor @Inject() (implicit oec: OntologyEngineContext, ss: StorageSe else { val response = ReviewManager.review(request, node) try { - NotificationManager.sendNotification("CONTENT_REVIEW_REQUEST", "ALERT", List(node.getMetadata.get("createdBy").toString), node.getMetadata.get("name").toString, Map.empty) + NotificationManager.sendNotification("CONTENT_REVIEW_REQUEST", "ALERT", List(node.getMetadata.get("reviewer").toString), node.getMetadata.get("name").toString, Map.empty) } catch { case e: Exception => logger.info("Error while sending notification ", e) } @@ -375,6 +375,11 @@ class ContentActor @Inject() (implicit oec: OntologyEngineContext, ss: StorageSe else DataNode.systemUpdate(request, response,"", None) }).map(node => { + try { + NotificationManager.sendNotification("CONTENT_EDITED", "UPDATE", List(node.getMetadata.get("createdBy").toString), node.getMetadata.get("name").toString, Map.empty) + } catch { + case e: Exception => logger.info("Error while sending notification ", e) + } ResponseHandler.OK.put("identifier", identifier).put("status", "success") }) } @@ -400,6 +405,11 @@ class ContentActor @Inject() (implicit oec: OntologyEngineContext, ss: StorageSe RequestUtil.restrictProperties(request) DataNode.update(request).map(node => { val identifier: String = node.getIdentifier.replace(".img", "") + try { + NotificationManager.sendNotification("CONTENT_REJECTED", "UPDATE", List(node.getMetadata.get("createdBy").toString), node.getMetadata.get("name").toString, Map.empty) + } catch { + case e: Exception => logger.info("Error while sending notification ", e) + } ResponseHandler.OK.put("node_id", identifier).put("identifier", identifier) }) }).flatMap(f => f) From e3d58f24b1dc41bc8ae9ed91fbddcf08d872b405 Mon Sep 17 00:00:00 2001 From: Sahil-tarento <140611066+Sahil-tarento@users.noreply.github.com> Date: Fri, 13 Jun 2025 10:07:40 +0530 Subject: [PATCH 112/126] 4.8.26 dev v1 (#60) * initial change for notification (#54) * initial change for notification * contenct review notification * Dev 4.8.26 notification (#55) * initial change for notification * contenct review notification * small fix for body change * Dev 4.8.26 notification (#56) * initial change for notification * contenct review notification * small fix for body change * more fields added to body * Dev 4.8.26 notification (#57) * initial change for notification * contenct review notification * small fix for body change * more fields added to body * json creation made better * Dev 4.8.26 notification (#58) * initial change for notification * contenct review notification * small fix for body change * more fields added to body * json creation made better * fix for message body * Dev 4.8.26 notification (#59) * initial change for notification * contenct review notification * small fix for body change * more fields added to body * json creation made better * fix for message body * add more notification --------- Co-authored-by: DeepikaMalav28 --- .../sunbird/content/actors/ContentActor.scala | 24 ++++++++++-- .../content/util/NotificationManager.scala | 38 +++++++++++++++++++ .../content-service/conf/application.conf | 1 + 3 files changed, 60 insertions(+), 3 deletions(-) create mode 100644 content-api/content-actors/src/main/scala/org/sunbird/content/util/NotificationManager.scala diff --git a/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala b/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala index e5cb762b7..e07bb4bc3 100644 --- a/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala +++ b/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala @@ -13,7 +13,7 @@ import org.slf4j.{Logger, LoggerFactory} import org.sunbird.`object`.importer.{ImportConfig, ImportManager} import org.sunbird.actor.core.BaseActor import org.sunbird.cache.impl.RedisCache -import org.sunbird.content.util.{AcceptFlagManager, ContentConstants, CopyManager, DiscardManager, FlagManager, RetireManager} +import org.sunbird.content.util.{AcceptFlagManager, ContentConstants, CopyManager, DiscardManager, FlagManager, NotificationManager, RetireManager} import org.sunbird.cloudstore.StorageService import org.sunbird.common.{ContentParams, JsonUtils, Platform, Slug} import org.sunbird.common.dto.{Request, Response, ResponseHandler} @@ -31,7 +31,7 @@ import org.sunbird.managers.HierarchyManager.hierarchyPrefix import java.time.{ZoneId, ZonedDateTime} import java.time.format.DateTimeFormatter -import scala.collection.JavaConverters +import scala.collection.{JavaConverters, Map} import scala.collection.JavaConverters._ import scala.concurrent.{ExecutionContext, Future} @@ -279,7 +279,15 @@ class ContentActor @Inject() (implicit oec: OntologyEngineContext, ss: StorageSe request.getContext.put("schemaName", node.getObjectType.toLowerCase()) if (StringUtils.equalsAnyIgnoreCase("Processing", node.getMetadata.getOrDefault("status", "").asInstanceOf[String])) throw new ClientException("ERR_NODE_ACCESS_DENIED", "Review Operation Can't Be Applied On Node Under Processing State") - else ReviewManager.review(request, node) + else { + val response = ReviewManager.review(request, node) + try { + NotificationManager.sendNotification("CONTENT_REVIEW_REQUEST", "ALERT", List(node.getMetadata.get("reviewer").toString), node.getMetadata.get("name").toString, Map.empty) + } catch { + case e: Exception => logger.info("Error while sending notification ", e) + } + response + } }).flatMap(f => f) } @@ -367,6 +375,11 @@ class ContentActor @Inject() (implicit oec: OntologyEngineContext, ss: StorageSe else DataNode.systemUpdate(request, response,"", None) }).map(node => { + try { + NotificationManager.sendNotification("CONTENT_EDITED", "UPDATE", List(node.getMetadata.get("createdBy").toString), node.getMetadata.get("name").toString, Map.empty) + } catch { + case e: Exception => logger.info("Error while sending notification ", e) + } ResponseHandler.OK.put("identifier", identifier).put("status", "success") }) } @@ -392,6 +405,11 @@ class ContentActor @Inject() (implicit oec: OntologyEngineContext, ss: StorageSe RequestUtil.restrictProperties(request) DataNode.update(request).map(node => { val identifier: String = node.getIdentifier.replace(".img", "") + try { + NotificationManager.sendNotification("CONTENT_REJECTED", "UPDATE", List(node.getMetadata.get("createdBy").toString), node.getMetadata.get("name").toString, Map.empty) + } catch { + case e: Exception => logger.info("Error while sending notification ", e) + } ResponseHandler.OK.put("node_id", identifier).put("identifier", identifier) }) }).flatMap(f => f) diff --git a/content-api/content-actors/src/main/scala/org/sunbird/content/util/NotificationManager.scala b/content-api/content-actors/src/main/scala/org/sunbird/content/util/NotificationManager.scala new file mode 100644 index 000000000..fe6538f8d --- /dev/null +++ b/content-api/content-actors/src/main/scala/org/sunbird/content/util/NotificationManager.scala @@ -0,0 +1,38 @@ +package org.sunbird.content.util + +import com.mashape.unirest.http.Unirest +import org.slf4j.{Logger, LoggerFactory} +import org.sunbird.common.{JsonUtils, Platform} +import org.sunbird.util.HTTPResponse +import scala.collection.JavaConverters._ + +object NotificationManager { + + private val logger: Logger = LoggerFactory.getLogger("NotificationManager") + + def sendNotification(subCategory: String, subType: String, userIds: List[String], title: String, data: Map[String, Any]): Unit = { + + logger.info("Notification construction started") + + val bodyMap = Map( + "subCategory" -> subCategory, + "subType" -> subType, + "userIds" -> userIds.asJava, + "message" -> Map("placeholders" -> Map("title" -> title).asJava, "data" -> data.asJava).asJava + ).asJava + + val body = JsonUtils.serialize(bodyMap) + + val url = Platform.getString("notification.api.url", "http://cb-notification-wrapper-service:8081/notifications/create") + logger.info("Started sending notification with body {}", body) + val response = Unirest.post(url) + .header("Content-Type", "application/json") + .body(body) + .asString() + + logger.info("Successfully sent notification status: {}, body: {}", response.getStatus, response.getBody) + + HTTPResponse(response.getStatus, response.getBody) + } + +} \ No newline at end of file diff --git a/content-api/content-service/conf/application.conf b/content-api/content-service/conf/application.conf index fa3214e9f..2cfa3342c 100644 --- a/content-api/content-service/conf/application.conf +++ b/content-api/content-service/conf/application.conf @@ -588,6 +588,7 @@ collection.image.migration.enabled=true cloud_storage.upload.url.ttl=600 composite.search.url="https://dev.sunbirded.org/action/composite/v3/search" +notification.api.url="http://cb-notification-wrapper-service:8081/notifications/create" # Enable Suggested Framework in Get Channel API. channel.fetch.suggested_frameworks=true From 3966641027bbbf5d35a64acae86af0fc21311e53 Mon Sep 17 00:00:00 2001 From: DeepikaMalav28 Date: Fri, 13 Jun 2025 10:56:58 +0530 Subject: [PATCH 113/126] add notification --- .../sunbird/content/actors/ContentActor.scala | 24 ++++++++++++++++--- .../content/util/NotificationManager.scala | 2 +- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala b/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala index e07bb4bc3..fa2807294 100644 --- a/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala +++ b/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala @@ -282,7 +282,13 @@ class ContentActor @Inject() (implicit oec: OntologyEngineContext, ss: StorageSe else { val response = ReviewManager.review(request, node) try { - NotificationManager.sendNotification("CONTENT_REVIEW_REQUEST", "ALERT", List(node.getMetadata.get("reviewer").toString), node.getMetadata.get("name").toString, Map.empty) + NotificationManager.sendNotification( + "CONTENT_REVIEW_REQUEST", + "ALERT", + List(node.getMetadata.get("reviewer").asInstanceOf[java.util.Map[String, AnyRef]].get("id").asInstanceOf[String]), + node.getMetadata.get("name").asInstanceOf[String], + Map[String, Any]("id" -> node.getMetadata.get("identifier").asInstanceOf[String]) + ) } catch { case e: Exception => logger.info("Error while sending notification ", e) } @@ -376,7 +382,13 @@ class ContentActor @Inject() (implicit oec: OntologyEngineContext, ss: StorageSe DataNode.systemUpdate(request, response,"", None) }).map(node => { try { - NotificationManager.sendNotification("CONTENT_EDITED", "UPDATE", List(node.getMetadata.get("createdBy").toString), node.getMetadata.get("name").toString, Map.empty) + NotificationManager.sendNotification( + "CONTENT_EDITED", + "UPDATE", + List(node.getMetadata.get("createdBy").asInstanceOf[String]), + node.getMetadata.get("name").asInstanceOf[String], + Map[String, Any]("id" -> node.getMetadata.get("identifier").asInstanceOf[String]) + ) } catch { case e: Exception => logger.info("Error while sending notification ", e) } @@ -406,7 +418,13 @@ class ContentActor @Inject() (implicit oec: OntologyEngineContext, ss: StorageSe DataNode.update(request).map(node => { val identifier: String = node.getIdentifier.replace(".img", "") try { - NotificationManager.sendNotification("CONTENT_REJECTED", "UPDATE", List(node.getMetadata.get("createdBy").toString), node.getMetadata.get("name").toString, Map.empty) + NotificationManager.sendNotification( + "CONTENT_REJECTED", + "UPDATE", + List(node.getMetadata.get("createdBy").asInstanceOf[String]), + node.getMetadata.get("name").asInstanceOf[String], + Map[String, Any]("id" -> node.getMetadata.get("identifier").asInstanceOf[String]) + ) } catch { case e: Exception => logger.info("Error while sending notification ", e) } diff --git a/content-api/content-actors/src/main/scala/org/sunbird/content/util/NotificationManager.scala b/content-api/content-actors/src/main/scala/org/sunbird/content/util/NotificationManager.scala index fe6538f8d..e59df56ad 100644 --- a/content-api/content-actors/src/main/scala/org/sunbird/content/util/NotificationManager.scala +++ b/content-api/content-actors/src/main/scala/org/sunbird/content/util/NotificationManager.scala @@ -10,7 +10,7 @@ object NotificationManager { private val logger: Logger = LoggerFactory.getLogger("NotificationManager") - def sendNotification(subCategory: String, subType: String, userIds: List[String], title: String, data: Map[String, Any]): Unit = { + def sendNotification(subCategory: String, subType: String, userIds: List[String], title: String, data: collection.Map[String, Any]): Unit = { logger.info("Notification construction started") From ed56fb719fd7d0de0d7294fd987418b414c62b58 Mon Sep 17 00:00:00 2001 From: DeepikaMalav28 Date: Fri, 13 Jun 2025 11:36:46 +0530 Subject: [PATCH 114/126] Dev 4.8.26 notification (#61) * initial change for notification * contenct review notification * initial change for notification (#54) * initial change for notification * contenct review notification * small fix for body change * Dev 4.8.26 notification (#55) * initial change for notification * contenct review notification * small fix for body change * more fields added to body * Dev 4.8.26 notification (#56) * initial change for notification * contenct review notification * small fix for body change * more fields added to body * json creation made better * Dev 4.8.26 notification (#57) * initial change for notification * contenct review notification * small fix for body change * more fields added to body * json creation made better * fix for message body * Dev 4.8.26 notification (#58) * initial change for notification * contenct review notification * small fix for body change * more fields added to body * json creation made better * fix for message body * add more notification * add notification --- .../sunbird/content/actors/ContentActor.scala | 24 ++++++++++++++++--- .../content/util/NotificationManager.scala | 2 +- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala b/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala index e07bb4bc3..fa2807294 100644 --- a/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala +++ b/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala @@ -282,7 +282,13 @@ class ContentActor @Inject() (implicit oec: OntologyEngineContext, ss: StorageSe else { val response = ReviewManager.review(request, node) try { - NotificationManager.sendNotification("CONTENT_REVIEW_REQUEST", "ALERT", List(node.getMetadata.get("reviewer").toString), node.getMetadata.get("name").toString, Map.empty) + NotificationManager.sendNotification( + "CONTENT_REVIEW_REQUEST", + "ALERT", + List(node.getMetadata.get("reviewer").asInstanceOf[java.util.Map[String, AnyRef]].get("id").asInstanceOf[String]), + node.getMetadata.get("name").asInstanceOf[String], + Map[String, Any]("id" -> node.getMetadata.get("identifier").asInstanceOf[String]) + ) } catch { case e: Exception => logger.info("Error while sending notification ", e) } @@ -376,7 +382,13 @@ class ContentActor @Inject() (implicit oec: OntologyEngineContext, ss: StorageSe DataNode.systemUpdate(request, response,"", None) }).map(node => { try { - NotificationManager.sendNotification("CONTENT_EDITED", "UPDATE", List(node.getMetadata.get("createdBy").toString), node.getMetadata.get("name").toString, Map.empty) + NotificationManager.sendNotification( + "CONTENT_EDITED", + "UPDATE", + List(node.getMetadata.get("createdBy").asInstanceOf[String]), + node.getMetadata.get("name").asInstanceOf[String], + Map[String, Any]("id" -> node.getMetadata.get("identifier").asInstanceOf[String]) + ) } catch { case e: Exception => logger.info("Error while sending notification ", e) } @@ -406,7 +418,13 @@ class ContentActor @Inject() (implicit oec: OntologyEngineContext, ss: StorageSe DataNode.update(request).map(node => { val identifier: String = node.getIdentifier.replace(".img", "") try { - NotificationManager.sendNotification("CONTENT_REJECTED", "UPDATE", List(node.getMetadata.get("createdBy").toString), node.getMetadata.get("name").toString, Map.empty) + NotificationManager.sendNotification( + "CONTENT_REJECTED", + "UPDATE", + List(node.getMetadata.get("createdBy").asInstanceOf[String]), + node.getMetadata.get("name").asInstanceOf[String], + Map[String, Any]("id" -> node.getMetadata.get("identifier").asInstanceOf[String]) + ) } catch { case e: Exception => logger.info("Error while sending notification ", e) } diff --git a/content-api/content-actors/src/main/scala/org/sunbird/content/util/NotificationManager.scala b/content-api/content-actors/src/main/scala/org/sunbird/content/util/NotificationManager.scala index fe6538f8d..e59df56ad 100644 --- a/content-api/content-actors/src/main/scala/org/sunbird/content/util/NotificationManager.scala +++ b/content-api/content-actors/src/main/scala/org/sunbird/content/util/NotificationManager.scala @@ -10,7 +10,7 @@ object NotificationManager { private val logger: Logger = LoggerFactory.getLogger("NotificationManager") - def sendNotification(subCategory: String, subType: String, userIds: List[String], title: String, data: Map[String, Any]): Unit = { + def sendNotification(subCategory: String, subType: String, userIds: List[String], title: String, data: collection.Map[String, Any]): Unit = { logger.info("Notification construction started") From 45c2b12265466958e92e01b2c826df9e6dc7de12 Mon Sep 17 00:00:00 2001 From: DeepikaMalav28 Date: Fri, 13 Jun 2025 12:21:43 +0530 Subject: [PATCH 115/126] convert in array --- .../main/scala/org/sunbird/content/actors/ContentActor.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala b/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala index fa2807294..40b27b58b 100644 --- a/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala +++ b/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala @@ -285,7 +285,7 @@ class ContentActor @Inject() (implicit oec: OntologyEngineContext, ss: StorageSe NotificationManager.sendNotification( "CONTENT_REVIEW_REQUEST", "ALERT", - List(node.getMetadata.get("reviewer").asInstanceOf[java.util.Map[String, AnyRef]].get("id").asInstanceOf[String]), + node.getMetadata.get("reviewerIDs").asInstanceOf[java.util.List[String]].asScala.toList, node.getMetadata.get("name").asInstanceOf[String], Map[String, Any]("id" -> node.getMetadata.get("identifier").asInstanceOf[String]) ) From 0ea259353ac31d5cd173f02ce0ec26af89fac150 Mon Sep 17 00:00:00 2001 From: DeepikaMalav28 Date: Fri, 13 Jun 2025 15:25:40 +0530 Subject: [PATCH 116/126] Dev 4.8.26 notification (#62) * initial change for notification * contenct review notification * initial change for notification (#54) * initial change for notification * contenct review notification * small fix for body change * Dev 4.8.26 notification (#55) * initial change for notification * contenct review notification * small fix for body change * more fields added to body * Dev 4.8.26 notification (#56) * initial change for notification * contenct review notification * small fix for body change * more fields added to body * json creation made better * Dev 4.8.26 notification (#57) * initial change for notification * contenct review notification * small fix for body change * more fields added to body * json creation made better * fix for message body * Dev 4.8.26 notification (#58) * initial change for notification * contenct review notification * small fix for body change * more fields added to body * json creation made better * fix for message body * add more notification * add notification * convert in array --- .../main/scala/org/sunbird/content/actors/ContentActor.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala b/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala index fa2807294..40b27b58b 100644 --- a/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala +++ b/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala @@ -285,7 +285,7 @@ class ContentActor @Inject() (implicit oec: OntologyEngineContext, ss: StorageSe NotificationManager.sendNotification( "CONTENT_REVIEW_REQUEST", "ALERT", - List(node.getMetadata.get("reviewer").asInstanceOf[java.util.Map[String, AnyRef]].get("id").asInstanceOf[String]), + node.getMetadata.get("reviewerIDs").asInstanceOf[java.util.List[String]].asScala.toList, node.getMetadata.get("name").asInstanceOf[String], Map[String, Any]("id" -> node.getMetadata.get("identifier").asInstanceOf[String]) ) From 7853472666c56af60b1f370a7b6c97d14f0a7a94 Mon Sep 17 00:00:00 2001 From: DeepikaMalav28 Date: Fri, 13 Jun 2025 16:49:47 +0530 Subject: [PATCH 117/126] convert in list reviewersId --- .../scala/org/sunbird/content/actors/ContentActor.scala | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala b/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala index 40b27b58b..eaf1a7184 100644 --- a/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala +++ b/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala @@ -282,10 +282,15 @@ class ContentActor @Inject() (implicit oec: OntologyEngineContext, ss: StorageSe else { val response = ReviewManager.review(request, node) try { + val reviewers = node.getMetadata.get("reviewerIDs") match { + case arr: Array[String] => arr.toList + case list: java.util.List[_] => list.asScala.toList.map(_.toString) + case other => throw new RuntimeException(s"Unexpected type for reviewerIDs: ${other.getClass}") + } NotificationManager.sendNotification( "CONTENT_REVIEW_REQUEST", "ALERT", - node.getMetadata.get("reviewerIDs").asInstanceOf[java.util.List[String]].asScala.toList, + reviewers, node.getMetadata.get("name").asInstanceOf[String], Map[String, Any]("id" -> node.getMetadata.get("identifier").asInstanceOf[String]) ) From 517255e59daa00499e65463257bbc223a003c93a Mon Sep 17 00:00:00 2001 From: DeepikaMalav28 Date: Sun, 15 Jun 2025 13:59:32 +0530 Subject: [PATCH 118/126] Dev 4.8.26 notification (#64) * initial change for notification * contenct review notification * initial change for notification (#54) * initial change for notification * contenct review notification * small fix for body change * Dev 4.8.26 notification (#55) * initial change for notification * contenct review notification * small fix for body change * more fields added to body * Dev 4.8.26 notification (#56) * initial change for notification * contenct review notification * small fix for body change * more fields added to body * json creation made better * Dev 4.8.26 notification (#57) * initial change for notification * contenct review notification * small fix for body change * more fields added to body * json creation made better * fix for message body * Dev 4.8.26 notification (#58) * initial change for notification * contenct review notification * small fix for body change * more fields added to body * json creation made better * fix for message body * add more notification * add notification * convert in array * convert in list reviewersId --- .../scala/org/sunbird/content/actors/ContentActor.scala | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala b/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala index 40b27b58b..eaf1a7184 100644 --- a/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala +++ b/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala @@ -282,10 +282,15 @@ class ContentActor @Inject() (implicit oec: OntologyEngineContext, ss: StorageSe else { val response = ReviewManager.review(request, node) try { + val reviewers = node.getMetadata.get("reviewerIDs") match { + case arr: Array[String] => arr.toList + case list: java.util.List[_] => list.asScala.toList.map(_.toString) + case other => throw new RuntimeException(s"Unexpected type for reviewerIDs: ${other.getClass}") + } NotificationManager.sendNotification( "CONTENT_REVIEW_REQUEST", "ALERT", - node.getMetadata.get("reviewerIDs").asInstanceOf[java.util.List[String]].asScala.toList, + reviewers, node.getMetadata.get("name").asInstanceOf[String], Map[String, Any]("id" -> node.getMetadata.get("identifier").asInstanceOf[String]) ) From 9bce56df8821a768e79ab616dd199ad53fbf8883 Mon Sep 17 00:00:00 2001 From: DeepikaMalav28 Date: Mon, 16 Jun 2025 12:53:23 +0530 Subject: [PATCH 119/126] #KB-10158 mutilple notification trigger fixes (#65) --- .../sunbird/content/actors/ContentActor.scala | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala b/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala index eaf1a7184..3a5d13e5d 100644 --- a/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala +++ b/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala @@ -423,13 +423,15 @@ class ContentActor @Inject() (implicit oec: OntologyEngineContext, ss: StorageSe DataNode.update(request).map(node => { val identifier: String = node.getIdentifier.replace(".img", "") try { - NotificationManager.sendNotification( - "CONTENT_REJECTED", - "UPDATE", - List(node.getMetadata.get("createdBy").asInstanceOf[String]), - node.getMetadata.get("name").asInstanceOf[String], - Map[String, Any]("id" -> node.getMetadata.get("identifier").asInstanceOf[String]) - ) + if(node.getMetadata.containsKey("reviewerIDs")) { + NotificationManager.sendNotification( + "CONTENT_REJECTED", + "UPDATE", + List(node.getMetadata.get("createdBy").asInstanceOf[String]), + node.getMetadata.get("name").asInstanceOf[String], + Map[String, Any]("id" -> node.getMetadata.get("identifier").asInstanceOf[String]) + ) + } } catch { case e: Exception => logger.info("Error while sending notification ", e) } From fa50370bb4793968898073492aae05be04deddd4 Mon Sep 17 00:00:00 2001 From: DeepikaMalav28 Date: Mon, 16 Jun 2025 15:51:13 +0530 Subject: [PATCH 120/126] Content reject fix (#71) * initial change for notification * contenct review notification * initial change for notification (#54) * initial change for notification * contenct review notification * small fix for body change * Dev 4.8.26 notification (#55) * initial change for notification * contenct review notification * small fix for body change * more fields added to body * Dev 4.8.26 notification (#56) * initial change for notification * contenct review notification * small fix for body change * more fields added to body * json creation made better * Dev 4.8.26 notification (#57) * initial change for notification * contenct review notification * small fix for body change * more fields added to body * json creation made better * fix for message body * Dev 4.8.26 notification (#58) * initial change for notification * contenct review notification * small fix for body change * more fields added to body * json creation made better * fix for message body * add more notification * add notification * convert in array * convert in list reviewersId * #KB-10158 mutilple notification trigger fixes * #KB-10158 mutilple notification trigger fixes (#65) --------- Co-authored-by: Rajeev Sathish --- .../sunbird/content/actors/ContentActor.scala | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala b/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala index eaf1a7184..3a5d13e5d 100644 --- a/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala +++ b/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala @@ -423,13 +423,15 @@ class ContentActor @Inject() (implicit oec: OntologyEngineContext, ss: StorageSe DataNode.update(request).map(node => { val identifier: String = node.getIdentifier.replace(".img", "") try { - NotificationManager.sendNotification( - "CONTENT_REJECTED", - "UPDATE", - List(node.getMetadata.get("createdBy").asInstanceOf[String]), - node.getMetadata.get("name").asInstanceOf[String], - Map[String, Any]("id" -> node.getMetadata.get("identifier").asInstanceOf[String]) - ) + if(node.getMetadata.containsKey("reviewerIDs")) { + NotificationManager.sendNotification( + "CONTENT_REJECTED", + "UPDATE", + List(node.getMetadata.get("createdBy").asInstanceOf[String]), + node.getMetadata.get("name").asInstanceOf[String], + Map[String, Any]("id" -> node.getMetadata.get("identifier").asInstanceOf[String]) + ) + } } catch { case e: Exception => logger.info("Error while sending notification ", e) } From 303c808129955864f735d1cc6c3344b3f19741ec Mon Sep 17 00:00:00 2001 From: DeepikaMalav28 Date: Wed, 18 Jun 2025 13:06:25 +0530 Subject: [PATCH 121/126] add content-edit and content-review notification --- .../sunbird/content/actors/ContentActor.scala | 42 +++++++++++++++---- .../controllers/v4/ContentController.scala | 1 + 2 files changed, 34 insertions(+), 9 deletions(-) diff --git a/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala b/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala index 3a5d13e5d..08269a148 100644 --- a/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala +++ b/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala @@ -209,6 +209,19 @@ class ContentActor @Inject() (implicit oec: OntologyEngineContext, ss: StorageSe request.getRequest.put("cqfVersion", System.currentTimeMillis().toString) } DataNode.update(request, dataModifier).map(node => { + try { + if (request.getContext.getOrDefault("sendNotification", false).asInstanceOf[Boolean]) { + NotificationManager.sendNotification( + "CONTENT_EDITED", + "UPDATE", + List(node.getMetadata.get("crcoeatedBy").asInstanceOf[String]), + node.getMetadata.get("name").asInstanceOf[String], + Map[String, Any]("id" -> node.getMetadata.get("identifier").asInstanceOf[String]) + ) + } + } catch { + case e: Exception => logger.info("Error while sending notification ", e) + } val identifier: String = node.getIdentifier.replace(".img", "") ResponseHandler.OK.put("node_id", identifier).put("identifier", identifier) .put("versionKey", node.getMetadata.get("versionKey")) @@ -274,34 +287,45 @@ class ContentActor @Inject() (implicit oec: OntologyEngineContext, ss: StorageSe val readReq = new Request(request) readReq.put("identifier", identifier) readReq.put("mode", "edit") + DataNode.read(readReq).map(node => { - if (null != node & StringUtils.isNotBlank(node.getObjectType)) + if (null != node && StringUtils.isNotBlank(node.getObjectType)) request.getContext.put("schemaName", node.getObjectType.toLowerCase()) + if (StringUtils.equalsAnyIgnoreCase("Processing", node.getMetadata.getOrDefault("status", "").asInstanceOf[String])) throw new ClientException("ERR_NODE_ACCESS_DENIED", "Review Operation Can't Be Applied On Node Under Processing State") else { val response = ReviewManager.review(request, node) try { + val nodeIdOpt = Option(node.getMetadata.get("identifier")).map(_.toString).filter(StringUtils.isNotBlank) + val reviewers = node.getMetadata.get("reviewerIDs") match { case arr: Array[String] => arr.toList case list: java.util.List[_] => list.asScala.toList.map(_.toString) case other => throw new RuntimeException(s"Unexpected type for reviewerIDs: ${other.getClass}") } - NotificationManager.sendNotification( - "CONTENT_REVIEW_REQUEST", - "ALERT", - reviewers, - node.getMetadata.get("name").asInstanceOf[String], - Map[String, Any]("id" -> node.getMetadata.get("identifier").asInstanceOf[String]) - ) + + nodeIdOpt match { + case Some(nodeId) => + NotificationManager.sendNotification( + "CONTENT_REVIEW_REQUEST", + "ALERT", + reviewers, + Option(node.getMetadata.get("name")).map(_.toString).getOrElse("Unnamed Content"), + Map[String, Any]("id" -> nodeId) + ) + case None => + logger.warn(s"Skipping notification: 'identifier' is missing or blank for node: ${node.getIdentifier}") + } } catch { - case e: Exception => logger.info("Error while sending notification ", e) + case e: Exception => logger.error("Error while sending notification ", e) } response } }).flatMap(f => f) } + def populateDefaultersForCreation(request: Request) = { setDefaultsBasedOnMimeType(request, ContentParams.create.name) setDefaultLicense(request) diff --git a/content-api/content-service/app/controllers/v4/ContentController.scala b/content-api/content-service/app/controllers/v4/ContentController.scala index 3657b2fc5..8c2088324 100644 --- a/content-api/content-service/app/controllers/v4/ContentController.scala +++ b/content-api/content-service/app/controllers/v4/ContentController.scala @@ -75,6 +75,7 @@ class ContentController @Inject()(@Named(ActorNames.CONTENT_ACTOR) contentActor: val contentRequest = getRequest(content, headers, "updateContent") setRequestContext(contentRequest, version, objectType, schemaName) contentRequest.getContext.put("identifier", identifier); + contentRequest.getContext.put("sendNotification", true) getResult(ApiId.UPDATE_CONTENT, contentActor, contentRequest, version = apiVersion) } From 31876b7081b3aec652cdae673e75aae722558bf3 Mon Sep 17 00:00:00 2001 From: DeepikaMalav28 Date: Wed, 18 Jun 2025 15:10:16 +0530 Subject: [PATCH 122/126] fix (#73) --- .../sunbird/content/actors/ContentActor.scala | 28 ++++++------------- 1 file changed, 9 insertions(+), 19 deletions(-) diff --git a/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala b/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala index 8734a5993..df35f0b69 100644 --- a/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala +++ b/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala @@ -287,38 +287,28 @@ class ContentActor @Inject() (implicit oec: OntologyEngineContext, ss: StorageSe val readReq = new Request(request) readReq.put("identifier", identifier) readReq.put("mode", "edit") - DataNode.read(readReq).map(node => { - if (null != node && StringUtils.isNotBlank(node.getObjectType)) + if (null != node & StringUtils.isNotBlank(node.getObjectType)) request.getContext.put("schemaName", node.getObjectType.toLowerCase()) - if (StringUtils.equalsAnyIgnoreCase("Processing", node.getMetadata.getOrDefault("status", "").asInstanceOf[String])) throw new ClientException("ERR_NODE_ACCESS_DENIED", "Review Operation Can't Be Applied On Node Under Processing State") else { val response = ReviewManager.review(request, node) try { - val nodeIdOpt = Option(node.getMetadata.get("identifier")).map(_.toString).filter(StringUtils.isNotBlank) val reviewers = node.getMetadata.get("reviewerIDs") match { case arr: Array[String] => arr.toList case list: java.util.List[_] => list.asScala.toList.map(_.toString) case other => throw new RuntimeException(s"Unexpected type for reviewerIDs: ${other.getClass}") } - - - nodeIdOpt match { - case Some(nodeId) => - NotificationManager.sendNotification( - "CONTENT_REVIEW_REQUEST", - "ALERT", - reviewers, - Option(node.getMetadata.get("name")).map(_.toString).getOrElse("Unnamed Content"), - Map[String, Any]("id" -> nodeId) - ) - case None => - logger.warn(s"Skipping notification: 'identifier' is missing or blank for node: ${node.getIdentifier}") - } + NotificationManager.sendNotification( + "CONTENT_REVIEW_REQUEST", + "ALERT", + reviewers, + node.getMetadata.get("name").asInstanceOf[String], + Map[String, Any]("id" -> node.getMetadata.get("identifier").asInstanceOf[String]) + ) } catch { - case e: Exception => logger.error("Error while sending notification ", e) + case e: Exception => logger.info("Error while sending notification ", e) } response } From c53ce1e48d5de78cbdad6848180679ed792cfc2c Mon Sep 17 00:00:00 2001 From: DeepikaMalav28 Date: Wed, 18 Jun 2025 16:46:03 +0530 Subject: [PATCH 123/126] fix --- .../org/sunbird/content/actors/ContentActor.scala | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala b/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala index df35f0b69..44c5ff73e 100644 --- a/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala +++ b/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala @@ -209,20 +209,21 @@ class ContentActor @Inject() (implicit oec: OntologyEngineContext, ss: StorageSe request.getRequest.put("cqfVersion", System.currentTimeMillis().toString) } DataNode.update(request, dataModifier).map(node => { - try { - if (request.getContext.getOrDefault("sendNotification", false).asInstanceOf[Boolean]) { + val identifier: String = node.getIdentifier.replace(".img", "") + if (request.getContext.getOrDefault("sendNotification", false).asInstanceOf[Boolean]) { + try { NotificationManager.sendNotification( "CONTENT_EDITED", "UPDATE", - List(node.getMetadata.get("crcoeatedBy").asInstanceOf[String]), + List(node.getMetadata.get("createdBy").asInstanceOf[String]), node.getMetadata.get("name").asInstanceOf[String], Map[String, Any]("id" -> node.getMetadata.get("identifier").asInstanceOf[String]) ) + + } catch { + case e: Exception => logger.info("Error while sending notification ", e) } - } catch { - case e: Exception => logger.info("Error while sending notification ", e) } - val identifier: String = node.getIdentifier.replace(".img", "") ResponseHandler.OK.put("node_id", identifier).put("identifier", identifier) .put("versionKey", node.getMetadata.get("versionKey")) }) From e0c1ab93635e3ca79db4bf22f4013d8d1be12da8 Mon Sep 17 00:00:00 2001 From: DeepikaMalav28 Date: Wed, 18 Jun 2025 16:49:09 +0530 Subject: [PATCH 124/126] fix (#79) --- .../org/sunbird/content/actors/ContentActor.scala | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala b/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala index df35f0b69..44c5ff73e 100644 --- a/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala +++ b/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala @@ -209,20 +209,21 @@ class ContentActor @Inject() (implicit oec: OntologyEngineContext, ss: StorageSe request.getRequest.put("cqfVersion", System.currentTimeMillis().toString) } DataNode.update(request, dataModifier).map(node => { - try { - if (request.getContext.getOrDefault("sendNotification", false).asInstanceOf[Boolean]) { + val identifier: String = node.getIdentifier.replace(".img", "") + if (request.getContext.getOrDefault("sendNotification", false).asInstanceOf[Boolean]) { + try { NotificationManager.sendNotification( "CONTENT_EDITED", "UPDATE", - List(node.getMetadata.get("crcoeatedBy").asInstanceOf[String]), + List(node.getMetadata.get("createdBy").asInstanceOf[String]), node.getMetadata.get("name").asInstanceOf[String], Map[String, Any]("id" -> node.getMetadata.get("identifier").asInstanceOf[String]) ) + + } catch { + case e: Exception => logger.info("Error while sending notification ", e) } - } catch { - case e: Exception => logger.info("Error while sending notification ", e) } - val identifier: String = node.getIdentifier.replace(".img", "") ResponseHandler.OK.put("node_id", identifier).put("identifier", identifier) .put("versionKey", node.getMetadata.get("versionKey")) }) From 48cf6aad446c4d512624120bf0a8443b4eaed0ca Mon Sep 17 00:00:00 2001 From: DeepikaMalav28 Date: Wed, 18 Jun 2025 17:10:58 +0530 Subject: [PATCH 125/126] change --- .../main/scala/org/sunbird/content/actors/ContentActor.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala b/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala index 44c5ff73e..c545be727 100644 --- a/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala +++ b/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala @@ -210,7 +210,7 @@ class ContentActor @Inject() (implicit oec: OntologyEngineContext, ss: StorageSe } DataNode.update(request, dataModifier).map(node => { val identifier: String = node.getIdentifier.replace(".img", "") - if (request.getContext.getOrDefault("sendNotification", false).asInstanceOf[Boolean]) { + if (request.getContext.getOrDefault("sendNotification", Boolean.box(false)).asInstanceOf[Boolean]) { try { NotificationManager.sendNotification( "CONTENT_EDITED", From 925e0601b531599d6f5f530f6e5ccba1d607bfc0 Mon Sep 17 00:00:00 2001 From: DeepikaMalav28 Date: Wed, 18 Jun 2025 17:19:51 +0530 Subject: [PATCH 126/126] fix --- .../content-service/app/controllers/v4/ContentController.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content-api/content-service/app/controllers/v4/ContentController.scala b/content-api/content-service/app/controllers/v4/ContentController.scala index 8c2088324..65cb1b6c3 100644 --- a/content-api/content-service/app/controllers/v4/ContentController.scala +++ b/content-api/content-service/app/controllers/v4/ContentController.scala @@ -75,7 +75,7 @@ class ContentController @Inject()(@Named(ActorNames.CONTENT_ACTOR) contentActor: val contentRequest = getRequest(content, headers, "updateContent") setRequestContext(contentRequest, version, objectType, schemaName) contentRequest.getContext.put("identifier", identifier); - contentRequest.getContext.put("sendNotification", true) + contentRequest.getContext.put("sendNotification", Boolean.box(true)) getResult(ApiId.UPDATE_CONTENT, contentActor, contentRequest, version = apiVersion) }