diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt b/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt index 702f98d38..c46bf77b4 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt @@ -27,6 +27,7 @@ import org.opensearch.alerting.core.settings.LegacyOpenDistroScheduledJobSetting import org.opensearch.alerting.core.settings.ScheduledJobSettings import org.opensearch.alerting.resthandler.RestAcknowledgeAlertAction import org.opensearch.alerting.resthandler.RestDeleteMonitorAction +import org.opensearch.alerting.resthandler.RestDeleteWorkflowAction import org.opensearch.alerting.resthandler.RestExecuteMonitorAction import org.opensearch.alerting.resthandler.RestExecuteWorkflowAction import org.opensearch.alerting.resthandler.RestGetAlertsAction @@ -35,11 +36,14 @@ import org.opensearch.alerting.resthandler.RestGetEmailAccountAction import org.opensearch.alerting.resthandler.RestGetEmailGroupAction import org.opensearch.alerting.resthandler.RestGetFindingsAction import org.opensearch.alerting.resthandler.RestGetMonitorAction +import org.opensearch.alerting.resthandler.RestGetWorkflowAction import org.opensearch.alerting.resthandler.RestIndexMonitorAction +import org.opensearch.alerting.resthandler.RestIndexWorkflowAction import org.opensearch.alerting.resthandler.RestSearchEmailAccountAction import org.opensearch.alerting.resthandler.RestSearchEmailGroupAction import org.opensearch.alerting.resthandler.RestSearchMonitorAction import org.opensearch.alerting.script.TriggerScript +import org.opensearch.alerting.service.DeleteMonitorService import org.opensearch.alerting.settings.AlertingSettings import org.opensearch.alerting.settings.DestinationSettings import org.opensearch.alerting.settings.LegacyOpenDistroAlertingSettings @@ -173,6 +177,7 @@ internal class AlertingPlugin : PainlessExtension, ActionPlugin, ScriptPlugin, R RestGetMonitorAction(), RestDeleteMonitorAction(), RestIndexMonitorAction(), + RestIndexWorkflowAction(), RestSearchMonitorAction(settings, clusterService), RestExecuteMonitorAction(), RestExecuteWorkflowAction(), @@ -184,7 +189,9 @@ internal class AlertingPlugin : PainlessExtension, ActionPlugin, ScriptPlugin, R RestGetEmailGroupAction(), RestGetDestinationsAction(), RestGetAlertsAction(), - RestGetFindingsAction() + RestGetFindingsAction(), + RestGetWorkflowAction(), + RestDeleteWorkflowAction() ) } @@ -293,6 +300,8 @@ internal class AlertingPlugin : PainlessExtension, ActionPlugin, ScriptPlugin, R settings ) + DeleteMonitorService.initialize(client) + return listOf(sweeper, scheduler, runner, scheduledJobIndices, docLevelMonitorQueries, destinationMigrationCoordinator) } diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/DeleteMonitorService.kt b/alerting/src/main/kotlin/org/opensearch/alerting/DeleteMonitorService.kt new file mode 100644 index 000000000..ac795dfcc --- /dev/null +++ b/alerting/src/main/kotlin/org/opensearch/alerting/DeleteMonitorService.kt @@ -0,0 +1,170 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.alerting + +import kotlinx.coroutines.CoroutineName +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.SupervisorJob +import org.apache.logging.log4j.LogManager +import org.apache.lucene.search.join.ScoreMode +import org.opensearch.action.ActionListener +import org.opensearch.action.admin.indices.delete.DeleteIndexRequest +import org.opensearch.action.admin.indices.exists.indices.IndicesExistsRequest +import org.opensearch.action.admin.indices.exists.indices.IndicesExistsResponse +import org.opensearch.action.delete.DeleteRequest +import org.opensearch.action.delete.DeleteResponse +import org.opensearch.action.search.SearchRequest +import org.opensearch.action.search.SearchResponse +import org.opensearch.action.support.IndicesOptions +import org.opensearch.action.support.WriteRequest.RefreshPolicy +import org.opensearch.action.support.master.AcknowledgedResponse +import org.opensearch.alerting.opensearchapi.suspendUntil +import org.opensearch.alerting.util.AlertingException +import org.opensearch.client.Client +import org.opensearch.commons.alerting.action.DeleteMonitorResponse +import org.opensearch.commons.alerting.model.Monitor +import org.opensearch.commons.alerting.model.ScheduledJob +import org.opensearch.commons.alerting.model.Workflow +import org.opensearch.index.query.QueryBuilders +import org.opensearch.index.reindex.BulkByScrollResponse +import org.opensearch.index.reindex.DeleteByQueryAction +import org.opensearch.index.reindex.DeleteByQueryRequestBuilder +import org.opensearch.search.builder.SearchSourceBuilder +import kotlin.coroutines.resume +import kotlin.coroutines.resumeWithException +import kotlin.coroutines.suspendCoroutine + +/** + * Component used when deleting the monitors + */ +object DeleteMonitorService : + CoroutineScope by CoroutineScope(SupervisorJob() + Dispatchers.Default + CoroutineName("WorkflowMetadataService")) { + private val log = LogManager.getLogger(this.javaClass) + + private lateinit var client: Client + + fun initialize( + client: Client + ) { + this.client = client + } + + /** + * Deletes the monitor, docLevelQueries and monitor metadata + * @param monitor monitor to be deleted + * @param refreshPolicy + */ + suspend fun deleteMonitor(monitor: Monitor, refreshPolicy: RefreshPolicy): DeleteMonitorResponse { + val deleteResponse = deleteMonitor(monitor.id, refreshPolicy) + deleteDocLevelMonitorQueriesAndIndices(monitor) + deleteMetadata(monitor) + return DeleteMonitorResponse(deleteResponse.id, deleteResponse.version) + } + + private suspend fun deleteMonitor(monitorId: String, refreshPolicy: RefreshPolicy): DeleteResponse { + val deleteMonitorRequest = DeleteRequest(ScheduledJob.SCHEDULED_JOBS_INDEX, monitorId) + .setRefreshPolicy(refreshPolicy) + return client.suspendUntil { delete(deleteMonitorRequest, it) } + } + + private suspend fun deleteMetadata(monitor: Monitor) { + val deleteRequest = DeleteRequest(ScheduledJob.SCHEDULED_JOBS_INDEX, "${monitor.id}-metadata") + val deleteResponse: DeleteResponse = client.suspendUntil { delete(deleteRequest, it) } + } + + private suspend fun deleteDocLevelMonitorQueriesAndIndices(monitor: Monitor) { + val metadata = MonitorMetadataService.getMetadata(monitor) + metadata?.sourceToQueryIndexMapping?.forEach { (_, queryIndex) -> + + val indicesExistsResponse: IndicesExistsResponse = + client.suspendUntil { + client.admin().indices().exists(IndicesExistsRequest(queryIndex), it) + } + if (indicesExistsResponse.isExists == false) { + return + } + // Check if there's any queries from other monitors in this queryIndex, + // to avoid unnecessary doc deletion, if we could just delete index completely + val searchResponse: SearchResponse = client.suspendUntil { + search( + SearchRequest(queryIndex).source( + SearchSourceBuilder() + .size(0) + .query( + QueryBuilders.boolQuery().mustNot( + QueryBuilders.matchQuery("monitor_id", monitor.id) + ) + ) + ).indicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN_HIDDEN), + it + ) + } + if (searchResponse.hits.totalHits.value == 0L) { + val ack: AcknowledgedResponse = client.suspendUntil { + client.admin().indices().delete( + DeleteIndexRequest(queryIndex).indicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN_HIDDEN), + it + ) + } + if (ack.isAcknowledged == false) { + log.error("Deletion of concrete queryIndex:$queryIndex is not ack'd!") + } + } else { + // Delete all queries added by this monitor + val response: BulkByScrollResponse = suspendCoroutine { cont -> + DeleteByQueryRequestBuilder(client, DeleteByQueryAction.INSTANCE) + .source(queryIndex) + .filter(QueryBuilders.matchQuery("monitor_id", monitor.id)) + .refresh(true) + .execute( + object : ActionListener { + override fun onResponse(response: BulkByScrollResponse) = cont.resume(response) + override fun onFailure(t: Exception) = cont.resumeWithException(t) + } + ) + } + } + } + } + + /** + * Checks if the monitor is part of the workflow + * + * @param monitorId id of monitor that is checked if it is a workflow delegate + */ + suspend fun monitorIsWorkflowDelegate(monitorId: String): Boolean { + val queryBuilder = QueryBuilders.nestedQuery( + Workflow.WORKFLOW_DELEGATE_PATH, + QueryBuilders.boolQuery().must( + QueryBuilders.matchQuery( + Workflow.WORKFLOW_MONITOR_PATH, + monitorId + ) + ), + ScoreMode.None + ) + try { + val searchRequest = SearchRequest() + .indices(ScheduledJob.SCHEDULED_JOBS_INDEX) + .source(SearchSourceBuilder().query(queryBuilder)) + + client.threadPool().threadContext.stashContext().use { + val searchResponse: SearchResponse = client.suspendUntil { search(searchRequest, it) } + if (searchResponse.hits.totalHits?.value == 0L) { + return false + } + + val workflowIds = searchResponse.hits.hits.map { it.id }.joinToString() + log.info("Monitor $monitorId can't be deleted since it belongs to $workflowIds") + return true + } + } catch (ex: Exception) { + log.error("Error getting the monitor workflows", ex) + throw AlertingException.wrap(ex) + } + } +} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/MonitorMetadataService.kt b/alerting/src/main/kotlin/org/opensearch/alerting/MonitorMetadataService.kt index 024bcdc50..d4c52a433 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/MonitorMetadataService.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/MonitorMetadataService.kt @@ -186,7 +186,11 @@ object MonitorMetadataService : } } - private suspend fun createNewMetadata(monitor: Monitor, createWithRunContext: Boolean, workflowMetadataId: String? = null): MonitorMetadata { + private suspend fun createNewMetadata( + monitor: Monitor, + createWithRunContext: Boolean, + workflowMetadataId: String? = null, + ): MonitorMetadata { val monitorIndex = if (monitor.monitorType == Monitor.MonitorType.DOC_LEVEL_MONITOR) { (monitor.inputs[0] as DocLevelMonitorInput).indices[0] } else null diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/WorkflowMetadataService.kt b/alerting/src/main/kotlin/org/opensearch/alerting/WorkflowMetadataService.kt index 056830af7..7020732fd 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/WorkflowMetadataService.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/WorkflowMetadataService.kt @@ -98,7 +98,10 @@ object WorkflowMetadataService : } catch (e: Exception) { // If the update is set to false and id is set conflict exception will be thrown if (e is OpenSearchException && e.status() == RestStatus.CONFLICT && !updating) { - log.debug("Metadata with ${metadata.id} for workflow ${metadata.workflowId} already exist. Instead of creating new, updating existing metadata will be performed") + log.debug( + "Metadata with ${metadata.id} for workflow ${metadata.workflowId} already exist." + + " Instead of creating new, updating existing metadata will be performed" + ) return upsertWorkflowMetadata(metadata, true) } log.error("Error saving metadata", e) @@ -157,6 +160,8 @@ object WorkflowMetadataService : } private fun createNewWorkflowMetadata(workflow: Workflow, executionId: String, isTempWorkflow: Boolean): WorkflowMetadata { + // In the case of temp workflow (ie. workflow is in dry-run) use timestampWithUUID-metadata format + // In the case of regular workflow execution, use the workflowId-metadata format val id = if (isTempWorkflow) "${LocalDateTime.now(ZoneOffset.UTC)}${UUID.randomUUID()}" else workflow.id return WorkflowMetadata( id = WorkflowMetadata.getId(id), diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/model/MonitorMetadata.kt b/alerting/src/main/kotlin/org/opensearch/alerting/model/MonitorMetadata.kt index 3acc2e318..565e5f9d6 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/model/MonitorMetadata.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/model/MonitorMetadata.kt @@ -120,9 +120,17 @@ data class MonitorMetadata( return MonitorMetadata(sin) } - fun getId(monitor: Monitor, workflowId: String? = null): String { - return if (workflowId.isNullOrEmpty()) "${monitor.id}-metadata" - else "${monitor.id}-$workflowId-metadata" + /** workflowMetadataId is used as key for monitor metadata in the case when the workflow execution happens + so the monitor lastRunContext (in the case of doc level monitor) is not interfering with the monitor execution + WorkflowMetadataId will be either workflowId-metadata (when executing the workflow as it is scheduled) + or timestampWithUUID-metadata (when a workflow is executed in a dry-run mode) + In the case of temp workflow, doc level monitors must have lastRunContext created from scratch + That's why we are using workflowMetadataId - in order to ensure that the doc level monitor metadata is created from scratch + **/ + fun getId(monitor: Monitor, workflowMetadataId: String? = null): String { + return if (workflowMetadataId.isNullOrEmpty()) "${monitor.id}-metadata" + // WorkflowMetadataId already contains -metadata suffix + else "$workflowMetadataId-${monitor.id}-metadata" } } } diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/resthandler/RestDeleteWorkflowAction.kt b/alerting/src/main/kotlin/org/opensearch/alerting/resthandler/RestDeleteWorkflowAction.kt new file mode 100644 index 000000000..a61a9b51c --- /dev/null +++ b/alerting/src/main/kotlin/org/opensearch/alerting/resthandler/RestDeleteWorkflowAction.kt @@ -0,0 +1,60 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.alerting.resthandler + +import org.apache.logging.log4j.LogManager +import org.opensearch.action.support.WriteRequest +import org.opensearch.alerting.AlertingPlugin +import org.opensearch.alerting.util.REFRESH +import org.opensearch.client.node.NodeClient +import org.opensearch.commons.alerting.action.AlertingActions +import org.opensearch.commons.alerting.action.DeleteWorkflowRequest +import org.opensearch.rest.BaseRestHandler +import org.opensearch.rest.RestHandler +import org.opensearch.rest.RestRequest +import org.opensearch.rest.action.RestToXContentListener +import java.io.IOException + +/** + * This class consists of the REST handler to delete workflows. + */ +class RestDeleteWorkflowAction : BaseRestHandler() { + + private val log = LogManager.getLogger(javaClass) + + override fun getName(): String { + return "delete_workflow_action" + } + + override fun routes(): List { + return listOf( + RestHandler.Route( + RestRequest.Method.DELETE, + "${AlertingPlugin.WORKFLOW_BASE_URI}/{workflowID}" + ) + ) + } + + @Throws(IOException::class) + override fun prepareRequest(request: RestRequest, client: NodeClient): RestChannelConsumer { + log.debug("${request.method()} ${AlertingPlugin.WORKFLOW_BASE_URI}/{workflowID}") + + val workflowId = request.param("workflowID") + val deleteDelegateMonitors = request.paramAsBoolean("deleteDelegateMonitors", false) + log.debug("${request.method()} ${request.uri()}") + + val refreshPolicy = + WriteRequest.RefreshPolicy.parse(request.param(REFRESH, WriteRequest.RefreshPolicy.IMMEDIATE.value)) + val deleteWorkflowRequest = DeleteWorkflowRequest(workflowId, deleteDelegateMonitors) + + return RestChannelConsumer { channel -> + client.execute( + AlertingActions.DELETE_WORKFLOW_ACTION_TYPE, deleteWorkflowRequest, + RestToXContentListener(channel) + ) + } + } +} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/resthandler/RestGetWorkflowAction.kt b/alerting/src/main/kotlin/org/opensearch/alerting/resthandler/RestGetWorkflowAction.kt new file mode 100644 index 000000000..3b44c2050 --- /dev/null +++ b/alerting/src/main/kotlin/org/opensearch/alerting/resthandler/RestGetWorkflowAction.kt @@ -0,0 +1,63 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.alerting.resthandler + +import org.apache.logging.log4j.LogManager +import org.opensearch.alerting.AlertingPlugin +import org.opensearch.alerting.util.context +import org.opensearch.client.node.NodeClient +import org.opensearch.commons.alerting.action.AlertingActions +import org.opensearch.commons.alerting.action.GetWorkflowRequest +import org.opensearch.rest.BaseRestHandler +import org.opensearch.rest.RestHandler +import org.opensearch.rest.RestRequest +import org.opensearch.rest.action.RestToXContentListener +import org.opensearch.search.fetch.subphase.FetchSourceContext + +/** + * This class consists of the REST handler to retrieve a workflow . + */ +class RestGetWorkflowAction : BaseRestHandler() { + + private val log = LogManager.getLogger(javaClass) + + override fun getName(): String { + return "get_workflow_action" + } + + override fun routes(): List { + return listOf( + RestHandler.Route( + RestRequest.Method.GET, + "${AlertingPlugin.WORKFLOW_BASE_URI}/{workflowID}" + ), + RestHandler.Route( + RestRequest.Method.HEAD, + "${AlertingPlugin.WORKFLOW_BASE_URI}/{workflowID}" + ) + ) + } + + override fun prepareRequest(request: RestRequest, client: NodeClient): RestChannelConsumer { + log.debug("${request.method()} ${AlertingPlugin.WORKFLOW_BASE_URI}/{workflowID}") + + val workflowId = request.param("workflowID") + if (workflowId == null || workflowId.isEmpty()) { + throw IllegalArgumentException("missing id") + } + + var srcContext = context(request) + if (request.method() == RestRequest.Method.HEAD) { + srcContext = FetchSourceContext.DO_NOT_FETCH_SOURCE + } + val getWorkflowRequest = + GetWorkflowRequest(workflowId, request.method()) + return RestChannelConsumer { + channel -> + client.execute(AlertingActions.GET_WORKFLOW_ACTION_TYPE, getWorkflowRequest, RestToXContentListener(channel)) + } + } +} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/resthandler/RestIndexWorkflowAction.kt b/alerting/src/main/kotlin/org/opensearch/alerting/resthandler/RestIndexWorkflowAction.kt new file mode 100644 index 000000000..faa0ae065 --- /dev/null +++ b/alerting/src/main/kotlin/org/opensearch/alerting/resthandler/RestIndexWorkflowAction.kt @@ -0,0 +1,99 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.alerting.resthandler + +import org.opensearch.action.support.WriteRequest +import org.opensearch.alerting.AlertingPlugin +import org.opensearch.alerting.util.AlertingException +import org.opensearch.alerting.util.IF_PRIMARY_TERM +import org.opensearch.alerting.util.IF_SEQ_NO +import org.opensearch.alerting.util.REFRESH +import org.opensearch.client.node.NodeClient +import org.opensearch.common.xcontent.XContentParserUtils +import org.opensearch.commons.alerting.action.AlertingActions +import org.opensearch.commons.alerting.action.IndexWorkflowRequest +import org.opensearch.commons.alerting.action.IndexWorkflowResponse +import org.opensearch.commons.alerting.model.Workflow +import org.opensearch.core.xcontent.ToXContent +import org.opensearch.core.xcontent.XContentParser +import org.opensearch.index.seqno.SequenceNumbers +import org.opensearch.rest.BaseRestHandler +import org.opensearch.rest.BaseRestHandler.RestChannelConsumer +import org.opensearch.rest.BytesRestResponse +import org.opensearch.rest.RestChannel +import org.opensearch.rest.RestHandler +import org.opensearch.rest.RestRequest +import org.opensearch.rest.RestResponse +import org.opensearch.rest.RestStatus +import org.opensearch.rest.action.RestResponseListener +import java.io.IOException +import java.time.Instant + +/** + * Rest handlers to create and update workflows. + */ +class RestIndexWorkflowAction : BaseRestHandler() { + + override fun getName(): String { + return "index_workflow_action" + } + + override fun routes(): List { + return listOf( + RestHandler.Route(RestRequest.Method.POST, AlertingPlugin.WORKFLOW_BASE_URI), + RestHandler.Route( + RestRequest.Method.PUT, + "${AlertingPlugin.WORKFLOW_BASE_URI}/{workflowID}" + ) + ) + } + + @Throws(IOException::class) + override fun prepareRequest(request: RestRequest, client: NodeClient): RestChannelConsumer { + val id = request.param("workflowID", Workflow.NO_ID) + if (request.method() == RestRequest.Method.PUT && Workflow.NO_ID == id) { + throw AlertingException.wrap(IllegalArgumentException("Missing workflow ID")) + } + + // Validate request by parsing JSON to Monitor + val xcp = request.contentParser() + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.nextToken(), xcp) + val workflow = Workflow.parse(xcp, id).copy(lastUpdateTime = Instant.now()) + val rbacRoles = request.contentParser().map()["rbac_roles"] as List? + + val seqNo = request.paramAsLong(IF_SEQ_NO, SequenceNumbers.UNASSIGNED_SEQ_NO) + val primaryTerm = request.paramAsLong(IF_PRIMARY_TERM, SequenceNumbers.UNASSIGNED_PRIMARY_TERM) + val refreshPolicy = if (request.hasParam(REFRESH)) { + WriteRequest.RefreshPolicy.parse(request.param(REFRESH)) + } else { + WriteRequest.RefreshPolicy.IMMEDIATE + } + val workflowRequest = + IndexWorkflowRequest(id, seqNo, primaryTerm, refreshPolicy, request.method(), workflow, rbacRoles) + + return RestChannelConsumer { channel -> + client.execute(AlertingActions.INDEX_WORKFLOW_ACTION_TYPE, workflowRequest, indexMonitorResponse(channel, request.method())) + } + } + + private fun indexMonitorResponse(channel: RestChannel, restMethod: RestRequest.Method): RestResponseListener { + return object : RestResponseListener(channel) { + @Throws(Exception::class) + override fun buildResponse(response: IndexWorkflowResponse): RestResponse { + var returnStatus = RestStatus.CREATED + if (restMethod == RestRequest.Method.PUT) + returnStatus = RestStatus.OK + + val restResponse = + BytesRestResponse(returnStatus, response.toXContent(channel.newBuilder(), ToXContent.EMPTY_PARAMS)) + if (returnStatus == RestStatus.CREATED) { + val location = "${AlertingPlugin.WORKFLOW_BASE_URI}/${response.id}" + restResponse.addHeader("Location", location) + } + return restResponse + } + } + } +} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/service/DeleteMonitorService.kt b/alerting/src/main/kotlin/org/opensearch/alerting/service/DeleteMonitorService.kt new file mode 100644 index 000000000..b4a92c100 --- /dev/null +++ b/alerting/src/main/kotlin/org/opensearch/alerting/service/DeleteMonitorService.kt @@ -0,0 +1,172 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.alerting.service + +import kotlinx.coroutines.CoroutineName +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.SupervisorJob +import org.apache.logging.log4j.LogManager +import org.apache.lucene.search.join.ScoreMode +import org.opensearch.action.ActionListener +import org.opensearch.action.admin.indices.delete.DeleteIndexRequest +import org.opensearch.action.admin.indices.exists.indices.IndicesExistsRequest +import org.opensearch.action.admin.indices.exists.indices.IndicesExistsResponse +import org.opensearch.action.delete.DeleteRequest +import org.opensearch.action.delete.DeleteResponse +import org.opensearch.action.search.SearchRequest +import org.opensearch.action.search.SearchResponse +import org.opensearch.action.support.IndicesOptions +import org.opensearch.action.support.WriteRequest.RefreshPolicy +import org.opensearch.action.support.master.AcknowledgedResponse +import org.opensearch.alerting.MonitorMetadataService +import org.opensearch.alerting.opensearchapi.suspendUntil +import org.opensearch.alerting.transport.TransportDeleteWorkflowAction.Companion.WORKFLOW_DELEGATE_PATH +import org.opensearch.alerting.transport.TransportDeleteWorkflowAction.Companion.WORKFLOW_MONITOR_PATH +import org.opensearch.alerting.util.AlertingException +import org.opensearch.client.Client +import org.opensearch.commons.alerting.action.DeleteMonitorResponse +import org.opensearch.commons.alerting.model.Monitor +import org.opensearch.commons.alerting.model.ScheduledJob +import org.opensearch.index.query.QueryBuilders +import org.opensearch.index.reindex.BulkByScrollResponse +import org.opensearch.index.reindex.DeleteByQueryAction +import org.opensearch.index.reindex.DeleteByQueryRequestBuilder +import org.opensearch.search.builder.SearchSourceBuilder +import kotlin.coroutines.resume +import kotlin.coroutines.resumeWithException +import kotlin.coroutines.suspendCoroutine + +/** + * Component used when deleting the monitors + */ +object DeleteMonitorService : + CoroutineScope by CoroutineScope(SupervisorJob() + Dispatchers.Default + CoroutineName("WorkflowMetadataService")) { + private val log = LogManager.getLogger(this.javaClass) + + private lateinit var client: Client + + fun initialize( + client: Client + ) { + DeleteMonitorService.client = client + } + + /** + * Deletes the monitor, docLevelQueries and monitor metadata + * @param monitor monitor to be deleted + * @param refreshPolicy + */ + suspend fun deleteMonitor(monitor: Monitor, refreshPolicy: RefreshPolicy): DeleteMonitorResponse { + val deleteResponse = deleteMonitor(monitor.id, refreshPolicy) + deleteDocLevelMonitorQueriesAndIndices(monitor) + deleteMetadata(monitor) + return DeleteMonitorResponse(deleteResponse.id, deleteResponse.version) + } + + private suspend fun deleteMonitor(monitorId: String, refreshPolicy: RefreshPolicy): DeleteResponse { + val deleteMonitorRequest = DeleteRequest(ScheduledJob.SCHEDULED_JOBS_INDEX, monitorId) + .setRefreshPolicy(refreshPolicy) + return client.suspendUntil { delete(deleteMonitorRequest, it) } + } + + private suspend fun deleteMetadata(monitor: Monitor) { + val deleteRequest = DeleteRequest(ScheduledJob.SCHEDULED_JOBS_INDEX, "${monitor.id}-metadata") + val deleteResponse: DeleteResponse = client.suspendUntil { delete(deleteRequest, it) } + } + + private suspend fun deleteDocLevelMonitorQueriesAndIndices(monitor: Monitor) { + val metadata = MonitorMetadataService.getMetadata(monitor) + metadata?.sourceToQueryIndexMapping?.forEach { (_, queryIndex) -> + + val indicesExistsResponse: IndicesExistsResponse = + client.suspendUntil { + client.admin().indices().exists(IndicesExistsRequest(queryIndex), it) + } + if (indicesExistsResponse.isExists == false) { + return + } + // Check if there's any queries from other monitors in this queryIndex, + // to avoid unnecessary doc deletion, if we could just delete index completely + val searchResponse: SearchResponse = client.suspendUntil { + search( + SearchRequest(queryIndex).source( + SearchSourceBuilder() + .size(0) + .query( + QueryBuilders.boolQuery().mustNot( + QueryBuilders.matchQuery("monitor_id", monitor.id) + ) + ) + ).indicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN_HIDDEN), + it + ) + } + if (searchResponse.hits.totalHits.value == 0L) { + val ack: AcknowledgedResponse = client.suspendUntil { + client.admin().indices().delete( + DeleteIndexRequest(queryIndex).indicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN_HIDDEN), + it + ) + } + if (ack.isAcknowledged == false) { + log.error("Deletion of concrete queryIndex:$queryIndex is not ack'd!") + } + } else { + // Delete all queries added by this monitor + val response: BulkByScrollResponse = suspendCoroutine { cont -> + DeleteByQueryRequestBuilder(client, DeleteByQueryAction.INSTANCE) + .source(queryIndex) + .filter(QueryBuilders.matchQuery("monitor_id", monitor.id)) + .refresh(true) + .execute( + object : ActionListener { + override fun onResponse(response: BulkByScrollResponse) = cont.resume(response) + override fun onFailure(t: Exception) = cont.resumeWithException(t) + } + ) + } + } + } + } + + /** + * Checks if the monitor is part of the workflow + * + * @param monitorId id of monitor that is checked if it is a workflow delegate + */ + suspend fun monitorIsWorkflowDelegate(monitorId: String): Boolean { + val queryBuilder = QueryBuilders.nestedQuery( + WORKFLOW_DELEGATE_PATH, + QueryBuilders.boolQuery().must( + QueryBuilders.matchQuery( + WORKFLOW_MONITOR_PATH, + monitorId + ) + ), + ScoreMode.None + ) + try { + val searchRequest = SearchRequest() + .indices(ScheduledJob.SCHEDULED_JOBS_INDEX) + .source(SearchSourceBuilder().query(queryBuilder)) + + client.threadPool().threadContext.stashContext().use { + val searchResponse: SearchResponse = client.suspendUntil { search(searchRequest, it) } + if (searchResponse.hits.totalHits?.value == 0L) { + return false + } + + val workflowIds = searchResponse.hits.hits.map { it.id }.joinToString() + log.info("Monitor $monitorId can't be deleted since it belongs to $workflowIds") + return true + } + } catch (ex: Exception) { + log.error("Error getting the monitor workflows", ex) + throw AlertingException.wrap(ex) + } + } +} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/transport/SecureTransportAction.kt b/alerting/src/main/kotlin/org/opensearch/alerting/transport/SecureTransportAction.kt index 671775113..3914ce909 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/transport/SecureTransportAction.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/transport/SecureTransportAction.kt @@ -81,8 +81,7 @@ interface SecureTransportAction { actionListener.onFailure( AlertingException.wrap( OpenSearchStatusException( - "Filter by user backend roles is enabled with security disabled.", - RestStatus.FORBIDDEN + "Filter by user backend roles is enabled with security disabled.", RestStatus.FORBIDDEN ) ) ) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportDeleteMonitorAction.kt b/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportDeleteMonitorAction.kt index 0d5648cc1..a28fad768 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportDeleteMonitorAction.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportDeleteMonitorAction.kt @@ -9,26 +9,16 @@ import kotlinx.coroutines.CoroutineName import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.GlobalScope import kotlinx.coroutines.launch -import org.apache.logging.log4j.LogManager -import org.apache.lucene.search.join.ScoreMode import org.opensearch.OpenSearchStatusException import org.opensearch.action.ActionListener import org.opensearch.action.ActionRequest -import org.opensearch.action.admin.indices.delete.DeleteIndexRequest -import org.opensearch.action.admin.indices.exists.indices.IndicesExistsRequest -import org.opensearch.action.admin.indices.exists.indices.IndicesExistsResponse -import org.opensearch.action.delete.DeleteRequest -import org.opensearch.action.delete.DeleteResponse import org.opensearch.action.get.GetRequest import org.opensearch.action.get.GetResponse -import org.opensearch.action.search.SearchRequest -import org.opensearch.action.search.SearchResponse import org.opensearch.action.support.ActionFilters import org.opensearch.action.support.HandledTransportAction -import org.opensearch.action.support.IndicesOptions -import org.opensearch.action.support.master.AcknowledgedResponse -import org.opensearch.alerting.MonitorMetadataService +import org.opensearch.action.support.WriteRequest.RefreshPolicy import org.opensearch.alerting.opensearchapi.suspendUntil +import org.opensearch.alerting.service.DeleteMonitorService import org.opensearch.alerting.settings.AlertingSettings import org.opensearch.alerting.util.AlertingException import org.opensearch.client.Client @@ -43,23 +33,12 @@ import org.opensearch.commons.alerting.action.DeleteMonitorRequest import org.opensearch.commons.alerting.action.DeleteMonitorResponse import org.opensearch.commons.alerting.model.Monitor import org.opensearch.commons.alerting.model.ScheduledJob -import org.opensearch.commons.alerting.model.Workflow import org.opensearch.commons.authuser.User import org.opensearch.commons.utils.recreateObject import org.opensearch.core.xcontent.NamedXContentRegistry -import org.opensearch.index.query.QueryBuilders -import org.opensearch.index.reindex.BulkByScrollResponse -import org.opensearch.index.reindex.DeleteByQueryAction -import org.opensearch.index.reindex.DeleteByQueryRequestBuilder import org.opensearch.rest.RestStatus -import org.opensearch.search.builder.SearchSourceBuilder import org.opensearch.tasks.Task import org.opensearch.transport.TransportService -import kotlin.coroutines.resume -import kotlin.coroutines.resumeWithException -import kotlin.coroutines.suspendCoroutine - -private val log = LogManager.getLogger(TransportDeleteMonitorAction::class.java) class TransportDeleteMonitorAction @Inject constructor( transportService: TransportService, @@ -86,41 +65,46 @@ class TransportDeleteMonitorAction @Inject constructor( val transformedRequest = request as? DeleteMonitorRequest ?: recreateObject(request) { DeleteMonitorRequest(it) } val user = readUserFromThreadContext(client) - val deleteRequest = DeleteRequest(ScheduledJob.SCHEDULED_JOBS_INDEX, transformedRequest.monitorId) - .setRefreshPolicy(transformedRequest.refreshPolicy) if (!validateUserBackendRoles(user, actionListener)) { return } GlobalScope.launch(Dispatchers.IO + CoroutineName("DeleteMonitorAction")) { - DeleteMonitorHandler(client, actionListener, deleteRequest, user, transformedRequest.monitorId).resolveUserAndStart() + DeleteMonitorHandler( + client, + actionListener, + user, + transformedRequest.monitorId + ).resolveUserAndStart(transformedRequest.refreshPolicy) } } inner class DeleteMonitorHandler( private val client: Client, private val actionListener: ActionListener, - private val deleteRequest: DeleteRequest, private val user: User?, private val monitorId: String ) { - suspend fun resolveUserAndStart() { + suspend fun resolveUserAndStart(refreshPolicy: RefreshPolicy) { try { val monitor = getMonitor() val canDelete = user == null || !doFilterForUser(user) || checkUserPermissionsWithResource(user, monitor.user, actionListener, "monitor", monitorId) - if (monitorIsWorkflowDelegate(monitor.id)) { + if (DeleteMonitorService.monitorIsWorkflowDelegate(monitor.id)) { actionListener.onFailure( - AlertingException("Monitor can't be deleted because it is a part of workflow(s)", RestStatus.FORBIDDEN, IllegalStateException()) + AlertingException( + "Monitor can't be deleted because it is a part of workflow(s)", + RestStatus.FORBIDDEN, + IllegalStateException() + ) ) } else if (canDelete) { - val deleteResponse = deleteMonitor(monitor) - deleteDocLevelMonitorQueriesAndIndices(monitor) - deleteMetadata(monitor) - actionListener.onResponse(DeleteMonitorResponse(deleteResponse.id, deleteResponse.version)) + actionListener.onResponse( + DeleteMonitorService.deleteMonitor(monitor, refreshPolicy) + ) } else { actionListener.onFailure( AlertingException("Not allowed to delete this monitor!", RestStatus.FORBIDDEN, IllegalStateException()) @@ -131,43 +115,6 @@ class TransportDeleteMonitorAction @Inject constructor( } } - /** - * Checks if the monitor is part of the workflow - * - * @param monitorId id of monitor that is checked if it is a workflow delegate - */ - private suspend fun monitorIsWorkflowDelegate(monitorId: String): Boolean { - val queryBuilder = QueryBuilders.nestedQuery( - Workflow.WORKFLOW_DELEGATE_PATH, - QueryBuilders.boolQuery().must( - QueryBuilders.matchQuery( - Workflow.WORKFLOW_MONITOR_PATH, - monitorId - ) - ), - ScoreMode.None - ) - try { - val searchRequest = SearchRequest() - .indices(ScheduledJob.SCHEDULED_JOBS_INDEX) - .source(SearchSourceBuilder().query(queryBuilder)) - - client.threadPool().threadContext.stashContext().use { - val searchResponse: SearchResponse = client.suspendUntil { search(searchRequest, it) } - if (searchResponse.hits.totalHits?.value == 0L) { - return false - } - - val workflowIds = searchResponse.hits.hits.map { it.id }.joinToString() - log.info("Monitor $monitorId can't be deleted since it belongs to $workflowIds") - return true - } - } catch (ex: Exception) { - log.error("Error getting the monitor workflows", ex) - throw AlertingException.wrap(ex) - } - } - private suspend fun getMonitor(): Monitor { val getRequest = GetRequest(ScheduledJob.SCHEDULED_JOBS_INDEX, monitorId) @@ -187,70 +134,5 @@ class TransportDeleteMonitorAction @Inject constructor( ) return ScheduledJob.parse(xcp, getResponse.id, getResponse.version) as Monitor } - - private suspend fun deleteMonitor(monitor: Monitor): DeleteResponse { - return client.suspendUntil { delete(deleteRequest, it) } - } - - private suspend fun deleteMetadata(monitor: Monitor) { - val deleteRequest = DeleteRequest(ScheduledJob.SCHEDULED_JOBS_INDEX, "${monitor.id}-metadata") - val deleteResponse: DeleteResponse = client.suspendUntil { delete(deleteRequest, it) } - } - - private suspend fun deleteDocLevelMonitorQueriesAndIndices(monitor: Monitor) { - val clusterState = clusterService.state() - val metadata = MonitorMetadataService.getMetadata(monitor) - metadata?.sourceToQueryIndexMapping?.forEach { (_, queryIndex) -> - - val indicesExistsResponse: IndicesExistsResponse = - client.suspendUntil { - client.admin().indices().exists(IndicesExistsRequest(queryIndex), it) - } - if (indicesExistsResponse.isExists == false) { - return - } - // Check if there's any queries from other monitors in this queryIndex, - // to avoid unnecessary doc deletion, if we could just delete index completely - val searchResponse: SearchResponse = client.suspendUntil { - search( - SearchRequest(queryIndex).source( - SearchSourceBuilder() - .size(0) - .query( - QueryBuilders.boolQuery().mustNot( - QueryBuilders.matchQuery("monitor_id", monitorId) - ) - ) - ).indicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN_HIDDEN), - it - ) - } - if (searchResponse.hits.totalHits.value == 0L) { - val ack: AcknowledgedResponse = client.suspendUntil { - client.admin().indices().delete( - DeleteIndexRequest(queryIndex).indicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN_HIDDEN), - it - ) - } - if (ack.isAcknowledged == false) { - log.error("Deletion of concrete queryIndex:$queryIndex is not ack'd!") - } - } else { - // Delete all queries added by this monitor - val response: BulkByScrollResponse = suspendCoroutine { cont -> - DeleteByQueryRequestBuilder(client, DeleteByQueryAction.INSTANCE) - .source(queryIndex) - .filter(QueryBuilders.matchQuery("monitor_id", monitorId)) - .refresh(true) - .execute( - object : ActionListener { - override fun onResponse(response: BulkByScrollResponse) = cont.resume(response) - override fun onFailure(t: Exception) = cont.resumeWithException(t) - } - ) - } - } - } - } } } diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportDeleteWorkflowAction.kt b/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportDeleteWorkflowAction.kt index 0bc223cfa..c8249d9ec 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportDeleteWorkflowAction.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportDeleteWorkflowAction.kt @@ -10,6 +10,7 @@ import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.launch import org.apache.logging.log4j.LogManager import org.apache.lucene.search.join.ScoreMode +import org.opensearch.OpenSearchException import org.opensearch.OpenSearchStatusException import org.opensearch.action.ActionListener import org.opensearch.action.ActionRequest @@ -22,27 +23,26 @@ import org.opensearch.action.search.SearchResponse import org.opensearch.action.support.ActionFilters import org.opensearch.action.support.HandledTransportAction import org.opensearch.action.support.WriteRequest.RefreshPolicy -import org.opensearch.alerting.opensearchapi.InjectorContextElement +import org.opensearch.alerting.DeleteMonitorService +import org.opensearch.alerting.model.MonitorMetadata +import org.opensearch.alerting.model.WorkflowMetadata import org.opensearch.alerting.opensearchapi.addFilter import org.opensearch.alerting.opensearchapi.suspendUntil -import org.opensearch.alerting.opensearchapi.withClosableContext +import org.opensearch.alerting.service.DeleteMonitorService import org.opensearch.alerting.settings.AlertingSettings import org.opensearch.alerting.util.AlertingException import org.opensearch.client.Client -import org.opensearch.client.node.NodeClient import org.opensearch.cluster.service.ClusterService import org.opensearch.common.inject.Inject import org.opensearch.common.settings.Settings import org.opensearch.common.xcontent.LoggingDeprecationHandler import org.opensearch.common.xcontent.XContentHelper import org.opensearch.common.xcontent.XContentType -import org.opensearch.commons.alerting.AlertingPluginInterface import org.opensearch.commons.alerting.action.AlertingActions -import org.opensearch.commons.alerting.action.DeleteMonitorRequest -import org.opensearch.commons.alerting.action.DeleteMonitorResponse import org.opensearch.commons.alerting.action.DeleteWorkflowRequest import org.opensearch.commons.alerting.action.DeleteWorkflowResponse import org.opensearch.commons.alerting.model.CompositeInput +import org.opensearch.commons.alerting.model.Monitor import org.opensearch.commons.alerting.model.ScheduledJob import org.opensearch.commons.alerting.model.Workflow import org.opensearch.commons.authuser.User @@ -51,11 +51,13 @@ import org.opensearch.core.xcontent.NamedXContentRegistry import org.opensearch.core.xcontent.XContentParser import org.opensearch.index.IndexNotFoundException import org.opensearch.index.query.QueryBuilders +import org.opensearch.index.reindex.BulkByScrollResponse +import org.opensearch.index.reindex.DeleteByQueryAction +import org.opensearch.index.reindex.DeleteByQueryRequestBuilder import org.opensearch.rest.RestStatus import org.opensearch.search.builder.SearchSourceBuilder import org.opensearch.tasks.Task import org.opensearch.transport.TransportService -import java.util.UUID private val scope: CoroutineScope = CoroutineScope(Dispatchers.IO) /** @@ -68,12 +70,11 @@ class TransportDeleteWorkflowAction @Inject constructor( actionFilters: ActionFilters, val clusterService: ClusterService, val settings: Settings, - val xContentRegistry: NamedXContentRegistry + val xContentRegistry: NamedXContentRegistry, ) : HandledTransportAction( AlertingActions.DELETE_WORKFLOW_ACTION_NAME, transportService, actionFilters, ::DeleteWorkflowRequest ), SecureTransportAction { - private val log = LogManager.getLogger(javaClass) @Volatile override var filterByEnabled = AlertingSettings.FILTER_BY_BACKEND_ROLES.get(settings) @@ -112,7 +113,7 @@ class TransportDeleteWorkflowAction @Inject constructor( private val deleteRequest: DeleteRequest, private val deleteDelegateMonitors: Boolean?, private val user: User?, - private val workflowId: String + private val workflowId: String, ) { suspend fun resolveUserAndStart() { try { @@ -130,13 +131,14 @@ class TransportDeleteWorkflowAction @Inject constructor( if (canDelete) { val delegateMonitorIds = (workflow.inputs[0] as CompositeInput).getMonitorIds() - + var deletableMonitors = listOf() // User can only delete the delegate monitors only in the case if all monitors can be deleted - // Partial monitor deletion is not available + // if there are monitors in this workflow that are referenced in other workflows, we cannot delete the monitors. + // We will not partially delete monitors. we delete them all or fail the request. if (deleteDelegateMonitors == true) { - val monitorIdsToBeDeleted = getDeletableDelegates(workflowId, delegateMonitorIds, user) + deletableMonitors = getDeletableDelegates(workflowId, delegateMonitorIds, user) val monitorsDiff = delegateMonitorIds.toMutableList() - monitorsDiff.removeAll(monitorIdsToBeDeleted) + monitorsDiff.removeAll(deletableMonitors.map { it.id }) if (monitorsDiff.isNotEmpty()) { actionListener.onFailure( @@ -150,26 +152,33 @@ class TransportDeleteWorkflowAction @Inject constructor( } } - val deleteResponse = deleteWorkflow(workflow) + val deleteResponse = deleteWorkflow(deleteRequest) + var deleteWorkflowResponse = DeleteWorkflowResponse(deleteResponse.id, deleteResponse.version) + + val workflowMetadataId = WorkflowMetadata.getId(workflow.id) + + val metadataIdsToDelete = mutableListOf(workflowMetadataId) + if (deleteDelegateMonitors == true) { - if (user != null && filterByEnabled) { - // Un-stash the context - withClosableContext( - InjectorContextElement( - user.name.plus(UUID.randomUUID().toString()), - settings, - client.threadPool().threadContext, - user.roles, - user - ) - ) { - deleteMonitors(delegateMonitorIds, RefreshPolicy.IMMEDIATE) - } - } else { - deleteMonitors(delegateMonitorIds, RefreshPolicy.IMMEDIATE) + val failedMonitorIds = tryDeletingMonitors(deletableMonitors, RefreshPolicy.IMMEDIATE) + // Update delete workflow response + deleteWorkflowResponse.nonDeletedMonitors = failedMonitorIds + // Delete monitors workflow metadata + // Monitor metadata will be in workflowId-monitorId-metadata format + metadataIdsToDelete.addAll(deletableMonitors.map { MonitorMetadata.getId(it, workflowMetadataId) }) + } + try { + // Delete the monitors workflow metadata + val deleteMonitorWorkflowMetadataResponse: BulkByScrollResponse = client.suspendUntil { + DeleteByQueryRequestBuilder(this, DeleteByQueryAction.INSTANCE) + .source(ScheduledJob.SCHEDULED_JOBS_INDEX) + .filter(QueryBuilders.idsQuery().addIds(*metadataIdsToDelete.toTypedArray())) + .execute(it) } + } catch (t: Exception) { + log.error("Failed to delete delegate monitor metadata. But proceeding with workflow deletion $workflowId", t) } - actionListener.onResponse(DeleteWorkflowResponse(deleteResponse.id, deleteResponse.version)) + actionListener.onResponse(deleteWorkflowResponse) } else { actionListener.onFailure( AlertingException( @@ -188,36 +197,45 @@ class TransportDeleteWorkflowAction @Inject constructor( ) ) } else { + log.error("Failed to delete workflow $workflowId", t) actionListener.onFailure(AlertingException.wrap(t)) } } } - private suspend fun deleteMonitors(monitorIds: List, refreshPolicy: RefreshPolicy) { - if (monitorIds.isEmpty()) - return - - for (monitorId in monitorIds) { - val deleteRequest = DeleteMonitorRequest(monitorId, refreshPolicy) - val searchResponse: DeleteMonitorResponse = client.suspendUntil { - AlertingPluginInterface.deleteMonitor(this as NodeClient, deleteRequest, it) + /** + * Tries to delete the given list of the monitors. Return value contains all the monitorIds for which deletion failed + * @param monitorIds list of monitor ids to be deleted + * @param refreshPolicy + * @return list of the monitors that were not deleted + */ + private suspend fun tryDeletingMonitors(monitors: List, refreshPolicy: RefreshPolicy): List { + val nonDeletedMonitorIds = mutableListOf() + for (monitor in monitors) { + try { + DeleteMonitorService.deleteMonitor(monitor, refreshPolicy) + } catch (ex: Exception) { + log.error("failed to delete delegate monitor ${monitor.id} for $workflowId") + nonDeletedMonitorIds.add(monitor.id) } } + return nonDeletedMonitorIds } /** - * Returns lit of monitor ids belonging only to a given workflow + * Returns lit of monitor ids belonging only to a given workflow. + * if filterBy is enabled, it filters and returns only those monitors which user has permission to delete. * @param workflowIdToBeDeleted Id of the workflow that should be deleted * @param monitorIds List of delegate monitor ids (underlying monitor ids) */ - private suspend fun getDeletableDelegates(workflowIdToBeDeleted: String, monitorIds: List, user: User?): List { + private suspend fun getDeletableDelegates(workflowIdToBeDeleted: String, monitorIds: List, user: User?): List { // Retrieve monitors belonging to another workflows val queryBuilder = QueryBuilders.boolQuery().mustNot(QueryBuilders.termQuery("_id", workflowIdToBeDeleted)).filter( QueryBuilders.nestedQuery( - Workflow.WORKFLOW_DELEGATE_PATH, + WORKFLOW_DELEGATE_PATH, QueryBuilders.boolQuery().must( QueryBuilders.termsQuery( - Workflow.WORKFLOW_MONITOR_PATH, + WORKFLOW_MONITOR_PATH, monitorIds ) ), @@ -229,11 +247,6 @@ class TransportDeleteWorkflowAction @Inject constructor( .indices(ScheduledJob.SCHEDULED_JOBS_INDEX) .source(SearchSourceBuilder().query(queryBuilder)) - // Check if user can access the monitors(since the monitors could get modified later and the user might not have the backend roles to access the monitors) - if (user != null && filterByEnabled) { - addFilter(user, searchRequest.source(), "monitor.user.backend_roles.keyword") - } - val searchResponse: SearchResponse = client.suspendUntil { search(searchRequest, it) } val workflows = searchResponse.hits.hits.map { hit -> @@ -250,9 +263,35 @@ class TransportDeleteWorkflowAction @Inject constructor( } workflow.copy(id = hit.id, version = hit.version) } - val workflowMonitors = workflows.filter { it.id != workflowIdToBeDeleted }.flatMap { (it.inputs[0] as CompositeInput).getMonitorIds() }.distinct() + val workflowMonitors = workflows.flatMap { (it.inputs[0] as CompositeInput).getMonitorIds() }.distinct() // Monitors that can be deleted -> all workflow delegates - monitors belonging to different workflows - return monitorIds.minus(workflowMonitors.toSet()) + val deletableMonitorIds = monitorIds.minus(workflowMonitors.toSet()) + + // filtering further to get the list of monitors that user has permission to delete if filterby is enabled and user is not null + val query = QueryBuilders.boolQuery().filter(QueryBuilders.termsQuery("_id", deletableMonitorIds)) + val searchSource = SearchSourceBuilder().query(query) + val monitorSearchRequest = SearchRequest(ScheduledJob.SCHEDULED_JOBS_INDEX).source(searchSource) + + if (user != null && filterByEnabled) { + addFilter(user, monitorSearchRequest.source(), "monitor.user.backend_roles.keyword") + } + + val searchMonitorResponse: SearchResponse = client.suspendUntil { search(monitorSearchRequest, it) } + if (searchMonitorResponse.isTimedOut) { + throw OpenSearchException("Cannot determine that the ${ScheduledJob.SCHEDULED_JOBS_INDEX} index is healthy") + } + val deletableMonitors = mutableListOf() + for (hit in searchMonitorResponse.hits) { + XContentType.JSON.xContent().createParser( + xContentRegistry, + LoggingDeprecationHandler.INSTANCE, hit.sourceAsString + ).use { hitsParser -> + val monitor = ScheduledJob.parse(hitsParser, hit.id, hit.version) as Monitor + deletableMonitors.add(monitor) + } + } + + return deletableMonitors } private suspend fun getWorkflow(): Workflow { @@ -273,14 +312,19 @@ class TransportDeleteWorkflowAction @Inject constructor( return ScheduledJob.parse(xcp, getResponse.id, getResponse.version) as Workflow } - private suspend fun deleteWorkflow(workflow: Workflow): DeleteResponse { + private suspend fun deleteWorkflow(deleteRequest: DeleteRequest): DeleteResponse { log.debug("Deleting the workflow with id ${deleteRequest.id()}") return client.suspendUntil { delete(deleteRequest, it) } } - // TODO - use once the workflow metadata concept is introduced - private suspend fun deleteMetadata(workflow: Workflow) { - val deleteRequest = DeleteRequest(ScheduledJob.SCHEDULED_JOBS_INDEX, "${workflow.id}-metadata") + + private suspend fun deleteWorkflowMetadata(workflow: Workflow) { + val deleteRequest = DeleteRequest(ScheduledJob.SCHEDULED_JOBS_INDEX, WorkflowMetadata.getId(workflow.id)) val deleteResponse: DeleteResponse = client.suspendUntil { delete(deleteRequest, it) } } } + + companion object { + const val WORKFLOW_DELEGATE_PATH = "workflow.inputs.composite_input.sequence.delegates" + const val WORKFLOW_MONITOR_PATH = "workflow.inputs.composite_input.sequence.delegates.monitor_id" + } } diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportExecuteWorkflowAction.kt b/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportExecuteWorkflowAction.kt index bd90d569a..45c2a5cd1 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportExecuteWorkflowAction.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportExecuteWorkflowAction.kt @@ -47,7 +47,11 @@ class TransportExecuteWorkflowAction @Inject constructor( ) : HandledTransportAction( ExecuteWorkflowAction.NAME, transportService, actionFilters, ::ExecuteWorkflowRequest ) { - override fun doExecute(task: Task, execWorkflowRequest: ExecuteWorkflowRequest, actionListener: ActionListener) { + override fun doExecute( + task: Task, + execWorkflowRequest: ExecuteWorkflowRequest, + actionListener: ActionListener, + ) { val userStr = client.threadPool().threadContext.getTransient(ConfigConstants.OPENSEARCH_SECURITY_USER_INFO_THREAD_CONTEXT) log.debug("User and roles string from thread context: $userStr") val user: User? = User.parse(userStr) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportIndexWorkflowAction.kt b/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportIndexWorkflowAction.kt index a68a32ab8..60f39a063 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportIndexWorkflowAction.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportIndexWorkflowAction.kt @@ -140,7 +140,8 @@ class TransportIndexWorkflowAction @Inject constructor( ) { if (transformedRequest.rbacRoles?.stream()?.anyMatch { !user.backendRoles.contains(it) } == true) { log.error( - "User specified backend roles, ${transformedRequest.rbacRoles}, that they don' have access to. User backend roles: ${user.backendRoles}" + "User specified backend roles, ${transformedRequest.rbacRoles}, " + + "that they don' have access to. User backend roles: ${user.backendRoles}" ) actionListener.onFailure( AlertingException.wrap( @@ -153,7 +154,8 @@ class TransportIndexWorkflowAction @Inject constructor( return } else if (transformedRequest.rbacRoles?.isEmpty() == true) { log.error( - "Non-admin user are not allowed to specify an empty set of backend roles. Please don't pass in the parameter or pass in at least one backend role." + "Non-admin user are not allowed to specify an empty set of backend roles. " + + "Please don't pass in the parameter or pass in at least one backend role." ) actionListener.onFailure( AlertingException.wrap( @@ -543,7 +545,9 @@ class TransportIndexWorkflowAction @Inject constructor( val chainedMonitorIndices = getMonitorIndices(chainedFindingMonitor) if (!delegateMonitorIndices.equalsIgnoreOrder(chainedMonitorIndices)) { - throw AlertingException.wrap(IllegalArgumentException("Delegate monitor and it's chained finding monitor must query the same indices")) + throw AlertingException.wrap( + IllegalArgumentException("Delegate monitor and it's chained finding monitor must query the same indices") + ) } } } @@ -604,7 +608,7 @@ class TransportIndexWorkflowAction @Inject constructor( val searchSource = SearchSourceBuilder().query(query) val searchRequest = SearchRequest(SCHEDULED_JOBS_INDEX).source(searchSource) - if (user != null && filterByEnabled) { + if (user != null && !isAdmin(user) && filterByEnabled) { addFilter(user, searchRequest.source(), "monitor.user.backend_roles.keyword") } @@ -703,7 +707,11 @@ class TransportIndexWorkflowAction @Inject constructor( val indices = mutableListOf() val searchInputs = - monitors.flatMap { monitor -> monitor.inputs.filter { it.name() == SearchInput.SEARCH_FIELD || it.name() == DocLevelMonitorInput.DOC_LEVEL_INPUT_FIELD } } + monitors.flatMap { monitor -> + monitor.inputs.filter { + it.name() == SearchInput.SEARCH_FIELD || it.name() == DocLevelMonitorInput.DOC_LEVEL_INPUT_FIELD + } + } searchInputs.forEach { val inputIndices = if (it.name() == SearchInput.SEARCH_FIELD) (it as SearchInput).indices else (it as DocLevelMonitorInput).indices diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/workflow/CompositeWorkflowRunner.kt b/alerting/src/main/kotlin/org/opensearch/alerting/workflow/CompositeWorkflowRunner.kt index 006cd6e5b..013ee3056 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/workflow/CompositeWorkflowRunner.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/workflow/CompositeWorkflowRunner.kt @@ -6,6 +6,7 @@ package org.opensearch.alerting.workflow import org.apache.logging.log4j.LogManager +import org.opensearch.ExceptionsHelper import org.opensearch.alerting.BucketLevelMonitorRunner import org.opensearch.alerting.DocumentLevelMonitorRunner import org.opensearch.alerting.MonitorRunnerExecutionContext @@ -83,8 +84,14 @@ object CompositeWorkflowRunner : WorkflowRunner() { try { indexToDocIds = monitorCtx.workflowService!!.getFindingDocIdsByExecutionId(chainedMonitor, executionId) } catch (e: Exception) { - logger.error("Failed to execute workflow. Error: ${e.message}", e) - return workflowResult.copy(error = AlertingException.wrap(e)) + val unwrappedException = ExceptionsHelper.unwrapCause(e) as Exception + // If it is not IndexNotFound exception return the result + if (unwrappedException.message?.contains("Configured indices are not found") == false) { + logger.error("Failed to execute workflow. Error: ${e.message}", e) + return workflowResult.copy(error = AlertingException.wrap(e)) + } + // Log that finding index is not found and proceed with the execution + logger.error("Finding index ${chainedMonitor.dataSources.findingsIndex} doesn't exist") } } diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/AccessRoles.kt b/alerting/src/test/kotlin/org/opensearch/alerting/AccessRoles.kt index d3b73779c..7f415a8ac 100644 --- a/alerting/src/test/kotlin/org/opensearch/alerting/AccessRoles.kt +++ b/alerting/src/test/kotlin/org/opensearch/alerting/AccessRoles.kt @@ -5,6 +5,9 @@ package org.opensearch.alerting +import org.opensearch.alerting.action.ExecuteWorkflowAction +import org.opensearch.commons.alerting.action.AlertingActions + val ALL_ACCESS_ROLE = "all_access" val READALL_AND_MONITOR_ROLE = "readall_and_monitor" val ALERTING_FULL_ACCESS_ROLE = "alerting_full_access" @@ -16,11 +19,15 @@ val ALERTING_GET_EMAIL_GROUP_ACCESS = "alerting_get_email_group_access" val ALERTING_SEARCH_EMAIL_GROUP_ACCESS = "alerting_search_email_group_access" val ALERTING_INDEX_MONITOR_ACCESS = "alerting_index_monitor_access" val ALERTING_GET_MONITOR_ACCESS = "alerting_get_monitor_access" +val ALERTING_GET_WORKFLOW_ACCESS = "alerting_get_workflow_access" +val ALERTING_DELETE_WORKFLOW_ACCESS = "alerting_delete_workflow_access" val ALERTING_SEARCH_MONITOR_ONLY_ACCESS = "alerting_search_monitor_access" val ALERTING_EXECUTE_MONITOR_ACCESS = "alerting_execute_monitor_access" +val ALERTING_EXECUTE_WORKFLOW_ACCESS = "alerting_execute_workflow_access" val ALERTING_DELETE_MONITOR_ACCESS = "alerting_delete_monitor_access" val ALERTING_GET_DESTINATION_ACCESS = "alerting_get_destination_access" val ALERTING_GET_ALERTS_ACCESS = "alerting_get_alerts_access" +val ALERTING_INDEX_WORKFLOW_ACCESS = "alerting_index_workflow_access" val ROLE_TO_PERMISSION_MAPPING = mapOf( ALERTING_NO_ACCESS_ROLE to "", @@ -30,9 +37,13 @@ val ROLE_TO_PERMISSION_MAPPING = mapOf( ALERTING_SEARCH_EMAIL_GROUP_ACCESS to "cluster:admin/opendistro/alerting/destination/email_group/search", ALERTING_INDEX_MONITOR_ACCESS to "cluster:admin/opendistro/alerting/monitor/write", ALERTING_GET_MONITOR_ACCESS to "cluster:admin/opendistro/alerting/monitor/get", + ALERTING_GET_WORKFLOW_ACCESS to AlertingActions.GET_WORKFLOW_ACTION_NAME, ALERTING_SEARCH_MONITOR_ONLY_ACCESS to "cluster:admin/opendistro/alerting/monitor/search", ALERTING_EXECUTE_MONITOR_ACCESS to "cluster:admin/opendistro/alerting/monitor/execute", + ALERTING_EXECUTE_WORKFLOW_ACCESS to ExecuteWorkflowAction.NAME, ALERTING_DELETE_MONITOR_ACCESS to "cluster:admin/opendistro/alerting/monitor/delete", ALERTING_GET_DESTINATION_ACCESS to "cluster:admin/opendistro/alerting/destination/get", - ALERTING_GET_ALERTS_ACCESS to "cluster:admin/opendistro/alerting/alerts/get" + ALERTING_GET_ALERTS_ACCESS to "cluster:admin/opendistro/alerting/alerts/get", + ALERTING_INDEX_WORKFLOW_ACCESS to AlertingActions.INDEX_WORKFLOW_ACTION_NAME, + ALERTING_DELETE_WORKFLOW_ACCESS to AlertingActions.DELETE_WORKFLOW_ACTION_NAME ) diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/AlertingRestTestCase.kt b/alerting/src/test/kotlin/org/opensearch/alerting/AlertingRestTestCase.kt index 07c115b95..2827d5b3b 100644 --- a/alerting/src/test/kotlin/org/opensearch/alerting/AlertingRestTestCase.kt +++ b/alerting/src/test/kotlin/org/opensearch/alerting/AlertingRestTestCase.kt @@ -55,6 +55,7 @@ import org.opensearch.commons.alerting.model.Monitor import org.opensearch.commons.alerting.model.QueryLevelTrigger import org.opensearch.commons.alerting.model.ScheduledJob import org.opensearch.commons.alerting.model.SearchInput +import org.opensearch.commons.alerting.model.Workflow import org.opensearch.commons.alerting.util.string import org.opensearch.core.xcontent.NamedXContentRegistry import org.opensearch.core.xcontent.ToXContent @@ -70,6 +71,7 @@ import java.time.format.DateTimeFormatter import java.time.temporal.ChronoUnit import java.util.Locale import java.util.UUID +import java.util.stream.Collectors import javax.management.MBeanServerInvocationHandler import javax.management.ObjectName import javax.management.remote.JMXConnectorFactory @@ -156,6 +158,29 @@ abstract class AlertingRestTestCase : ODFERestTestCase() { return response } + protected fun deleteWorkflow(workflow: Workflow, deleteDelegates: Boolean = false, refresh: Boolean = true): Response { + val response = client().makeRequest( + "DELETE", + "$WORKFLOW_ALERTING_BASE_URI/${workflow.id}?refresh=$refresh&deleteDelegateMonitors=$deleteDelegates", + emptyMap(), + workflow.toHttpEntity() + ) + assertEquals("Unable to delete a workflow", RestStatus.OK, response.restStatus()) + return response + } + + protected fun deleteWorkflowWithClient(client: RestClient, workflow: Workflow, deleteDelegates: Boolean = false, refresh: Boolean = true): Response { + val response = client.makeRequest( + "DELETE", + "$WORKFLOW_ALERTING_BASE_URI/${workflow.id}?refresh=$refresh&deleteDelegateMonitors=$deleteDelegates", + emptyMap(), + workflow.toHttpEntity() + ) + assertEquals("Unable to delete a workflow", RestStatus.OK, response.restStatus()) + + return response + } + /** * Destinations are now deprecated in favor of the Notification plugin's configs. * This method should only be used for checking legacy behavior/Notification migration scenarios. @@ -529,6 +554,19 @@ abstract class AlertingRestTestCase : ODFERestTestCase() { return getMonitor(monitorId = monitor.id) } + @Suppress("UNCHECKED_CAST") + protected fun updateWorkflow(workflow: Workflow, refresh: Boolean = false): Workflow { + val response = client().makeRequest( + "PUT", + "${workflow.relativeUrl()}?refresh=$refresh", + emptyMap(), + workflow.toHttpEntity() + ) + assertEquals("Unable to update a workflow", RestStatus.OK, response.restStatus()) + assertUserNull(response.asMap()["workflow"] as Map) + return getWorkflow(workflowId = workflow.id) + } + protected fun updateMonitorWithClient( client: RestClient, monitor: Monitor, @@ -546,6 +584,23 @@ abstract class AlertingRestTestCase : ODFERestTestCase() { return getMonitor(monitorId = monitor.id) } + protected fun updateWorkflowWithClient( + client: RestClient, + workflow: Workflow, + rbacRoles: List = emptyList(), + refresh: Boolean = true + ): Workflow { + val response = client.makeRequest( + "PUT", + "${workflow.relativeUrl()}?refresh=$refresh", + emptyMap(), + createWorkflowEntityWithBackendRoles(workflow, rbacRoles) + ) + assertEquals("Unable to update a workflow", RestStatus.OK, response.restStatus()) + assertUserNull(response.asMap()["workflow"] as Map) + return getWorkflow(workflowId = workflow.id) + } + protected fun getMonitor(monitorId: String, header: BasicHeader = BasicHeader(HttpHeaders.CONTENT_TYPE, "application/json")): Monitor { val response = client().makeRequest("GET", "$ALERTING_BASE_URI/$monitorId", null, header) assertEquals("Unable to get monitor $monitorId", RestStatus.OK, response.restStatus()) @@ -719,10 +774,18 @@ abstract class AlertingRestTestCase : ODFERestTestCase() { return executeMonitor(client(), monitorId, params) } + protected fun executeWorkflow(workflowId: String, params: Map = mutableMapOf()): Response { + return executeWorkflow(client(), workflowId, params) + } + protected fun executeMonitor(client: RestClient, monitorId: String, params: Map = mutableMapOf()): Response { return client.makeRequest("POST", "$ALERTING_BASE_URI/$monitorId/_execute", params) } + protected fun executeWorkflow(client: RestClient, workflowId: String, params: Map = mutableMapOf()): Response { + return client.makeRequest("POST", "$WORKFLOW_ALERTING_BASE_URI/$workflowId/_execute", params) + } + protected fun executeMonitor(monitor: Monitor, params: Map = mapOf()): Response { return executeMonitor(client(), monitor, params) } @@ -1206,6 +1269,37 @@ abstract class AlertingRestTestCase : ODFERestTestCase() { client().performRequest(request) } + private fun createCustomIndexRole(name: String, index: String, clusterPermissions: List) { + val request = Request("PUT", "/_plugins/_security/api/roles/$name") + + val clusterPermissionsStr = + clusterPermissions.stream().map { p: String? -> "\"" + p + "\"" }.collect( + Collectors.joining(",") + ) + + var entity = "{\n" + + "\"cluster_permissions\": [\n" + + "$clusterPermissionsStr\n" + + "],\n" + + "\"index_permissions\": [\n" + + "{\n" + + "\"index_patterns\": [\n" + + "\"$index\"\n" + + "],\n" + + "\"dls\": \"\",\n" + + "\"fls\": [],\n" + + "\"masked_fields\": [],\n" + + "\"allowed_actions\": [\n" + + "\"crud\"\n" + + "]\n" + + "}\n" + + "],\n" + + "\"tenant_permissions\": []\n" + + "}" + request.setJsonEntity(entity) + client().performRequest(request) + } + fun createIndexRoleWithDocLevelSecurity(name: String, index: String, dlsQuery: String, clusterPermissions: String? = "") { val request = Request("PUT", "/_plugins/_security/api/roles/$name") var entity = "{\n" + @@ -1231,6 +1325,36 @@ abstract class AlertingRestTestCase : ODFERestTestCase() { client().performRequest(request) } + fun createIndexRoleWithDocLevelSecurity(name: String, index: String, dlsQuery: String, clusterPermissions: List) { + val clusterPermissionsStr = + clusterPermissions.stream().map { p: String -> "\"" + getClusterPermissionsFromCustomRole(p) + "\"" }.collect( + Collectors.joining(",") + ) + + val request = Request("PUT", "/_plugins/_security/api/roles/$name") + var entity = "{\n" + + "\"cluster_permissions\": [\n" + + "$clusterPermissionsStr\n" + + "],\n" + + "\"index_permissions\": [\n" + + "{\n" + + "\"index_patterns\": [\n" + + "\"$index\"\n" + + "],\n" + + "\"dls\": \"$dlsQuery\",\n" + + "\"fls\": [],\n" + + "\"masked_fields\": [],\n" + + "\"allowed_actions\": [\n" + + "\"crud\"\n" + + "]\n" + + "}\n" + + "],\n" + + "\"tenant_permissions\": []\n" + + "}" + request.setJsonEntity(entity) + client().performRequest(request) + } + fun createUserRolesMapping(role: String, users: Array) { val request = Request("PUT", "/_plugins/_security/api/rolesmapping/$role") val usersStr = users.joinToString { it -> "\"$it\"" } @@ -1296,6 +1420,19 @@ abstract class AlertingRestTestCase : ODFERestTestCase() { createUserRolesMapping(role, arrayOf(user)) } + fun createUserWithTestDataAndCustomRole( + user: String, + index: String, + role: String, + backendRoles: List, + clusterPermissions: List + ) { + createUser(user, user, backendRoles.toTypedArray()) + createTestIndex(index) + createCustomIndexRole(role, index, clusterPermissions) + createUserRolesMapping(role, arrayOf(user)) + } + fun createUserWithRoles( user: String, roles: List, @@ -1383,4 +1520,80 @@ abstract class AlertingRestTestCase : ODFERestTestCase() { } } } + + protected fun createRandomWorkflow(monitorIds: List, refresh: Boolean = false): Workflow { + val workflow = randomWorkflow(monitorIds = monitorIds) + return createWorkflow(workflow, refresh) + } + + private fun createWorkflowEntityWithBackendRoles(workflow: Workflow, rbacRoles: List?): HttpEntity { + if (rbacRoles == null) { + return workflow.toHttpEntity() + } + val temp = workflow.toJsonString() + val toReplace = temp.lastIndexOf("}") + val rbacString = rbacRoles.joinToString { "\"$it\"" } + val jsonString = temp.substring(0, toReplace) + ", \"rbac_roles\": [$rbacString] }" + return StringEntity(jsonString, ContentType.APPLICATION_JSON) + } + + protected fun createWorkflowWithClient( + client: RestClient, + workflow: Workflow, + rbacRoles: List? = null, + refresh: Boolean = true + ): Workflow { + val response = client.makeRequest( + "POST", "$WORKFLOW_ALERTING_BASE_URI?refresh=$refresh", emptyMap(), + createWorkflowEntityWithBackendRoles(workflow, rbacRoles) + ) + assertEquals("Unable to create a new monitor", RestStatus.CREATED, response.restStatus()) + + val workflowJson = JsonXContent.jsonXContent.createParser( + NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, + response.entity.content + ).map() + assertUserNull(workflowJson as HashMap) + return workflow.copy(id = workflowJson["_id"] as String) + } + + protected fun createWorkflow(workflow: Workflow, refresh: Boolean = true): Workflow { + return createWorkflowWithClient(client(), workflow, emptyList(), refresh) + } + + protected fun Workflow.toHttpEntity(): HttpEntity { + return StringEntity(toJsonString(), APPLICATION_JSON) + } + + private fun Workflow.toJsonString(): String { + val builder = XContentFactory.jsonBuilder() + return shuffleXContent(toXContent(builder, ToXContent.EMPTY_PARAMS)).string() + } + + protected fun getWorkflow(workflowId: String, header: BasicHeader = BasicHeader(HttpHeaders.CONTENT_TYPE, "application/json")): Workflow { + val response = client().makeRequest("GET", "$WORKFLOW_ALERTING_BASE_URI/$workflowId", null, header) + assertEquals("Unable to get workflow $workflowId", RestStatus.OK, response.restStatus()) + + val parser = createParser(XContentType.JSON.xContent(), response.entity.content) + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser) + + lateinit var id: String + var version: Long = 0 + lateinit var workflow: Workflow + + while (parser.nextToken() != XContentParser.Token.END_OBJECT) { + parser.nextToken() + + when (parser.currentName()) { + "_id" -> id = parser.text() + "_version" -> version = parser.longValue() + "workflow" -> workflow = Workflow.parse(parser) + } + } + + assertUserNull(workflow) + return workflow.copy(id = id, version = version) + } + + protected fun Workflow.relativeUrl() = "$WORKFLOW_ALERTING_BASE_URI/$id" } diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/TestHelpers.kt b/alerting/src/test/kotlin/org/opensearch/alerting/TestHelpers.kt index d785179d1..548f67fda 100644 --- a/alerting/src/test/kotlin/org/opensearch/alerting/TestHelpers.kt +++ b/alerting/src/test/kotlin/org/opensearch/alerting/TestHelpers.kt @@ -252,7 +252,10 @@ fun randomWorkflow( enabledTime = enabledTime, workflowType = WorkflowType.COMPOSITE, user = user, - inputs = listOf(CompositeInput(Sequence(delegates))) + inputs = listOf(CompositeInput(Sequence(delegates))), + version = -1L, + schemaVersion = 0, + triggers = emptyList(), ) } @@ -275,7 +278,10 @@ fun randomWorkflowWithDelegates( enabledTime = enabledTime, workflowType = WorkflowType.COMPOSITE, user = user, - inputs = listOf(CompositeInput(Sequence(delegates))) + inputs = listOf(CompositeInput(Sequence(delegates))), + version = -1L, + schemaVersion = 0, + triggers = emptyList() ) } @@ -387,6 +393,7 @@ fun randomScript(source: String = "return " + OpenSearchRestTestCase.randomBoole val ADMIN = "admin" val ALERTING_BASE_URI = "/_plugins/_alerting/monitors" +val WORKFLOW_ALERTING_BASE_URI = "/_plugins/_alerting/workflows" val DESTINATION_BASE_URI = "/_plugins/_alerting/destinations" val LEGACY_OPENDISTRO_ALERTING_BASE_URI = "/_opendistro/_alerting/monitors" val LEGACY_OPENDISTRO_DESTINATION_BASE_URI = "/_opendistro/_alerting/destinations" diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/WorkflowMonitorIT.kt b/alerting/src/test/kotlin/org/opensearch/alerting/WorkflowMonitorIT.kt index 77515e975..343b9b45c 100644 --- a/alerting/src/test/kotlin/org/opensearch/alerting/WorkflowMonitorIT.kt +++ b/alerting/src/test/kotlin/org/opensearch/alerting/WorkflowMonitorIT.kt @@ -5,7 +5,9 @@ package org.opensearch.alerting +import org.opensearch.alerting.model.WorkflowMetadata import org.opensearch.alerting.transport.WorkflowSingleNodeTestCase +import org.opensearch.commons.alerting.action.IndexMonitorResponse import org.opensearch.commons.alerting.model.ChainedMonitorFindings import org.opensearch.commons.alerting.model.CompositeInput import org.opensearch.commons.alerting.model.DataSources @@ -482,6 +484,46 @@ class WorkflowMonitorIT : WorkflowSingleNodeTestCase() { } } + fun `test delete workflow keeping delegate monitor`() { + val docLevelInput = DocLevelMonitorInput( + "description", listOf(index), listOf(DocLevelQuery(query = "source.ip.v6.v1:12345", name = "3")) + ) + val trigger = randomDocumentLevelTrigger(condition = ALWAYS_RUN) + + val monitor = randomDocumentLevelMonitor( + inputs = listOf(docLevelInput), + triggers = listOf(trigger) + ) + + val monitorResponse = createMonitor(monitor)!! + + val workflowRequest = randomWorkflow( + monitorIds = listOf(monitorResponse.id) + ) + val workflowResponse = upsertWorkflow(workflowRequest)!! + val workflowId = workflowResponse.id + val getWorkflowResponse = getWorkflowById(id = workflowResponse.id) + + assertNotNull(getWorkflowResponse) + assertEquals(workflowId, getWorkflowResponse.id) + + deleteWorkflow(workflowId, false) + // Verify that the workflow is deleted + try { + getWorkflowById(workflowId) + } catch (e: Exception) { + e.message?.let { + assertTrue( + "Exception not returning GetWorkflow Action error ", + it.contains("Workflow not found.") + ) + } + } + // Verify that the monitor is not deleted + val existingDelegate = getMonitorResponse(monitorResponse.id) + assertNotNull(existingDelegate) + } + fun `test delete workflow delegate monitor deleted`() { val docLevelInput = DocLevelMonitorInput( "description", listOf(index), listOf(DocLevelQuery(query = "source.ip.v6.v1:12345", name = "3")) @@ -572,6 +614,19 @@ class WorkflowMonitorIT : WorkflowSingleNodeTestCase() { val monitorsRunResults = executeWorkflowResponse.workflowRunResult.workflowRunResult assertEquals(2, monitorsRunResults.size) + val workflowMetadata = searchWorkflowMetadata(workflowId) + assertNotNull(workflowMetadata) + + val monitorMetadataId1 = getDelegateMonitorMetadataId(workflowMetadata, monitorResponse) + val monitorMetadata1 = searchMonitorMetadata(monitorMetadataId1) + assertNotNull(monitorMetadata1) + + val monitorMetadataId2 = getDelegateMonitorMetadataId(workflowMetadata, monitorResponse2) + val monitorMetadata2 = searchMonitorMetadata(monitorMetadataId2) + assertNotNull(monitorMetadata2) + + assertFalse(monitorMetadata1!!.id == monitorMetadata2!!.id) + deleteWorkflow(workflowId, true) // Verify that the workflow is deleted try { @@ -587,6 +642,31 @@ class WorkflowMonitorIT : WorkflowSingleNodeTestCase() { // Verify that the workflow metadata is deleted try { searchWorkflowMetadata(workflowId) + fail("expected searchWorkflowMetadata method to throw exception") + } catch (e: Exception) { + e.message?.let { + assertTrue( + "Exception not returning GetMonitor Action error ", + it.contains("List is empty") + ) + } + } + // Verify that the monitors metadata are deleted + try { + searchMonitorMetadata(monitorMetadataId1) + fail("expected searchMonitorMetadata method to throw exception") + } catch (e: Exception) { + e.message?.let { + assertTrue( + "Exception not returning GetMonitor Action error ", + it.contains("List is empty") + ) + } + } + + try { + searchMonitorMetadata(monitorMetadataId2) + fail("expected searchMonitorMetadata method to throw exception") } catch (e: Exception) { e.message?.let { assertTrue( @@ -597,7 +677,12 @@ class WorkflowMonitorIT : WorkflowSingleNodeTestCase() { } } - fun `test delete workflow delegate monitor not deleted`() { + private fun getDelegateMonitorMetadataId( + workflowMetadata: WorkflowMetadata?, + monitorResponse: IndexMonitorResponse, + ) = "${workflowMetadata!!.id}-${monitorResponse.id}-metadata" + + fun `test delete workflow delegate monitor part of another workflow not deleted`() { val docLevelInput = DocLevelMonitorInput( "description", listOf(index), listOf(DocLevelQuery(query = "source.ip.v6.v1:12345", name = "3")) ) diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/WorkflowRunnerIT.kt b/alerting/src/test/kotlin/org/opensearch/alerting/WorkflowRunnerIT.kt index 76371c3a3..21c0c335e 100644 --- a/alerting/src/test/kotlin/org/opensearch/alerting/WorkflowRunnerIT.kt +++ b/alerting/src/test/kotlin/org/opensearch/alerting/WorkflowRunnerIT.kt @@ -7,13 +7,16 @@ package org.opensearch.alerting import org.junit.Assert import org.opensearch.action.support.WriteRequest +import org.opensearch.alerting.alerts.AlertIndices import org.opensearch.alerting.model.DocumentLevelTriggerRunResult import org.opensearch.alerting.transport.WorkflowSingleNodeTestCase import org.opensearch.alerting.util.AlertingException import org.opensearch.commons.alerting.action.AcknowledgeAlertRequest import org.opensearch.commons.alerting.action.AlertingActions import org.opensearch.commons.alerting.action.GetAlertsRequest +import org.opensearch.commons.alerting.action.GetAlertsResponse import org.opensearch.commons.alerting.aggregation.bucketselectorext.BucketSelectorExtAggregationBuilder +import org.opensearch.commons.alerting.model.Alert import org.opensearch.commons.alerting.model.DataSources import org.opensearch.commons.alerting.model.DocLevelMonitorInput import org.opensearch.commons.alerting.model.DocLevelQuery @@ -118,14 +121,16 @@ class WorkflowRunnerIT : WorkflowSingleNodeTestCase() { Assert.assertEquals(monitor2.name, monitorsRunResults[1].monitorName) Assert.assertEquals(1, monitorsRunResults[1].triggerResults.size) - assertAlerts(monitorResponse.id, customAlertsIndex1, 2) + val getAlertsResponse = assertAlerts(monitorResponse.id, customAlertsIndex1, 2) + assertAcknowledges(getAlertsResponse.alerts, monitorResponse.id, 2) assertFindings(monitorResponse.id, customFindingsIndex1, 2, 2, listOf("1", "2")) - assertAlerts(monitorResponse2.id, customAlertsIndex2, 1) + val getAlertsResponse2 = assertAlerts(monitorResponse2.id, customAlertsIndex2, 1) + assertAcknowledges(getAlertsResponse2.alerts, monitorResponse2.id, 1) assertFindings(monitorResponse2.id, customFindingsIndex2, 1, 1, listOf("2")) } - fun `test execute workflows with shared monitor delegates`() { + fun `test execute workflows with shared doc level monitor delegate`() { val docQuery = DocLevelQuery(query = "test_field_1:\"us-west-2\"", name = "3") val docLevelInput = DocLevelMonitorInput("description", listOf(index), listOf(docQuery)) val trigger = randomDocumentLevelTrigger(condition = ALWAYS_RUN) @@ -184,9 +189,22 @@ class WorkflowRunnerIT : WorkflowSingleNodeTestCase() { assertEquals(monitor.name, monitorsRunResults[0].monitorName) assertEquals(1, monitorsRunResults[0].triggerResults.size) + // Assert and not ack the alerts (in order to verify later on that all the alerts are generated) assertAlerts(monitorResponse.id, customAlertsIndex, 2) assertFindings(monitorResponse.id, customFindingsIndex, 2, 2, listOf("1", "2")) + // Verify workflow and monitor delegate metadata + val workflowMetadata = searchWorkflowMetadata(id = workflowId) + assertNotNull("Workflow metadata not initialized", workflowMetadata) + assertEquals( + "Workflow metadata execution id not correct", + executeWorkflowResponse.workflowRunResult.executionId, + workflowMetadata!!.latestExecutionId + ) + val monitorMetadataId = "${monitorResponse.id}-${workflowMetadata.id}" + val monitorMetadata = searchMonitorMetadata(monitorMetadataId) + assertNotNull(monitorMetadata) + // Execute second workflow val workflowId1 = workflowResponse1.id val executeWorkflowResponse1 = executeWorkflow(workflowById1, workflowId1, false)!! val monitorsRunResults1 = executeWorkflowResponse1.workflowRunResult.workflowRunResult @@ -195,8 +213,134 @@ class WorkflowRunnerIT : WorkflowSingleNodeTestCase() { assertEquals(monitor.name, monitorsRunResults1[0].monitorName) assertEquals(1, monitorsRunResults1[0].triggerResults.size) - assertAlerts(monitorResponse.id, customAlertsIndex, 2) + val getAlertsResponse = assertAlerts(monitorResponse.id, customAlertsIndex, 4) + assertAcknowledges(getAlertsResponse.alerts, monitorResponse.id, 4) assertFindings(monitorResponse.id, customFindingsIndex, 4, 4, listOf("1", "2", "1", "2")) + // Verify workflow and monitor delegate metadata + val workflowMetadata1 = searchWorkflowMetadata(id = workflowId1) + assertNotNull("Workflow metadata not initialized", workflowMetadata1) + assertEquals( + "Workflow metadata execution id not correct", + executeWorkflowResponse1.workflowRunResult.executionId, + workflowMetadata1!!.latestExecutionId + ) + val monitorMetadataId1 = "${monitorResponse.id}-${workflowMetadata1.id}" + val monitorMetadata1 = searchMonitorMetadata(monitorMetadataId1) + assertNotNull(monitorMetadata1) + // Verify that for two workflows two different doc level monitor metadata has been created + assertTrue("Different monitor is used in workflows", monitorMetadata!!.monitorId == monitorMetadata1!!.monitorId) + assertTrue(monitorMetadata.id != monitorMetadata1.id) + } + + fun `test execute workflows with shared doc level monitor delegate updating delegate datasource`() { + val docQuery = DocLevelQuery(query = "test_field_1:\"us-west-2\"", name = "3") + val docLevelInput = DocLevelMonitorInput("description", listOf(index), listOf(docQuery)) + val trigger = randomDocumentLevelTrigger(condition = ALWAYS_RUN) + + var monitor = randomDocumentLevelMonitor( + inputs = listOf(docLevelInput), + triggers = listOf(trigger) + ) + val monitorResponse = createMonitor(monitor)!! + + val workflow = randomWorkflow( + monitorIds = listOf(monitorResponse.id) + ) + val workflowResponse = upsertWorkflow(workflow)!! + val workflowById = searchWorkflow(workflowResponse.id) + assertNotNull(workflowById) + + val workflow1 = randomWorkflow( + monitorIds = listOf(monitorResponse.id) + ) + val workflowResponse1 = upsertWorkflow(workflow1)!! + val workflowById1 = searchWorkflow(workflowResponse1.id) + assertNotNull(workflowById1) + + var testTime = DateTimeFormatter.ISO_OFFSET_DATE_TIME.format(ZonedDateTime.now().truncatedTo(ChronoUnit.MILLIS)) + // Matches monitor1 + val testDoc1 = """{ + "message" : "This is an error from IAD region", + "source.ip.v6.v2" : 16644, + "test_strict_date_time" : "$testTime", + "test_field_1" : "us-west-2" + }""" + indexDoc(index, "1", testDoc1) + + testTime = DateTimeFormatter.ISO_OFFSET_DATE_TIME.format(ZonedDateTime.now().truncatedTo(ChronoUnit.MILLIS)) + val testDoc2 = """{ + "message" : "This is an error from IAD region", + "source.ip.v6.v2" : 16645, + "test_strict_date_time" : "$testTime", + "test_field_1" : "us-west-2" + }""" + indexDoc(index, "2", testDoc2) + + val workflowId = workflowResponse.id + val executeWorkflowResponse = executeWorkflow(workflowById, workflowId, false)!! + val monitorsRunResults = executeWorkflowResponse.workflowRunResult.workflowRunResult + assertEquals(1, monitorsRunResults.size) + + assertEquals(monitor.name, monitorsRunResults[0].monitorName) + assertEquals(1, monitorsRunResults[0].triggerResults.size) + + assertAlerts(monitorResponse.id, AlertIndices.ALERT_INDEX, 2) + assertFindings(monitorResponse.id, AlertIndices.FINDING_HISTORY_WRITE_INDEX, 2, 2, listOf("1", "2")) + // Verify workflow and monitor delegate metadata + val workflowMetadata = searchWorkflowMetadata(id = workflowId) + assertNotNull("Workflow metadata not initialized", workflowMetadata) + assertEquals( + "Workflow metadata execution id not correct", + executeWorkflowResponse.workflowRunResult.executionId, + workflowMetadata!!.latestExecutionId + ) + val monitorMetadataId = "${monitorResponse.id}-${workflowMetadata.id}" + val monitorMetadata = searchMonitorMetadata(monitorMetadataId) + assertNotNull(monitorMetadata) + + val customAlertsIndex = "custom_alerts_index" + val customFindingsIndex = "custom_findings_index" + val customFindingsIndexPattern = "custom_findings_index-1" + val monitorId = monitorResponse.id + updateMonitor( + monitor = monitor.copy( + dataSources = DataSources( + alertsIndex = customAlertsIndex, + findingsIndex = customFindingsIndex, + findingsIndexPattern = customFindingsIndexPattern + ) + ), + monitorId + ) + + // Execute second workflow + val workflowId1 = workflowResponse1.id + val executeWorkflowResponse1 = executeWorkflow(workflowById1, workflowId1, false)!! + val monitorsRunResults1 = executeWorkflowResponse1.workflowRunResult.workflowRunResult + assertEquals(1, monitorsRunResults1.size) + + assertEquals(monitor.name, monitorsRunResults1[0].monitorName) + assertEquals(1, monitorsRunResults1[0].triggerResults.size) + + // Verify alerts for the custom index + val getAlertsResponse = assertAlerts(monitorResponse.id, customAlertsIndex, 2) + assertAcknowledges(getAlertsResponse.alerts, monitorResponse.id, 2) + assertFindings(monitorResponse.id, customFindingsIndex, 2, 2, listOf("1", "2")) + + // Verify workflow and monitor delegate metadata + val workflowMetadata1 = searchWorkflowMetadata(id = workflowId1) + assertNotNull("Workflow metadata not initialized", workflowMetadata1) + assertEquals( + "Workflow metadata execution id not correct", + executeWorkflowResponse1.workflowRunResult.executionId, + workflowMetadata1!!.latestExecutionId + ) + val monitorMetadataId1 = "${monitorResponse.id}-${workflowMetadata1.id}" + val monitorMetadata1 = searchMonitorMetadata(monitorMetadataId1) + assertNotNull(monitorMetadata1) + // Verify that for two workflows two different doc level monitor metadata has been created + assertTrue("Different monitor is used in workflows", monitorMetadata!!.monitorId == monitorMetadata1!!.monitorId) + assertTrue(monitorMetadata.id != monitorMetadata1.id) } fun `test execute workflow verify workflow metadata`() { @@ -248,6 +392,10 @@ class WorkflowRunnerIT : WorkflowSingleNodeTestCase() { executeWorkflowResponse.workflowRunResult.executionId, workflowMetadata!!.latestExecutionId ) + val monitorMetadataId = "${monitorResponse.id}-${workflowMetadata.id}" + val monitorMetadata = searchMonitorMetadata(monitorMetadataId) + assertNotNull(monitorMetadata) + // Second execution val executeWorkflowResponse1 = executeWorkflow(workflowById, workflowId, false)!! val monitorsRunResults1 = executeWorkflowResponse1.workflowRunResult.workflowRunResult @@ -260,6 +408,10 @@ class WorkflowRunnerIT : WorkflowSingleNodeTestCase() { executeWorkflowResponse1.workflowRunResult.executionId, workflowMetadata1!!.latestExecutionId ) + val monitorMetadataId1 = "${monitorResponse.id}-${workflowMetadata1.id}" + assertTrue(monitorMetadataId == monitorMetadataId1) + val monitorMetadata1 = searchMonitorMetadata(monitorMetadataId1) + assertNotNull(monitorMetadata1) } fun `test execute workflow dryrun verify workflow metadata not created`() { @@ -301,6 +453,7 @@ class WorkflowRunnerIT : WorkflowSingleNodeTestCase() { // First execution val workflowId = workflowResponse.id val executeWorkflowResponse = executeWorkflow(workflowById, workflowId, true) + assertNotNull("Workflow run result is null", executeWorkflowResponse) val monitorsRunResults = executeWorkflowResponse!!.workflowRunResult.workflowRunResult assertEquals(2, monitorsRunResults.size) @@ -409,7 +562,8 @@ class WorkflowRunnerIT : WorkflowSingleNodeTestCase() { val buckets = searchResult.stringMap("aggregations")?.stringMap("composite_agg")?.get("buckets") as List> assertEquals("Incorrect search result", 3, buckets.size) - assertAlerts(bucketLevelMonitorResponse.id, bucketCustomAlertsIndex, 2) + val getAlertsResponse = assertAlerts(bucketLevelMonitorResponse.id, bucketCustomAlertsIndex, 2) + assertAcknowledges(getAlertsResponse.alerts, bucketLevelMonitorResponse.id, 2) assertFindings(bucketLevelMonitorResponse.id, bucketCustomFindingsIndex, 1, 4, listOf("1", "2", "3", "4")) } else { assertEquals(1, monitorRunResults.inputResults.results.size) @@ -421,7 +575,8 @@ class WorkflowRunnerIT : WorkflowSingleNodeTestCase() { val expectedTriggeredDocIds = listOf("1", "2", "3", "4") assertEquals(expectedTriggeredDocIds, triggeredDocIds.sorted()) - assertAlerts(docLevelMonitorResponse.id, docCustomAlertsIndex, 4) + val getAlertsResponse = assertAlerts(docLevelMonitorResponse.id, docCustomAlertsIndex, 4) + assertAcknowledges(getAlertsResponse.alerts, docLevelMonitorResponse.id, 4) assertFindings(docLevelMonitorResponse.id, docCustomFindingsIndex, 4, 4, listOf("1", "2", "3", "4")) } } @@ -511,11 +666,17 @@ class WorkflowRunnerIT : WorkflowSingleNodeTestCase() { """.trimIndent() val queryLevelTrigger = randomQueryLevelTrigger(condition = Script(queryTriggerScript)) - val queryMonitorResponse = createMonitor(randomQueryLevelMonitor(inputs = listOf(queryMonitorInput), triggers = listOf(queryLevelTrigger)))!! + val queryMonitorResponse = + createMonitor(randomQueryLevelMonitor(inputs = listOf(queryMonitorInput), triggers = listOf(queryLevelTrigger)))!! // 1. docMonitor (chainedFinding = null) 2. bucketMonitor (chainedFinding = docMonitor) 3. docMonitor (chainedFinding = bucketMonitor) 4. queryMonitor (chainedFinding = docMonitor 3) var workflow = randomWorkflow( - monitorIds = listOf(docLevelMonitorResponse.id, bucketLevelMonitorResponse.id, docLevelMonitorResponse1.id, queryMonitorResponse.id) + monitorIds = listOf( + docLevelMonitorResponse.id, + bucketLevelMonitorResponse.id, + docLevelMonitorResponse1.id, + queryMonitorResponse.id + ) ) val workflowResponse = upsertWorkflow(workflow)!! val workflowById = searchWorkflow(workflowResponse.id) @@ -553,18 +714,36 @@ class WorkflowRunnerIT : WorkflowSingleNodeTestCase() { val expectedTriggeredDocIds = listOf("3", "4", "5", "6") assertEquals(expectedTriggeredDocIds, triggeredDocIds.sorted()) - assertAlerts(docLevelMonitorResponse.id, docLevelMonitorResponse.monitor.dataSources.alertsIndex, 4) - assertFindings(docLevelMonitorResponse.id, docLevelMonitorResponse.monitor.dataSources.findingsIndex, 4, 4, listOf("3", "4", "5", "6")) + val getAlertsResponse = + assertAlerts(docLevelMonitorResponse.id, docLevelMonitorResponse.monitor.dataSources.alertsIndex, 4) + assertAcknowledges(getAlertsResponse.alerts, docLevelMonitorResponse.id, 4) + assertFindings( + docLevelMonitorResponse.id, + docLevelMonitorResponse.monitor.dataSources.findingsIndex, + 4, + 4, + listOf("3", "4", "5", "6") + ) } // Verify second bucket level monitor execution, alerts and findings bucketLevelMonitorResponse.monitor.name -> { val searchResult = monitorRunResults.inputResults.results.first() @Suppress("UNCHECKED_CAST") - val buckets = searchResult.stringMap("aggregations")?.stringMap("composite_agg")?.get("buckets") as List> + val buckets = + searchResult + .stringMap("aggregations")?.stringMap("composite_agg")?.get("buckets") as List> assertEquals("Incorrect search result", 2, buckets.size) - assertAlerts(bucketLevelMonitorResponse.id, bucketLevelMonitorResponse.monitor.dataSources.alertsIndex, 2) - assertFindings(bucketLevelMonitorResponse.id, bucketLevelMonitorResponse.monitor.dataSources.findingsIndex, 1, 4, listOf("3", "4", "5", "6")) + val getAlertsResponse = + assertAlerts(bucketLevelMonitorResponse.id, bucketLevelMonitorResponse.monitor.dataSources.alertsIndex, 2) + assertAcknowledges(getAlertsResponse.alerts, bucketLevelMonitorResponse.id, 2) + assertFindings( + bucketLevelMonitorResponse.id, + bucketLevelMonitorResponse.monitor.dataSources.findingsIndex, + 1, + 4, + listOf("3", "4", "5", "6") + ) } // Verify third doc level monitor execution, alerts and findings docLevelMonitorResponse1.monitor.name -> { @@ -577,8 +756,16 @@ class WorkflowRunnerIT : WorkflowSingleNodeTestCase() { val expectedTriggeredDocIds = listOf("5", "6") assertEquals(expectedTriggeredDocIds, triggeredDocIds.sorted()) - assertAlerts(docLevelMonitorResponse1.id, docLevelMonitorResponse1.monitor.dataSources.alertsIndex, 2) - assertFindings(docLevelMonitorResponse1.id, docLevelMonitorResponse1.monitor.dataSources.findingsIndex, 2, 2, listOf("5", "6")) + val getAlertsResponse = + assertAlerts(docLevelMonitorResponse1.id, docLevelMonitorResponse1.monitor.dataSources.alertsIndex, 2) + assertAcknowledges(getAlertsResponse.alerts, docLevelMonitorResponse1.id, 2) + assertFindings( + docLevelMonitorResponse1.id, + docLevelMonitorResponse1.monitor.dataSources.findingsIndex, + 2, + 2, + listOf("5", "6") + ) } // Verify fourth query level monitor execution queryMonitorResponse.monitor.name -> { @@ -586,10 +773,14 @@ class WorkflowRunnerIT : WorkflowSingleNodeTestCase() { val values = monitorRunResults.triggerResults.values assertEquals(1, values.size) @Suppress("UNCHECKED_CAST") - val totalHits = ((monitorRunResults.inputResults.results[0]["hits"] as Map)["total"] as Map) ["value"] + val totalHits = + ((monitorRunResults.inputResults.results[0]["hits"] as Map)["total"] as Map)["value"] assertEquals(2, totalHits) @Suppress("UNCHECKED_CAST") - val docIds = ((monitorRunResults.inputResults.results[0]["hits"] as Map)["hits"] as List>).map { it["_id"]!! } + val docIds = + ( + (monitorRunResults.inputResults.results[0]["hits"] as Map)["hits"] as List> + ).map { it["_id"]!! } assertEquals(listOf("5", "6"), docIds.sorted()) } } @@ -622,7 +813,7 @@ class WorkflowRunnerIT : WorkflowSingleNodeTestCase() { assertNotNull(error) assertTrue(error is AlertingException) assertEquals(RestStatus.NOT_FOUND, (error as AlertingException).status) - assertEquals("Configured indices are not found: [$index]", (error as AlertingException).message) + assertEquals("Configured indices are not found: [$index]", error.message) } fun `test execute workflow wrong workflow id`() { @@ -666,7 +857,7 @@ class WorkflowRunnerIT : WorkflowSingleNodeTestCase() { customFindingsIndex: String, findingSize: Int, matchedQueryNumber: Int, - relatedDocIds: List + relatedDocIds: List, ) { val findings = searchFindings(monitorId, customFindingsIndex) assertEquals("Findings saved for test monitor", findingSize, findings.size) @@ -680,8 +871,8 @@ class WorkflowRunnerIT : WorkflowSingleNodeTestCase() { private fun assertAlerts( monitorId: String, customAlertsIndex: String, - alertSize: Int - ) { + alertSize: Int, + ): GetAlertsResponse { val alerts = searchAlerts(monitorId, customAlertsIndex) assertEquals("Alert saved for test monitor", alertSize, alerts.size) val table = Table("asc", "id", null, alertSize, 0, "") @@ -699,7 +890,15 @@ class WorkflowRunnerIT : WorkflowSingleNodeTestCase() { assertTrue(getAlertsResponse != null) assertTrue(getAlertsResponse.alerts.size == alertSize) - val alertIds = getAlertsResponse.alerts.map { it.id } + return getAlertsResponse + } + + private fun assertAcknowledges( + alerts: List, + monitorId: String, + alertSize: Int, + ) { + val alertIds = alerts.map { it.id } val acknowledgeAlertResponse = client().execute( AlertingActions.ACKNOWLEDGE_ALERTS_ACTION_TYPE, AcknowledgeAlertRequest(monitorId, alertIds, WriteRequest.RefreshPolicy.IMMEDIATE) @@ -707,4 +906,91 @@ class WorkflowRunnerIT : WorkflowSingleNodeTestCase() { assertEquals(alertSize, acknowledgeAlertResponse.acknowledged.size) } + + fun `test execute workflow with bucket-level and doc-level chained monitors`() { + createTestIndex(TEST_HR_INDEX) + + val compositeSources = listOf( + TermsValuesSourceBuilder("test_field").field("test_field") + ) + val compositeAgg = CompositeAggregationBuilder("composite_agg", compositeSources) + val input = SearchInput( + indices = listOf(TEST_HR_INDEX), + query = SearchSourceBuilder().size(0).query(QueryBuilders.matchAllQuery()).aggregation(compositeAgg) + ) + val triggerScript = """ + params.docCount > 0 + """.trimIndent() + + var trigger = randomBucketLevelTrigger() + trigger = trigger.copy( + bucketSelector = BucketSelectorExtAggregationBuilder( + name = trigger.id, + bucketsPathsMap = mapOf("docCount" to "_count"), + script = Script(triggerScript), + parentBucketPath = "composite_agg", + filter = null + ), + actions = listOf() + ) + val bucketMonitor = createMonitor( + randomBucketLevelMonitor( + inputs = listOf(input), + enabled = false, + triggers = listOf(trigger) + ) + ) + assertNotNull("The bucket monitor was not created", bucketMonitor) + + val docQuery1 = DocLevelQuery(query = "test_field:\"a\"", name = "3") + var monitor1 = randomDocumentLevelMonitor( + inputs = listOf(DocLevelMonitorInput("description", listOf(TEST_HR_INDEX), listOf(docQuery1))), + triggers = listOf(randomDocumentLevelTrigger(condition = ALWAYS_RUN)) + ) + val docMonitor = createMonitor(monitor1)!! + assertNotNull("The doc level monitor was not created", docMonitor) + + val workflow = randomWorkflow(monitorIds = listOf(bucketMonitor!!.id, docMonitor.id)) + val workflowResponse = upsertWorkflow(workflow) + assertNotNull("The workflow was not created", workflowResponse) + + // Add a doc that is accessible to the user + indexDoc( + TEST_HR_INDEX, + "1", + """ + { + "test_field": "a", + "accessible": true + } + """.trimIndent() + ) + + // Add a second doc that is not accessible to the user + indexDoc( + TEST_HR_INDEX, + "2", + """ + { + "test_field": "b", + "accessible": false + } + """.trimIndent() + ) + + indexDoc( + TEST_HR_INDEX, + "3", + """ + { + "test_field": "c", + "accessible": true + } + """.trimIndent() + ) + + val executeResult = executeWorkflow(id = workflowResponse!!.id) + assertNotNull(executeResult) + assertEquals(2, executeResult!!.workflowRunResult.workflowRunResult.size) + } } diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/resthandler/SecureWorkflowRestApiIT.kt b/alerting/src/test/kotlin/org/opensearch/alerting/resthandler/SecureWorkflowRestApiIT.kt new file mode 100644 index 000000000..615eea4ca --- /dev/null +++ b/alerting/src/test/kotlin/org/opensearch/alerting/resthandler/SecureWorkflowRestApiIT.kt @@ -0,0 +1,1406 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.alerting.resthandler + +import org.apache.http.HttpHeaders +import org.apache.http.entity.ContentType +import org.apache.http.message.BasicHeader +import org.apache.http.nio.entity.NStringEntity +import org.junit.After +import org.junit.Before +import org.junit.BeforeClass +import org.opensearch.alerting.ALERTING_BASE_URI +import org.opensearch.alerting.ALERTING_DELETE_WORKFLOW_ACCESS +import org.opensearch.alerting.ALERTING_EXECUTE_WORKFLOW_ACCESS +import org.opensearch.alerting.ALERTING_FULL_ACCESS_ROLE +import org.opensearch.alerting.ALERTING_GET_WORKFLOW_ACCESS +import org.opensearch.alerting.ALERTING_INDEX_MONITOR_ACCESS +import org.opensearch.alerting.ALERTING_INDEX_WORKFLOW_ACCESS +import org.opensearch.alerting.ALERTING_NO_ACCESS_ROLE +import org.opensearch.alerting.ALERTING_READ_ONLY_ACCESS +import org.opensearch.alerting.ALWAYS_RUN +import org.opensearch.alerting.AlertingRestTestCase +import org.opensearch.alerting.READALL_AND_MONITOR_ROLE +import org.opensearch.alerting.TERM_DLS_QUERY +import org.opensearch.alerting.TEST_HR_BACKEND_ROLE +import org.opensearch.alerting.TEST_HR_INDEX +import org.opensearch.alerting.TEST_HR_ROLE +import org.opensearch.alerting.TEST_NON_HR_INDEX +import org.opensearch.alerting.WORKFLOW_ALERTING_BASE_URI +import org.opensearch.alerting.assertUserNull +import org.opensearch.alerting.makeRequest +import org.opensearch.alerting.randomBucketLevelMonitor +import org.opensearch.alerting.randomBucketLevelTrigger +import org.opensearch.alerting.randomDocLevelQuery +import org.opensearch.alerting.randomDocumentLevelMonitor +import org.opensearch.alerting.randomDocumentLevelTrigger +import org.opensearch.alerting.randomQueryLevelMonitor +import org.opensearch.alerting.randomWorkflow +import org.opensearch.client.Response +import org.opensearch.client.ResponseException +import org.opensearch.client.RestClient +import org.opensearch.common.xcontent.LoggingDeprecationHandler +import org.opensearch.common.xcontent.XContentType +import org.opensearch.common.xcontent.json.JsonXContent +import org.opensearch.commons.alerting.aggregation.bucketselectorext.BucketSelectorExtAggregationBuilder +import org.opensearch.commons.alerting.model.DataSources +import org.opensearch.commons.alerting.model.DocLevelMonitorInput +import org.opensearch.commons.alerting.model.DocLevelQuery +import org.opensearch.commons.alerting.model.SearchInput +import org.opensearch.commons.rest.SecureRestClientBuilder +import org.opensearch.core.xcontent.NamedXContentRegistry +import org.opensearch.index.query.QueryBuilders +import org.opensearch.rest.RestStatus +import org.opensearch.script.Script +import org.opensearch.search.aggregations.bucket.composite.CompositeAggregationBuilder +import org.opensearch.search.aggregations.bucket.composite.TermsValuesSourceBuilder +import org.opensearch.search.builder.SearchSourceBuilder +import org.opensearch.test.junit.annotations.TestLogging +import java.time.Instant + +@TestLogging("level:DEBUG", reason = "Debug for tests.") +@Suppress("UNCHECKED_CAST") +class SecureWorkflowRestApiIT : AlertingRestTestCase() { + + companion object { + + @BeforeClass + @JvmStatic + fun setup() { + // things to execute once and keep around for the class + org.junit.Assume.assumeTrue(System.getProperty("security", "false")!!.toBoolean()) + } + } + + val user = "userD" + var userClient: RestClient? = null + + @Before + fun create() { + if (userClient == null) { + createUser(user, user, arrayOf()) + userClient = SecureRestClientBuilder(clusterHosts.toTypedArray(), isHttps(), user, user).setSocketTimeout(60000).build() + } + } + + @After + fun cleanup() { + userClient?.close() + deleteUser(user) + } + + // Create Workflow related security tests + fun `test create workflow with an user with alerting role`() { + val clusterPermissions = listOf( + getClusterPermissionsFromCustomRole(ALERTING_INDEX_WORKFLOW_ACCESS) + ) + + createUserWithTestDataAndCustomRole( + user, + TEST_HR_INDEX, + TEST_HR_ROLE, + listOf(TEST_HR_BACKEND_ROLE), + clusterPermissions + ) + try { + val monitor = createMonitor( + randomQueryLevelMonitor( + inputs = listOf(SearchInput(listOf(TEST_HR_INDEX), SearchSourceBuilder().query(QueryBuilders.matchAllQuery()))), + ), + true + ) + + val workflow = randomWorkflow( + monitorIds = listOf(monitor.id) + ) + + val createResponse = userClient?.makeRequest("POST", WORKFLOW_ALERTING_BASE_URI, emptyMap(), workflow.toHttpEntity()) + assertEquals("Create workflow failed", RestStatus.CREATED, createResponse?.restStatus()) + + assertUserNull(createResponse?.asMap()!!["workflow"] as HashMap) + } finally { + deleteRoleAndRoleMapping(TEST_HR_ROLE) + } + } + + fun `test create workflow with an user without alerting role`() { + createUserWithTestDataAndCustomRole( + user, + TEST_HR_INDEX, + TEST_HR_ROLE, + listOf(TEST_HR_BACKEND_ROLE), + getClusterPermissionsFromCustomRole(ALERTING_NO_ACCESS_ROLE) + ) + try { + val monitor = createRandomMonitor(true) + + val workflow = randomWorkflow( + monitorIds = listOf(monitor.id) + ) + + userClient?.makeRequest("POST", WORKFLOW_ALERTING_BASE_URI, emptyMap(), workflow.toHttpEntity()) + fail("Expected 403 Method FORBIDDEN response") + } catch (e: ResponseException) { + assertEquals("Unexpected status", RestStatus.FORBIDDEN, e.response.restStatus()) + } finally { + deleteRoleAndRoleMapping(TEST_HR_ROLE) + } + } + + fun `test create workflow with an user with read-only role`() { + createUserWithTestData(user, TEST_HR_INDEX, TEST_HR_ROLE, TEST_HR_BACKEND_ROLE) + createUserRolesMapping(ALERTING_READ_ONLY_ACCESS, arrayOf(user)) + + try { + val monitor = createRandomMonitor(true) + val workflow = randomWorkflow( + monitorIds = listOf(monitor.id) + ) + userClient?.makeRequest("POST", WORKFLOW_ALERTING_BASE_URI, emptyMap(), workflow.toHttpEntity()) + fail("Expected 403 Method FORBIDDEN response") + } catch (e: ResponseException) { + assertEquals("Unexpected status", RestStatus.FORBIDDEN, e.response.restStatus()) + } finally { + deleteRoleAndRoleMapping(TEST_HR_ROLE) + deleteRoleMapping(ALERTING_READ_ONLY_ACCESS) + } + } + + fun `test create workflow with delegate with an user without index read role`() { + createTestIndex(TEST_NON_HR_INDEX) + val clusterPermissions = listOf( + getClusterPermissionsFromCustomRole(ALERTING_INDEX_WORKFLOW_ACCESS) + ) + createUserWithTestDataAndCustomRole( + user, + TEST_HR_INDEX, + TEST_HR_ROLE, + listOf(TEST_HR_BACKEND_ROLE), + clusterPermissions + ) + try { + val query = randomDocLevelQuery(tags = listOf()) + val triggers = listOf(randomDocumentLevelTrigger(condition = Script("query[id=\"${query.id}\"]"))) + + val monitor = createMonitor( + randomDocumentLevelMonitor( + inputs = listOf( + DocLevelMonitorInput( + indices = listOf(TEST_NON_HR_INDEX), + queries = listOf(query) + ) + ), + triggers = triggers + ), + true + ) + + val workflow = randomWorkflow( + monitorIds = listOf(monitor.id) + ) + + userClient?.makeRequest("POST", WORKFLOW_ALERTING_BASE_URI, emptyMap(), workflow.toHttpEntity()) + } catch (e: ResponseException) { + assertEquals("Unexpected status", RestStatus.FORBIDDEN, e.response.restStatus()) + } finally { + deleteRoleAndRoleMapping(TEST_HR_ROLE) + deleteIndex(TEST_NON_HR_INDEX) + } + } + + fun `test create workflow with disable filter by`() { + disableFilterBy() + val monitor = createRandomMonitor(true) + val workflow = randomWorkflow( + monitorIds = listOf(monitor.id) + ) + val createResponse = client().makeRequest("POST", WORKFLOW_ALERTING_BASE_URI, emptyMap(), workflow.toHttpEntity()) + assertEquals("Create workflow failed", RestStatus.CREATED, createResponse.restStatus()) + assertUserNull(createResponse.asMap()["workflow"] as HashMap) + } + + fun `test get workflow with an user with get workflow role`() { + createUserWithTestDataAndCustomRole( + user, + TEST_HR_INDEX, + TEST_HR_ROLE, + listOf(TEST_HR_BACKEND_ROLE), + getClusterPermissionsFromCustomRole(ALERTING_GET_WORKFLOW_ACCESS) + ) + + val monitor = createRandomMonitor(true) + val workflow = createWorkflow(randomWorkflow(monitorIds = listOf(monitor.id))) + + try { + val getWorkflowResponse = userClient?.makeRequest( + "GET", + "$WORKFLOW_ALERTING_BASE_URI/${workflow.id}", + null, + BasicHeader(HttpHeaders.CONTENT_TYPE, "application/json") + ) + assertEquals("Get workflow failed", RestStatus.OK, getWorkflowResponse?.restStatus()) + } finally { + deleteRoleAndRoleMapping(TEST_HR_ROLE) + } + } + + /* + TODO: https://github.com/opensearch-project/alerting/issues/300 + */ + fun `test get workflow with an user without get monitor role`() { + createUserWithTestDataAndCustomRole( + user, + TEST_HR_INDEX, + TEST_HR_ROLE, + listOf(TEST_HR_BACKEND_ROLE), + getClusterPermissionsFromCustomRole(ALERTING_NO_ACCESS_ROLE) + ) + + val monitor = createRandomMonitor(true) + val workflow = createWorkflow(randomWorkflow(monitorIds = listOf(monitor.id))) + + try { + userClient?.makeRequest( + "GET", + "$WORKFLOW_ALERTING_BASE_URI/${workflow.id}", + null, + BasicHeader(HttpHeaders.CONTENT_TYPE, "application/json") + ) + fail("Expected 403 Method FORBIDDEN response") + } catch (e: ResponseException) { + assertEquals("Unexpected status", RestStatus.FORBIDDEN, e.response.restStatus()) + } finally { + deleteRoleAndRoleMapping(TEST_HR_ROLE) + } + } + + fun getDocs(response: Response?): Any? { + val hits = createParser( + XContentType.JSON.xContent(), + response?.entity?.content + ).map()["hits"]!! as Map> + return hits["total"]?.get("value") + } + + // Query Monitors related security tests + fun `test update workflow with disable filter by`() { + disableFilterBy() + + val createdMonitor = createMonitor(monitor = randomQueryLevelMonitor(enabled = true)) + val createdWorkflow = createWorkflow( + randomWorkflow(monitorIds = listOf(createdMonitor.id), enabled = true, enabledTime = Instant.now()) + ) + + assertNotNull("The workflow was not created", createdWorkflow) + assertTrue("The workflow was not enabled", createdWorkflow.enabled) + + val workflowV2 = createdWorkflow.copy(enabled = false, enabledTime = null) + val updatedWorkflow = updateWorkflow(workflowV2) + + assertFalse("The monitor was not disabled", updatedWorkflow.enabled) + } + + fun `test update workflow with enable filter by`() { + enableFilterBy() + if (!isHttps()) { + // if security is disabled and filter by is enabled, we can't create monitor + // refer: `test create monitor with enable filter by` + return + } + + val createdMonitor = createMonitorWithClient( + client = client(), + monitor = randomQueryLevelMonitor(enabled = true), + rbacRoles = listOf("admin") + ) + val createdWorkflow = createWorkflow( + randomWorkflow(monitorIds = listOf(createdMonitor.id), enabled = true, enabledTime = Instant.now()) + ) + + assertNotNull("The workflow was not created", createdWorkflow) + assertTrue("The workflow was not enabled", createdWorkflow.enabled) + + val workflowV2 = createdWorkflow.copy(enabled = false, enabledTime = null) + val updatedWorkflow = updateWorkflow(workflow = workflowV2) + + assertFalse("The monitor was not disabled", updatedWorkflow.enabled) + } + + fun `test create workflow with enable filter by with a user have access and without role has no access`() { + enableFilterBy() + if (!isHttps()) { + return + } + + createUserWithRoles( + user, + listOf(ALERTING_FULL_ACCESS_ROLE, READALL_AND_MONITOR_ROLE), + listOf(TEST_HR_BACKEND_ROLE, "role2"), + false + ) + + val createdMonitor = createMonitorWithClient( + userClient!!, + monitor = randomQueryLevelMonitor(enabled = true), + listOf(TEST_HR_BACKEND_ROLE, "role2") + ) + + assertNotNull("The monitor was not created", createdMonitor) + + val createdWorkflow = createWorkflowWithClient( + userClient!!, + workflow = randomWorkflow(monitorIds = listOf(createdMonitor.id), enabled = true), + listOf(TEST_HR_BACKEND_ROLE, "role2") + ) + assertNotNull("The workflow was not created", createdWorkflow) + + createUserRolesMapping(ALERTING_FULL_ACCESS_ROLE, arrayOf()) + createUserRolesMapping(READALL_AND_MONITOR_ROLE, arrayOf()) + + // getUser should have access to the monitor + val getUser = "getUser" + createUserWithTestDataAndCustomRole( + getUser, + TEST_HR_INDEX, + TEST_HR_ROLE, + listOf("role2"), + getClusterPermissionsFromCustomRole(ALERTING_GET_WORKFLOW_ACCESS) + ) + val getUserClient = SecureRestClientBuilder(clusterHosts.toTypedArray(), isHttps(), getUser, getUser) + .setSocketTimeout(60000).build() + + val getWorkflowResponse = getUserClient?.makeRequest( + "GET", + "$WORKFLOW_ALERTING_BASE_URI/${createdWorkflow.id}", + null, + BasicHeader(HttpHeaders.CONTENT_TYPE, "application/json") + ) + assertEquals("Get workflow failed", RestStatus.OK, getWorkflowResponse?.restStatus()) + + // Remove backend role and ensure no access is granted after + patchUserBackendRoles(getUser, arrayOf("role1")) + try { + getUserClient?.makeRequest( + "GET", + "$WORKFLOW_ALERTING_BASE_URI/${createdWorkflow.id}", + null, + BasicHeader(HttpHeaders.CONTENT_TYPE, "application/json") + ) + fail("Expected Forbidden exception") + } catch (e: ResponseException) { + assertEquals("Get workflow failed", RestStatus.FORBIDDEN.status, e.response.statusLine.statusCode) + } finally { + deleteRoleAndRoleMapping(TEST_HR_ROLE) + deleteUser(getUser) + getUserClient?.close() + } + } + + fun `test create workflow with enable filter by with a user with a backend role doesn't have access to monitor`() { + enableFilterBy() + if (!isHttps()) { + return + } + + createUserWithRoles( + user, + listOf(ALERTING_FULL_ACCESS_ROLE, READALL_AND_MONITOR_ROLE), + listOf(TEST_HR_BACKEND_ROLE, "role2"), + false + ) + + val createdMonitor = createMonitorWithClient( + userClient!!, + monitor = randomQueryLevelMonitor(enabled = true), + listOf("role2") + ) + + assertNotNull("The monitor was not created", createdMonitor) + + val userWithDifferentRole = "role3User" + + createUserWithRoles( + userWithDifferentRole, + listOf(ALERTING_FULL_ACCESS_ROLE, READALL_AND_MONITOR_ROLE), + listOf(TEST_HR_BACKEND_ROLE, "role3"), + false + ) + + val userWithDifferentRoleClient = SecureRestClientBuilder(clusterHosts.toTypedArray(), isHttps(), userWithDifferentRole, userWithDifferentRole) + .setSocketTimeout(60000).build() + + try { + createWorkflowWithClient( + userWithDifferentRoleClient!!, + workflow = randomWorkflow(monitorIds = listOf(createdMonitor.id), enabled = true), + listOf("role3") + ) + fail("Expected Forbidden exception") + } catch (e: ResponseException) { + assertEquals("Create workflow failed", RestStatus.FORBIDDEN.status, e.response.statusLine.statusCode) + } finally { + deleteUser(userWithDifferentRole) + userWithDifferentRoleClient?.close() + } + } + + fun `test create workflow with enable filter by with no backend roles`() { + enableFilterBy() + if (!isHttps()) { + // if security is disabled and filter by is enabled, we can't create monitor + // refer: `test create monitor with enable filter by` + return + } + val monitor = createMonitor(randomQueryLevelMonitor(enabled = true)) + + val workflow = randomWorkflow(monitorIds = listOf(monitor.id)) + + createUserWithRoles( + user, + listOf(ALERTING_FULL_ACCESS_ROLE, READALL_AND_MONITOR_ROLE), + listOf(TEST_HR_BACKEND_ROLE, "role2"), + false + ) + + try { + createWorkflowWithClient(userClient!!, workflow, listOf()) + fail("Expected exception since a non-admin user is trying to create a workflow with no backend roles") + } catch (e: ResponseException) { + assertEquals("Create workflow failed", RestStatus.FORBIDDEN.status, e.response.statusLine.statusCode) + } finally { + createUserRolesMapping(ALERTING_FULL_ACCESS_ROLE, arrayOf()) + createUserRolesMapping(READALL_AND_MONITOR_ROLE, arrayOf()) + } + } + + fun `test create workflow as admin with enable filter by with no backend roles`() { + enableFilterBy() + if (!isHttps()) { + // if security is disabled and filter by is enabled, we can't create monitor + // refer: `test create monitor with enable filter by` + return + } + val monitor = randomQueryLevelMonitor(enabled = true) + + createUserWithRoles( + user, + listOf(ALERTING_FULL_ACCESS_ROLE, READALL_AND_MONITOR_ROLE), + listOf(TEST_HR_BACKEND_ROLE, "role2"), + false + ) + + val createdMonitor = createMonitor(monitor = monitor) + val createdWorkflow = createWorkflow(randomWorkflow(monitorIds = listOf(createdMonitor.id))) + assertNotNull("The workflow was not created", createdWorkflow) + + try { + + userClient?.makeRequest( + "GET", + "$WORKFLOW_ALERTING_BASE_URI/${createdWorkflow.id}", + null, + BasicHeader(HttpHeaders.CONTENT_TYPE, "application/json") + ) + fail("Expected Forbidden exception") + } catch (e: ResponseException) { + assertEquals("Get workflow failed", RestStatus.FORBIDDEN.status, e.response.statusLine.statusCode) + } finally { + createUserRolesMapping(ALERTING_FULL_ACCESS_ROLE, arrayOf()) + createUserRolesMapping(READALL_AND_MONITOR_ROLE, arrayOf()) + } + } + + fun `test create workflow with enable filter by with roles user has no access and throw exception`() { + enableFilterBy() + if (!isHttps()) { + // if security is disabled and filter by is enabled, we can't create monitor + // refer: `test create monitor with enable filter by` + return + } + val monitor = createMonitor(randomQueryLevelMonitor(enabled = true)) + val workflow = randomWorkflow(monitorIds = listOf(monitor.id)) + + createUserWithRoles( + user, + listOf(ALERTING_FULL_ACCESS_ROLE, READALL_AND_MONITOR_ROLE), + listOf(TEST_HR_BACKEND_ROLE, "role2"), + false + ) + + try { + createWorkflowWithClient(userClient!!, workflow = workflow, listOf(TEST_HR_BACKEND_ROLE, "role1", "role2")) + fail("Expected create workflow to fail as user does not have role1 backend role") + } catch (e: ResponseException) { + assertEquals("Create workflow failed", RestStatus.FORBIDDEN.status, e.response.statusLine.statusCode) + } finally { + createUserRolesMapping(ALERTING_FULL_ACCESS_ROLE, arrayOf()) + createUserRolesMapping(READALL_AND_MONITOR_ROLE, arrayOf()) + } + } + + fun `test create workflow as admin with enable filter by with a user have access and without role has no access`() { + enableFilterBy() + if (!isHttps()) { + // if security is disabled and filter by is enabled, we can't create monitor + // refer: `test create monitor with enable filter by` + return + } + val monitor = randomQueryLevelMonitor(enabled = true) + + val createdMonitor = createMonitorWithClient(client(), monitor = monitor, listOf(TEST_HR_BACKEND_ROLE, "role1", "role2")) + val createdWorkflow = createWorkflowWithClient(client(), randomWorkflow(monitorIds = listOf(createdMonitor.id)), listOf(TEST_HR_BACKEND_ROLE, "role1", "role2")) + assertNotNull("The workflow was not created", createdWorkflow) + + // user should have access to the admin monitor + createUserWithTestDataAndCustomRole( + user, + TEST_HR_INDEX, + TEST_HR_ROLE, + listOf(TEST_HR_BACKEND_ROLE), + getClusterPermissionsFromCustomRole(ALERTING_GET_WORKFLOW_ACCESS) + ) + + val getWorkflowResponse = userClient?.makeRequest( + "GET", + "$WORKFLOW_ALERTING_BASE_URI/${createdWorkflow.id}", + null, + BasicHeader(HttpHeaders.CONTENT_TYPE, "application/json") + ) + assertEquals("Get workflow failed", RestStatus.OK, getWorkflowResponse?.restStatus()) + + // Remove good backend role and ensure no access is granted after + patchUserBackendRoles(user, arrayOf("role5")) + try { + userClient?.makeRequest( + "GET", + "$WORKFLOW_ALERTING_BASE_URI/${createdWorkflow.id}", + null, + BasicHeader(HttpHeaders.CONTENT_TYPE, "application/json") + ) + fail("Expected Forbidden exception") + } catch (e: ResponseException) { + assertEquals("Get workflow failed", RestStatus.FORBIDDEN.status, e.response.statusLine.statusCode) + } finally { + deleteRoleAndRoleMapping(TEST_HR_ROLE) + } + } + + fun `test update workflow with enable filter by with removing a permission`() { + enableFilterBy() + if (!isHttps()) { + // if security is disabled and filter by is enabled, we can't create monitor + // refer: `test create monitor with enable filter by` + return + } + + createUserWithRoles( + user, + listOf(ALERTING_FULL_ACCESS_ROLE, READALL_AND_MONITOR_ROLE), + listOf(TEST_HR_BACKEND_ROLE, "role2"), + false + ) + + val createdMonitor = createMonitorWithClient(userClient!!, randomQueryLevelMonitor(), listOf(TEST_HR_BACKEND_ROLE, "role2")) + val createdWorkflow = createWorkflowWithClient(userClient!!, workflow = randomWorkflow(enabled = true, monitorIds = listOf(createdMonitor.id)), listOf(TEST_HR_BACKEND_ROLE, "role2")) + assertNotNull("The workflow was not created", createdWorkflow) + + // getUser should have access to the monitor + val getUser = "getUser" + createUserWithTestDataAndCustomRole( + getUser, + TEST_HR_INDEX, + TEST_HR_ROLE, + listOf("role2"), + getClusterPermissionsFromCustomRole(ALERTING_GET_WORKFLOW_ACCESS) + ) + val getUserClient = SecureRestClientBuilder(clusterHosts.toTypedArray(), isHttps(), getUser, getUser) + .setSocketTimeout(60000).build() + + val getWorkflowResponse = getUserClient?.makeRequest( + "GET", + "$WORKFLOW_ALERTING_BASE_URI/${createdWorkflow.id}", + null, + BasicHeader(HttpHeaders.CONTENT_TYPE, "application/json") + ) + assertEquals("Get workflow failed", RestStatus.OK, getWorkflowResponse?.restStatus()) + + // Remove backend role from monitor + val updatedWorkflow = updateWorkflowWithClient(userClient!!, createdWorkflow, listOf(TEST_HR_BACKEND_ROLE)) + + // getUser should no longer have access + try { + getUserClient?.makeRequest( + "GET", + "$WORKFLOW_ALERTING_BASE_URI/${updatedWorkflow.id}", + null, + BasicHeader(HttpHeaders.CONTENT_TYPE, "application/json") + ) + fail("Expected Forbidden exception") + } catch (e: ResponseException) { + assertEquals("Get monitor failed", RestStatus.FORBIDDEN.status, e.response.statusLine.statusCode) + } finally { + deleteRoleAndRoleMapping(TEST_HR_ROLE) + deleteUser(getUser) + getUserClient?.close() + createUserRolesMapping(ALERTING_FULL_ACCESS_ROLE, arrayOf()) + createUserRolesMapping(READALL_AND_MONITOR_ROLE, arrayOf()) + } + } + + fun `test update workflow with enable filter by with no backend roles`() { + enableFilterBy() + if (!isHttps()) { + // if security is disabled and filter by is enabled, we can't create monitor + // refer: `test create monitor with enable filter by` + return + } + val monitor = randomQueryLevelMonitor(enabled = true) + + createUserWithRoles( + user, + listOf(ALERTING_FULL_ACCESS_ROLE, READALL_AND_MONITOR_ROLE), + listOf(TEST_HR_BACKEND_ROLE, "role2"), + false + ) + + val createdMonitor = createMonitorWithClient(userClient!!, monitor = monitor, listOf("role2")) + assertNotNull("The monitor was not created", createdMonitor) + + val createdWorkflow = createWorkflowWithClient( + userClient!!, + randomWorkflow(monitorIds = listOf(createdMonitor.id)), + listOf("role2") + ) + + assertNotNull("The workflow was not created", createdWorkflow) + + try { + updateWorkflowWithClient(userClient!!, createdWorkflow, listOf()) + } catch (e: ResponseException) { + assertEquals("Update monitor failed", RestStatus.FORBIDDEN.status, e.response.statusLine.statusCode) + } finally { + createUserRolesMapping(ALERTING_FULL_ACCESS_ROLE, arrayOf()) + createUserRolesMapping(READALL_AND_MONITOR_ROLE, arrayOf()) + } + } + + fun `test update workflow as admin with enable filter by with no backend roles`() { + enableFilterBy() + if (!isHttps()) { + // if security is disabled and filter by is enabled, we can't create monitor + // refer: `test create monitor with enable filter by` + return + } + val monitor = randomQueryLevelMonitor(enabled = true) + val createdMonitorResponse = createMonitor(monitor, true) + assertNotNull("The monitor was not created", createdMonitorResponse) + + createUserWithRoles( + user, + listOf(ALERTING_FULL_ACCESS_ROLE, READALL_AND_MONITOR_ROLE), + listOf(TEST_HR_BACKEND_ROLE, "role2"), + false + ) + val workflow = randomWorkflow( + monitorIds = listOf(createdMonitorResponse.id) + ) + + val createdWorkflow = createWorkflowWithClient( + client(), + workflow = workflow, + rbacRoles = listOf(TEST_HR_BACKEND_ROLE) + ) + + assertNotNull("The workflow was not created", createdWorkflow) + + val getWorkflowResponse = userClient?.makeRequest( + "GET", + "$WORKFLOW_ALERTING_BASE_URI/${createdWorkflow.id}", + null, + BasicHeader(HttpHeaders.CONTENT_TYPE, "application/json") + ) + assertEquals("Get workflow failed", RestStatus.OK, getWorkflowResponse?.restStatus()) + + val updatedWorkflow = updateWorkflowWithClient(client(), createdWorkflow, listOf()) + + try { + userClient?.makeRequest( + "GET", + "$WORKFLOW_ALERTING_BASE_URI/${updatedWorkflow.id}", + null, + BasicHeader(HttpHeaders.CONTENT_TYPE, "application/json") + ) + fail("Expected Forbidden exception") + } catch (e: ResponseException) { + assertEquals("Get workflow failed", RestStatus.FORBIDDEN.status, e.response.statusLine.statusCode) + } finally { + createUserRolesMapping(ALERTING_FULL_ACCESS_ROLE, arrayOf()) + createUserRolesMapping(READALL_AND_MONITOR_ROLE, arrayOf()) + } + } + + fun `test update workflow with enable filter by with updating with a permission user has no access to and throw exception`() { + enableFilterBy() + if (!isHttps()) { + // if security is disabled and filter by is enabled, we can't create monitor + // refer: `test create monitor with enable filter by` + return + } + val monitor = randomQueryLevelMonitor(enabled = true) + + createUserWithRoles( + user, + listOf(ALERTING_FULL_ACCESS_ROLE, READALL_AND_MONITOR_ROLE), + listOf(TEST_HR_BACKEND_ROLE, "role2"), + false + ) + + val createdMonitor = createMonitorWithClient(userClient!!, monitor = monitor, listOf(TEST_HR_BACKEND_ROLE, "role2")) + assertNotNull("The monitor was not created", createdMonitor) + + val createdWorkflow = createWorkflowWithClient( + userClient!!, + workflow = randomWorkflow(monitorIds = listOf(createdMonitor.id)), listOf(TEST_HR_BACKEND_ROLE, "role2") + ) + + assertNotNull("The workflow was not created", createdWorkflow) + + // getUser should have access to the monitor + val getUser = "getUser" + createUserWithTestDataAndCustomRole( + getUser, + TEST_HR_INDEX, + TEST_HR_ROLE, + listOf("role2"), + getClusterPermissionsFromCustomRole(ALERTING_GET_WORKFLOW_ACCESS) + ) + val getUserClient = SecureRestClientBuilder(clusterHosts.toTypedArray(), isHttps(), getUser, getUser) + .setSocketTimeout(60000).build() + + val getWorkflowResponse = getUserClient?.makeRequest( + "GET", + "$WORKFLOW_ALERTING_BASE_URI/${createdWorkflow.id}", + null, + BasicHeader(HttpHeaders.CONTENT_TYPE, "application/json") + ) + assertEquals("Get workflow failed", RestStatus.OK, getWorkflowResponse?.restStatus()) + + try { + updateWorkflowWithClient(userClient!!, createdWorkflow, listOf(TEST_HR_BACKEND_ROLE, "role1")) + fail("Expected update workflow to fail as user doesn't have access to role1") + } catch (e: ResponseException) { + assertEquals("Update workflow failed", RestStatus.FORBIDDEN.status, e.response.statusLine.statusCode) + } finally { + deleteRoleAndRoleMapping(TEST_HR_ROLE) + deleteUser(getUser) + getUserClient?.close() + createUserRolesMapping(ALERTING_FULL_ACCESS_ROLE, arrayOf()) + createUserRolesMapping(READALL_AND_MONITOR_ROLE, arrayOf()) + } + } + + fun `test update workflow as another user with enable filter by with removing a permission and adding permission`() { + enableFilterBy() + if (!isHttps()) { + // if security is disabled and filter by is enabled, we can't create monitor + // refer: `test create monitor with enable filter by` + return + } + val monitor = randomQueryLevelMonitor(enabled = true) + + createUserWithRoles( + user, + listOf(ALERTING_FULL_ACCESS_ROLE, READALL_AND_MONITOR_ROLE), + listOf(TEST_HR_BACKEND_ROLE, "role2"), + false + ) + + val createdMonitor = createMonitorWithClient(userClient!!, monitor = monitor, listOf(TEST_HR_BACKEND_ROLE)) + assertNotNull("The monitor was not created", createdMonitor) + + val createdWorkflow = createWorkflowWithClient( + userClient!!, + workflow = randomWorkflow(monitorIds = listOf(createdMonitor.id), enabled = true) + ) + + assertNotNull("The workflow was not created", createdWorkflow) + + // Remove backend role from workflow with new user and add role5 + val updateUser = "updateUser" + createUserWithRoles( + updateUser, + listOf(ALERTING_FULL_ACCESS_ROLE, READALL_AND_MONITOR_ROLE), + listOf(TEST_HR_BACKEND_ROLE, "role5"), + false + ) + + val updateUserClient = SecureRestClientBuilder(clusterHosts.toTypedArray(), isHttps(), updateUser, updateUser) + .setSocketTimeout(60000).build() + val updatedWorkflow = updateWorkflowWithClient(updateUserClient, createdWorkflow, listOf("role5")) + + // old user should no longer have access + try { + userClient?.makeRequest( + "GET", + "$WORKFLOW_ALERTING_BASE_URI/${updatedWorkflow.id}", + null, + BasicHeader(HttpHeaders.CONTENT_TYPE, "application/json") + ) + fail("Expected Forbidden exception") + } catch (e: ResponseException) { + assertEquals("Get workflow failed", RestStatus.FORBIDDEN.status, e.response.statusLine.statusCode) + } finally { + deleteUser(updateUser) + updateUserClient?.close() + createUserRolesMapping(ALERTING_FULL_ACCESS_ROLE, arrayOf()) + createUserRolesMapping(READALL_AND_MONITOR_ROLE, arrayOf()) + } + } + + fun `test update workflow as admin with enable filter by with removing a permission`() { + enableFilterBy() + if (!isHttps()) { + // if security is disabled and filter by is enabled, we can't create monitor + // refer: `test create monitor with enable filter by` + return + } + val monitor = randomQueryLevelMonitor(enabled = true) + + createUserWithRoles( + user, + listOf(ALERTING_FULL_ACCESS_ROLE, READALL_AND_MONITOR_ROLE), + listOf(TEST_HR_BACKEND_ROLE, "role2"), + false + ) + + val createdMonitor = createMonitorWithClient(userClient!!, monitor = monitor, listOf(TEST_HR_BACKEND_ROLE, "role2")) + assertNotNull("The monitor was not created", createdMonitor) + + val createdWorkflow = createWorkflowWithClient( + userClient!!, + workflow = randomWorkflow(monitorIds = listOf(createdMonitor.id)), + listOf(TEST_HR_BACKEND_ROLE, "role2") + ) + assertNotNull("The workflow was not created", createdWorkflow) + + // getUser should have access to the monitor + val getUser = "getUser" + createUserWithTestDataAndCustomRole( + getUser, + TEST_HR_INDEX, + TEST_HR_ROLE, + listOf("role1", "role2"), + getClusterPermissionsFromCustomRole(ALERTING_GET_WORKFLOW_ACCESS) + ) + val getUserClient = SecureRestClientBuilder(clusterHosts.toTypedArray(), isHttps(), getUser, getUser) + .setSocketTimeout(60000).build() + + val getWorkflowResponse = getUserClient?.makeRequest( + "GET", + "$WORKFLOW_ALERTING_BASE_URI/${createdWorkflow.id}", + null, + BasicHeader(HttpHeaders.CONTENT_TYPE, "application/json") + ) + assertEquals("Get workflow failed", RestStatus.OK, getWorkflowResponse?.restStatus()) + + // Remove backend role from monitor + val updatedWorkflow = updateWorkflowWithClient(client(), createdWorkflow, listOf("role4")) + + // original user should no longer have access + try { + userClient?.makeRequest( + "GET", + "$WORKFLOW_ALERTING_BASE_URI/${updatedWorkflow.id}", + null, + BasicHeader(HttpHeaders.CONTENT_TYPE, "application/json") + ) + fail("Expected Forbidden exception") + } catch (e: ResponseException) { + assertEquals("Get workflow failed", RestStatus.FORBIDDEN.status, e.response.statusLine.statusCode) + } finally { + createUserRolesMapping(ALERTING_FULL_ACCESS_ROLE, arrayOf()) + createUserRolesMapping(READALL_AND_MONITOR_ROLE, arrayOf()) + } + + // get user should no longer have access + try { + getUserClient?.makeRequest( + "GET", + "$WORKFLOW_ALERTING_BASE_URI/${updatedWorkflow.id}", + null, + BasicHeader(HttpHeaders.CONTENT_TYPE, "application/json") + ) + fail("Expected Forbidden exception") + } catch (e: ResponseException) { + assertEquals("Get workflow failed", RestStatus.FORBIDDEN.status, e.response.statusLine.statusCode) + } finally { + deleteRoleAndRoleMapping(TEST_HR_ROLE) + deleteUser(getUser) + getUserClient?.close() + } + } + + fun `test delete workflow with disable filter by`() { + disableFilterBy() + val monitor = randomQueryLevelMonitor(enabled = true) + + val createdMonitor = createMonitor(monitor = monitor) + val createdWorkflow = createWorkflow(workflow = randomWorkflow(monitorIds = listOf(createdMonitor.id), enabled = true)) + + assertNotNull("The workflow was not created", createdWorkflow) + assertTrue("The workflow was not enabled", createdWorkflow.enabled) + + deleteWorkflow(workflow = createdWorkflow, deleteDelegates = true) + + val searchMonitor = SearchSourceBuilder().query(QueryBuilders.termQuery("_id", createdMonitor.id)).toString() + // Verify if the delegate monitors are deleted + // search as "admin" - must get 0 docs + val adminMonitorSearchResponse = client().makeRequest( + "POST", + "$ALERTING_BASE_URI/_search", + emptyMap(), + NStringEntity(searchMonitor, ContentType.APPLICATION_JSON) + ) + assertEquals("Search monitor failed", RestStatus.OK, adminMonitorSearchResponse.restStatus()) + + val adminMonitorHits = createParser( + XContentType.JSON.xContent(), + adminMonitorSearchResponse.entity.content + ).map()["hits"]!! as Map> + val adminMonitorDocsFound = adminMonitorHits["total"]?.get("value") + assertEquals("Monitor found during search", 0, adminMonitorDocsFound) + + // Verify workflow deletion + try { + client().makeRequest( + "GET", + "$WORKFLOW_ALERTING_BASE_URI/${createdWorkflow.id}", + emptyMap(), + null + ) + fail("Workflow found during search") + } catch (e: ResponseException) { + assertEquals("Get workflow failed", RestStatus.NOT_FOUND.status, e.response.statusLine.statusCode) + } + } + + fun `test delete workflow with enable filter by`() { + enableFilterBy() + if (!isHttps()) { + // if security is disabled and filter by is enabled, we can't create monitor + // refer: `test create monitor with enable filter by` + return + } + val createdMonitor = createMonitorWithClient( + monitor = randomQueryLevelMonitor(), + client = client(), + rbacRoles = listOf("admin") + ) + + assertNotNull("The monitor was not created", createdMonitor) + + val createdWorkflow = createWorkflow(workflow = randomWorkflow(monitorIds = listOf(createdMonitor.id), enabled = true)) + assertNotNull("The workflow was not created", createdWorkflow) + assertTrue("The workflow was not enabled", createdWorkflow.enabled) + + deleteWorkflow(workflow = createdWorkflow, true) + + // Verify underlying delegates deletion + val search = SearchSourceBuilder().query(QueryBuilders.termQuery("_id", createdMonitor.id)).toString() + // search as "admin" - must get 0 docs + val adminSearchResponse = client().makeRequest( + "POST", + "$ALERTING_BASE_URI/_search", + emptyMap(), + NStringEntity(search, ContentType.APPLICATION_JSON) + ) + assertEquals("Search monitor failed", RestStatus.OK, adminSearchResponse.restStatus()) + + val adminHits = createParser( + XContentType.JSON.xContent(), + adminSearchResponse.entity.content + ).map()["hits"]!! as Map> + val adminDocsFound = adminHits["total"]?.get("value") + assertEquals("Monitor found during search", 0, adminDocsFound) + + // Verify workflow deletion + try { + client().makeRequest( + "GET", + "$WORKFLOW_ALERTING_BASE_URI/${createdWorkflow.id}", + emptyMap(), + null + ) + fail("Workflow found during search") + } catch (e: ResponseException) { + assertEquals("Get workflow failed", RestStatus.NOT_FOUND.status, e.response.statusLine.statusCode) + } + } + + fun `test delete workflow with enable filter with user that doesn't have delete_monitor cluster privilege failed`() { + enableFilterBy() + if (!isHttps()) { + // if security is disabled and filter by is enabled, we can't create monitor + // refer: `test create monitor with enable filter by` + return + } + createUserWithRoles( + user, + listOf(ALERTING_FULL_ACCESS_ROLE, READALL_AND_MONITOR_ROLE), + listOf(TEST_HR_BACKEND_ROLE, "role2"), + false + ) + + val deleteUser = "deleteUser" + createUserWithTestDataAndCustomRole( + deleteUser, + TEST_HR_INDEX, + TEST_HR_ROLE, + listOf("role1", "role3"), + listOf( + getClusterPermissionsFromCustomRole(ALERTING_DELETE_WORKFLOW_ACCESS), + getClusterPermissionsFromCustomRole(ALERTING_GET_WORKFLOW_ACCESS) + ) + ) + val deleteUserClient = SecureRestClientBuilder(clusterHosts.toTypedArray(), isHttps(), deleteUser, deleteUser) + .setSocketTimeout(60000).build() + + try { + val createdMonitor = createMonitorWithClient(userClient!!, monitor = randomQueryLevelMonitor()) + + assertNotNull("The monitor was not created", createdMonitor) + + val createdWorkflow = createWorkflowWithClient(userClient!!, workflow = randomWorkflow(monitorIds = listOf(createdMonitor.id), enabled = true)) + assertNotNull("The workflow was not created", createdWorkflow) + assertTrue("The workflow was not enabled", createdWorkflow.enabled) + + try { + deleteWorkflowWithClient(deleteUserClient, workflow = createdWorkflow, true) + fail("Expected Forbidden exception") + } catch (e: ResponseException) { + assertEquals("Get workflow failed", RestStatus.FORBIDDEN.status, e.response.statusLine.statusCode) + } + patchUserBackendRoles(deleteUser, arrayOf("role2")) + + val response = deleteWorkflowWithClient(deleteUserClient!!, workflow = createdWorkflow, true) + assertEquals("Delete workflow failed", RestStatus.OK, response?.restStatus()) + } finally { + deleteRoleAndRoleMapping(TEST_HR_ROLE) + deleteUser(deleteUser) + deleteUserClient?.close() + } + } + + fun `test execute workflow with an user with execute workflow access`() { + createUserWithTestDataAndCustomRole( + user, + TEST_HR_INDEX, + TEST_HR_ROLE, + listOf(TEST_HR_BACKEND_ROLE), + getClusterPermissionsFromCustomRole(ALERTING_EXECUTE_WORKFLOW_ACCESS) + ) + + val monitor = createRandomMonitor(true) + val workflow = createRandomWorkflow(listOf(monitor.id), true) + + try { + val executeWorkflowResponse = userClient?.makeRequest( + "POST", + "$WORKFLOW_ALERTING_BASE_URI/${workflow.id}/_execute", + mutableMapOf() + ) + assertEquals("Executing workflow failed", RestStatus.OK, executeWorkflowResponse?.restStatus()) + } finally { + deleteRoleAndRoleMapping(TEST_HR_ROLE) + } + } + + fun `test execute workflow with an user without execute workflow access`() { + createUserWithTestDataAndCustomRole( + user, + TEST_HR_INDEX, + TEST_HR_ROLE, + listOf(TEST_HR_BACKEND_ROLE), + getClusterPermissionsFromCustomRole(ALERTING_NO_ACCESS_ROLE) + ) + + val monitor = createRandomMonitor(true) + val workflow = createRandomWorkflow(listOf(monitor.id), true) + + try { + userClient?.makeRequest( + "POST", + "$WORKFLOW_ALERTING_BASE_URI/${workflow.id}/_execute", + mutableMapOf() + ) + fail("Expected 403 Method FORBIDDEN response") + } catch (e: ResponseException) { + assertEquals("Execute workflow failed", RestStatus.FORBIDDEN, e.response.restStatus()) + } finally { + deleteRoleAndRoleMapping(TEST_HR_ROLE) + } + } + + fun `test delete workflow with an user with delete workflow access`() { + createUserWithTestDataAndCustomRole( + user, + TEST_HR_INDEX, + TEST_HR_ROLE, + listOf(TEST_HR_BACKEND_ROLE), + getClusterPermissionsFromCustomRole(ALERTING_DELETE_WORKFLOW_ACCESS) + ) + + val monitor = createRandomMonitor(true) + val workflow = createRandomWorkflow(monitorIds = listOf(monitor.id)) + val refresh = true + + try { + val deleteWorkflowResponse = userClient?.makeRequest( + "DELETE", + "$WORKFLOW_ALERTING_BASE_URI/${workflow.id}?refresh=$refresh", + emptyMap(), + monitor.toHttpEntity() + ) + assertEquals("DELETE workflow failed", RestStatus.OK, deleteWorkflowResponse?.restStatus()) + } finally { + deleteRoleAndRoleMapping(TEST_HR_ROLE) + } + } + + fun `test delete workflow with deleting delegates with an user with delete workflow access`() { + createUserWithTestDataAndCustomRole( + user, + TEST_HR_INDEX, + TEST_HR_ROLE, + listOf(TEST_HR_BACKEND_ROLE), + getClusterPermissionsFromCustomRole(ALERTING_DELETE_WORKFLOW_ACCESS) + ) + + val monitor = createRandomMonitor(true) + val workflow = createRandomWorkflow(monitorIds = listOf(monitor.id)) + + try { + val deleteWorkflowResponse = deleteWorkflowWithClient( + userClient!!, + workflow, + deleteDelegates = true, + refresh = true + ) + assertEquals("DELETE workflow failed", RestStatus.OK, deleteWorkflowResponse?.restStatus()) + } finally { + deleteRoleAndRoleMapping(TEST_HR_ROLE) + } + // Verify delegate deletion + val search = SearchSourceBuilder().query(QueryBuilders.termQuery("_id", monitor.id)).toString() + // search as "admin" - must get 0 docs + val adminSearchResponse = client().makeRequest( + "POST", + "$ALERTING_BASE_URI/_search", + emptyMap(), + NStringEntity(search, ContentType.APPLICATION_JSON) + ) + assertEquals("Search monitor failed", RestStatus.OK, adminSearchResponse.restStatus()) + + val adminHits = createParser( + XContentType.JSON.xContent(), + adminSearchResponse.entity.content + ).map()["hits"]!! as Map> + val adminDocsFound = adminHits["total"]?.get("value") + assertEquals("Monitor found during search", 0, adminDocsFound) + } + + fun `test delete workflow with an user without delete monitor access`() { + createUserWithTestDataAndCustomRole( + user, + TEST_HR_INDEX, + TEST_HR_ROLE, + listOf(TEST_HR_BACKEND_ROLE), + getClusterPermissionsFromCustomRole(ALERTING_NO_ACCESS_ROLE) + ) + + val monitor = createRandomMonitor(true) + val workflow = createRandomWorkflow(monitorIds = listOf(monitor.id)) + + try { + userClient?.makeRequest( + "DELETE", + "$WORKFLOW_ALERTING_BASE_URI/${workflow.id}?refresh=true", + emptyMap(), + monitor.toHttpEntity() + ) + fail("Expected 403 Method FORBIDDEN response") + } catch (e: ResponseException) { + assertEquals("DELETE workflow failed", RestStatus.FORBIDDEN, e.response.restStatus()) + } finally { + deleteRoleAndRoleMapping(TEST_HR_ROLE) + } + } + + fun `test admin all access with enable filter by`() { + enableFilterBy() + createUserWithTestData(user, TEST_HR_INDEX, TEST_HR_ROLE, TEST_HR_BACKEND_ROLE) + createUserRolesMapping(ALERTING_FULL_ACCESS_ROLE, arrayOf(user)) + try { + // randomMonitor has a dummy user, api ignores the User passed as part of monitor, it picks user info from the logged-in user. + val monitor = randomQueryLevelMonitor().copy( + inputs = listOf( + SearchInput( + indices = listOf(TEST_HR_INDEX), + query = SearchSourceBuilder().query(QueryBuilders.matchAllQuery()) + ) + ) + ) + + val createResponse = userClient?.makeRequest("POST", ALERTING_BASE_URI, emptyMap(), monitor.toHttpEntity()) + assertEquals("Create monitor failed", RestStatus.CREATED, createResponse?.restStatus()) + val monitorJson = JsonXContent.jsonXContent.createParser( + NamedXContentRegistry.EMPTY, + LoggingDeprecationHandler.INSTANCE, + createResponse?.entity?.content + ).map() + val monitorId = monitorJson["_id"] as String + + val workflow = randomWorkflow(monitorIds = listOf(monitorId)) + val createWorkflowResponse = userClient?.makeRequest("POST", WORKFLOW_ALERTING_BASE_URI, emptyMap(), workflow.toHttpEntity()) + assertEquals("Create workflow failed", RestStatus.CREATED, createWorkflowResponse?.restStatus()) + + val workflowJson = JsonXContent.jsonXContent.createParser( + NamedXContentRegistry.EMPTY, + LoggingDeprecationHandler.INSTANCE, + createWorkflowResponse?.entity?.content + ).map() + + val id: String = workflowJson["_id"] as String + val search = SearchSourceBuilder().query(QueryBuilders.termQuery("_id", id)).toString() + + // get as "admin" - must get 1 docs + val adminGetResponse = client().makeRequest( + "GET", + "$WORKFLOW_ALERTING_BASE_URI/$id", + emptyMap(), + NStringEntity(search, ContentType.APPLICATION_JSON) + ) + assertEquals("Get workflow failed", RestStatus.OK, adminGetResponse.restStatus()) + + // delete as "admin" + val adminDeleteResponse = client().makeRequest( + "DELETE", + "$WORKFLOW_ALERTING_BASE_URI/$id", + emptyMap(), + NStringEntity(search, ContentType.APPLICATION_JSON) + ) + assertEquals("Delete workflow failed", RestStatus.OK, adminDeleteResponse.restStatus()) + } finally { + deleteRoleAndRoleMapping(TEST_HR_ROLE) + deleteRoleMapping(ALERTING_FULL_ACCESS_ROLE) + } + } + + fun `test execute workflow with bucket-level and doc-level chained monitors with user having partial index permissions`() { + createUser(user, user, arrayOf(TEST_HR_BACKEND_ROLE)) + createTestIndex(TEST_HR_INDEX) + + createIndexRoleWithDocLevelSecurity( + TEST_HR_ROLE, + TEST_HR_INDEX, + TERM_DLS_QUERY, + listOf(ALERTING_INDEX_WORKFLOW_ACCESS, ALERTING_INDEX_MONITOR_ACCESS) + ) + createUserRolesMapping(TEST_HR_ROLE, arrayOf(user)) + + // Add a doc that is accessible to the user + indexDoc( + TEST_HR_INDEX, + "1", + """ + { + "test_field": "a", + "accessible": true + } + """.trimIndent() + ) + + // Add a second doc that is not accessible to the user + indexDoc( + TEST_HR_INDEX, + "2", + """ + { + "test_field": "b", + "accessible": false + } + """.trimIndent() + ) + + indexDoc( + TEST_HR_INDEX, + "3", + """ + { + "test_field": "c", + "accessible": true + } + """.trimIndent() + ) + + val compositeSources = listOf( + TermsValuesSourceBuilder("test_field").field("test_field") + ) + val compositeAgg = CompositeAggregationBuilder("composite_agg", compositeSources) + val input = SearchInput( + indices = listOf(TEST_HR_INDEX), + query = SearchSourceBuilder().size(0).query(QueryBuilders.matchAllQuery()).aggregation(compositeAgg) + ) + val triggerScript = """ + params.docCount > 0 + """.trimIndent() + + var trigger = randomBucketLevelTrigger() + trigger = trigger.copy( + bucketSelector = BucketSelectorExtAggregationBuilder( + name = trigger.id, + bucketsPathsMap = mapOf("docCount" to "_count"), + script = Script(triggerScript), + parentBucketPath = "composite_agg", + filter = null + ), + actions = listOf() + ) + val bucketMonitor = createMonitorWithClient( + userClient!!, + randomBucketLevelMonitor( + inputs = listOf(input), + enabled = false, + triggers = listOf(trigger), + dataSources = DataSources(findingsEnabled = true) + ) + ) + assertNotNull("The bucket monitor was not created", bucketMonitor) + + val docQuery1 = DocLevelQuery(query = "test_field:\"a\"", name = "3") + var monitor1 = randomDocumentLevelMonitor( + inputs = listOf(DocLevelMonitorInput("description", listOf(TEST_HR_INDEX), listOf(docQuery1))), + triggers = listOf(randomDocumentLevelTrigger(condition = ALWAYS_RUN)) + ) + val docMonitor = createMonitorWithClient(userClient!!, monitor1)!! + assertNotNull("The doc level monitor was not created", docMonitor) + + val workflow = randomWorkflow(monitorIds = listOf(bucketMonitor.id, docMonitor.id)) + val workflowResponse = createWorkflowWithClient(userClient!!, workflow) + assertNotNull("The workflow was not created", workflowResponse) + + try { + executeWorkflow(workflowId = workflowResponse.id) + val bucketAlerts = searchAlerts(bucketMonitor) + assertEquals("Incorrect number of alerts", 2, bucketAlerts.size) + + val docAlerts = searchAlerts(docMonitor) + assertEquals("Incorrect number of alerts", 1, docAlerts.size) + } finally { + deleteRoleAndRoleMapping(TEST_HR_ROLE) + } + } +} diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/resthandler/WorkflowRestApiIT.kt b/alerting/src/test/kotlin/org/opensearch/alerting/resthandler/WorkflowRestApiIT.kt new file mode 100644 index 000000000..00a81c442 --- /dev/null +++ b/alerting/src/test/kotlin/org/opensearch/alerting/resthandler/WorkflowRestApiIT.kt @@ -0,0 +1,1027 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.alerting.resthandler + +import org.opensearch.alerting.ALWAYS_RUN +import org.opensearch.alerting.AlertingRestTestCase +import org.opensearch.alerting.WORKFLOW_ALERTING_BASE_URI +import org.opensearch.alerting.makeRequest +import org.opensearch.alerting.randomBucketLevelMonitor +import org.opensearch.alerting.randomDocumentLevelMonitor +import org.opensearch.alerting.randomDocumentLevelTrigger +import org.opensearch.alerting.randomQueryLevelMonitor +import org.opensearch.alerting.randomWorkflow +import org.opensearch.alerting.randomWorkflowWithDelegates +import org.opensearch.client.ResponseException +import org.opensearch.commons.alerting.model.ChainedMonitorFindings +import org.opensearch.commons.alerting.model.CompositeInput +import org.opensearch.commons.alerting.model.Delegate +import org.opensearch.commons.alerting.model.DocLevelMonitorInput +import org.opensearch.commons.alerting.model.DocLevelQuery +import org.opensearch.commons.alerting.model.Monitor +import org.opensearch.commons.alerting.model.SearchInput +import org.opensearch.commons.alerting.model.Workflow +import org.opensearch.index.query.QueryBuilders +import org.opensearch.rest.RestStatus +import org.opensearch.search.aggregations.bucket.terms.TermsAggregationBuilder +import org.opensearch.search.builder.SearchSourceBuilder +import org.opensearch.test.junit.annotations.TestLogging +import java.util.Collections +import java.util.Locale +import java.util.UUID + +@TestLogging("level:DEBUG", reason = "Debug for tests.") +@Suppress("UNCHECKED_CAST") +class WorkflowRestApiIT : AlertingRestTestCase() { + + fun `test create workflow success`() { + val index = createTestIndex() + val docQuery1 = DocLevelQuery(query = "source.ip.v6.v1:12345", name = "3") + val docLevelInput = DocLevelMonitorInput( + "description", listOf(index), listOf(docQuery1) + ) + val trigger = randomDocumentLevelTrigger(condition = ALWAYS_RUN) + + val monitor = randomDocumentLevelMonitor( + inputs = listOf(docLevelInput), + triggers = listOf(trigger) + ) + val monitorResponse = createMonitor(monitor) + + val workflow = randomWorkflow( + monitorIds = listOf(monitorResponse.id) + ) + + val createResponse = client().makeRequest("POST", WORKFLOW_ALERTING_BASE_URI, emptyMap(), workflow.toHttpEntity()) + + assertEquals("Create workflow failed", RestStatus.CREATED, createResponse.restStatus()) + + val responseBody = createResponse.asMap() + val createdId = responseBody["_id"] as String + val createdVersion = responseBody["_version"] as Int + + assertNotEquals("response is missing Id", Workflow.NO_ID, createdId) + assertTrue("incorrect version", createdVersion > 0) + assertEquals("Incorrect Location header", "$WORKFLOW_ALERTING_BASE_URI/$createdId", createResponse.getHeader("Location")) + } + + fun `test create workflow with different monitor types success`() { + val index = createTestIndex() + val docQuery = DocLevelQuery(query = "source.ip.v6.v1:12345", name = "3") + val docLevelInput = DocLevelMonitorInput( + "description", listOf(index), listOf(docQuery) + ) + val trigger = randomDocumentLevelTrigger(condition = ALWAYS_RUN) + + val monitor = randomDocumentLevelMonitor( + inputs = listOf(docLevelInput), + triggers = listOf(trigger) + ) + val docLevelMonitorResponse = createMonitor(monitor) + + val bucketLevelMonitor = randomBucketLevelMonitor( + inputs = listOf( + SearchInput( + listOf(index), + SearchSourceBuilder().query(QueryBuilders.matchAllQuery()) + .aggregation(TermsAggregationBuilder("test_agg").field("test_field")) + ) + ) + ) + val bucketLevelMonitorResponse = createMonitor(bucketLevelMonitor) + + val workflow = randomWorkflow( + monitorIds = listOf(docLevelMonitorResponse.id, bucketLevelMonitorResponse.id) + ) + + val createResponse = client().makeRequest("POST", WORKFLOW_ALERTING_BASE_URI, emptyMap(), workflow.toHttpEntity()) + + assertEquals("Create workflow failed", RestStatus.CREATED, createResponse.restStatus()) + + val responseBody = createResponse.asMap() + val createdId = responseBody["_id"] as String + val createdVersion = responseBody["_version"] as Int + + assertNotEquals("response is missing Id", Workflow.NO_ID, createdId) + assertTrue("incorrect version", createdVersion > 0) + assertEquals("Incorrect Location header", "$WORKFLOW_ALERTING_BASE_URI/$createdId", createResponse.getHeader("Location")) + + val workflowById = getWorkflow(createdId) + assertNotNull(workflowById) + + // Verify workflow + assertNotEquals("response is missing Id", Monitor.NO_ID, workflowById.id) + assertTrue("incorrect version", workflowById.version > 0) + assertEquals("Workflow name not correct", workflow.name, workflowById.name) + assertEquals("Workflow owner not correct", workflow.owner, workflowById.owner) + assertEquals("Workflow input not correct", workflow.inputs, workflowById.inputs) + + // Delegate verification + @Suppress("UNCHECKED_CAST") + val delegates = (workflowById.inputs as List)[0].sequence.delegates.sortedBy { it.order } + assertEquals("Delegates size not correct", 2, delegates.size) + + val delegate1 = delegates[0] + assertNotNull(delegate1) + assertEquals("Delegate1 order not correct", 1, delegate1.order) + assertEquals("Delegate1 id not correct", docLevelMonitorResponse.id, delegate1.monitorId) + + val delegate2 = delegates[1] + assertNotNull(delegate2) + assertEquals("Delegate2 order not correct", 2, delegate2.order) + assertEquals("Delegate2 id not correct", bucketLevelMonitorResponse.id, delegate2.monitorId) + assertEquals( + "Delegate2 Chained finding not correct", docLevelMonitorResponse.id, delegate2.chainedMonitorFindings!!.monitorId + ) + } + + fun `test create workflow without delegate failure`() { + val workflow = randomWorkflow( + monitorIds = Collections.emptyList() + ) + try { + createWorkflow(workflow) + } catch (e: ResponseException) { + assertEquals("Unexpected status", RestStatus.BAD_REQUEST, e.response.restStatus()) + e.message?.let { + assertTrue( + "Exception not returning IndexWorkflow Action error ", + it.contains("Delegates list can not be empty.") + ) + } + } + } + + fun `test create workflow duplicate delegate failure`() { + val workflow = randomWorkflow( + monitorIds = listOf("1", "1", "2") + ) + try { + createWorkflow(workflow) + } catch (e: ResponseException) { + assertEquals("Unexpected status", RestStatus.BAD_REQUEST, e.response.restStatus()) + e.message?.let { + assertTrue( + "Exception not returning IndexWorkflow Action error ", + it.contains("Duplicate delegates not allowed") + ) + } + } + } + + fun `test create workflow delegate monitor doesn't exist failure`() { + val index = createTestIndex() + val docQuery = DocLevelQuery(query = "source.ip.v6.v1:12345", name = "3") + val docLevelInput = DocLevelMonitorInput( + "description", listOf(index), listOf(docQuery) + ) + val trigger = randomDocumentLevelTrigger(condition = ALWAYS_RUN) + + val monitor = randomDocumentLevelMonitor( + inputs = listOf(docLevelInput), + triggers = listOf(trigger) + ) + val docLevelMonitorResponse = createMonitor(monitor) + + val workflow = randomWorkflow( + monitorIds = listOf("-1", docLevelMonitorResponse.id) + ) + try { + createWorkflow(workflow) + } catch (e: ResponseException) { + assertEquals("Unexpected status", RestStatus.BAD_REQUEST, e.response.restStatus()) + e.message?.let { + assertTrue( + "Exception not returning IndexWorkflow Action error ", + it.contains("are not valid monitor ids") + ) + } + } + } + + fun `test create workflow sequence order not correct failure`() { + val delegates = listOf( + Delegate(1, "monitor-1"), + Delegate(1, "monitor-2"), + Delegate(2, "monitor-3") + ) + val workflow = randomWorkflowWithDelegates( + delegates = delegates + ) + try { + createWorkflow(workflow) + } catch (e: ResponseException) { + assertEquals("Unexpected status", RestStatus.BAD_REQUEST, e.response.restStatus()) + e.message?.let { + assertTrue( + "Exception not returning IndexWorkflow Action error ", + it.contains("Sequence ordering of delegate monitor shouldn't contain duplicate order values") + ) + } + } + } + + fun `test create workflow chained findings monitor not in sequence failure`() { + val delegates = listOf( + Delegate(1, "monitor-1"), + Delegate(2, "monitor-2", ChainedMonitorFindings("monitor-1")), + Delegate(3, "monitor-3", ChainedMonitorFindings("monitor-x")) + ) + val workflow = randomWorkflowWithDelegates( + delegates = delegates + ) + + try { + createWorkflow(workflow) + } catch (e: ResponseException) { + assertEquals("Unexpected status", RestStatus.BAD_REQUEST, e.response.restStatus()) + e.message?.let { + assertTrue( + "Exception not returning IndexWorkflow Action error ", + it.contains("Chained Findings Monitor monitor-x doesn't exist in sequence") + ) + } + } + } + + fun `test create workflow chained findings order not correct failure`() { + val delegates = listOf( + Delegate(1, "monitor-1"), + Delegate(3, "monitor-2", ChainedMonitorFindings("monitor-1")), + Delegate(2, "monitor-3", ChainedMonitorFindings("monitor-2")) + ) + val workflow = randomWorkflowWithDelegates( + delegates = delegates + ) + + try { + createWorkflow(workflow) + } catch (e: ResponseException) { + assertEquals("Unexpected status", RestStatus.BAD_REQUEST, e.response.restStatus()) + e.message?.let { + assertTrue( + "Exception not returning IndexWorkflow Action error ", + it.contains("Chained Findings Monitor monitor-2 should be executed before monitor monitor-3") + ) + } + } + } + + fun `test create workflow when monitor index not initialized failure`() { + val delegates = listOf( + Delegate(1, "monitor-1") + ) + val workflow = randomWorkflowWithDelegates( + delegates = delegates + ) + + try { + createWorkflow(workflow) + } catch (e: ResponseException) { + assertEquals("Unexpected status", RestStatus.NOT_FOUND, e.response.restStatus()) + e.message?.let { + assertTrue( + "Exception not returning IndexWorkflow Action error ", + it.contains("Monitors not found") + ) + } + } + } + + fun `test create workflow delegate and chained finding monitor different indices failure`() { + val index = randomAlphaOfLength(10).lowercase(Locale.ROOT) + createTestIndex(index) + + val docLevelInput = DocLevelMonitorInput( + "description", listOf(index), listOf(DocLevelQuery(query = "source.ip.v6.v1:12345", name = "3")) + ) + val trigger = randomDocumentLevelTrigger(condition = ALWAYS_RUN) + + val docMonitor = randomDocumentLevelMonitor( + inputs = listOf(docLevelInput), + triggers = listOf(trigger) + ) + val docMonitorResponse = createMonitor(docMonitor) + + val index1 = "$index-1" + createTestIndex(index1) + + val docLevelInput1 = DocLevelMonitorInput( + "description", listOf(index1), listOf(DocLevelQuery(query = "source.ip.v6.v1:12345", name = "3")) + ) + + val docMonitor1 = randomDocumentLevelMonitor( + inputs = listOf(docLevelInput1), + triggers = listOf(trigger) + ) + val docMonitorResponse1 = createMonitor(docMonitor1) + + val workflow = randomWorkflow( + monitorIds = listOf(docMonitorResponse1.id, docMonitorResponse.id) + ) + try { + createWorkflow(workflow) + } catch (e: ResponseException) { + assertEquals("Unexpected status", RestStatus.BAD_REQUEST, e.response.restStatus()) + e.message?.let { + assertTrue( + "Exception not returning IndexWorkflow Action error ", + it.contains("Delegate monitor and it's chained finding monitor must query the same indices") + ) + } + } + } + + fun `test create workflow query monitor chained findings monitor failure`() { + val index = createTestIndex() + val docLevelInput = DocLevelMonitorInput( + "description", listOf(index), listOf(DocLevelQuery(query = "source.ip.v6.v1:12345", name = "3")) + ) + val trigger = randomDocumentLevelTrigger(condition = ALWAYS_RUN) + + val docMonitor = randomDocumentLevelMonitor( + inputs = listOf(docLevelInput), + triggers = listOf(trigger) + ) + val docMonitorResponse = createMonitor(docMonitor) + + val queryMonitor = randomQueryLevelMonitor() + val queryMonitorResponse = createMonitor(queryMonitor) + + val workflow = randomWorkflow( + monitorIds = listOf(queryMonitorResponse.id, docMonitorResponse.id) + ) + try { + createWorkflow(workflow) + } catch (e: ResponseException) { + assertEquals("Unexpected status", RestStatus.BAD_REQUEST, e.response.restStatus()) + e.message?.let { + assertTrue( + "Exception not returning IndexWorkflow Action error ", + it.contains("Query level monitor can't be part of chained findings") + ) + } + } + } + + fun `test create workflow with 26 delegates failure`() { + val monitorsIds = mutableListOf() + for (i in 0..25) { + monitorsIds.add(UUID.randomUUID().toString()) + } + val workflow = randomWorkflow( + monitorIds = monitorsIds + ) + try { + createWorkflow(workflow) + } catch (e: ResponseException) { + assertEquals("Unexpected status", RestStatus.BAD_REQUEST, e.response.restStatus()) + e.message?.let { + assertTrue( + "Exception not returning IndexWorkflow Action error ", + it.contains("Delegates list can not be larger then 25.") + ) + } + } + } + + fun `test update workflow add monitor success`() { + val index = createTestIndex() + val docQuery1 = DocLevelQuery(query = "source.ip.v6.v1:12345", name = "3") + val docLevelInput = DocLevelMonitorInput( + "description", listOf(index), listOf(docQuery1) + ) + val trigger = randomDocumentLevelTrigger(condition = ALWAYS_RUN) + + val monitor = randomDocumentLevelMonitor( + inputs = listOf(docLevelInput), + triggers = listOf(trigger) + ) + val monitorResponse = createMonitor(monitor) + + val workflow = randomWorkflow( + monitorIds = listOf(monitorResponse.id) + ) + + val createResponse = client().makeRequest("POST", WORKFLOW_ALERTING_BASE_URI, emptyMap(), workflow.toHttpEntity()) + + assertEquals("Create workflow failed", RestStatus.CREATED, createResponse.restStatus()) + + val responseBody = createResponse.asMap() + val createdId = responseBody["_id"] as String + val createdVersion = responseBody["_version"] as Int + + assertNotEquals("response is missing Id", Workflow.NO_ID, createdId) + assertTrue("incorrect version", createdVersion > 0) + assertEquals("Incorrect Location header", "$WORKFLOW_ALERTING_BASE_URI/$createdId", createResponse.getHeader("Location")) + + val monitor2 = randomDocumentLevelMonitor( + inputs = listOf(docLevelInput), + triggers = listOf(trigger) + ) + + val monitorResponse2 = createMonitor(monitor2) + + val updatedWorkflow = randomWorkflow( + id = createdId, + monitorIds = listOf(monitorResponse.id, monitorResponse2.id) + ) + + val updateResponse = client().makeRequest("PUT", updatedWorkflow.relativeUrl(), emptyMap(), updatedWorkflow.toHttpEntity()) + + assertEquals("Update workflow failed", RestStatus.OK, updateResponse.restStatus()) + + val updateResponseBody = updateResponse.asMap() + val updatedId = updateResponseBody["_id"] as String + val updatedVersion = updateResponseBody["_version"] as Int + + assertNotEquals("response is missing Id", Workflow.NO_ID, updatedId) + assertTrue("incorrect version", updatedVersion > 0) + + val workflowById = getWorkflow(updatedId) + assertNotNull(workflowById) + // Delegate verification + @Suppress("UNCHECKED_CAST") + val delegates = (workflowById.inputs as List)[0].sequence.delegates.sortedBy { it.order } + assertEquals("Delegates size not correct", 2, delegates.size) + + val delegate1 = delegates[0] + assertNotNull(delegate1) + assertEquals("Delegate1 order not correct", 1, delegate1.order) + assertEquals("Delegate1 id not correct", monitorResponse.id, delegate1.monitorId) + + val delegate2 = delegates[1] + assertNotNull(delegate2) + assertEquals("Delegate2 order not correct", 2, delegate2.order) + assertEquals("Delegate2 id not correct", monitorResponse2.id, delegate2.monitorId) + assertEquals( + "Delegate2 Chained finding not correct", monitorResponse.id, delegate2.chainedMonitorFindings!!.monitorId + ) + } + + fun `test update workflow remove monitor success`() { + val index = createTestIndex() + val docQuery1 = DocLevelQuery(query = "source.ip.v6.v1:12345", name = "3") + val docLevelInput = DocLevelMonitorInput( + "description", listOf(index), listOf(docQuery1) + ) + val trigger = randomDocumentLevelTrigger(condition = ALWAYS_RUN) + + val monitor = randomDocumentLevelMonitor( + inputs = listOf(docLevelInput), + triggers = listOf(trigger) + ) + val monitorResponse = createMonitor(monitor) + + val monitor2 = randomDocumentLevelMonitor( + inputs = listOf(docLevelInput), + triggers = listOf(trigger) + ) + + val monitorResponse2 = createMonitor(monitor2) + + val workflow = randomWorkflow( + monitorIds = listOf(monitorResponse.id, monitorResponse2.id) + ) + + val createResponse = client().makeRequest("POST", WORKFLOW_ALERTING_BASE_URI, emptyMap(), workflow.toHttpEntity()) + + assertEquals("Create workflow failed", RestStatus.CREATED, createResponse.restStatus()) + + val responseBody = createResponse.asMap() + val createdId = responseBody["_id"] as String + val createdVersion = responseBody["_version"] as Int + + assertNotEquals("response is missing Id", Workflow.NO_ID, createdId) + assertTrue("incorrect version", createdVersion > 0) + assertEquals("Incorrect Location header", "$WORKFLOW_ALERTING_BASE_URI/$createdId", createResponse.getHeader("Location")) + + var workflowById = getWorkflow(createdId) + assertNotNull(workflowById) + // Delegate verification + @Suppress("UNCHECKED_CAST") + var delegates = (workflowById.inputs as List)[0].sequence.delegates.sortedBy { it.order } + assertEquals("Delegates size not correct", 2, delegates.size) + + val updatedWorkflow = randomWorkflow( + id = createdId, + monitorIds = listOf(monitorResponse.id) + ) + + val updateResponse = client().makeRequest("PUT", updatedWorkflow.relativeUrl(), emptyMap(), updatedWorkflow.toHttpEntity()) + + assertEquals("Update workflow failed", RestStatus.OK, updateResponse.restStatus()) + + val updateResponseBody = updateResponse.asMap() + val updatedId = updateResponseBody["_id"] as String + val updatedVersion = updateResponseBody["_version"] as Int + + assertNotEquals("response is missing Id", Workflow.NO_ID, updatedId) + assertTrue("incorrect version", updatedVersion > 0) + + workflowById = getWorkflow(updatedId) + assertNotNull(workflowById) + // Delegate verification + @Suppress("UNCHECKED_CAST") + delegates = (workflowById.inputs as List)[0].sequence.delegates.sortedBy { it.order } + assertEquals("Delegates size not correct", 1, delegates.size) + + val delegate1 = delegates[0] + assertNotNull(delegate1) + assertEquals("Delegate1 order not correct", 1, delegate1.order) + assertEquals("Delegate1 id not correct", monitorResponse.id, delegate1.monitorId) + } + + fun `test update workflow change order of delegate monitors`() { + val index = createTestIndex() + val docQuery1 = DocLevelQuery(query = "source.ip.v6.v1:12345", name = "3") + val docLevelInput = DocLevelMonitorInput( + "description", listOf(index), listOf(docQuery1) + ) + val trigger = randomDocumentLevelTrigger(condition = ALWAYS_RUN) + val monitor1 = randomDocumentLevelMonitor( + inputs = listOf(docLevelInput), + triggers = listOf(trigger) + ) + + val monitor2 = randomDocumentLevelMonitor( + inputs = listOf(docLevelInput), + triggers = listOf(trigger) + ) + + val monitorResponse1 = createMonitor(monitor1) + val monitorResponse2 = createMonitor(monitor2) + + val workflow = randomWorkflow( + monitorIds = listOf(monitorResponse1.id, monitorResponse2.id) + ) + + val workflowResponse = createWorkflow(workflow) + assertNotNull("Workflow creation failed", workflowResponse) + assertNotNull(workflow) + assertNotEquals("response is missing Id", Monitor.NO_ID, workflowResponse.id) + assertTrue("incorrect version", workflowResponse.version > 0) + + var workflowById = getWorkflow(workflowResponse.id) + assertNotNull(workflowById) + + val updatedWorkflowResponse = updateWorkflow( + randomWorkflow( + id = workflowById.id, + monitorIds = listOf(monitorResponse2.id, monitorResponse1.id) + ) + ) + + assertNotNull("Workflow creation failed", updatedWorkflowResponse) + assertNotNull(updatedWorkflowResponse) + assertEquals( + "Workflow id changed", + workflowResponse.id, + updatedWorkflowResponse.id + ) + assertTrue("incorrect version", updatedWorkflowResponse.version > 0) + + workflowById = getWorkflow(updatedWorkflowResponse.id) + + // Verify workflow + assertNotEquals("response is missing Id", Monitor.NO_ID, workflowById.id) + assertTrue("incorrect version", workflowById.version > 0) + assertEquals( + "Workflow name not correct", + updatedWorkflowResponse.name, + workflowById.name + ) + assertEquals( + "Workflow owner not correct", + updatedWorkflowResponse.owner, + workflowById.owner + ) + assertEquals( + "Workflow input not correct", + updatedWorkflowResponse.inputs, + workflowById.inputs + ) + + // Delegate verification + @Suppress("UNCHECKED_CAST") + val delegates = (workflowById.inputs as List)[0].sequence.delegates.sortedBy { it.order } + assertEquals("Delegates size not correct", 2, delegates.size) + + val delegate1 = delegates[0] + assertNotNull(delegate1) + assertEquals("Delegate1 order not correct", 1, delegate1.order) + assertEquals("Delegate1 id not correct", monitorResponse2.id, delegate1.monitorId) + + val delegate2 = delegates[1] + assertNotNull(delegate2) + assertEquals("Delegate2 order not correct", 2, delegate2.order) + assertEquals("Delegate2 id not correct", monitorResponse1.id, delegate2.monitorId) + assertEquals( + "Delegate2 Chained finding not correct", monitorResponse2.id, delegate2.chainedMonitorFindings!!.monitorId + ) + } + + fun `test update workflow doesn't exist failure`() { + val index = createTestIndex() + val docQuery1 = DocLevelQuery(query = "source.ip.v6.v1:12345", name = "3") + val docLevelInput = DocLevelMonitorInput( + "description", listOf(index), listOf(docQuery1) + ) + val monitor1 = randomDocumentLevelMonitor( + inputs = listOf(docLevelInput), + triggers = listOf(randomDocumentLevelTrigger(condition = ALWAYS_RUN)) + ) + + val monitorResponse1 = createMonitor(monitor1) + + val workflow = randomWorkflow( + monitorIds = listOf(monitorResponse1.id) + ) + val workflowResponse = createWorkflow(workflow) + assertNotNull("Workflow creation failed", workflowResponse) + + try { + updateWorkflow(workflow.copy(id = "testId")) + } catch (e: ResponseException) { + assertEquals("Unexpected status", RestStatus.NOT_FOUND, e.response.restStatus()) + e.message?.let { + assertTrue( + "Exception not returning GetWorkflow Action error ", + it.contains("Workflow with testId is not found") + ) + } + } + val updatedWorkflow = updateWorkflow(workflowResponse.copy(enabled = true)) + assertNotNull(updatedWorkflow) + val getWorkflow = getWorkflow(workflowId = updatedWorkflow.id) + assertTrue(getWorkflow.enabled) + } + + fun `test update workflow duplicate delegate failure`() { + val index = createTestIndex() + val docLevelInput = DocLevelMonitorInput( + "description", listOf(index), listOf(DocLevelQuery(query = "source.ip.v6.v1:12345", name = "3")) + ) + val trigger = randomDocumentLevelTrigger(condition = ALWAYS_RUN) + val monitor = randomDocumentLevelMonitor( + inputs = listOf(docLevelInput), + triggers = listOf(trigger) + ) + + val monitorResponse = createMonitor(monitor) + + var workflow = randomWorkflow( + monitorIds = listOf(monitorResponse.id) + ) + + val workflowResponse = createWorkflow(workflow) + assertNotNull("Workflow creation failed", workflowResponse) + + workflow = randomWorkflow( + id = workflowResponse.id, + monitorIds = listOf("1", "1", "2") + ) + try { + updateWorkflow(workflow) + } catch (e: ResponseException) { + assertEquals("Unexpected status", RestStatus.BAD_REQUEST, e.response.restStatus()) + e.message?.let { + assertTrue( + "Exception not returning IndexWorkflow Action error ", + it.contains("Duplicate delegates not allowed") + ) + } + } + } + + fun `test update workflow delegate monitor doesn't exist failure`() { + val index = createTestIndex() + val docLevelInput = DocLevelMonitorInput( + "description", listOf(index), listOf(DocLevelQuery(query = "source.ip.v6.v1:12345", name = "3")) + ) + val trigger = randomDocumentLevelTrigger(condition = ALWAYS_RUN) + + val monitor = randomDocumentLevelMonitor( + inputs = listOf(docLevelInput), + triggers = listOf(trigger) + ) + val monitorResponse = createMonitor(monitor) + + var workflow = randomWorkflow( + monitorIds = listOf(monitorResponse.id) + ) + val workflowResponse = createWorkflow(workflow) + assertNotNull("Workflow creation failed", workflowResponse) + + workflow = randomWorkflow( + id = workflowResponse.id, + monitorIds = listOf("-1", monitorResponse.id) + ) + + try { + updateWorkflow(workflow) + } catch (e: ResponseException) { + assertEquals("Unexpected status", RestStatus.BAD_REQUEST, e.response.restStatus()) + e.message?.let { + assertTrue( + "Exception not returning IndexWorkflow Action error ", + it.contains("are not valid monitor ids") + ) + } + } + } + + fun `test update workflow sequence order not correct failure`() { + val index = createTestIndex() + val docLevelInput = DocLevelMonitorInput( + "description", listOf(index), listOf(DocLevelQuery(query = "source.ip.v6.v1:12345", name = "3")) + ) + val trigger = randomDocumentLevelTrigger(condition = ALWAYS_RUN) + + val monitor = randomDocumentLevelMonitor( + inputs = listOf(docLevelInput), + triggers = listOf(trigger) + ) + val monitorResponse = createMonitor(monitor) + + var workflow = randomWorkflow( + monitorIds = listOf(monitorResponse.id) + ) + val workflowResponse = createWorkflow(workflow) + assertNotNull("Workflow creation failed", workflowResponse) + + val delegates = listOf( + Delegate(1, "monitor-1"), + Delegate(1, "monitor-2"), + Delegate(2, "monitor-3") + ) + workflow = randomWorkflowWithDelegates( + id = workflowResponse.id, + delegates = delegates + ) + try { + updateWorkflow(workflow) + } catch (e: ResponseException) { + assertEquals("Unexpected status", RestStatus.BAD_REQUEST, e.response.restStatus()) + e.message?.let { + assertTrue( + "Exception not returning IndexWorkflow Action error ", + it.contains("Sequence ordering of delegate monitor shouldn't contain duplicate order values") + ) + } + } + } + + fun `test update workflow chained findings monitor not in sequence failure`() { + val index = createTestIndex() + val docLevelInput = DocLevelMonitorInput( + "description", listOf(index), listOf(DocLevelQuery(query = "source.ip.v6.v1:12345", name = "3")) + ) + val trigger = randomDocumentLevelTrigger(condition = ALWAYS_RUN) + + val monitor = randomDocumentLevelMonitor( + inputs = listOf(docLevelInput), + triggers = listOf(trigger) + ) + val monitorResponse = createMonitor(monitor) + + var workflow = randomWorkflow( + monitorIds = listOf(monitorResponse.id) + ) + val workflowResponse = createWorkflow(workflow) + assertNotNull("Workflow creation failed", workflowResponse) + + val delegates = listOf( + Delegate(1, "monitor-1"), + Delegate(2, "monitor-2", ChainedMonitorFindings("monitor-1")), + Delegate(3, "monitor-3", ChainedMonitorFindings("monitor-x")) + ) + workflow = randomWorkflowWithDelegates( + id = workflowResponse.id, + delegates = delegates + ) + + try { + updateWorkflow(workflow) + } catch (e: ResponseException) { + assertEquals("Unexpected status", RestStatus.BAD_REQUEST, e.response.restStatus()) + e.message?.let { + assertTrue( + "Exception not returning IndexWorkflow Action error ", + it.contains("Chained Findings Monitor monitor-x doesn't exist in sequence") + ) + } + } + } + + fun `test update workflow chained findings order not correct failure`() { + val index = createTestIndex() + val docLevelInput = DocLevelMonitorInput( + "description", listOf(index), listOf(DocLevelQuery(query = "source.ip.v6.v1:12345", name = "3")) + ) + val trigger = randomDocumentLevelTrigger(condition = ALWAYS_RUN) + + val monitor = randomDocumentLevelMonitor( + inputs = listOf(docLevelInput), + triggers = listOf(trigger) + ) + val monitorResponse = createMonitor(monitor) + + var workflow = randomWorkflow( + monitorIds = listOf(monitorResponse.id) + ) + val workflowResponse = createWorkflow(workflow) + assertNotNull("Workflow creation failed", workflowResponse) + + val delegates = listOf( + Delegate(1, "monitor-1"), + Delegate(3, "monitor-2", ChainedMonitorFindings("monitor-1")), + Delegate(2, "monitor-3", ChainedMonitorFindings("monitor-2")) + ) + workflow = randomWorkflowWithDelegates( + id = workflowResponse.id, + delegates = delegates + ) + + try { + updateWorkflow(workflow) + } catch (e: ResponseException) { + assertEquals("Unexpected status", RestStatus.BAD_REQUEST, e.response.restStatus()) + e.message?.let { + assertTrue( + "Exception not returning IndexWorkflow Action error ", + it.contains("Chained Findings Monitor monitor-2 should be executed before monitor monitor-3") + ) + } + } + } + + @Throws(Exception::class) + fun `test getting a workflow`() { + val query = randomQueryLevelMonitor() + val monitor = createMonitor(query) + val storedMonitor = getMonitor(monitor.id) + + assertEquals("Indexed and retrieved monitor differ", monitor, storedMonitor) + + val workflow = createRandomWorkflow(monitorIds = listOf(monitor.id)) + + val storedWorkflow = getWorkflow(workflow.id) + + assertEquals("Indexed and retrieved workflow differ", workflow.id, storedWorkflow.id) + val delegates = (storedWorkflow.inputs[0] as CompositeInput).sequence.delegates + assertEquals("Delegate list not correct", 1, delegates.size) + assertEquals("Delegate order id not correct", 1, delegates[0].order) + assertEquals("Delegate id list not correct", monitor.id, delegates[0].monitorId) + } + + @Throws(Exception::class) + fun `test getting a workflow that doesn't exist`() { + try { + getWorkflow(randomAlphaOfLength(20)) + fail("expected response exception") + } catch (e: ResponseException) { + assertEquals(RestStatus.NOT_FOUND, e.response.restStatus()) + } + } + + @Throws(Exception::class) + fun `test checking if a workflow exists`() { + val query = randomQueryLevelMonitor() + val monitor = createMonitor(query) + + // val monitor = createMonitor(docLevelMonitor) + val storedMonitor = getMonitor(monitor.id) + assertEquals("Indexed and retrieved monitor differ", monitor, storedMonitor) + val workflow = createRandomWorkflow(monitorIds = listOf(monitor.id)) + + val headResponse = client().makeRequest("HEAD", workflow.relativeUrl()) + assertEquals("Unable to HEAD workflow", RestStatus.OK, headResponse.restStatus()) + assertNull("Workflow response contains unexpected body", headResponse.entity) + } + + fun `test checking if a non-existent workflow exists`() { + val headResponse = client().makeRequest("HEAD", "$WORKFLOW_ALERTING_BASE_URI/foobarbaz") + assertEquals("Unexpected status", RestStatus.NOT_FOUND, headResponse.restStatus()) + } + + fun `test delete workflow`() { + val query = randomQueryLevelMonitor() + val monitor = createMonitor(query) + + val workflowRequest = randomWorkflow( + monitorIds = listOf(monitor.id) + ) + val workflowResponse = createWorkflow(workflowRequest) + val workflowId = workflowResponse.id + val getWorkflowResponse = getWorkflow(workflowResponse.id) + + assertNotNull(getWorkflowResponse) + assertEquals(workflowId, getWorkflowResponse.id) + + client().makeRequest("DELETE", getWorkflowResponse.relativeUrl()) + + // Verify that the workflow is deleted + try { + getWorkflow(workflowId) + } catch (e: ResponseException) { + assertEquals(RestStatus.NOT_FOUND, e.response.restStatus()) + e.message?.let { + assertTrue( + "Exception not returning GetWorkflow Action error ", + it.contains("Workflow not found.") + ) + } + } + } + + fun `test delete workflow delete delegate monitors`() { + val query = randomQueryLevelMonitor() + val monitor = createMonitor(query) + + val workflowRequest = randomWorkflow( + monitorIds = listOf(monitor.id) + ) + val workflowResponse = createWorkflow(workflowRequest) + val workflowId = workflowResponse.id + val getWorkflowResponse = getWorkflow(workflowResponse.id) + + assertNotNull(getWorkflowResponse) + assertEquals(workflowId, getWorkflowResponse.id) + + client().makeRequest("DELETE", getWorkflowResponse.relativeUrl().plus("?deleteDelegateMonitors=true")) + + // Verify that the workflow is deleted + try { + getWorkflow(workflowId) + } catch (e: ResponseException) { + assertEquals(RestStatus.NOT_FOUND, e.response.restStatus()) + e.message?.let { + assertTrue( + "Exception not returning GetWorkflow Action error ", + it.contains("Workflow not found.") + ) + } + } + + // Verify that delegate monitor is deleted + try { + getMonitor(monitor.id) + } catch (e: ResponseException) { + assertEquals(RestStatus.NOT_FOUND, e.response.restStatus()) + e.message?.let { + assertTrue( + "Exception not returning GetWorkflow Action error ", + it.contains("Monitor not found.") + ) + } + } + } + + fun `test delete workflow preserve delegate monitors`() { + val query = randomQueryLevelMonitor() + val monitor = createMonitor(query) + + val workflowRequest = randomWorkflow( + monitorIds = listOf(monitor.id) + ) + val workflowResponse = createWorkflow(workflowRequest) + val workflowId = workflowResponse.id + val getWorkflowResponse = getWorkflow(workflowResponse.id) + + assertNotNull(getWorkflowResponse) + assertEquals(workflowId, getWorkflowResponse.id) + + client().makeRequest("DELETE", getWorkflowResponse.relativeUrl().plus("?deleteDelegateMonitors=false")) + + // Verify that the workflow is deleted + try { + getWorkflow(workflowId) + } catch (e: ResponseException) { + assertEquals(RestStatus.NOT_FOUND, e.response.restStatus()) + e.message?.let { + assertTrue( + "Exception not returning GetWorkflow Action error ", + it.contains("Workflow not found.") + ) + } + } + + // Verify that delegate monitor is not deleted + val delegateMonitor = getMonitor(monitor.id) + assertNotNull(delegateMonitor) + } + + @Throws(Exception::class) + fun `test deleting a workflow that doesn't exist`() { + try { + client().makeRequest("DELETE", "$WORKFLOW_ALERTING_BASE_URI/foobarbaz") + fail("expected 404 ResponseException") + } catch (e: ResponseException) { + assertEquals(RestStatus.NOT_FOUND, e.response.restStatus()) + } + } +} diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/transport/WorkflowSingleNodeTestCase.kt b/alerting/src/test/kotlin/org/opensearch/alerting/transport/WorkflowSingleNodeTestCase.kt index 4b3bf4e81..699a3d565 100644 --- a/alerting/src/test/kotlin/org/opensearch/alerting/transport/WorkflowSingleNodeTestCase.kt +++ b/alerting/src/test/kotlin/org/opensearch/alerting/transport/WorkflowSingleNodeTestCase.kt @@ -10,6 +10,7 @@ import org.opensearch.action.support.WriteRequest import org.opensearch.alerting.action.ExecuteWorkflowAction import org.opensearch.alerting.action.ExecuteWorkflowRequest import org.opensearch.alerting.action.ExecuteWorkflowResponse +import org.opensearch.alerting.model.MonitorMetadata import org.opensearch.alerting.model.WorkflowMetadata import org.opensearch.common.unit.TimeValue import org.opensearch.common.xcontent.json.JsonXContent @@ -94,6 +95,35 @@ abstract class WorkflowSingleNodeTestCase : AlertingSingleNodeTestCase() { }.first() } + protected fun searchMonitorMetadata( + id: String, + indices: String = ScheduledJob.SCHEDULED_JOBS_INDEX, + refresh: Boolean = true, + ): MonitorMetadata? { + try { + if (refresh) refreshIndex(indices) + } catch (e: Exception) { + logger.warn("Could not refresh index $indices because: ${e.message}") + return null + } + val ssb = SearchSourceBuilder() + ssb.version(true) + ssb.query(TermQueryBuilder("_id", id)) + val searchResponse = client().prepareSearch(indices).setRouting(id).setSource(ssb).get() + + return searchResponse.hits.hits.map { it -> + val xcp = createParser(JsonXContent.jsonXContent, it.sourceRef).also { it.nextToken() } + lateinit var monitorMetadata: MonitorMetadata + while (xcp.nextToken() != XContentParser.Token.END_OBJECT) { + xcp.nextToken() + when (xcp.currentName()) { + "metadata" -> monitorMetadata = MonitorMetadata.parse(xcp) + } + } + monitorMetadata.copy(id = it.id) + }.first() + } + protected fun upsertWorkflow( workflow: Workflow, id: String = Workflow.NO_ID,