diff --git a/alerting/build.gradle b/alerting/build.gradle index 6c77a1db3..678a1749b 100644 --- a/alerting/build.gradle +++ b/alerting/build.gradle @@ -167,7 +167,10 @@ dependencies { implementation "org.jetbrains.kotlin:kotlin-stdlib-common:${kotlin_version}" implementation "org.jetbrains:annotations:13.0" + // SQL/PPL plugin dependencies are included in alerting-core api project(":alerting-core") + implementation 'org.json:json:20240303' + implementation "com.github.seancfoley:ipaddress:5.4.1" implementation project(path: ":alerting-spi", configuration: 'shadow') diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt b/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt index 4ad7b6361..94d2e5f83 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt @@ -14,8 +14,16 @@ import org.opensearch.alerting.action.GetEmailGroupAction import org.opensearch.alerting.action.GetRemoteIndexesAction import org.opensearch.alerting.action.SearchEmailAccountAction import org.opensearch.alerting.action.SearchEmailGroupAction +import org.opensearch.alerting.actionv2.DeleteMonitorV2Action +import org.opensearch.alerting.actionv2.ExecuteMonitorV2Action +import org.opensearch.alerting.actionv2.GetAlertsV2Action +import org.opensearch.alerting.actionv2.GetMonitorV2Action +import org.opensearch.alerting.actionv2.IndexMonitorV2Action +import org.opensearch.alerting.actionv2.SearchMonitorV2Action import org.opensearch.alerting.alerts.AlertIndices import org.opensearch.alerting.alerts.AlertIndices.Companion.ALL_ALERT_INDEX_PATTERN +import org.opensearch.alerting.alertsv2.AlertV2Indices +import org.opensearch.alerting.alertsv2.AlertV2Mover import org.opensearch.alerting.comments.CommentsIndices import org.opensearch.alerting.comments.CommentsIndices.Companion.ALL_COMMENTS_INDEX_PATTERN import org.opensearch.alerting.core.JobSweeper @@ -23,7 +31,9 @@ import org.opensearch.alerting.core.ScheduledJobIndices import org.opensearch.alerting.core.action.node.ScheduledJobsStatsAction import org.opensearch.alerting.core.action.node.ScheduledJobsStatsTransportAction import org.opensearch.alerting.core.lock.LockService +import org.opensearch.alerting.core.modelv2.MonitorV2 import org.opensearch.alerting.core.resthandler.RestScheduledJobStatsHandler +import org.opensearch.alerting.core.resthandler.RestScheduledJobStatsV2Handler import org.opensearch.alerting.core.schedule.JobScheduler import org.opensearch.alerting.core.settings.LegacyOpenDistroScheduledJobSettings import org.opensearch.alerting.core.settings.ScheduledJobSettings @@ -32,25 +42,31 @@ import org.opensearch.alerting.resthandler.RestAcknowledgeAlertAction import org.opensearch.alerting.resthandler.RestAcknowledgeChainedAlertAction import org.opensearch.alerting.resthandler.RestDeleteAlertingCommentAction import org.opensearch.alerting.resthandler.RestDeleteMonitorAction +import org.opensearch.alerting.resthandler.RestDeleteMonitorV2Action import org.opensearch.alerting.resthandler.RestDeleteWorkflowAction import org.opensearch.alerting.resthandler.RestExecuteMonitorAction +import org.opensearch.alerting.resthandler.RestExecuteMonitorV2Action import org.opensearch.alerting.resthandler.RestExecuteWorkflowAction import org.opensearch.alerting.resthandler.RestGetAlertsAction +import org.opensearch.alerting.resthandler.RestGetAlertsV2Action import org.opensearch.alerting.resthandler.RestGetDestinationsAction import org.opensearch.alerting.resthandler.RestGetEmailAccountAction import org.opensearch.alerting.resthandler.RestGetEmailGroupAction import org.opensearch.alerting.resthandler.RestGetFindingsAction import org.opensearch.alerting.resthandler.RestGetMonitorAction +import org.opensearch.alerting.resthandler.RestGetMonitorV2Action import org.opensearch.alerting.resthandler.RestGetRemoteIndexesAction import org.opensearch.alerting.resthandler.RestGetWorkflowAction import org.opensearch.alerting.resthandler.RestGetWorkflowAlertsAction import org.opensearch.alerting.resthandler.RestIndexAlertingCommentAction import org.opensearch.alerting.resthandler.RestIndexMonitorAction +import org.opensearch.alerting.resthandler.RestIndexMonitorV2Action import org.opensearch.alerting.resthandler.RestIndexWorkflowAction import org.opensearch.alerting.resthandler.RestSearchAlertingCommentAction import org.opensearch.alerting.resthandler.RestSearchEmailAccountAction import org.opensearch.alerting.resthandler.RestSearchEmailGroupAction import org.opensearch.alerting.resthandler.RestSearchMonitorAction +import org.opensearch.alerting.resthandler.RestSearchMonitorV2Action import org.opensearch.alerting.script.TriggerScript import org.opensearch.alerting.service.DeleteMonitorService import org.opensearch.alerting.settings.AlertingSettings @@ -63,26 +79,32 @@ import org.opensearch.alerting.transport.TransportAcknowledgeAlertAction import org.opensearch.alerting.transport.TransportAcknowledgeChainedAlertAction import org.opensearch.alerting.transport.TransportDeleteAlertingCommentAction import org.opensearch.alerting.transport.TransportDeleteMonitorAction +import org.opensearch.alerting.transport.TransportDeleteMonitorV2Action import org.opensearch.alerting.transport.TransportDeleteWorkflowAction import org.opensearch.alerting.transport.TransportDocLevelMonitorFanOutAction import org.opensearch.alerting.transport.TransportExecuteMonitorAction +import org.opensearch.alerting.transport.TransportExecuteMonitorV2Action import org.opensearch.alerting.transport.TransportExecuteWorkflowAction import org.opensearch.alerting.transport.TransportGetAlertsAction +import org.opensearch.alerting.transport.TransportGetAlertsV2Action import org.opensearch.alerting.transport.TransportGetDestinationsAction import org.opensearch.alerting.transport.TransportGetEmailAccountAction import org.opensearch.alerting.transport.TransportGetEmailGroupAction import org.opensearch.alerting.transport.TransportGetFindingsSearchAction import org.opensearch.alerting.transport.TransportGetMonitorAction +import org.opensearch.alerting.transport.TransportGetMonitorV2Action import org.opensearch.alerting.transport.TransportGetRemoteIndexesAction import org.opensearch.alerting.transport.TransportGetWorkflowAction import org.opensearch.alerting.transport.TransportGetWorkflowAlertsAction import org.opensearch.alerting.transport.TransportIndexAlertingCommentAction import org.opensearch.alerting.transport.TransportIndexMonitorAction +import org.opensearch.alerting.transport.TransportIndexMonitorV2Action import org.opensearch.alerting.transport.TransportIndexWorkflowAction import org.opensearch.alerting.transport.TransportSearchAlertingCommentAction import org.opensearch.alerting.transport.TransportSearchEmailAccountAction import org.opensearch.alerting.transport.TransportSearchEmailGroupAction import org.opensearch.alerting.transport.TransportSearchMonitorAction +import org.opensearch.alerting.transport.TransportSearchMonitorV2Action import org.opensearch.alerting.util.DocLevelMonitorQueries import org.opensearch.alerting.util.destinationmigration.DestinationMigrationCoordinator import org.opensearch.cluster.metadata.IndexNameExpressionResolver @@ -157,6 +179,7 @@ internal class AlertingPlugin : PainlessExtension, ActionPlugin, ScriptPlugin, R @JvmField val OPEN_SEARCH_DASHBOARDS_USER_AGENT = "OpenSearch-Dashboards" @JvmField val UI_METADATA_EXCLUDE = arrayOf("monitor.${Monitor.UI_METADATA_FIELD}") @JvmField val MONITOR_BASE_URI = "/_plugins/_alerting/monitors" + @JvmField val MONITOR_V2_BASE_URI = "/_plugins/_alerting/v2/monitors" @JvmField val WORKFLOW_BASE_URI = "/_plugins/_alerting/workflows" @JvmField val REMOTE_BASE_URI = "/_plugins/_alerting/remote" @JvmField val DESTINATION_BASE_URI = "/_plugins/_alerting/destinations" @@ -169,7 +192,7 @@ internal class AlertingPlugin : PainlessExtension, ActionPlugin, ScriptPlugin, R @JvmField val FINDING_BASE_URI = "/_plugins/_alerting/findings" @JvmField val COMMENTS_BASE_URI = "/_plugins/_alerting/comments" - @JvmField val ALERTING_JOB_TYPES = listOf("monitor", "workflow") + @JvmField val ALERTING_JOB_TYPES = listOf("monitor", "workflow", "monitor_v2") } lateinit var runner: MonitorRunnerService @@ -180,8 +203,10 @@ internal class AlertingPlugin : PainlessExtension, ActionPlugin, ScriptPlugin, R lateinit var docLevelMonitorQueries: DocLevelMonitorQueries lateinit var threadPool: ThreadPool lateinit var alertIndices: AlertIndices + lateinit var alertV2Indices: AlertV2Indices lateinit var clusterService: ClusterService lateinit var destinationMigrationCoordinator: DestinationMigrationCoordinator + lateinit var alertV2Mover: AlertV2Mover var monitorTypeToMonitorRunners: MutableMap = mutableMapOf() override fun getRestHandlers( @@ -194,6 +219,7 @@ internal class AlertingPlugin : PainlessExtension, ActionPlugin, ScriptPlugin, R nodesInCluster: Supplier ): List { return listOf( + // Alerting V1 RestGetMonitorAction(), RestDeleteMonitorAction(), RestIndexMonitorAction(), @@ -218,11 +244,21 @@ internal class AlertingPlugin : PainlessExtension, ActionPlugin, ScriptPlugin, R RestIndexAlertingCommentAction(), RestSearchAlertingCommentAction(), RestDeleteAlertingCommentAction(), + + // Alerting V2 + RestIndexMonitorV2Action(), + RestExecuteMonitorV2Action(), + RestDeleteMonitorV2Action(), + RestGetMonitorV2Action(), + RestSearchMonitorV2Action(settings, clusterService), + RestGetAlertsV2Action(), + RestScheduledJobStatsV2Handler() ) } override fun getActions(): List> { return listOf( + // Alerting V1 ActionPlugin.ActionHandler(ScheduledJobsStatsAction.INSTANCE, ScheduledJobsStatsTransportAction::class.java), ActionPlugin.ActionHandler(AlertingActions.INDEX_MONITOR_ACTION_TYPE, TransportIndexMonitorAction::class.java), ActionPlugin.ActionHandler(AlertingActions.GET_MONITOR_ACTION_TYPE, TransportGetMonitorAction::class.java), @@ -249,13 +285,22 @@ internal class AlertingPlugin : PainlessExtension, ActionPlugin, ScriptPlugin, R ActionPlugin.ActionHandler(AlertingActions.DELETE_COMMENT_ACTION_TYPE, TransportDeleteAlertingCommentAction::class.java), ActionPlugin.ActionHandler(ExecuteWorkflowAction.INSTANCE, TransportExecuteWorkflowAction::class.java), ActionPlugin.ActionHandler(GetRemoteIndexesAction.INSTANCE, TransportGetRemoteIndexesAction::class.java), - ActionPlugin.ActionHandler(DocLevelMonitorFanOutAction.INSTANCE, TransportDocLevelMonitorFanOutAction::class.java) + ActionPlugin.ActionHandler(DocLevelMonitorFanOutAction.INSTANCE, TransportDocLevelMonitorFanOutAction::class.java), + + // Alerting V2 + ActionPlugin.ActionHandler(IndexMonitorV2Action.INSTANCE, TransportIndexMonitorV2Action::class.java), + ActionPlugin.ActionHandler(GetMonitorV2Action.INSTANCE, TransportGetMonitorV2Action::class.java), + ActionPlugin.ActionHandler(SearchMonitorV2Action.INSTANCE, TransportSearchMonitorV2Action::class.java), + ActionPlugin.ActionHandler(DeleteMonitorV2Action.INSTANCE, TransportDeleteMonitorV2Action::class.java), + ActionPlugin.ActionHandler(ExecuteMonitorV2Action.INSTANCE, TransportExecuteMonitorV2Action::class.java), + ActionPlugin.ActionHandler(GetAlertsV2Action.INSTANCE, TransportGetAlertsV2Action::class.java) ) } override fun getNamedXContent(): List { return listOf( Monitor.XCONTENT_REGISTRY, + MonitorV2.XCONTENT_REGISTRY, SearchInput.XCONTENT_REGISTRY, DocLevelMonitorInput.XCONTENT_REGISTRY, QueryLevelTrigger.XCONTENT_REGISTRY, @@ -285,6 +330,7 @@ internal class AlertingPlugin : PainlessExtension, ActionPlugin, ScriptPlugin, R val settings = environment.settings() val lockService = LockService(client, clusterService) alertIndices = AlertIndices(settings, client, threadPool, clusterService) + alertV2Indices = AlertV2Indices(settings, client, threadPool, clusterService) val alertService = AlertService(client, xContentRegistry, alertIndices) val triggerService = TriggerService(scriptService) runner = MonitorRunnerService @@ -296,6 +342,7 @@ internal class AlertingPlugin : PainlessExtension, ActionPlugin, ScriptPlugin, R .registerSettings(settings) .registerThreadPool(threadPool) .registerAlertIndices(alertIndices) + .registerAlertV2Indices(alertV2Indices) .registerInputService( InputService( client, @@ -322,6 +369,7 @@ internal class AlertingPlugin : PainlessExtension, ActionPlugin, ScriptPlugin, R scheduler = JobScheduler(threadPool, runner) sweeper = JobSweeper(environment.settings(), client, clusterService, threadPool, xContentRegistry, scheduler, ALERTING_JOB_TYPES) destinationMigrationCoordinator = DestinationMigrationCoordinator(client, clusterService, threadPool, scheduledJobIndices) + alertV2Mover = AlertV2Mover(environment.settings(), client, threadPool, clusterService) this.threadPool = threadPool this.clusterService = clusterService @@ -349,6 +397,7 @@ internal class AlertingPlugin : PainlessExtension, ActionPlugin, ScriptPlugin, R commentsIndices, docLevelMonitorQueries, destinationMigrationCoordinator, + alertV2Mover, lockService, alertService, triggerService @@ -431,7 +480,14 @@ internal class AlertingPlugin : PainlessExtension, ActionPlugin, ScriptPlugin, R AlertingSettings.COMMENTS_HISTORY_RETENTION_PERIOD, AlertingSettings.COMMENTS_MAX_CONTENT_SIZE, AlertingSettings.MAX_COMMENTS_PER_ALERT, - AlertingSettings.MAX_COMMENTS_PER_NOTIFICATION + AlertingSettings.MAX_COMMENTS_PER_NOTIFICATION, + AlertingSettings.ALERT_V2_HISTORY_ENABLED, + AlertingSettings.ALERT_V2_HISTORY_ROLLOVER_PERIOD, + AlertingSettings.ALERT_V2_HISTORY_INDEX_MAX_AGE, + AlertingSettings.ALERT_V2_HISTORY_MAX_DOCS, + AlertingSettings.ALERT_V2_HISTORY_RETENTION_PERIOD, + AlertingSettings.ALERT_V2_NOTIF_QUERY_RESULTS_MAX_SIZE, + AlertingSettings.ALERT_V2_PER_RESULT_TRIGGER_MAX_ALERTS ) } diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/AlertingV2Utils.kt b/alerting/src/main/kotlin/org/opensearch/alerting/AlertingV2Utils.kt new file mode 100644 index 000000000..70df08ffe --- /dev/null +++ b/alerting/src/main/kotlin/org/opensearch/alerting/AlertingV2Utils.kt @@ -0,0 +1,30 @@ +package org.opensearch.alerting + +import org.opensearch.alerting.core.modelv2.MonitorV2 +import org.opensearch.commons.alerting.model.Monitor +import org.opensearch.commons.alerting.model.ScheduledJob + +object AlertingV2Utils { + + // Validates that the given scheduled job is a Monitor + // returns the exception to pass into actionListener.onFailure if not. + fun validateMonitorV1(scheduledJob: ScheduledJob): Exception? { + if (scheduledJob is MonitorV2) { + return IllegalArgumentException("The ID given corresponds to a V2 Monitor, please pass in the ID of a V1 Monitor") + } else if (scheduledJob !is Monitor) { + return IllegalArgumentException("The ID given corresponds to a scheduled job of unknown type: ${scheduledJob.javaClass.name}") + } + return null + } + + // Validates that the given scheduled job is a MonitorV2 + // returns the exception to pass into actionListener.onFailure if not. + fun validateMonitorV2(scheduledJob: ScheduledJob): Exception? { + if (scheduledJob is Monitor) { + return IllegalArgumentException("The ID given corresponds to a V1 Monitor, please pass in the ID of a V2 Monitor") + } else if (scheduledJob !is MonitorV2) { + return IllegalArgumentException("The ID given corresponds to a scheduled job of unknown type: ${scheduledJob.javaClass.name}") + } + return null + } +} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/MonitorRunner.kt b/alerting/src/main/kotlin/org/opensearch/alerting/MonitorRunner.kt index 4e6cdbc02..7acfdff8b 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/MonitorRunner.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/MonitorRunner.kt @@ -94,7 +94,7 @@ abstract class MonitorRunner { } } - protected suspend fun getConfigAndSendNotification( + suspend fun getConfigAndSendNotification( action: Action, monitorCtx: MonitorRunnerExecutionContext, subject: String?, diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/MonitorRunnerExecutionContext.kt b/alerting/src/main/kotlin/org/opensearch/alerting/MonitorRunnerExecutionContext.kt index a890ec1a6..5c5e24070 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/MonitorRunnerExecutionContext.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/MonitorRunnerExecutionContext.kt @@ -7,6 +7,7 @@ package org.opensearch.alerting import org.opensearch.action.bulk.BackoffPolicy import org.opensearch.alerting.alerts.AlertIndices +import org.opensearch.alerting.alertsv2.AlertV2Indices import org.opensearch.alerting.core.lock.LockService import org.opensearch.alerting.model.destination.DestinationContextFactory import org.opensearch.alerting.remote.monitors.RemoteMonitorRegistry @@ -35,6 +36,7 @@ data class MonitorRunnerExecutionContext( var settings: Settings? = null, var threadPool: ThreadPool? = null, var alertIndices: AlertIndices? = null, + var alertV2Indices: AlertV2Indices? = null, var inputService: InputService? = null, var triggerService: TriggerService? = null, var alertService: AlertService? = null, diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/MonitorRunnerService.kt b/alerting/src/main/kotlin/org/opensearch/alerting/MonitorRunnerService.kt index f8703aec2..27dee9537 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/MonitorRunnerService.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/MonitorRunnerService.kt @@ -21,18 +21,27 @@ import org.opensearch.alerting.action.ExecuteMonitorResponse import org.opensearch.alerting.action.ExecuteWorkflowAction import org.opensearch.alerting.action.ExecuteWorkflowRequest import org.opensearch.alerting.action.ExecuteWorkflowResponse +import org.opensearch.alerting.actionv2.ExecuteMonitorV2Action +import org.opensearch.alerting.actionv2.ExecuteMonitorV2Request +import org.opensearch.alerting.actionv2.ExecuteMonitorV2Response import org.opensearch.alerting.alerts.AlertIndices import org.opensearch.alerting.alerts.AlertMover.Companion.moveAlerts +import org.opensearch.alerting.alertsv2.AlertV2Indices import org.opensearch.alerting.core.JobRunner import org.opensearch.alerting.core.ScheduledJobIndices import org.opensearch.alerting.core.lock.LockModel import org.opensearch.alerting.core.lock.LockService +import org.opensearch.alerting.core.modelv2.MonitorV2 +import org.opensearch.alerting.core.modelv2.MonitorV2RunResult +import org.opensearch.alerting.core.modelv2.PPLMonitor +import org.opensearch.alerting.core.modelv2.PPLMonitor.Companion.PPL_MONITOR_TYPE import org.opensearch.alerting.model.destination.DestinationContextFactory import org.opensearch.alerting.opensearchapi.retry import org.opensearch.alerting.opensearchapi.suspendUntil import org.opensearch.alerting.remote.monitors.RemoteDocumentLevelMonitorRunner import org.opensearch.alerting.remote.monitors.RemoteMonitorRegistry import org.opensearch.alerting.script.TriggerExecutionContext +import org.opensearch.alerting.script.TriggerV2ExecutionContext import org.opensearch.alerting.settings.AlertingSettings import org.opensearch.alerting.settings.AlertingSettings.Companion.ALERT_BACKOFF_COUNT import org.opensearch.alerting.settings.AlertingSettings.Companion.ALERT_BACKOFF_MILLIS @@ -137,6 +146,11 @@ object MonitorRunnerService : JobRunner, CoroutineScope, AbstractLifecycleCompon return this } + fun registerAlertV2Indices(alertV2Indices: AlertV2Indices): MonitorRunnerService { + this.monitorCtx.alertV2Indices = alertV2Indices + return this + } + fun registerInputService(inputService: InputService): MonitorRunnerService { this.monitorCtx.inputService = inputService return this @@ -316,11 +330,16 @@ object MonitorRunnerService : JobRunner, CoroutineScope, AbstractLifecycleCompon logger.error("Failed to move active alerts for monitor [${job.id}].", e) } } + } else if (job is MonitorV2) { + return } else { throw IllegalArgumentException("Invalid job type") } } + // TODO: if MonitorV2 was deleted, skip trying to move alerts + // cluster throws failed to move alerts exception whenever a MonitorV2 is deleted + // because Alerting V2's stateless alerts don't need to be moved override fun postDelete(jobId: String) { launch { try { @@ -408,6 +427,45 @@ object MonitorRunnerService : JobRunner, CoroutineScope, AbstractLifecycleCompon } } } + is MonitorV2 -> { + if (job !is PPLMonitor) { + throw IllegalStateException("Unexpected invalid MonitorV2 type: ${job.javaClass.name}") + } + + launch { + var monitorLock: LockModel? = null + try { + monitorLock = monitorCtx.client!!.suspendUntil { + monitorCtx.lockService!!.acquireLock(job, it) + } ?: return@launch + logger.debug("lock ${monitorLock!!.lockId} acquired") + logger.debug( + "PERF_DEBUG: executing $PPL_MONITOR_TYPE ${job.id} on node " + + monitorCtx.clusterService!!.state().nodes().localNode.id + ) + val executeMonitorV2Request = ExecuteMonitorV2Request( + false, + false, + job.id, // only need to pass in MonitorV2 ID + null, // no need to pass in MonitorV2 object itself + TimeValue(periodStart.toEpochMilli()), + TimeValue(periodEnd.toEpochMilli()) + ) + monitorCtx.client!!.suspendUntil { + monitorCtx.client!!.execute( + ExecuteMonitorV2Action.INSTANCE, + executeMonitorV2Request, + it + ) + } + } catch (e: Exception) { + logger.error("MonitorV2 run failed for monitor with id ${job.id}", e) + } finally { + monitorCtx.client!!.suspendUntil { monitorCtx.lockService!!.release(monitorLock, it) } + logger.debug("lock ${monitorLock?.lockId} released") + } + } + } else -> { throw IllegalArgumentException("Invalid job type") } @@ -433,20 +491,7 @@ object MonitorRunnerService : JobRunner, CoroutineScope, AbstractLifecycleCompon ): MonitorRunResult<*> { // Updating the scheduled job index at the start of monitor execution runs for when there is an upgrade the the schema mapping // has not been updated. - if (!IndexUtils.scheduledJobIndexUpdated && monitorCtx.clusterService != null && monitorCtx.client != null) { - IndexUtils.updateIndexMapping( - ScheduledJob.SCHEDULED_JOBS_INDEX, - ScheduledJobIndices.scheduledJobMappings(), monitorCtx.clusterService!!.state(), monitorCtx.client!!.admin().indices(), - object : ActionListener { - override fun onResponse(response: AcknowledgedResponse) { - } - - override fun onFailure(t: Exception) { - logger.error("Failed to update config index schema", t) - } - } - ) - } + updateAlertingConfigIndexSchema() if (job is Workflow) { logger.info("Executing scheduled workflow - id: ${job.id}, periodStart: $periodStart, periodEnd: $periodEnd, dryrun: $dryrun") @@ -539,6 +584,46 @@ object MonitorRunnerService : JobRunner, CoroutineScope, AbstractLifecycleCompon } } + // after the above JobRunner interface override runJob calls ExecuteMonitorV2 API, + // the ExecuteMonitorV2 transport action calls this function to call the PPLMonitorRunner, + // where the core PPL Monitor execution logic resides + suspend fun runJobV2( + monitorV2: MonitorV2, + periodStart: Instant, + periodEnd: Instant, + dryrun: Boolean, + manual: Boolean, + transportService: TransportService, + ): MonitorV2RunResult<*> { + updateAlertingConfigIndexSchema() + + val executionId = "${monitorV2.id}_${LocalDateTime.now(ZoneOffset.UTC)}_${UUID.randomUUID()}" + val monitorV2Type = when (monitorV2) { + is PPLMonitor -> PPL_MONITOR_TYPE + else -> throw IllegalStateException("Unexpected MonitorV2 type: ${monitorV2.javaClass.name}") + } + + logger.info( + "Executing scheduled monitor - id: ${monitorV2.id}, type: $monitorV2Type, periodStart: $periodStart, " + + "periodEnd: $periodEnd, dryrun: $dryrun, manual: $manual, executionId: $executionId" + ) + + // for now, always call PPLMonitorRunner since only PPL Monitors are initially supported + // to introduce new MonitorV2 type, create its MonitorRunner, and if/else branch + // to the corresponding MonitorRunners based on type. For now, default to PPLMonitorRunner + val runResult = PPLMonitorRunner.runMonitorV2( + monitorV2, + monitorCtx, + periodStart, + periodEnd, + dryrun, + manual, + executionId = executionId, + transportService = transportService, + ) + return runResult + } + // TODO: See if we can move below methods (or few of these) to a common utils internal fun getRolesForMonitor(monitor: Monitor): List { /* @@ -582,4 +667,27 @@ object MonitorRunnerService : JobRunner, CoroutineScope, AbstractLifecycleCompon .newInstance(template.params + mapOf("ctx" to ctx.asTemplateArg())) .execute() } + + internal fun compileTemplateV2(template: Script, ctx: TriggerV2ExecutionContext): String { + return monitorCtx.scriptService!!.compile(template, TemplateScript.CONTEXT) + .newInstance(template.params + mapOf("ctx" to ctx.asTemplateArg())) + .execute() + } + + private fun updateAlertingConfigIndexSchema() { + if (!IndexUtils.scheduledJobIndexUpdated && monitorCtx.clusterService != null && monitorCtx.client != null) { + IndexUtils.updateIndexMapping( + ScheduledJob.SCHEDULED_JOBS_INDEX, + ScheduledJobIndices.scheduledJobMappings(), monitorCtx.clusterService!!.state(), monitorCtx.client!!.admin().indices(), + object : ActionListener { + override fun onResponse(response: AcknowledgedResponse) { + } + + override fun onFailure(t: Exception) { + logger.error("Failed to update config index schema", t) + } + } + ) + } + } } diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/MonitorV2Runner.kt b/alerting/src/main/kotlin/org/opensearch/alerting/MonitorV2Runner.kt new file mode 100644 index 000000000..ef2b3fcfa --- /dev/null +++ b/alerting/src/main/kotlin/org/opensearch/alerting/MonitorV2Runner.kt @@ -0,0 +1,19 @@ +package org.opensearch.alerting + +import org.opensearch.alerting.core.modelv2.MonitorV2 +import org.opensearch.alerting.core.modelv2.MonitorV2RunResult +import org.opensearch.transport.TransportService +import java.time.Instant + +interface MonitorV2Runner { + suspend fun runMonitorV2( + monitorV2: MonitorV2, + monitorCtx: MonitorRunnerExecutionContext, // MonitorV2 reads from same context as Monitor + periodStart: Instant, + periodEnd: Instant, + dryRun: Boolean, + manual: Boolean, + executionId: String, + transportService: TransportService + ): MonitorV2RunResult<*> +} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/PPLMonitorRunner.kt b/alerting/src/main/kotlin/org/opensearch/alerting/PPLMonitorRunner.kt new file mode 100644 index 000000000..e495800ef --- /dev/null +++ b/alerting/src/main/kotlin/org/opensearch/alerting/PPLMonitorRunner.kt @@ -0,0 +1,698 @@ +package org.opensearch.alerting + +import org.apache.logging.log4j.LogManager +import org.json.JSONArray +import org.json.JSONObject +import org.opensearch.ExceptionsHelper +import org.opensearch.action.DocWriteRequest +import org.opensearch.action.bulk.BackoffPolicy +import org.opensearch.action.bulk.BulkRequest +import org.opensearch.action.bulk.BulkResponse +import org.opensearch.action.index.IndexRequest +import org.opensearch.action.support.WriteRequest +import org.opensearch.alerting.QueryLevelMonitorRunner.getConfigAndSendNotification +import org.opensearch.alerting.alertsv2.AlertV2Indices +import org.opensearch.alerting.core.modelv2.AlertV2 +import org.opensearch.alerting.core.modelv2.MonitorV2 +import org.opensearch.alerting.core.modelv2.MonitorV2RunResult +import org.opensearch.alerting.core.modelv2.PPLMonitor +import org.opensearch.alerting.core.modelv2.PPLMonitorRunResult +import org.opensearch.alerting.core.modelv2.PPLTrigger +import org.opensearch.alerting.core.modelv2.PPLTrigger.ConditionType +import org.opensearch.alerting.core.modelv2.PPLTrigger.NumResultsCondition +import org.opensearch.alerting.core.modelv2.PPLTrigger.TriggerMode +import org.opensearch.alerting.core.modelv2.PPLTriggerRunResult +import org.opensearch.alerting.core.modelv2.TriggerV2.Severity +import org.opensearch.alerting.core.ppl.PPLPluginInterface +import org.opensearch.alerting.opensearchapi.InjectorContextElement +import org.opensearch.alerting.opensearchapi.retry +import org.opensearch.alerting.opensearchapi.suspendUntil +import org.opensearch.alerting.opensearchapi.withClosableContext +import org.opensearch.alerting.script.PPLTriggerExecutionContext +import org.opensearch.alerting.settings.AlertingSettings +import org.opensearch.common.unit.TimeValue +import org.opensearch.common.xcontent.XContentFactory +import org.opensearch.commons.alerting.alerts.AlertError +import org.opensearch.commons.alerting.model.Alert +import org.opensearch.commons.alerting.model.ScheduledJob.Companion.SCHEDULED_JOBS_INDEX +import org.opensearch.commons.alerting.model.action.Action +import org.opensearch.commons.alerting.model.userErrorMessage +import org.opensearch.core.common.Strings +import org.opensearch.core.rest.RestStatus +import org.opensearch.core.xcontent.ToXContent +import org.opensearch.sql.plugin.transport.TransportPPLQueryRequest +import org.opensearch.transport.TransportService +import org.opensearch.transport.client.node.NodeClient +import java.time.Instant +import java.time.ZoneOffset.UTC +import java.time.format.DateTimeFormatter +import java.time.temporal.ChronoUnit + +object PPLMonitorRunner : MonitorV2Runner { + private val logger = LogManager.getLogger(javaClass) + + private const val PPL_SQL_QUERY_FIELD = "query" // name of PPL query field when passing into PPL/SQL Execute API call + + private const val TIMESTAMP_FIELD = "timestamp" // TODO: this should be deleted once PPL plugin side time keywords are introduced + + override suspend fun runMonitorV2( + monitorV2: MonitorV2, + monitorCtx: MonitorRunnerExecutionContext, // MonitorV2 reads from same context as Monitor + periodStart: Instant, + periodEnd: Instant, + dryRun: Boolean, + manual: Boolean, + executionId: String, + transportService: TransportService, + ): MonitorV2RunResult<*> { + if (monitorV2 !is PPLMonitor) { + throw IllegalStateException("Unexpected monitor type: ${monitorV2.javaClass.name}") + } + + if (monitorV2.id == MonitorV2.NO_ID) { + throw IllegalStateException("Received PPL Monitor to execute that unexpectedly has no ID") + } + + if (periodStart == periodEnd) { + logger.warn("Start and end time are the same: $periodStart. This PPL Monitor will probably only run once.") + } + + logger.debug("Running PPL Monitor: ${monitorV2.name}. Thread: ${Thread.currentThread().name}") + + val pplMonitor = monitorV2 + val nodeClient = monitorCtx.client as NodeClient + + // create some objects that will be used later + val triggerResults = mutableMapOf() + val pplQueryResults = mutableMapOf>() + + // set the current execution time + // use threadpool time for cross node consistency + val timeOfCurrentExecution = Instant.ofEpochMilli(MonitorRunnerService.monitorCtx.threadPool!!.absoluteTimeInMillis()) + + try { + monitorCtx.alertV2Indices!!.createOrUpdateAlertV2Index() + monitorCtx.alertV2Indices!!.createOrUpdateInitialAlertV2HistoryIndex() + } catch (e: Exception) { + val id = if (pplMonitor.id.trim().isEmpty()) "_na_" else pplMonitor.id + logger.error("Error loading alerts for monitorV2: $id", e) + return PPLMonitorRunResult(pplMonitor.name, e, periodStart, periodEnd, mapOf(), mapOf()) + } + + // only query data between now and the last PPL Monitor execution + // unless a look back window is specified, in which case use that instead, + // then inject a time filter where statement into PPL Monitor query. + // if the given monitor query already has any time check whatsoever, this + // simply returns the original query itself + // TODO: get lookback window based start time and put that in execution results instead of periodStart + val timeFilteredQuery = addTimeFilter(pplMonitor.query, periodStart, periodEnd, pplMonitor.lookBackWindow) + logger.info("time filtered query: $timeFilteredQuery") + + // run each trigger + for (pplTrigger in pplMonitor.triggers) { + try { + // check for suppression and skip execution + // before even running the trigger itself + val suppressed = checkForSuppress(pplTrigger, timeOfCurrentExecution, manual) + if (suppressed) { + logger.info("suppressing trigger ${pplTrigger.name} from monitor ${pplMonitor.name}") + + // automatically set this trigger to untriggered + triggerResults[pplTrigger.id] = PPLTriggerRunResult(pplTrigger.name, false, null) + + continue + } + logger.info("suppression check passed, executing trigger ${pplTrigger.name} from monitor ${pplMonitor.name}") + + // if trigger uses custom condition, append the custom condition to query, otherwise simply proceed + val queryToExecute = if (pplTrigger.conditionType == ConditionType.NUMBER_OF_RESULTS) { // number of results trigger + timeFilteredQuery + } else { // custom condition trigger + appendCustomCondition(timeFilteredQuery, pplTrigger.customCondition!!) + } + + // execute the PPL query + val queryResponseJson = executePplQuery(queryToExecute, nodeClient) + logger.info("query execution results for trigger ${pplTrigger.name}: $queryResponseJson") + + // retrieve deep copies of only the relevant query response rows. + // for num_results triggers, that's the entire response + // for custom triggers, that's only rows that evaluated to true + val relevantQueryResultRows = if (pplTrigger.conditionType == ConditionType.NUMBER_OF_RESULTS) { + // number of results trigger + getQueryResponseWithoutSize(queryResponseJson) + } else { + // custom condition trigger + collectCustomConditionResults(queryResponseJson, pplTrigger) + } + + // retrieve the number of results + // for number of results triggers, this is simply the number of PPL query results + // for custom triggers, this is the number of rows in the query response's eval result column that evaluated to true + val numResults = relevantQueryResultRows.getLong("total") + logger.info("number of results: $numResults") + + // determine if the trigger condition has been met + val triggered = if (pplTrigger.conditionType == ConditionType.NUMBER_OF_RESULTS) { // number of results trigger + evaluateNumResultsTrigger(numResults, pplTrigger.numResultsCondition!!, pplTrigger.numResultsValue!!) + } else { // custom condition trigger + numResults > 0 // if any of the query results satisfied the custom condition, the trigger counts as triggered + } + + logger.info("PPLTrigger ${pplTrigger.name} triggered: $triggered") + + // store the trigger execution and ppl query results for + // trigger execution response and notification message context + triggerResults[pplTrigger.id] = PPLTriggerRunResult(pplTrigger.name, triggered, null) + pplQueryResults[pplTrigger.id] = queryResponseJson.toMap() + + if (triggered) { + // if trigger is on result set mode, this list will have exactly 1 element + // if trigger is on per result mode, this list will have as many elements as the query results had rows + // up to the max number of alerts a per result trigger can generate + val preparedQueryResults = prepareQueryResults(relevantQueryResultRows, pplTrigger.mode, monitorCtx) + + // generate alerts based on trigger mode + // if this trigger is on result_set mode, this list contains exactly 1 alert + // if this trigger is on per_result mode, this list has any alerts as there are relevant query results + val thisTriggersGeneratedAlerts = generateAlerts( + pplTrigger, + pplMonitor, + preparedQueryResults, + executionId, + timeOfCurrentExecution + ) + + // update the trigger's last execution time for future suppression checks + pplTrigger.lastTriggeredTime = timeOfCurrentExecution + + // send alert notifications + for (action in pplTrigger.actions) { + for (queryResult in preparedQueryResults) { + val pplTriggerExecutionContext = PPLTriggerExecutionContext( + pplMonitor, + periodStart, + periodEnd, + null, + pplTrigger, + queryResult + ) + + runAction( + action, + pplTriggerExecutionContext, + monitorCtx, + pplMonitor, + dryRun + ) + } + } + + // write the alerts to the alerts index + monitorCtx.retryPolicy?.let { + saveAlertsV2(thisTriggersGeneratedAlerts, pplMonitor, it, nodeClient) + } + } + } catch (e: Exception) { + logger.error("failed to run PPL Trigger ${pplTrigger.name} from PPL Monitor ${pplMonitor.name}", e) + + // generate an alert with an error message + monitorCtx.retryPolicy?.let { + saveAlertsV2( + generateErrorAlert(pplTrigger, pplMonitor, e, executionId, timeOfCurrentExecution), + pplMonitor, + it, + nodeClient + ) + } + + continue + } + } + + // for suppression checking purposes, reindex the PPL Monitor into the alerting-config index + // with updated last triggered times for each of its triggers + if (triggerResults.any { it.value.triggered }) { + updateMonitorWithLastTriggeredTimes(pplMonitor, nodeClient) + } + + return PPLMonitorRunResult( + pplMonitor.name, + null, + periodStart, + periodEnd, + triggerResults, + pplQueryResults + ) + } + + // returns true if the pplTrigger should be suppressed + private fun checkForSuppress(pplTrigger: PPLTrigger, timeOfCurrentExecution: Instant, manual: Boolean): Boolean { + // manual calls from the user to execute a monitor should never be suppressed + if (manual) { + return false + } + + // the interval between throttledTimeBound and now is the suppression window + // i.e. any PPLTrigger whose last trigger time is in this window must be suppressed + val suppressTimeBound = pplTrigger.suppressDuration?.let { + timeOfCurrentExecution.minus(pplTrigger.suppressDuration!!.millis, ChronoUnit.MILLIS) + } + + // the trigger must be suppressed if... + return pplTrigger.suppressDuration != null && // suppression is enabled on the PPLTrigger + pplTrigger.lastTriggeredTime != null && // and it has triggered before at least once + pplTrigger.lastTriggeredTime!!.isAfter(suppressTimeBound!!) // and it's not yet out of the suppression window + } + + // adds monitor schedule-based time filter + // query: the raw PPL Monitor query + // periodStart: the lower bound of the initially computed query interval based on monitor schedule + // periodEnd: the upper bound of the initially computed query interval based on monitor schedule + // lookBackWindow: customer's desired query look back window, overrides [periodStart, periodEnd] if not null + private fun addTimeFilter(query: String, periodStart: Instant, periodEnd: Instant, lookBackWindow: TimeValue): String { + // inject time filter into PPL query to only query for data within the (periodStart, periodEnd) interval + // TODO: if query contains "_time", "span", "earliest", "latest", skip adding filter + // pending https://github.com/opensearch-project/sql/issues/3969 + // for now assume TIMESTAMP_FIELD field is always present in customer data + + // TODO: delete this, add lookback window time filter always + // if the raw query contained any time check whatsoever, skip adding a time filter internally + // and return query as is, customer's in-query time checks instantly and automatically overrides + if (query.contains(TIMESTAMP_FIELD)) { // TODO: replace with PPL time keyword checks after that's GA + return query + } + + // if customer passed in a look back window, override the precomputed interval with it + val updatedPeriodStart = periodEnd.minus(lookBackWindow.millis, ChronoUnit.MILLIS) + + // PPL plugin only accepts timestamp strings in this format + val formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss").withZone(UTC) + + val periodStartPplTimestamp = formatter.format(updatedPeriodStart) + val periodEndPplTimeStamp = formatter.format(periodEnd) + + val timeFilterAppend = "| where $TIMESTAMP_FIELD > TIMESTAMP('$periodStartPplTimestamp') and " + + "$TIMESTAMP_FIELD < TIMESTAMP('$periodEndPplTimeStamp')" + val timeFilterReplace = "$timeFilterAppend |" + + val timeFilteredQuery: String = if (query.contains("|")) { + // if Monitor query contains piped statements, inject the time filter + // as the first piped statement (i.e. before more complex statements + // like aggregations can take effect later in the query) + query.replaceFirst("|", timeFilterReplace) + } else { + // otherwise the query contains no piped statements and is simply a + // `search source=` statement, simply append time filter at the end + query + timeFilterAppend + } + + return timeFilteredQuery + } + + private fun evaluateNumResultsTrigger(numResults: Long, numResultsCondition: NumResultsCondition, numResultsValue: Long): Boolean { + return when (numResultsCondition) { + NumResultsCondition.GREATER_THAN -> numResults > numResultsValue + NumResultsCondition.GREATER_THAN_EQUAL -> numResults >= numResultsValue + NumResultsCondition.LESS_THAN -> numResults < numResultsValue + NumResultsCondition.LESS_THAN_EQUAL -> numResults <= numResultsValue + NumResultsCondition.EQUAL -> numResults == numResultsValue + NumResultsCondition.NOT_EQUAL -> numResults != numResultsValue + } + } + + private fun getQueryResponseWithoutSize(queryResponseJson: JSONObject): JSONObject { + // this will eventually store a deep copy of just the rows that triggered the custom condition + val queryResponseDeepCopy = JSONObject() + + // first add a deep copy of the schema + queryResponseDeepCopy.put("schema", JSONArray(queryResponseJson.getJSONArray("schema").toList())) + + // append empty datarows list, to be populated later + queryResponseDeepCopy.put("datarows", JSONArray()) + + val dataRowList = queryResponseJson.getJSONArray("datarows") + for (i in 0 until dataRowList.length()) { + val dataRow = dataRowList.getJSONArray(i) + queryResponseDeepCopy.getJSONArray("datarows").put(JSONArray(dataRow.toList())) + } + + // include the total but not the size field of the PPL Query response + queryResponseDeepCopy.put("total", queryResponseJson.getLong("total")) + + return queryResponseDeepCopy + } + + private fun collectCustomConditionResults(customConditionQueryResponse: JSONObject, pplTrigger: PPLTrigger): JSONObject { + // a PPL query with custom condition returning 0 results should imply a valid but not useful query. + // do not trigger alert, but warn that query likely is not functioning as user intended + if (customConditionQueryResponse.getLong("total") == 0L) { + logger.warn( + "During execution of PPL Trigger ${pplTrigger.name}, PPL query with custom " + + "condition returned no results. Proceeding without generating alert." + ) + return customConditionQueryResponse + } + + // this will eventually store a deep copy of just the rows that triggered the custom condition + val relevantQueryResultRows = JSONObject() + + // first add a deep copy of the schema + relevantQueryResultRows.put("schema", JSONArray(customConditionQueryResponse.getJSONArray("schema").toList())) + + // append empty datarows list, to be populated later + relevantQueryResultRows.put("datarows", JSONArray()) + + // find the name of the eval result variable defined in custom condition + val evalResultVarName = findEvalResultVar(pplTrigger.customCondition!!) + + // find the index eval statement result variable in the PPL query response schema + val evalResultVarIdx = findEvalResultVarIdxInSchema(customConditionQueryResponse, evalResultVarName) + + val dataRowList = customConditionQueryResponse.getJSONArray("datarows") + for (i in 0 until dataRowList.length()) { + val dataRow = dataRowList.getJSONArray(i) + val evalResult = dataRow.getBoolean(evalResultVarIdx) + if (evalResult) { + // if the row triggered the custom condition + // add it to the relevant results deep copy + relevantQueryResultRows.getJSONArray("datarows").put(JSONArray(dataRow.toList())) + } + } + + // include the total but not the size field of the PPL Query response + relevantQueryResultRows.put("total", relevantQueryResultRows.getJSONArray("datarows").length()) + + // return only the rows that triggered the custom condition + return relevantQueryResultRows + } + + // prepares the query results to be passed into alerts and notifications based on trigger mode + // if result set, alert and notification simply stores all query results + // if per result, each alert and notification stores a single row of the query results + private fun prepareQueryResults( + relevantQueryResultRows: JSONObject, + triggerMode: TriggerMode, + monitorCtx: MonitorRunnerExecutionContext + ): List { + // case: result set + // return the results as a single set of all the results + if (triggerMode == TriggerMode.RESULT_SET) { + return listOf(relevantQueryResultRows) + } + + // case: per result + // prepare to generate an alert for each query result row + val individualRows = mutableListOf() + val numAlertsToGenerate = relevantQueryResultRows.getInt("total") + for (i in 0 until numAlertsToGenerate) { + val individualRow = JSONObject() + individualRow.put("schema", JSONArray(relevantQueryResultRows.getJSONArray("schema").toList())) + individualRow.put( + "datarows", + JSONArray().put( + JSONArray(relevantQueryResultRows.getJSONArray("datarows").getJSONArray(i).toList()) + ) + ) + individualRows.add(individualRow) + } + + logger.info("individualRows: $individualRows") + + // there may be many query result rows, and generating an alert for each of them could lead to cluster issues, + // so limit the number of per_result alerts that are generated + val maxAlerts = monitorCtx.clusterService!!.clusterSettings.get(AlertingSettings.ALERT_V2_PER_RESULT_TRIGGER_MAX_ALERTS) + val reducedIndividualRows = individualRows.take(maxAlerts) + + return reducedIndividualRows + } + + private fun generateAlerts( + pplTrigger: PPLTrigger, + pplMonitor: PPLMonitor, + preparedQueryResults: List, + executionId: String, + timeOfCurrentExecution: Instant + ): List { + val expirationTime = pplTrigger.expireDuration.millis.let { timeOfCurrentExecution.plus(it, ChronoUnit.MILLIS) } + + val alertV2s = mutableListOf() + for (queryResult in preparedQueryResults) { + val alertV2 = AlertV2( + monitorId = pplMonitor.id, + monitorName = pplMonitor.name, + monitorVersion = pplMonitor.version, + monitorUser = pplMonitor.user, + triggerId = pplTrigger.id, + triggerName = pplTrigger.name, + query = pplMonitor.query, + triggeredTime = timeOfCurrentExecution, + expirationTime = expirationTime, + severity = pplTrigger.severity, + executionId = executionId + ) + alertV2s.add(alertV2) + } + + return alertV2s.toList() // return as immutable list + } + + private fun generateErrorAlert( + pplTrigger: PPLTrigger, + pplMonitor: PPLMonitor, + exception: Exception, + executionId: String, + timeOfCurrentExecution: Instant + ): List { + val expirationTime = pplTrigger.expireDuration.millis.let { timeOfCurrentExecution.plus(it, ChronoUnit.MILLIS) } + + val errorMessage = "Failed to run PPL Trigger ${pplTrigger.name} from PPL Monitor ${pplMonitor.name}: " + + exception.userErrorMessage() + val obfuscatedErrorMessage = AlertError.obfuscateIPAddresses(errorMessage) + + val alertV2 = AlertV2( + monitorId = pplMonitor.id, + monitorName = pplMonitor.name, + monitorVersion = pplMonitor.version, + monitorUser = pplMonitor.user, + triggerId = pplTrigger.id, + triggerName = pplTrigger.name, + query = pplMonitor.query, + triggeredTime = timeOfCurrentExecution, + expirationTime = expirationTime, + errorMessage = obfuscatedErrorMessage, + severity = Severity.ERROR, + executionId = executionId + ) + + return listOf(alertV2) + } + + private suspend fun saveAlertsV2( + alerts: List, + pplMonitor: PPLMonitor, + retryPolicy: BackoffPolicy, + client: NodeClient + ) { + logger.info("received alerts: $alerts") + + var requestsToRetry = alerts.flatMap { alert -> + listOf>( + IndexRequest(AlertV2Indices.ALERT_V2_INDEX) + .routing(pplMonitor.id) // set routing ID to PPL Monitor ID + .source(alert.toXContentWithUser(XContentFactory.jsonBuilder())) + .id(if (alert.id != Alert.NO_ID) alert.id else null) + ) + } + + if (requestsToRetry.isEmpty()) return + // Retry Bulk requests if there was any 429 response + retryPolicy.retry(logger, listOf(RestStatus.TOO_MANY_REQUESTS)) { + val bulkRequest = BulkRequest().add(requestsToRetry).setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) + val bulkResponse: BulkResponse = client.suspendUntil { client.bulk(bulkRequest, it) } + val failedResponses = (bulkResponse.items ?: arrayOf()).filter { it.isFailed } + failedResponses.forEach { + logger.info("write alerts failed responses: ${it.failureMessage}") + } + requestsToRetry = failedResponses.filter { it.status() == RestStatus.TOO_MANY_REQUESTS } + .map { bulkRequest.requests()[it.itemId] as IndexRequest } + + if (requestsToRetry.isNotEmpty()) { + val retryCause = failedResponses.first { it.status() == RestStatus.TOO_MANY_REQUESTS }.failure.cause + throw ExceptionsHelper.convertToOpenSearchException(retryCause) + } + } + } + + // TODO: every time this is done, trigger and action IDs change, figure out how to retain IDs + private suspend fun updateMonitorWithLastTriggeredTimes(pplMonitor: PPLMonitor, client: NodeClient) { + val indexRequest = IndexRequest(SCHEDULED_JOBS_INDEX) + .id(pplMonitor.id) + .source( + pplMonitor.toXContentWithUser( + XContentFactory.jsonBuilder(), + ToXContent.MapParams( + mapOf("with_type" to "true") + ) + ) + ) + .routing(pplMonitor.id) + val indexResponse = client.suspendUntil { index(indexRequest, it) } + + logger.info("PPLMonitor update with last execution times index response: ${indexResponse.result}") + } + + suspend fun runAction( + action: Action, + triggerCtx: PPLTriggerExecutionContext, + monitorCtx: MonitorRunnerExecutionContext, + pplMonitor: PPLMonitor, + dryrun: Boolean + ) { + // this function can throw an exception, which is caught by the try + // catch in runMonitor() to generate an error alert +// val actionOutput = mutableMapOf() + +// JSONArray(relevantQueryResultRows.getJSONArray("datarows").getJSONArray(i).toList()) + + // these are the full query results we got from the monitor's + // query execution + val pplQueryFullResults = triggerCtx.pplQueryResults + + // make a deep copy of the original query results with only a single data row + // do this by serializing the full results into a string, then creating a new JSONObject from the string, + // then remove all but one row in the deep copy's datarows + val pplQueryResultsSingleRow = JSONObject(pplQueryFullResults.toString()) + pplQueryResultsSingleRow.getJSONArray("datarows").apply { + for (i in length() - 1 downTo 1) { + remove(i) + } + } + + // estimate byte size with string length + val size = pplQueryFullResults.toString().length + val oneRowSize = pplQueryResultsSingleRow.toString().length + + logger.info("pplQueryFullResults: $pplQueryFullResults") + logger.info("pplQueryResultsSingleRow: $pplQueryResultsSingleRow") + + // retrieve the size limit from cluster settings + val maxSize = monitorCtx.clusterService!!.clusterSettings.get(AlertingSettings.ALERT_V2_NOTIF_QUERY_RESULTS_MAX_SIZE) + + var truncatedToSingleRow = false + var truncatedEntirely = false + if (size > maxSize && oneRowSize <= maxSize) { + triggerCtx.pplQueryResults = pplQueryResultsSingleRow + truncatedToSingleRow = true + } else if (size > maxSize && oneRowSize > maxSize) { + triggerCtx.pplQueryResults = JSONObject() + truncatedEntirely = true + } + + val notifSubject = if (action.subjectTemplate != null) + MonitorRunnerService.compileTemplateV2(action.subjectTemplate!!, triggerCtx) + else "" + + var notifMessage = MonitorRunnerService.compileTemplateV2(action.messageTemplate, triggerCtx) + if (Strings.isNullOrEmpty(notifMessage)) { + throw IllegalStateException("Message content missing in the Destination with id: ${action.destinationId}") + } + + if (truncatedToSingleRow) { + notifMessage += "\n\n(Note from Alerting Plugin: the full query results were too large, " + + "only one query result row was passed into this notification)" + } else if (truncatedEntirely) { + notifMessage += "\n\n(Note from Alerting Plugin: the query results were too large, " + + "no query results were passed into this notification)" + } + + if (!dryrun) { + monitorCtx.client!!.threadPool().threadContext.stashContext().use { + withClosableContext( + InjectorContextElement( + pplMonitor.id, + monitorCtx.settings!!, + monitorCtx.threadPool!!.threadContext, + pplMonitor.user?.roles, + pplMonitor.user + ) + ) { + getConfigAndSendNotification( + action, + monitorCtx, + notifSubject, + notifMessage + ) + } + } + } + } + + /* public util functions */ + + // appends user-defined custom trigger condition to PPL query, only for custom condition Triggers + fun appendCustomCondition(query: String, customCondition: String): String { + return "$query | $customCondition" + } + + // returns PPL query response as parsable JSONObject + suspend fun executePplQuery(query: String, client: NodeClient): JSONObject { + // call PPL plugin to execute time filtered query + val transportPplQueryRequest = TransportPPLQueryRequest( + query, + JSONObject(mapOf(PPL_SQL_QUERY_FIELD to query)), + null // null path falls back to a default path internal to SQL/PPL Plugin + ) + + val transportPplQueryResponse = PPLPluginInterface.suspendUntil { + this.executeQuery( + client, + transportPplQueryRequest, + it + ) + } + + val queryResponseJson = JSONObject(transportPplQueryResponse.result) + + return queryResponseJson + } + + // TODO: is there maybe some PPL plugin util function we can use to replace this? + // searches a given custom condition eval statement for the name of + // the eval result variable and returns it + fun findEvalResultVar(customCondition: String): String { + // the PPL keyword "eval", followed by a whitespace must be present, otherwise a syntax error from PPL plugin would've + // been thrown when executing the query (without the whitespace, the query would've had something like "evalresult", + // which is invalid PPL + val startOfEvalStatement = "eval " + + val startIdx = customCondition.indexOf(startOfEvalStatement) + startOfEvalStatement.length + val endIdx = startIdx + customCondition.substring(startIdx).indexOfFirst { it == ' ' || it == '=' } + return customCondition.substring(startIdx, endIdx) + } + + fun findEvalResultVarIdxInSchema(customConditionQueryResponse: JSONObject, evalResultVarName: String): Int { + // find the index eval statement result variable in the PPL query response schema + val schemaList = customConditionQueryResponse.getJSONArray("schema") + var evalResultVarIdx = -1 + for (i in 0 until schemaList.length()) { + val schemaObj = schemaList.getJSONObject(i) + val columnName = schemaObj.getString("name") + + if (columnName == evalResultVarName) { + evalResultVarIdx = i + break + } + } + + // eval statement result variable should always be found + if (evalResultVarIdx == -1) { + throw IllegalStateException( + "expected to find eval statement results variable \"$evalResultVarName\" in results " + + "of PPL query with custom condition, but did not." + ) + } + + return evalResultVarIdx + } +} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/DeleteMonitorV2Action.kt b/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/DeleteMonitorV2Action.kt new file mode 100644 index 000000000..2cd1ba703 --- /dev/null +++ b/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/DeleteMonitorV2Action.kt @@ -0,0 +1,10 @@ +package org.opensearch.alerting.actionv2 + +import org.opensearch.action.ActionType + +class DeleteMonitorV2Action private constructor() : ActionType(NAME, ::DeleteMonitorV2Response) { + companion object { + val INSTANCE = DeleteMonitorV2Action() + const val NAME = "cluster:admin/opensearch/alerting/v2/monitor/delete" + } +} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/DeleteMonitorV2Request.kt b/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/DeleteMonitorV2Request.kt new file mode 100644 index 000000000..601d83588 --- /dev/null +++ b/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/DeleteMonitorV2Request.kt @@ -0,0 +1,34 @@ +package org.opensearch.alerting.actionv2 + +import org.opensearch.action.ActionRequest +import org.opensearch.action.ActionRequestValidationException +import org.opensearch.action.support.WriteRequest +import org.opensearch.core.common.io.stream.StreamInput +import org.opensearch.core.common.io.stream.StreamOutput +import java.io.IOException + +class DeleteMonitorV2Request : ActionRequest { + val monitorV2Id: String + val refreshPolicy: WriteRequest.RefreshPolicy + + constructor(monitorV2Id: String, refreshPolicy: WriteRequest.RefreshPolicy) : super() { + this.monitorV2Id = monitorV2Id + this.refreshPolicy = refreshPolicy + } + + @Throws(IOException::class) + constructor(sin: StreamInput) : this( + monitorV2Id = sin.readString(), + refreshPolicy = WriteRequest.RefreshPolicy.readFrom(sin) + ) + + override fun validate(): ActionRequestValidationException? { + return null + } + + @Throws(IOException::class) + override fun writeTo(out: StreamOutput) { + out.writeString(monitorV2Id) + refreshPolicy.writeTo(out) + } +} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/DeleteMonitorV2Response.kt b/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/DeleteMonitorV2Response.kt new file mode 100644 index 000000000..71dcfcbd4 --- /dev/null +++ b/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/DeleteMonitorV2Response.kt @@ -0,0 +1,38 @@ +package org.opensearch.alerting.actionv2 + +import org.opensearch.commons.alerting.util.IndexUtils +import org.opensearch.commons.notifications.action.BaseResponse +import org.opensearch.core.common.io.stream.StreamInput +import org.opensearch.core.common.io.stream.StreamOutput +import org.opensearch.core.xcontent.ToXContent +import org.opensearch.core.xcontent.XContentBuilder + +class DeleteMonitorV2Response : BaseResponse { + var id: String + var version: Long + + constructor( + id: String, + version: Long + ) : super() { + this.id = id + this.version = version + } + + constructor(sin: StreamInput) : this( + sin.readString(), // id + sin.readLong() // version + ) + + override fun writeTo(out: StreamOutput) { + out.writeString(id) + out.writeLong(version) + } + + override fun toXContent(builder: XContentBuilder, params: ToXContent.Params): XContentBuilder { + return builder.startObject() + .field(IndexUtils._ID, id) + .field(IndexUtils._VERSION, version) + .endObject() + } +} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/ExecuteMonitorV2Action.kt b/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/ExecuteMonitorV2Action.kt new file mode 100644 index 000000000..c3ba7968b --- /dev/null +++ b/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/ExecuteMonitorV2Action.kt @@ -0,0 +1,10 @@ +package org.opensearch.alerting.actionv2 + +import org.opensearch.action.ActionType + +class ExecuteMonitorV2Action private constructor() : ActionType(NAME, ::ExecuteMonitorV2Response) { + companion object { + val INSTANCE = ExecuteMonitorV2Action() + const val NAME = "cluster:admin/opensearch/alerting/v2/monitor/execute" + } +} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/ExecuteMonitorV2Request.kt b/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/ExecuteMonitorV2Request.kt new file mode 100644 index 000000000..dbf5d62b9 --- /dev/null +++ b/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/ExecuteMonitorV2Request.kt @@ -0,0 +1,71 @@ +package org.opensearch.alerting.actionv2 + +import org.opensearch.action.ActionRequest +import org.opensearch.action.ActionRequestValidationException +import org.opensearch.action.ValidateActions +import org.opensearch.alerting.core.modelv2.MonitorV2 +import org.opensearch.common.unit.TimeValue +import org.opensearch.core.common.io.stream.StreamInput +import org.opensearch.core.common.io.stream.StreamOutput +import java.io.IOException + +class ExecuteMonitorV2Request : ActionRequest { + val dryrun: Boolean + val manual: Boolean + val monitorId: String? // exactly one of monitorId or monitor must be non-null + val monitorV2: MonitorV2? + val requestStart: TimeValue? + val requestEnd: TimeValue + + constructor( + dryrun: Boolean, + manual: Boolean, // if execute was called by user or by scheduled job + monitorId: String?, + monitorV2: MonitorV2?, + requestStart: TimeValue? = null, + requestEnd: TimeValue + ) : super() { + this.dryrun = dryrun + this.manual = manual + this.monitorId = monitorId + this.monitorV2 = monitorV2 + this.requestStart = requestStart + this.requestEnd = requestEnd + } + + @Throws(IOException::class) + constructor(sin: StreamInput) : this( + sin.readBoolean(), // dryrun + sin.readBoolean(), // manual + sin.readOptionalString(), // monitorId + if (sin.readBoolean()) { + MonitorV2.readFrom(sin) // monitor + } else { + null + }, + sin.readOptionalTimeValue(), + sin.readTimeValue() // requestEnd + ) + + override fun validate(): ActionRequestValidationException? = + if (monitorV2 == null && monitorId == null) { + ValidateActions.addValidationError("Neither a monitor ID nor monitor object was supplied", null) + } else { + null + } + + @Throws(IOException::class) + override fun writeTo(out: StreamOutput) { + out.writeBoolean(dryrun) + out.writeBoolean(manual) + out.writeOptionalString(monitorId) + if (monitorV2 != null) { + out.writeBoolean(true) + monitorV2.writeTo(out) + } else { + out.writeBoolean(false) + } + out.writeOptionalTimeValue(requestStart) + out.writeTimeValue(requestEnd) + } +} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/ExecuteMonitorV2Response.kt b/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/ExecuteMonitorV2Response.kt new file mode 100644 index 000000000..6635f3791 --- /dev/null +++ b/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/ExecuteMonitorV2Response.kt @@ -0,0 +1,33 @@ +package org.opensearch.alerting.actionv2 + +import org.opensearch.alerting.core.modelv2.MonitorV2RunResult +import org.opensearch.core.action.ActionResponse +import org.opensearch.core.common.io.stream.StreamInput +import org.opensearch.core.common.io.stream.StreamOutput +import org.opensearch.core.xcontent.ToXContent +import org.opensearch.core.xcontent.ToXContentObject +import org.opensearch.core.xcontent.XContentBuilder +import java.io.IOException + +class ExecuteMonitorV2Response : ActionResponse, ToXContentObject { + val monitorV2RunResult: MonitorV2RunResult<*> + + constructor(monitorV2RunResult: MonitorV2RunResult<*>) : super() { + this.monitorV2RunResult = monitorV2RunResult + } + + @Throws(IOException::class) + constructor(sin: StreamInput) : this( + MonitorV2RunResult.readFrom(sin) // monitorRunResult + ) + + @Throws(IOException::class) + override fun writeTo(out: StreamOutput) { + monitorV2RunResult.writeTo(out) + } + + @Throws(IOException::class) + override fun toXContent(builder: XContentBuilder, params: ToXContent.Params): XContentBuilder { + return monitorV2RunResult.toXContent(builder, ToXContent.EMPTY_PARAMS) + } +} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/GetAlertsV2Action.kt b/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/GetAlertsV2Action.kt new file mode 100644 index 000000000..6255d6ddd --- /dev/null +++ b/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/GetAlertsV2Action.kt @@ -0,0 +1,10 @@ +package org.opensearch.alerting.actionv2 + +import org.opensearch.action.ActionType + +class GetAlertsV2Action private constructor() : ActionType(NAME, ::GetAlertsV2Response) { + companion object { + val INSTANCE = GetAlertsV2Action() + const val NAME = "cluster:admin/opensearch/alerting/v2/alerts/get" + } +} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/GetAlertsV2Request.kt b/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/GetAlertsV2Request.kt new file mode 100644 index 000000000..d85aec2e5 --- /dev/null +++ b/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/GetAlertsV2Request.kt @@ -0,0 +1,63 @@ +package org.opensearch.alerting.actionv2 + +import org.opensearch.action.ActionRequest +import org.opensearch.action.ActionRequestValidationException +import org.opensearch.commons.alerting.model.Table +import org.opensearch.core.common.io.stream.StreamInput +import org.opensearch.core.common.io.stream.StreamOutput +import org.opensearch.index.query.BoolQueryBuilder +import java.io.IOException + +class GetAlertsV2Request : ActionRequest { + val table: Table + val severityLevel: String + val monitorV2Id: String? + val monitorV2Ids: List? + val alertV2Ids: List? + val boolQueryBuilder: BoolQueryBuilder? + + constructor( + table: Table, + severityLevel: String, + monitorV2Id: String?, + monitorV2Ids: List? = null, + alertV2Ids: List? = null, + boolQueryBuilder: BoolQueryBuilder? = null + ) : super() { + this.table = table + this.severityLevel = severityLevel + this.monitorV2Id = monitorV2Id + this.monitorV2Ids = monitorV2Ids + this.alertV2Ids = alertV2Ids + this.boolQueryBuilder = boolQueryBuilder + } + + @Throws(IOException::class) + constructor(sin: StreamInput) : this( + table = Table.readFrom(sin), + severityLevel = sin.readString(), + monitorV2Id = sin.readOptionalString(), + monitorV2Ids = sin.readOptionalStringList(), + alertV2Ids = sin.readOptionalStringList(), + boolQueryBuilder = if (sin.readOptionalBoolean() == true) BoolQueryBuilder(sin) else null + ) + + override fun validate(): ActionRequestValidationException? { + return null + } + + @Throws(IOException::class) + override fun writeTo(out: StreamOutput) { + table.writeTo(out) + out.writeString(severityLevel) + out.writeOptionalString(monitorV2Id) + out.writeOptionalStringCollection(monitorV2Ids) + out.writeOptionalStringCollection(alertV2Ids) + if (boolQueryBuilder != null) { + out.writeOptionalBoolean(true) + boolQueryBuilder.writeTo(out) + } else { + out.writeOptionalBoolean(false) + } + } +} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/GetAlertsV2Response.kt b/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/GetAlertsV2Response.kt new file mode 100644 index 000000000..39b9faf53 --- /dev/null +++ b/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/GetAlertsV2Response.kt @@ -0,0 +1,47 @@ +package org.opensearch.alerting.actionv2 + +import org.opensearch.alerting.core.modelv2.AlertV2 +import org.opensearch.commons.notifications.action.BaseResponse +import org.opensearch.core.common.io.stream.StreamInput +import org.opensearch.core.common.io.stream.StreamOutput +import org.opensearch.core.xcontent.ToXContent +import org.opensearch.core.xcontent.XContentBuilder +import java.io.IOException +import java.util.Collections + +class GetAlertsV2Response : BaseResponse { + val alertV2s: List + + // totalAlertV2s is not the same as the size of alerts because there can be 30 alerts from the request, but + // the request only asked for 5 alerts, so totalAlertV2s will be 30, but alerts will only contain 5 alerts + val totalAlertV2s: Int? + + constructor( + alertV2s: List, + totalAlertV2s: Int? + ) : super() { + this.alertV2s = alertV2s + this.totalAlertV2s = totalAlertV2s + } + + @Throws(IOException::class) + constructor(sin: StreamInput) : this( + alertV2s = Collections.unmodifiableList(sin.readList(::AlertV2)), + totalAlertV2s = sin.readOptionalInt() + ) + + @Throws(IOException::class) + override fun writeTo(out: StreamOutput) { + out.writeCollection(alertV2s) + out.writeOptionalInt(totalAlertV2s) + } + + @Throws(IOException::class) + override fun toXContent(builder: XContentBuilder, params: ToXContent.Params): XContentBuilder { + builder.startObject() + .field("alertV2s", alertV2s) + .field("totalAlertV2s", totalAlertV2s) + + return builder.endObject() + } +} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/GetMonitorV2Action.kt b/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/GetMonitorV2Action.kt new file mode 100644 index 000000000..9fb0915c6 --- /dev/null +++ b/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/GetMonitorV2Action.kt @@ -0,0 +1,10 @@ +package org.opensearch.alerting.actionv2 + +import org.opensearch.action.ActionType + +class GetMonitorV2Action private constructor() : ActionType(NAME, ::GetMonitorV2Response) { + companion object { + val INSTANCE = GetMonitorV2Action() + const val NAME = "cluster:admin/opensearch/alerting/v2/monitor/get" + } +} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/GetMonitorV2Request.kt b/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/GetMonitorV2Request.kt new file mode 100644 index 000000000..a14f482e7 --- /dev/null +++ b/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/GetMonitorV2Request.kt @@ -0,0 +1,47 @@ +package org.opensearch.alerting.actionv2 + +import org.opensearch.action.ActionRequest +import org.opensearch.action.ActionRequestValidationException +import org.opensearch.core.common.io.stream.StreamInput +import org.opensearch.core.common.io.stream.StreamOutput +import org.opensearch.search.fetch.subphase.FetchSourceContext +import java.io.IOException + +class GetMonitorV2Request : ActionRequest { + val monitorV2Id: String + val version: Long + val srcContext: FetchSourceContext? + + constructor( + monitorV2Id: String, + version: Long, + srcContext: FetchSourceContext? + ) : super() { + this.monitorV2Id = monitorV2Id + this.version = version + this.srcContext = srcContext + } + + @Throws(IOException::class) + constructor(sin: StreamInput) : this( + sin.readString(), // monitorV2Id + sin.readLong(), // version + if (sin.readBoolean()) { + FetchSourceContext(sin) // srcContext + } else { + null + } + ) + + override fun validate(): ActionRequestValidationException? { + return null + } + + @Throws(IOException::class) + override fun writeTo(out: StreamOutput) { + out.writeString(monitorV2Id) + out.writeLong(version) + out.writeBoolean(srcContext != null) + srcContext?.writeTo(out) + } +} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/GetMonitorV2Response.kt b/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/GetMonitorV2Response.kt new file mode 100644 index 000000000..fe083f5e0 --- /dev/null +++ b/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/GetMonitorV2Response.kt @@ -0,0 +1,75 @@ +package org.opensearch.alerting.actionv2 + +import org.opensearch.alerting.core.modelv2.MonitorV2 +import org.opensearch.commons.alerting.util.IndexUtils.Companion._ID +import org.opensearch.commons.alerting.util.IndexUtils.Companion._PRIMARY_TERM +import org.opensearch.commons.alerting.util.IndexUtils.Companion._SEQ_NO +import org.opensearch.commons.alerting.util.IndexUtils.Companion._VERSION +import org.opensearch.commons.notifications.action.BaseResponse +import org.opensearch.core.common.io.stream.StreamInput +import org.opensearch.core.common.io.stream.StreamOutput +import org.opensearch.core.xcontent.ToXContent +import org.opensearch.core.xcontent.XContentBuilder +import java.io.IOException + +class GetMonitorV2Response : BaseResponse { + var id: String + var version: Long + var seqNo: Long + var primaryTerm: Long + var monitorV2: MonitorV2? + + constructor( + id: String, + version: Long, + seqNo: Long, + primaryTerm: Long, + monitorV2: MonitorV2? + ) : super() { + this.id = id + this.version = version + this.seqNo = seqNo + this.primaryTerm = primaryTerm + this.monitorV2 = monitorV2 + } + + @Throws(IOException::class) + constructor(sin: StreamInput) : this( + id = sin.readString(), // id + version = sin.readLong(), // version + seqNo = sin.readLong(), // seqNo + primaryTerm = sin.readLong(), // primaryTerm + monitorV2 = if (sin.readBoolean()) { + MonitorV2.readFrom(sin) // monitorV2 + } else { + null + } + ) + + @Throws(IOException::class) + override fun writeTo(out: StreamOutput) { + out.writeString(id) + out.writeLong(version) + out.writeLong(seqNo) + out.writeLong(primaryTerm) + if (monitorV2 != null) { + out.writeBoolean(true) + monitorV2?.writeTo(out) + } else { + out.writeBoolean(false) + } + } + + @Throws(IOException::class) + override fun toXContent(builder: XContentBuilder, params: ToXContent.Params): XContentBuilder { + builder.startObject() + .field(_ID, id) + .field(_VERSION, version) + .field(_SEQ_NO, seqNo) + .field(_PRIMARY_TERM, primaryTerm) + if (monitorV2 != null) { + builder.field("monitorV2", monitorV2) + } + return builder.endObject() + } +} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/IndexMonitorV2Action.kt b/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/IndexMonitorV2Action.kt new file mode 100644 index 000000000..cff851598 --- /dev/null +++ b/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/IndexMonitorV2Action.kt @@ -0,0 +1,10 @@ +package org.opensearch.alerting.actionv2 + +import org.opensearch.action.ActionType + +class IndexMonitorV2Action private constructor() : ActionType(NAME, ::IndexMonitorV2Response) { + companion object { + val INSTANCE = IndexMonitorV2Action() + const val NAME = "cluster:admin/opensearch/alerting/v2/monitor/write" + } +} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/IndexMonitorV2Request.kt b/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/IndexMonitorV2Request.kt new file mode 100644 index 000000000..a5ad591c1 --- /dev/null +++ b/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/IndexMonitorV2Request.kt @@ -0,0 +1,64 @@ +package org.opensearch.alerting.actionv2 + +import org.opensearch.action.ActionRequest +import org.opensearch.action.ActionRequestValidationException +import org.opensearch.action.support.WriteRequest +import org.opensearch.alerting.core.modelv2.MonitorV2 +import org.opensearch.core.common.io.stream.StreamInput +import org.opensearch.core.common.io.stream.StreamOutput +import org.opensearch.rest.RestRequest +import java.io.IOException + +class IndexMonitorV2Request : ActionRequest { + val monitorId: String + val seqNo: Long + val primaryTerm: Long + val refreshPolicy: WriteRequest.RefreshPolicy + val method: RestRequest.Method + var monitorV2: MonitorV2 + val rbacRoles: List? + + constructor( + monitorId: String, + seqNo: Long, + primaryTerm: Long, + refreshPolicy: WriteRequest.RefreshPolicy, + method: RestRequest.Method, + monitorV2: MonitorV2, + rbacRoles: List? = null + ) : super() { + this.monitorId = monitorId + this.seqNo = seqNo + this.primaryTerm = primaryTerm + this.refreshPolicy = refreshPolicy + this.method = method + this.monitorV2 = monitorV2 + this.rbacRoles = rbacRoles + } + + @Throws(IOException::class) + constructor(sin: StreamInput) : this( + monitorId = sin.readString(), + seqNo = sin.readLong(), + primaryTerm = sin.readLong(), + refreshPolicy = WriteRequest.RefreshPolicy.readFrom(sin), + method = sin.readEnum(RestRequest.Method::class.java), + monitorV2 = MonitorV2.readFrom(sin), + rbacRoles = sin.readOptionalStringList() + ) + + override fun validate(): ActionRequestValidationException? { + return null + } + + @Throws(IOException::class) + override fun writeTo(out: StreamOutput) { + out.writeString(monitorId) + out.writeLong(seqNo) + out.writeLong(primaryTerm) + refreshPolicy.writeTo(out) + out.writeEnum(method) + MonitorV2.writeTo(out, monitorV2) + out.writeOptionalStringCollection(rbacRoles) + } +} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/IndexMonitorV2Response.kt b/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/IndexMonitorV2Response.kt new file mode 100644 index 000000000..35640330b --- /dev/null +++ b/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/IndexMonitorV2Response.kt @@ -0,0 +1,68 @@ +package org.opensearch.alerting.actionv2 + +import org.opensearch.alerting.core.modelv2.MonitorV2 +import org.opensearch.commons.alerting.util.IndexUtils.Companion._ID +import org.opensearch.commons.alerting.util.IndexUtils.Companion._PRIMARY_TERM +import org.opensearch.commons.alerting.util.IndexUtils.Companion._SEQ_NO +import org.opensearch.commons.alerting.util.IndexUtils.Companion._VERSION +import org.opensearch.commons.notifications.action.BaseResponse +import org.opensearch.core.common.io.stream.StreamInput +import org.opensearch.core.common.io.stream.StreamOutput +import org.opensearch.core.xcontent.ToXContent +import org.opensearch.core.xcontent.XContentBuilder +import java.io.IOException + +class IndexMonitorV2Response : BaseResponse { + var id: String + var version: Long + var seqNo: Long + var primaryTerm: Long + var monitorV2: MonitorV2 + + constructor( + id: String, + version: Long, + seqNo: Long, + primaryTerm: Long, + monitorV2: MonitorV2 + ) : super() { + this.id = id + this.version = version + this.seqNo = seqNo + this.primaryTerm = primaryTerm + this.monitorV2 = monitorV2 + } + + @Throws(IOException::class) + constructor(sin: StreamInput) : this( + sin.readString(), // id + sin.readLong(), // version + sin.readLong(), // seqNo + sin.readLong(), // primaryTerm + MonitorV2.readFrom(sin) // monitorV2 + ) + + @Throws(IOException::class) + override fun writeTo(out: StreamOutput) { + out.writeString(id) + out.writeLong(version) + out.writeLong(seqNo) + out.writeLong(primaryTerm) + MonitorV2.writeTo(out, monitorV2) + } + + @Throws(IOException::class) + override fun toXContent(builder: XContentBuilder, params: ToXContent.Params): XContentBuilder { + return builder.startObject() + .field(_ID, id) + .field(_VERSION, version) + .field(_SEQ_NO, seqNo) + .field(_PRIMARY_TERM, primaryTerm) + .field(MONITOR_V2_FIELD, monitorV2) + .endObject() + } + + companion object { + const val MONITOR_V2_FIELD = "monitor_v2" + } +} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/SearchMonitorV2Action.kt b/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/SearchMonitorV2Action.kt new file mode 100644 index 000000000..d83ffd510 --- /dev/null +++ b/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/SearchMonitorV2Action.kt @@ -0,0 +1,11 @@ +package org.opensearch.alerting.actionv2 + +import org.opensearch.action.ActionType +import org.opensearch.action.search.SearchResponse + +class SearchMonitorV2Action private constructor() : ActionType(NAME, ::SearchResponse) { + companion object { + val INSTANCE = SearchMonitorV2Action() + const val NAME = "cluster:admin/opensearch/alerting/v2/monitor/search" + } +} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/SearchMonitorV2Request.kt b/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/SearchMonitorV2Request.kt new file mode 100644 index 000000000..51fba09aa --- /dev/null +++ b/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/SearchMonitorV2Request.kt @@ -0,0 +1,32 @@ +package org.opensearch.alerting.actionv2 + +import org.opensearch.action.ActionRequest +import org.opensearch.action.ActionRequestValidationException +import org.opensearch.action.search.SearchRequest +import org.opensearch.core.common.io.stream.StreamInput +import org.opensearch.core.common.io.stream.StreamOutput +import java.io.IOException + +class SearchMonitorV2Request : ActionRequest { + val searchRequest: SearchRequest + + constructor( + searchRequest: SearchRequest + ) : super() { + this.searchRequest = searchRequest + } + + @Throws(IOException::class) + constructor(sin: StreamInput) : this( + searchRequest = SearchRequest(sin) + ) + + override fun validate(): ActionRequestValidationException? { + return null + } + + @Throws(IOException::class) + override fun writeTo(out: StreamOutput) { + searchRequest.writeTo(out) + } +} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/alertsv2/AlertV2Indices.kt b/alerting/src/main/kotlin/org/opensearch/alerting/alertsv2/AlertV2Indices.kt new file mode 100644 index 000000000..3921be51d --- /dev/null +++ b/alerting/src/main/kotlin/org/opensearch/alerting/alertsv2/AlertV2Indices.kt @@ -0,0 +1,424 @@ +package org.opensearch.alerting.alertsv2 + +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.launch +import org.apache.logging.log4j.LogManager +import org.opensearch.ExceptionsHelper +import org.opensearch.ResourceAlreadyExistsException +import org.opensearch.action.admin.cluster.state.ClusterStateRequest +import org.opensearch.action.admin.cluster.state.ClusterStateResponse +import org.opensearch.action.admin.indices.alias.Alias +import org.opensearch.action.admin.indices.create.CreateIndexRequest +import org.opensearch.action.admin.indices.create.CreateIndexResponse +import org.opensearch.action.admin.indices.delete.DeleteIndexRequest +import org.opensearch.action.admin.indices.exists.indices.IndicesExistsRequest +import org.opensearch.action.admin.indices.exists.indices.IndicesExistsResponse +import org.opensearch.action.admin.indices.mapping.put.PutMappingRequest +import org.opensearch.action.admin.indices.rollover.RolloverRequest +import org.opensearch.action.admin.indices.rollover.RolloverResponse +import org.opensearch.action.support.IndicesOptions +import org.opensearch.action.support.clustermanager.AcknowledgedResponse +import org.opensearch.alerting.opensearchapi.suspendUntil +import org.opensearch.alerting.settings.AlertingSettings.Companion.ALERT_V2_HISTORY_ENABLED +import org.opensearch.alerting.settings.AlertingSettings.Companion.ALERT_V2_HISTORY_INDEX_MAX_AGE +import org.opensearch.alerting.settings.AlertingSettings.Companion.ALERT_V2_HISTORY_MAX_DOCS +import org.opensearch.alerting.settings.AlertingSettings.Companion.ALERT_V2_HISTORY_RETENTION_PERIOD +import org.opensearch.alerting.settings.AlertingSettings.Companion.ALERT_V2_HISTORY_ROLLOVER_PERIOD +import org.opensearch.alerting.settings.AlertingSettings.Companion.REQUEST_TIMEOUT +import org.opensearch.alerting.util.IndexUtils +import org.opensearch.cluster.ClusterChangedEvent +import org.opensearch.cluster.ClusterStateListener +import org.opensearch.cluster.metadata.IndexMetadata +import org.opensearch.cluster.service.ClusterService +import org.opensearch.common.settings.Settings +import org.opensearch.common.unit.TimeValue +import org.opensearch.common.xcontent.XContentType +import org.opensearch.commons.alerting.util.AlertingException +import org.opensearch.core.action.ActionListener +import org.opensearch.threadpool.Scheduler.Cancellable +import org.opensearch.threadpool.ThreadPool +import org.opensearch.transport.client.Client +import java.time.Instant + +private val scope: CoroutineScope = CoroutineScope(Dispatchers.IO) +private val logger = LogManager.getLogger(AlertV2Indices::class.java) + +// TODO: find what you can and factor into to IndexUtils.kt +class AlertV2Indices( + settings: Settings, + private val client: Client, + private val threadPool: ThreadPool, + private val clusterService: ClusterService +) : ClusterStateListener { + + init { + clusterService.addListener(this) + clusterService.clusterSettings.addSettingsUpdateConsumer(ALERT_V2_HISTORY_ENABLED) { alertV2HistoryEnabled = it } + clusterService.clusterSettings.addSettingsUpdateConsumer(ALERT_V2_HISTORY_MAX_DOCS) { alertV2HistoryMaxDocs = it } + clusterService.clusterSettings.addSettingsUpdateConsumer(ALERT_V2_HISTORY_INDEX_MAX_AGE) { alertV2HistoryMaxAge = it } + clusterService.clusterSettings.addSettingsUpdateConsumer(ALERT_V2_HISTORY_ROLLOVER_PERIOD) { + alertV2HistoryRolloverPeriod = it + rescheduleAlertRollover() + } + clusterService.clusterSettings.addSettingsUpdateConsumer(ALERT_V2_HISTORY_RETENTION_PERIOD) { + alertV2HistoryRetentionPeriod = it + } + clusterService.clusterSettings.addSettingsUpdateConsumer(REQUEST_TIMEOUT) { requestTimeout = it } + } + + companion object { + + /** The in progress alert history index. */ + const val ALERT_V2_INDEX = ".opensearch-alerting-v2-alerts" + + /** The alias of the index in which to write alert history */ + const val ALERT_V2_HISTORY_WRITE_INDEX = ".opensearch-alerting-v2-alert-history-write" + + /** The index name pattern referring to all alert history indices */ + const val ALERT_V2_HISTORY_ALL = ".opensearch-alerting-v2-alert-history*" + + /** The index name pattern to create alert history indices */ + const val ALERT_V2_HISTORY_INDEX_PATTERN = "<.opensearch-alerting-v2-alert-history-{now/d}-1>" + + /** The index name pattern to query all alerts, history and current alerts. */ + const val ALL_ALERT_V2_INDEX_PATTERN = ".opensearch-alerting-v2-alert*" + + @JvmStatic + fun alertV2Mapping() = + AlertV2Indices::class.java.getResource("alert_v2_mapping.json").readText() + } + + @Volatile private var alertV2HistoryEnabled = ALERT_V2_HISTORY_ENABLED.get(settings) + + @Volatile private var alertV2HistoryMaxDocs = ALERT_V2_HISTORY_MAX_DOCS.get(settings) + + @Volatile private var alertV2HistoryMaxAge = ALERT_V2_HISTORY_INDEX_MAX_AGE.get(settings) + + @Volatile private var alertV2HistoryRolloverPeriod = ALERT_V2_HISTORY_ROLLOVER_PERIOD.get(settings) + + @Volatile private var alertV2HistoryRetentionPeriod = ALERT_V2_HISTORY_RETENTION_PERIOD.get(settings) + + @Volatile private var requestTimeout = REQUEST_TIMEOUT.get(settings) + + @Volatile private var isClusterManager = false + + // for JobsMonitor to report + var lastRolloverTime: TimeValue? = null + + private var alertV2HistoryIndexInitialized: Boolean = false + + private var alertV2IndexInitialized: Boolean = false + + private var scheduledAlertV2Rollover: Cancellable? = null + + fun onClusterManager() { + try { + // try to rollover immediately as we might be restarting the cluster + rolloverAlertV2HistoryIndex() + + // schedule the next rollover for approx MAX_AGE later + scheduledAlertV2Rollover = threadPool + .scheduleWithFixedDelay({ rolloverAndDeleteAlertV2HistoryIndices() }, alertV2HistoryRolloverPeriod, executorName()) + } catch (e: Exception) { + logger.error( + "Error creating alert/finding indices. " + + "Alerts/Findings can't be recorded until clustermanager node is restarted.", + e + ) + } + } + + fun offClusterManager() { + scheduledAlertV2Rollover?.cancel() + } + + private fun executorName(): String { + return ThreadPool.Names.MANAGEMENT + } + + override fun clusterChanged(event: ClusterChangedEvent) { + // Instead of using a LocalNodeClusterManagerListener to track clustermanager changes, this service will + // track them here to avoid conditions where clustermanager listener events run after other + // listeners that depend on what happened in the clustermanager listener + if (this.isClusterManager != event.localNodeClusterManager()) { + this.isClusterManager = event.localNodeClusterManager() + if (this.isClusterManager) { + onClusterManager() + } else { + offClusterManager() + } + } + + // if the indexes have been deleted they need to be reinitialized + alertV2IndexInitialized = event.state().routingTable().hasIndex(ALERT_V2_INDEX) + alertV2HistoryIndexInitialized = event.state().metadata().hasAlias(ALERT_V2_HISTORY_WRITE_INDEX) + } + + private fun rescheduleAlertRollover() { + if (clusterService.state().nodes.isLocalNodeElectedClusterManager) { + scheduledAlertV2Rollover?.cancel() + scheduledAlertV2Rollover = threadPool + .scheduleWithFixedDelay({ rolloverAndDeleteAlertV2HistoryIndices() }, alertV2HistoryRolloverPeriod, executorName()) + } + } + + fun isAlertV2Initialized(): Boolean { + return alertV2IndexInitialized && alertV2HistoryIndexInitialized + } + + fun isAlertV2HistoryEnabled(): Boolean { + return alertV2HistoryEnabled + } + + suspend fun createOrUpdateAlertV2Index() { + if (!alertV2IndexInitialized) { + alertV2IndexInitialized = createIndex(ALERT_V2_INDEX, alertV2Mapping()) + if (alertV2IndexInitialized) IndexUtils.alertIndexUpdated() + } else { + if (!IndexUtils.alertIndexUpdated) updateIndexMapping(ALERT_V2_INDEX, alertV2Mapping()) + } + alertV2IndexInitialized + } + + suspend fun createOrUpdateInitialAlertV2HistoryIndex() { + if (!alertV2HistoryIndexInitialized) { + alertV2HistoryIndexInitialized = createIndex(ALERT_V2_HISTORY_INDEX_PATTERN, alertV2Mapping(), ALERT_V2_HISTORY_WRITE_INDEX) + if (alertV2HistoryIndexInitialized) + IndexUtils.lastUpdatedAlertV2HistoryIndex = IndexUtils.getIndexNameWithAlias( + clusterService.state(), + ALERT_V2_HISTORY_WRITE_INDEX + ) + } else { + updateIndexMapping(ALERT_V2_HISTORY_WRITE_INDEX, alertV2Mapping(), true) + } + alertV2HistoryIndexInitialized // TODO: potentially delete this + } + + private fun rolloverAndDeleteAlertV2HistoryIndices() { + if (alertV2HistoryEnabled) rolloverAlertV2HistoryIndex() + deleteOldIndices("History", ALERT_V2_HISTORY_ALL) + } + + private suspend fun createIndex(index: String, schemaMapping: String, alias: String? = null): Boolean { + // This should be a fast check of local cluster state. Should be exceedingly rare that the local cluster + // state does not contain the index and multiple nodes concurrently try to create the index. + // If it does happen that error is handled we catch the ResourceAlreadyExistsException + val existsResponse: IndicesExistsResponse = client.admin().indices().suspendUntil { + exists(IndicesExistsRequest(index).local(true), it) + } + if (existsResponse.isExists) return true + + logger.debug("index: [$index] schema mappings: [$schemaMapping]") + val request = CreateIndexRequest(index) + .mapping(schemaMapping) + .settings(Settings.builder().put("index.hidden", true).build()) + + if (alias != null) request.alias(Alias(alias)) + return try { + val createIndexResponse: CreateIndexResponse = client.admin().indices().suspendUntil { create(request, it) } + createIndexResponse.isAcknowledged + } catch (t: Exception) { + if (ExceptionsHelper.unwrapCause(t) is ResourceAlreadyExistsException) { + true + } else { + throw AlertingException.wrap(t) + } + } + } + + private suspend fun updateIndexMapping(index: String, mapping: String, alias: Boolean = false) { + val clusterState = clusterService.state() + var targetIndex = index + if (alias) { + targetIndex = IndexUtils.getIndexNameWithAlias(clusterState, index) + } + + // TODO call getMapping and compare actual mappings here instead of this + if (targetIndex == IndexUtils.lastUpdatedAlertV2HistoryIndex) { + return + } + + val putMappingRequest: PutMappingRequest = PutMappingRequest(targetIndex) + .source(mapping, XContentType.JSON) + val updateResponse: AcknowledgedResponse = client.admin().indices().suspendUntil { putMapping(putMappingRequest, it) } + if (updateResponse.isAcknowledged) { + logger.info("Index mapping of $targetIndex is updated") + setIndexUpdateFlag(index, targetIndex) + } else { + logger.info("Failed to update index mapping of $targetIndex") + } + } + + private fun setIndexUpdateFlag(index: String, targetIndex: String) { + when (index) { + ALERT_V2_INDEX -> IndexUtils.alertV2IndexUpdated() + ALERT_V2_HISTORY_WRITE_INDEX -> IndexUtils.lastUpdatedAlertV2HistoryIndex = targetIndex + } + } + + private fun rolloverAndDeleteAlertHistoryIndices() { + if (alertV2HistoryEnabled) rolloverAlertV2HistoryIndex() + deleteOldIndices("History", ALERT_V2_HISTORY_ALL) + } + + private fun rolloverIndex( + initialized: Boolean, + index: String, + pattern: String, + map: String, + docsCondition: Long, + ageCondition: TimeValue, + writeIndex: String + ) { + if (!initialized) { + return + } + + // We have to pass null for newIndexName in order to get Elastic to increment the index count. + val request = RolloverRequest(index, null) + request.createIndexRequest.index(pattern) + .mapping(map) + .settings(Settings.builder().put("index.hidden", true).build()) + request.addMaxIndexDocsCondition(docsCondition) + request.addMaxIndexAgeCondition(ageCondition) + client.admin().indices().rolloverIndex( + request, + object : ActionListener { + override fun onResponse(response: RolloverResponse) { + if (!response.isRolledOver) { + logger.info("$writeIndex not rolled over. Conditions were: ${response.conditionStatus}") + } else { + lastRolloverTime = TimeValue.timeValueMillis(threadPool.absoluteTimeInMillis()) + } + } + override fun onFailure(e: Exception) { + logger.error("$writeIndex not roll over failed.") + } + } + ) + } + + private fun rolloverAlertV2HistoryIndex() { + rolloverIndex( + alertV2HistoryIndexInitialized, + ALERT_V2_HISTORY_WRITE_INDEX, + ALERT_V2_HISTORY_INDEX_PATTERN, + alertV2Mapping(), + alertV2HistoryMaxDocs, + alertV2HistoryMaxAge, + ALERT_V2_HISTORY_WRITE_INDEX + ) + } + + private fun deleteOldIndices(tag: String, indices: String) { + logger.info("info deleteOldIndices") + val clusterStateRequest = ClusterStateRequest() + .clear() + .indices(indices) + .metadata(true) + .local(true) + .indicesOptions(IndicesOptions.strictExpand()) + client.admin().cluster().state( + clusterStateRequest, + object : ActionListener { + override fun onResponse(clusterStateResponse: ClusterStateResponse) { + if (clusterStateResponse.state.metadata.indices.isNotEmpty()) { + scope.launch { + val indicesToDelete = getIndicesToDelete(clusterStateResponse) + logger.info("Deleting old $tag indices viz $indicesToDelete") + deleteAllOldHistoryIndices(indicesToDelete) + } + } else { + logger.info("No Old $tag Indices to delete") + } + } + override fun onFailure(e: Exception) { + logger.error("Error fetching cluster state") + } + } + ) + } + + private fun getIndicesToDelete(clusterStateResponse: ClusterStateResponse): List { + val indicesToDelete = mutableListOf() + for (entry in clusterStateResponse.state.metadata.indices) { + val indexMetaData = entry.value + getHistoryIndexToDelete( + indexMetaData, + alertV2HistoryRetentionPeriod.millis, + ALERT_V2_HISTORY_WRITE_INDEX, + alertV2HistoryEnabled + )?.let { indicesToDelete.add(it) } + } + return indicesToDelete + } + + private fun getHistoryIndexToDelete( + indexMetadata: IndexMetadata, + retentionPeriodMillis: Long, + writeIndex: String, + historyEnabled: Boolean + ): String? { + val creationTime = indexMetadata.creationDate + if ((Instant.now().toEpochMilli() - creationTime) > retentionPeriodMillis) { + val alias = indexMetadata.aliases.entries.firstOrNull { writeIndex == it.value.alias } + if (alias != null) { + if (historyEnabled) { + // If the index has the write alias and history is enabled, don't delete the index + return null + } else if (writeIndex == ALERT_V2_HISTORY_WRITE_INDEX) { + // Otherwise reset alertHistoryIndexInitialized since index will be deleted + alertV2HistoryIndexInitialized = false + } + } + + return indexMetadata.index.name + } + return null + } + + private fun deleteAllOldHistoryIndices(indicesToDelete: List) { + if (indicesToDelete.isNotEmpty()) { + val deleteIndexRequest = DeleteIndexRequest(*indicesToDelete.toTypedArray()) + client.admin().indices().delete( + deleteIndexRequest, + object : ActionListener { + override fun onResponse(deleteIndicesResponse: AcknowledgedResponse) { + if (!deleteIndicesResponse.isAcknowledged) { + logger.error( + "Could not delete one or more Alerting V2 history indices: $indicesToDelete. Retrying one by one." + ) + deleteOldHistoryIndex(indicesToDelete) + } + } + override fun onFailure(e: Exception) { + logger.error("Delete for Alerting V2 History Indices $indicesToDelete Failed. Retrying one By one.") + deleteOldHistoryIndex(indicesToDelete) + } + } + ) + } + } + + private fun deleteOldHistoryIndex(indicesToDelete: List) { + for (index in indicesToDelete) { + val singleDeleteRequest = DeleteIndexRequest(*indicesToDelete.toTypedArray()) + client.admin().indices().delete( + singleDeleteRequest, + object : ActionListener { + override fun onResponse(acknowledgedResponse: AcknowledgedResponse?) { + if (acknowledgedResponse != null) { + if (!acknowledgedResponse.isAcknowledged) { + logger.error("Could not delete one or more Alerting V2 history indices: $index") + } + } + } + override fun onFailure(e: Exception) { + logger.debug("Exception ${e.message} while deleting the index $index") + } + } + ) + } + } +} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/alertsv2/AlertV2Mover.kt b/alerting/src/main/kotlin/org/opensearch/alerting/alertsv2/AlertV2Mover.kt new file mode 100644 index 000000000..ec72bddd7 --- /dev/null +++ b/alerting/src/main/kotlin/org/opensearch/alerting/alertsv2/AlertV2Mover.kt @@ -0,0 +1,218 @@ +package org.opensearch.alerting.alertsv2 + +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.launch +import org.apache.logging.log4j.LogManager +import org.opensearch.action.bulk.BulkRequest +import org.opensearch.action.bulk.BulkResponse +import org.opensearch.action.delete.DeleteRequest +import org.opensearch.action.index.IndexRequest +import org.opensearch.action.search.SearchRequest +import org.opensearch.action.search.SearchResponse +import org.opensearch.alerting.core.modelv2.AlertV2 +import org.opensearch.alerting.opensearchapi.suspendUntil +import org.opensearch.alerting.settings.AlertingSettings.Companion.ALERT_V2_HISTORY_ENABLED +import org.opensearch.cluster.ClusterChangedEvent +import org.opensearch.cluster.ClusterStateListener +import org.opensearch.cluster.service.ClusterService +import org.opensearch.common.settings.Settings +import org.opensearch.common.unit.TimeValue +import org.opensearch.common.xcontent.LoggingDeprecationHandler +import org.opensearch.common.xcontent.XContentFactory +import org.opensearch.common.xcontent.XContentHelper +import org.opensearch.common.xcontent.XContentType +import org.opensearch.core.common.bytes.BytesReference +import org.opensearch.core.rest.RestStatus +import org.opensearch.core.xcontent.NamedXContentRegistry +import org.opensearch.core.xcontent.ToXContent +import org.opensearch.core.xcontent.XContentParser +import org.opensearch.core.xcontent.XContentParserUtils +import org.opensearch.index.VersionType +import org.opensearch.index.query.QueryBuilders +import org.opensearch.search.builder.SearchSourceBuilder +import org.opensearch.threadpool.Scheduler +import org.opensearch.threadpool.ThreadPool +import org.opensearch.transport.client.Client +import java.time.Instant +import java.util.concurrent.TimeUnit + +private val scope: CoroutineScope = CoroutineScope(Dispatchers.IO) +private val logger = LogManager.getLogger(AlertV2Mover::class.java) + +class AlertV2Mover( + settings: Settings, + private val client: Client, + private val threadPool: ThreadPool, + private val clusterService: ClusterService, +) : ClusterStateListener { + init { + clusterService.addListener(this) + clusterService.clusterSettings.addSettingsUpdateConsumer(ALERT_V2_HISTORY_ENABLED) { alertV2HistoryEnabled = it } + } + + @Volatile private var isClusterManager = false + + private var alertV2IndexInitialized = false + + private var alertV2HistoryIndexInitialized = false + + private var alertV2HistoryEnabled = ALERT_V2_HISTORY_ENABLED.get(settings) + + private var scheduledAlertsV2CheckAndExpire: Scheduler.Cancellable? = null + + private val executorName = ThreadPool.Names.MANAGEMENT + + private val checkForExpirationInterval = TimeValue(1L, TimeUnit.MINUTES) + + override fun clusterChanged(event: ClusterChangedEvent) { + if (this.isClusterManager != event.localNodeClusterManager()) { + this.isClusterManager = event.localNodeClusterManager() + if (this.isClusterManager) { + onManager() + } else { + offManager() + } + } + + alertV2IndexInitialized = event.state().routingTable().hasIndex(AlertV2Indices.ALERT_V2_INDEX) + alertV2HistoryIndexInitialized = event.state().metadata().hasAlias(AlertV2Indices.ALERT_V2_HISTORY_WRITE_INDEX) + } + + fun onManager() { + try { + // try to sweep current AlertV2s for expiration immediately as we might be restarting the cluster + moveOrDeleteAlertV2s() + // schedule expiration checks and expirations to happen repeatedly at some interval + scheduledAlertsV2CheckAndExpire = threadPool + .scheduleWithFixedDelay({ moveOrDeleteAlertV2s() }, checkForExpirationInterval, executorName) + } catch (e: Exception) { + // This should be run on cluster startup + logger.error( + "Error sweeping AlertV2s for expiration. This cannot be done until clustermanager node is restarted.", + e + ) + } + } + + fun offManager() { + scheduledAlertsV2CheckAndExpire?.cancel() + } + + // if alertV2 history is enabled, move expired alerts to alertV2 history indices + // if alertV2 history is disabled, permanently delete expired alerts + private fun moveOrDeleteAlertV2s() { + if (!areAlertV2IndicesPresent()) { + return + } + + scope.launch { + val expiredAlertsSearchResponse = searchForExpiredAlerts() + + var copyResponse: BulkResponse? = null + val deleteResponse: BulkResponse? + if (!alertV2HistoryEnabled) { + deleteResponse = deleteExpiredAlerts(expiredAlertsSearchResponse) + } else { + copyResponse = copyExpiredAlerts(expiredAlertsSearchResponse) + deleteResponse = deleteExpiredAlertsThatWereCopied(copyResponse) + } + checkForFailures(copyResponse) + checkForFailures(deleteResponse) + } + } + + private suspend fun searchForExpiredAlerts(): SearchResponse { + val now = Instant.now().toEpochMilli() + val expiredAlertsQuery = QueryBuilders.rangeQuery(AlertV2.EXPIRATION_TIME_FIELD).lte(now) + + val expiredAlertsSearchQuery = SearchSourceBuilder.searchSource() + .query(expiredAlertsQuery) + .version(true) + + val activeAlertsRequest = SearchRequest(AlertV2Indices.ALERT_V2_INDEX) + .source(expiredAlertsSearchQuery) + val searchResponse: SearchResponse = client.suspendUntil { search(activeAlertsRequest, it) } + return searchResponse + } + + private suspend fun copyExpiredAlerts(expiredAlertsSearchResponse: SearchResponse): BulkResponse? { + // If no alerts are found, simply return + if (expiredAlertsSearchResponse.hits.totalHits?.value == 0L) { + return null + } + + val indexRequests = expiredAlertsSearchResponse.hits.map { hit -> + IndexRequest(AlertV2Indices.ALERT_V2_HISTORY_WRITE_INDEX) + .source( + AlertV2.parse(alertV2ContentParser(hit.sourceRef), hit.id, hit.version) + .toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS) + ) + .version(hit.version) + .versionType(VersionType.EXTERNAL_GTE) + .id(hit.id) + } + + val copyRequest = BulkRequest().add(indexRequests) + val copyResponse: BulkResponse = client.suspendUntil { bulk(copyRequest, it) } + + return copyResponse + } + + private suspend fun deleteExpiredAlerts(expiredAlertsSearchResponse: SearchResponse): BulkResponse { + val deleteRequests = expiredAlertsSearchResponse.hits.map { + DeleteRequest(AlertV2Indices.ALERT_V2_INDEX, it.id) + .version(it.version) + .versionType(VersionType.EXTERNAL_GTE) + } + + val deleteRequest = BulkRequest().add(deleteRequests) + val deleteResponse: BulkResponse = client.suspendUntil { bulk(deleteRequest, it) } + + return deleteResponse + } + + private suspend fun deleteExpiredAlertsThatWereCopied(copyResponse: BulkResponse?): BulkResponse? { + // if there were no expired alerts to copy, skip deleting anything + if (copyResponse == null) { + return null + } + + val deleteRequests = copyResponse.items.filterNot { it.isFailed }.map { + DeleteRequest(AlertV2Indices.ALERT_V2_INDEX, it.id) + .version(it.version) + .versionType(VersionType.EXTERNAL_GTE) + } + val deleteResponse: BulkResponse = client.suspendUntil { bulk(BulkRequest().add(deleteRequests), it) } + + return deleteResponse + } + + private fun checkForFailures(bulkResponse: BulkResponse?) { + bulkResponse?.let { + if (bulkResponse.hasFailures()) { + val retryCause = bulkResponse.items.filter { it.isFailed } + .firstOrNull { it.status() == RestStatus.TOO_MANY_REQUESTS } + ?.failure?.cause + throw RuntimeException( + "Failed to move or delete alert v2s: " + + bulkResponse.buildFailureMessage(), + retryCause + ) + } + } + } + + private fun alertV2ContentParser(bytesReference: BytesReference): XContentParser { + val xcp = XContentHelper.createParser( + NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, + bytesReference, XContentType.JSON + ) + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.nextToken(), xcp) + return xcp + } + + private fun areAlertV2IndicesPresent(): Boolean { + return alertV2IndexInitialized && alertV2HistoryIndexInitialized + } +} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/resthandler/RestDeleteMonitorV2Action.kt b/alerting/src/main/kotlin/org/opensearch/alerting/resthandler/RestDeleteMonitorV2Action.kt new file mode 100644 index 000000000..76b3c656e --- /dev/null +++ b/alerting/src/main/kotlin/org/opensearch/alerting/resthandler/RestDeleteMonitorV2Action.kt @@ -0,0 +1,47 @@ +package org.opensearch.alerting.resthandler + +import org.apache.logging.log4j.LogManager +import org.apache.logging.log4j.Logger +import org.opensearch.action.support.WriteRequest.RefreshPolicy +import org.opensearch.alerting.AlertingPlugin +import org.opensearch.alerting.actionv2.DeleteMonitorV2Action +import org.opensearch.alerting.actionv2.DeleteMonitorV2Request +import org.opensearch.alerting.util.REFRESH +import org.opensearch.rest.BaseRestHandler +import org.opensearch.rest.RestHandler.Route +import org.opensearch.rest.RestRequest +import org.opensearch.rest.RestRequest.Method.DELETE +import org.opensearch.rest.action.RestToXContentListener +import org.opensearch.transport.client.node.NodeClient +import java.io.IOException + +private val log: Logger = LogManager.getLogger(RestDeleteMonitorAction::class.java) + +class RestDeleteMonitorV2Action : BaseRestHandler() { + + override fun getName(): String { + return "delete_monitor_v2_action" + } + + override fun routes(): List { + return mutableListOf( + Route( + DELETE, + "${AlertingPlugin.MONITOR_V2_BASE_URI}/{monitorV2Id}" + ) + ) + } + + @Throws(IOException::class) + override fun prepareRequest(request: RestRequest, client: NodeClient): RestChannelConsumer { + val monitorV2Id = request.param("monitorV2Id") + log.info("${request.method()} ${AlertingPlugin.MONITOR_V2_BASE_URI}/$monitorV2Id") + + val refreshPolicy = RefreshPolicy.parse(request.param(REFRESH, RefreshPolicy.IMMEDIATE.value)) + val deleteMonitorV2Request = DeleteMonitorV2Request(monitorV2Id, refreshPolicy) + + return RestChannelConsumer { channel -> + client.execute(DeleteMonitorV2Action.INSTANCE, deleteMonitorV2Request, RestToXContentListener(channel)) + } + } +} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/resthandler/RestExecuteMonitorV2Action.kt b/alerting/src/main/kotlin/org/opensearch/alerting/resthandler/RestExecuteMonitorV2Action.kt new file mode 100644 index 000000000..058cd7a1f --- /dev/null +++ b/alerting/src/main/kotlin/org/opensearch/alerting/resthandler/RestExecuteMonitorV2Action.kt @@ -0,0 +1,70 @@ +package org.opensearch.alerting.resthandler + +import org.apache.logging.log4j.LogManager +import org.opensearch.alerting.AlertingPlugin +import org.opensearch.alerting.actionv2.ExecuteMonitorV2Action +import org.opensearch.alerting.actionv2.ExecuteMonitorV2Request +import org.opensearch.alerting.core.modelv2.MonitorV2 +import org.opensearch.common.unit.TimeValue +import org.opensearch.commons.alerting.util.AlertingException +import org.opensearch.core.xcontent.XContentParser.Token.START_OBJECT +import org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken +import org.opensearch.rest.BaseRestHandler +import org.opensearch.rest.RestHandler.Route +import org.opensearch.rest.RestRequest +import org.opensearch.rest.RestRequest.Method.POST +import org.opensearch.rest.action.RestToXContentListener +import org.opensearch.transport.client.node.NodeClient +import java.time.Instant + +private val log = LogManager.getLogger(RestExecuteMonitorV2Action::class.java) + +class RestExecuteMonitorV2Action : BaseRestHandler() { + + override fun getName(): String = "execute_monitor_v2_action" + + override fun routes(): List { + return listOf( + Route( + POST, + "${AlertingPlugin.MONITOR_V2_BASE_URI}/{monitorV2Id}/_execute" + ), + Route( + POST, + "${AlertingPlugin.MONITOR_V2_BASE_URI}/_execute" + ) + ) + } + + override fun prepareRequest(request: RestRequest, client: NodeClient): RestChannelConsumer { + log.debug("${request.method()} ${AlertingPlugin.MONITOR_V2_BASE_URI}/_execute") + + return RestChannelConsumer { channel -> + val dryrun = request.paramAsBoolean("dryrun", false) + val requestEnd = request.paramAsTime("period_end", TimeValue(Instant.now().toEpochMilli())) + + if (request.hasParam("monitorV2Id")) { + val monitorV2Id = request.param("monitorV2Id") + val execMonitorV2Request = ExecuteMonitorV2Request(dryrun, true, monitorV2Id, null, null, requestEnd) + client.execute(ExecuteMonitorV2Action.INSTANCE, execMonitorV2Request, RestToXContentListener(channel)) + } else { + val xcp = request.contentParser() + ensureExpectedToken(START_OBJECT, xcp.nextToken(), xcp) + + val monitorV2: MonitorV2 + try { + monitorV2 = MonitorV2.parse(xcp) + } catch (e: Exception) { + throw AlertingException.wrap(e) + } + + val execMonitorV2Request = ExecuteMonitorV2Request(dryrun, true, null, monitorV2, null, requestEnd) + client.execute(ExecuteMonitorV2Action.INSTANCE, execMonitorV2Request, RestToXContentListener(channel)) + } + } + } + + override fun responseParams(): Set { + return setOf("dryrun", "period_end", "monitorV2Id") + } +} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/resthandler/RestGetAlertsV2Action.kt b/alerting/src/main/kotlin/org/opensearch/alerting/resthandler/RestGetAlertsV2Action.kt new file mode 100644 index 000000000..246ac46c2 --- /dev/null +++ b/alerting/src/main/kotlin/org/opensearch/alerting/resthandler/RestGetAlertsV2Action.kt @@ -0,0 +1,63 @@ +package org.opensearch.alerting.resthandler + +import org.apache.logging.log4j.LogManager +import org.opensearch.alerting.AlertingPlugin +import org.opensearch.alerting.actionv2.GetAlertsV2Action +import org.opensearch.alerting.actionv2.GetAlertsV2Request +import org.opensearch.commons.alerting.model.Table +import org.opensearch.rest.BaseRestHandler +import org.opensearch.rest.RestHandler.Route +import org.opensearch.rest.RestRequest +import org.opensearch.rest.RestRequest.Method.GET +import org.opensearch.rest.action.RestToXContentListener +import org.opensearch.transport.client.node.NodeClient + +/** + * This class consists of the REST handler to retrieve alerts . + */ +class RestGetAlertsV2Action : BaseRestHandler() { + + private val log = LogManager.getLogger(RestGetAlertsV2Action::class.java) + + override fun getName(): String { + return "get_alerts_v2_action" + } + + override fun routes(): List { + return listOf( + Route( + GET, + "${AlertingPlugin.MONITOR_V2_BASE_URI}/alerts" + ) + ) + } + + // TODO: this is an Get Alerts V2 rest handler that points to the Get Alerts V1 Transport action + // TODO: for now for playground, separate the 2 for GA + override fun prepareRequest(request: RestRequest, client: NodeClient): RestChannelConsumer { + log.debug("${request.method()} ${AlertingPlugin.MONITOR_V2_BASE_URI}/alerts") + + val sortString = request.param("sortString", "monitor_name.keyword") + val sortOrder = request.param("sortOrder", "asc") + val missing: String? = request.param("missing") + val size = request.paramAsInt("size", 20) + val startIndex = request.paramAsInt("startIndex", 0) + val searchString = request.param("searchString", "") + val severityLevel = request.param("severityLevel", "ALL") + val monitorId: String? = request.param("monitorId") + val table = Table( + sortOrder, + sortString, + missing, + size, + startIndex, + searchString + ) + + val getAlertsV2Request = GetAlertsV2Request(table, severityLevel, monitorId, null) + return RestChannelConsumer { + channel -> + client.execute(GetAlertsV2Action.INSTANCE, getAlertsV2Request, RestToXContentListener(channel)) + } + } +} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/resthandler/RestGetMonitorV2Action.kt b/alerting/src/main/kotlin/org/opensearch/alerting/resthandler/RestGetMonitorV2Action.kt new file mode 100644 index 000000000..d053e42f0 --- /dev/null +++ b/alerting/src/main/kotlin/org/opensearch/alerting/resthandler/RestGetMonitorV2Action.kt @@ -0,0 +1,54 @@ +package org.opensearch.alerting.resthandler + +import org.apache.logging.log4j.LogManager +import org.opensearch.alerting.AlertingPlugin +import org.opensearch.alerting.actionv2.GetMonitorV2Action +import org.opensearch.alerting.actionv2.GetMonitorV2Request +import org.opensearch.alerting.util.context +import org.opensearch.rest.BaseRestHandler +import org.opensearch.rest.RestHandler.Route +import org.opensearch.rest.RestRequest +import org.opensearch.rest.RestRequest.Method.GET +import org.opensearch.rest.RestRequest.Method.HEAD +import org.opensearch.rest.action.RestActions +import org.opensearch.rest.action.RestToXContentListener +import org.opensearch.search.fetch.subphase.FetchSourceContext +import org.opensearch.transport.client.node.NodeClient + +private val log = LogManager.getLogger(RestGetMonitorV2Action::class.java) + +class RestGetMonitorV2Action : BaseRestHandler() { + + override fun getName(): String { + return "get_monitor_v2_action" + } + + override fun routes(): List { + return listOf( + Route( + GET, + "${AlertingPlugin.MONITOR_V2_BASE_URI}/{monitorV2Id}" + ) + ) + } + + override fun prepareRequest(request: RestRequest, client: NodeClient): RestChannelConsumer { + log.debug("${request.method()} ${AlertingPlugin.MONITOR_V2_BASE_URI}/{monitorV2Id}") + + val monitorV2Id = request.param("monitorV2Id") + if (monitorV2Id == null || monitorV2Id.isEmpty()) { + throw IllegalArgumentException("No MonitorV2 ID provided") + } + + var srcContext = context(request) + if (request.method() == HEAD) { + srcContext = FetchSourceContext.DO_NOT_FETCH_SOURCE + } + + val getMonitorV2Request = GetMonitorV2Request(monitorV2Id, RestActions.parseVersion(request), srcContext) + return RestChannelConsumer { + channel -> + client.execute(GetMonitorV2Action.INSTANCE, getMonitorV2Request, RestToXContentListener(channel)) + } + } +} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/resthandler/RestIndexMonitorV2Action.kt b/alerting/src/main/kotlin/org/opensearch/alerting/resthandler/RestIndexMonitorV2Action.kt new file mode 100644 index 000000000..0544a8b22 --- /dev/null +++ b/alerting/src/main/kotlin/org/opensearch/alerting/resthandler/RestIndexMonitorV2Action.kt @@ -0,0 +1,79 @@ +package org.opensearch.alerting.resthandler + +import org.apache.logging.log4j.LogManager +import org.opensearch.action.support.WriteRequest +import org.opensearch.alerting.AlertingPlugin +import org.opensearch.alerting.actionv2.IndexMonitorV2Action +import org.opensearch.alerting.actionv2.IndexMonitorV2Request +import org.opensearch.alerting.core.modelv2.MonitorV2 +import org.opensearch.alerting.util.IF_PRIMARY_TERM +import org.opensearch.alerting.util.IF_SEQ_NO +import org.opensearch.alerting.util.REFRESH +import org.opensearch.commons.alerting.util.AlertingException +import org.opensearch.core.xcontent.XContentParser.Token +import org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken +import org.opensearch.index.seqno.SequenceNumbers +import org.opensearch.rest.BaseRestHandler +import org.opensearch.rest.RestHandler.Route +import org.opensearch.rest.RestRequest +import org.opensearch.rest.RestRequest.Method.POST +import org.opensearch.rest.RestRequest.Method.PUT +import org.opensearch.rest.action.RestToXContentListener +import org.opensearch.transport.client.node.NodeClient +import java.io.IOException + +private val log = LogManager.getLogger(RestIndexMonitorV2Action::class.java) + +/** + * Rest handlers to create and update V2 Monitors like PPL Monitors + */ +class RestIndexMonitorV2Action : BaseRestHandler() { + override fun getName(): String { + return "index_monitor_v2_action" + } + + override fun routes(): List { + return listOf( + Route( + POST, + AlertingPlugin.MONITOR_V2_BASE_URI + ), + Route( + PUT, + "${AlertingPlugin.MONITOR_V2_BASE_URI}/{monitorV2Id}" + ) + ) + } + + @Throws(IOException::class) + override fun prepareRequest(request: RestRequest, client: NodeClient): RestChannelConsumer { + log.debug("${request.method()} ${request.path()}") + + val xcp = request.contentParser() + ensureExpectedToken(Token.START_OBJECT, xcp.nextToken(), xcp) + + val monitorV2: MonitorV2 + val rbacRoles: List? // TODO: do we want to support specifying rbac roles in monitor body? + try { + monitorV2 = MonitorV2.parse(xcp) + rbacRoles = request.contentParser().map()["rbac_roles"] as List? + } catch (e: Exception) { + throw AlertingException.wrap(e) + } + + val id = request.param("monitorV2Id", MonitorV2.NO_ID) + val seqNo = request.paramAsLong(IF_SEQ_NO, SequenceNumbers.UNASSIGNED_SEQ_NO) + val primaryTerm = request.paramAsLong(IF_PRIMARY_TERM, SequenceNumbers.UNASSIGNED_PRIMARY_TERM) + val refreshPolicy = if (request.hasParam(REFRESH)) { + WriteRequest.RefreshPolicy.parse(request.param(REFRESH)) + } else { + WriteRequest.RefreshPolicy.IMMEDIATE + } + + val indexMonitorV2Request = IndexMonitorV2Request(id, seqNo, primaryTerm, refreshPolicy, request.method(), monitorV2, rbacRoles) + + return RestChannelConsumer { channel -> + client.execute(IndexMonitorV2Action.INSTANCE, indexMonitorV2Request, RestToXContentListener(channel)) + } + } +} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/resthandler/RestSearchMonitorV2Action.kt b/alerting/src/main/kotlin/org/opensearch/alerting/resthandler/RestSearchMonitorV2Action.kt new file mode 100644 index 000000000..b15e5b95d --- /dev/null +++ b/alerting/src/main/kotlin/org/opensearch/alerting/resthandler/RestSearchMonitorV2Action.kt @@ -0,0 +1,118 @@ +package org.opensearch.alerting.resthandler + +import org.apache.logging.log4j.LogManager +import org.opensearch.action.search.SearchRequest +import org.opensearch.action.search.SearchResponse +import org.opensearch.alerting.AlertingPlugin +import org.opensearch.alerting.actionv2.SearchMonitorV2Action +import org.opensearch.alerting.actionv2.SearchMonitorV2Request +import org.opensearch.alerting.settings.AlertingSettings +import org.opensearch.alerting.util.context +import org.opensearch.cluster.service.ClusterService +import org.opensearch.common.settings.Settings +import org.opensearch.common.xcontent.LoggingDeprecationHandler +import org.opensearch.common.xcontent.XContentFactory.jsonBuilder +import org.opensearch.common.xcontent.XContentType +import org.opensearch.commons.alerting.model.ScheduledJob +import org.opensearch.commons.alerting.model.ScheduledJob.Companion.SCHEDULED_JOBS_INDEX +import org.opensearch.core.common.bytes.BytesReference +import org.opensearch.core.rest.RestStatus +import org.opensearch.core.xcontent.ToXContent.EMPTY_PARAMS +import org.opensearch.rest.BaseRestHandler +import org.opensearch.rest.BytesRestResponse +import org.opensearch.rest.RestChannel +import org.opensearch.rest.RestHandler.Route +import org.opensearch.rest.RestRequest +import org.opensearch.rest.RestRequest.Method.GET +import org.opensearch.rest.RestRequest.Method.POST +import org.opensearch.rest.RestResponse +import org.opensearch.rest.action.RestResponseListener +import org.opensearch.search.builder.SearchSourceBuilder +import org.opensearch.transport.client.node.NodeClient +import java.io.IOException + +private val log = LogManager.getLogger(RestSearchMonitorV2Action::class.java) + +class RestSearchMonitorV2Action( + val settings: Settings, + clusterService: ClusterService, +) : BaseRestHandler() { + + @Volatile private var filterBy = AlertingSettings.FILTER_BY_BACKEND_ROLES.get(settings) + + init { + clusterService.clusterSettings.addSettingsUpdateConsumer(AlertingSettings.FILTER_BY_BACKEND_ROLES) { filterBy = it } + } + + override fun getName(): String { + return "search_monitor_v2_action" + } + + override fun routes(): List { + return listOf( + Route( + POST, + "${AlertingPlugin.MONITOR_V2_BASE_URI}/_search" + ), + Route( + GET, + "${AlertingPlugin.MONITOR_V2_BASE_URI}/_search" + ) + ) + } + + @Throws(IOException::class) + override fun prepareRequest(request: RestRequest, client: NodeClient): RestChannelConsumer { + log.debug("${request.method()} ${AlertingPlugin.MONITOR_V2_BASE_URI}/_search") + + val searchSourceBuilder = SearchSourceBuilder() + searchSourceBuilder.parseXContent(request.contentOrSourceParamParser()) + searchSourceBuilder.fetchSource(context(request)) + + val searchRequest = SearchRequest() + .source(searchSourceBuilder) + .indices(SCHEDULED_JOBS_INDEX) + + val searchMonitorV2Request = SearchMonitorV2Request(searchRequest) + return RestChannelConsumer { channel -> + client.execute(SearchMonitorV2Action.INSTANCE, searchMonitorV2Request, searchMonitorResponse(channel)) + } + } + + // once the search response is received, rewrite the search hits to remove the extra "monitor_v2" JSON object wrapper + // that is used as ScheduledJob metadata + private fun searchMonitorResponse(channel: RestChannel): RestResponseListener { + return object : RestResponseListener(channel) { + @Throws(Exception::class) + override fun buildResponse(response: SearchResponse): RestResponse { + if (response.isTimedOut) { + return BytesRestResponse(RestStatus.REQUEST_TIMEOUT, response.toString()) + } + + try { + for (hit in response.hits) { + XContentType.JSON.xContent().createParser( + channel.request().xContentRegistry, + LoggingDeprecationHandler.INSTANCE, hit.sourceAsString + ).use { hitsParser -> + // when reconstructing XContent, intentionally leave out + // user field in response for security reasons by + // calling ScheduledJob.toXContent instead of + // a MonitorV2's toXContentWithUser + val monitorV2 = ScheduledJob.parse(hitsParser, hit.id, hit.version) + val xcb = monitorV2.toXContent(jsonBuilder(), EMPTY_PARAMS) + + // rewrite the search hit as just the MonitorV2 source, + // without the extra "monitor_v2" JSON object wrapper + hit.sourceRef(BytesReference.bytes(xcb)) + } + } + } catch (e: Exception) { + // Swallow exception and return response as is + log.error("The monitor_v2 parsing failed. Will return response as is.") + } + return BytesRestResponse(RestStatus.OK, response.toXContent(channel.newBuilder(), EMPTY_PARAMS)) + } + } + } +} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/script/PPLTriggerExecutionContext.kt b/alerting/src/main/kotlin/org/opensearch/alerting/script/PPLTriggerExecutionContext.kt new file mode 100644 index 000000000..23f746ace --- /dev/null +++ b/alerting/src/main/kotlin/org/opensearch/alerting/script/PPLTriggerExecutionContext.kt @@ -0,0 +1,25 @@ +package org.opensearch.alerting.script + +import org.json.JSONObject +import org.opensearch.alerting.core.modelv2.PPLMonitor +import org.opensearch.alerting.core.modelv2.PPLMonitorRunResult.Companion.PPL_QUERY_RESULTS_FIELD +import org.opensearch.alerting.core.modelv2.PPLTrigger +import org.opensearch.alerting.core.modelv2.PPLTrigger.Companion.PPL_TRIGGER_FIELD +import java.time.Instant + +data class PPLTriggerExecutionContext( + override val monitorV2: PPLMonitor, + override val periodStart: Instant, + override val periodEnd: Instant, + override val error: Exception? = null, + val pplTrigger: PPLTrigger, + var pplQueryResults: JSONObject // can be a full set of PPL query results, or an individual result row +) : TriggerV2ExecutionContext(monitorV2, periodStart, periodEnd, error) { + + override fun asTemplateArg(): Map { + val templateArg = super.asTemplateArg().toMutableMap() + templateArg[PPL_TRIGGER_FIELD] = pplTrigger.asTemplateArg() + templateArg[PPL_QUERY_RESULTS_FIELD] = pplQueryResults.toMap() + return templateArg.toMap() + } +} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/script/TriggerV2ExecutionContext.kt b/alerting/src/main/kotlin/org/opensearch/alerting/script/TriggerV2ExecutionContext.kt new file mode 100644 index 000000000..ebbbbfc2d --- /dev/null +++ b/alerting/src/main/kotlin/org/opensearch/alerting/script/TriggerV2ExecutionContext.kt @@ -0,0 +1,31 @@ +package org.opensearch.alerting.script + +import org.opensearch.alerting.core.modelv2.MonitorV2 +import org.opensearch.alerting.core.modelv2.MonitorV2RunResult +import org.opensearch.alerting.core.modelv2.TriggerV2 +import java.time.Instant + +abstract class TriggerV2ExecutionContext( + open val monitorV2: MonitorV2, + open val periodStart: Instant, + open val periodEnd: Instant, + open val error: Exception? = null +) { + + constructor(monitorV2: MonitorV2, triggerV2: TriggerV2, monitorV2RunResult: MonitorV2RunResult<*>) : + this( + monitorV2, + monitorV2RunResult.periodStart, + monitorV2RunResult.periodEnd, + monitorV2RunResult.triggerResults[triggerV2.id]?.error + ) + + open fun asTemplateArg(): Map { + return mapOf( + "monitorV2" to monitorV2.asTemplateArg(), + "periodStart" to periodStart, + "periodEnd" to periodEnd, + "error" to error + ) + } +} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/service/DeleteMonitorService.kt b/alerting/src/main/kotlin/org/opensearch/alerting/service/DeleteMonitorService.kt index fbc655543..c1a56a4c6 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/service/DeleteMonitorService.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/service/DeleteMonitorService.kt @@ -22,6 +22,7 @@ import org.opensearch.action.support.IndicesOptions import org.opensearch.action.support.WriteRequest.RefreshPolicy import org.opensearch.action.support.clustermanager.AcknowledgedResponse import org.opensearch.alerting.MonitorMetadataService +import org.opensearch.alerting.actionv2.DeleteMonitorV2Response import org.opensearch.alerting.core.lock.LockModel import org.opensearch.alerting.core.lock.LockService import org.opensearch.alerting.opensearchapi.suspendUntil @@ -74,6 +75,19 @@ object DeleteMonitorService : return DeleteMonitorResponse(deleteResponse.id, deleteResponse.version) } + /** + * Deletes the monitorV2, which does not come with other metadata and queries + * like doc level monitors + * @param monitorV2Id monitorV2 ID to be deleted + * @param refreshPolicy + */ + suspend fun deleteMonitorV2(monitorV2Id: String, refreshPolicy: RefreshPolicy): DeleteMonitorV2Response { + val deleteResponse = deleteMonitor(monitorV2Id, refreshPolicy) + deleteLock(monitorV2Id) + return DeleteMonitorV2Response(deleteResponse.id, deleteResponse.version) + } + + // both Alerting v1 and v2 workflows flow through this function private suspend fun deleteMonitor(monitorId: String, refreshPolicy: RefreshPolicy): DeleteResponse { val deleteMonitorRequest = DeleteRequest(ScheduledJob.SCHEDULED_JOBS_INDEX, monitorId) .setRefreshPolicy(refreshPolicy) @@ -167,7 +181,12 @@ object DeleteMonitorService : } private suspend fun deleteLock(monitor: Monitor) { - client.suspendUntil { lockService.deleteLock(LockModel.generateLockId(monitor.id), it) } + deleteLock(monitor.id) + } + + // both Alerting v1 and v2 workflows flow through this function + private suspend fun deleteLock(monitorId: String) { + client.suspendUntil { lockService.deleteLock(LockModel.generateLockId(monitorId), it) } } /** diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/settings/AlertingSettings.kt b/alerting/src/main/kotlin/org/opensearch/alerting/settings/AlertingSettings.kt index 5a50ce632..21dfba936 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/settings/AlertingSettings.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/settings/AlertingSettings.kt @@ -293,5 +293,49 @@ class AlertingSettings { 0, Setting.Property.NodeScope, Setting.Property.Dynamic ) + + val ALERT_V2_HISTORY_ENABLED = Setting.boolSetting( + "plugins.alerting_v2.alert_history_enabled", + true, + Setting.Property.NodeScope, Setting.Property.Dynamic + ) + + val ALERT_V2_HISTORY_ROLLOVER_PERIOD = Setting.positiveTimeSetting( + "plugins.alerting_v2.alert_history_rollover_period", + TimeValue(12, TimeUnit.HOURS), + Setting.Property.NodeScope, Setting.Property.Dynamic + ) + + val ALERT_V2_HISTORY_INDEX_MAX_AGE = Setting.positiveTimeSetting( + "plugins.alerting_v2.alert_history_max_age", + TimeValue(30, TimeUnit.DAYS), + Setting.Property.NodeScope, Setting.Property.Dynamic + ) + + val ALERT_V2_HISTORY_MAX_DOCS = Setting.longSetting( + "plugins.alerting_v2.alert_history_max_docs", + 1000L, 0L, + Setting.Property.NodeScope, Setting.Property.Dynamic + ) + + val ALERT_V2_HISTORY_RETENTION_PERIOD = Setting.positiveTimeSetting( + "plugins.alerting_v2.alert_history_retention_period", + TimeValue(60, TimeUnit.DAYS), + Setting.Property.NodeScope, Setting.Property.Dynamic + ) + + val ALERT_V2_NOTIF_QUERY_RESULTS_MAX_SIZE = Setting.longSetting( + "plugins.alerting_v2.notif_query_results_max_size", + 3000L, + 0L, + Setting.Property.NodeScope, Setting.Property.Dynamic + ) + + val ALERT_V2_PER_RESULT_TRIGGER_MAX_ALERTS = Setting.intSetting( + "plugins.alerting_v2.per_result_trigger_max_alerts", + 10, + 1, + Setting.Property.NodeScope, Setting.Property.Dynamic + ) } } diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/transport/SecureTransportAction.kt b/alerting/src/main/kotlin/org/opensearch/alerting/transport/SecureTransportAction.kt index 54667e125..627181188 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/transport/SecureTransportAction.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/transport/SecureTransportAction.kt @@ -45,7 +45,7 @@ interface SecureTransportAction { fun readUserFromThreadContext(client: Client): User? { val userStr = client.threadPool().threadContext.getTransient(ConfigConstants.OPENSEARCH_SECURITY_USER_INFO_THREAD_CONTEXT) - log.debug("User and roles string from thread context: $userStr") + log.info("User and roles string from thread context: $userStr") return User.parse(userStr) } diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportDeleteMonitorAction.kt b/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportDeleteMonitorAction.kt index b28311bd0..39b96f2b0 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportDeleteMonitorAction.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportDeleteMonitorAction.kt @@ -16,6 +16,7 @@ import org.opensearch.action.get.GetResponse import org.opensearch.action.support.ActionFilters import org.opensearch.action.support.HandledTransportAction import org.opensearch.action.support.WriteRequest.RefreshPolicy +import org.opensearch.alerting.AlertingV2Utils.validateMonitorV1 import org.opensearch.alerting.opensearchapi.suspendUntil import org.opensearch.alerting.service.DeleteMonitorService import org.opensearch.alerting.settings.AlertingSettings @@ -87,7 +88,7 @@ class TransportDeleteMonitorAction @Inject constructor( ) { suspend fun resolveUserAndStart(refreshPolicy: RefreshPolicy) { try { - val monitor = getMonitor() + val monitor = getMonitor() ?: return // null means there was an issue retrieving the Monitor val canDelete = user == null || !doFilterForUser(user) || checkUserPermissionsWithResource(user, monitor.user, actionListener, "monitor", monitorId) @@ -115,11 +116,11 @@ class TransportDeleteMonitorAction @Inject constructor( } } - private suspend fun getMonitor(): Monitor { + private suspend fun getMonitor(): Monitor? { val getRequest = GetRequest(ScheduledJob.SCHEDULED_JOBS_INDEX, monitorId) val getResponse: GetResponse = client.suspendUntil { get(getRequest, it) } - if (getResponse.isExists == false) { + if (!getResponse.isExists) { actionListener.onFailure( AlertingException.wrap( OpenSearchStatusException("Monitor with $monitorId is not found", RestStatus.NOT_FOUND) @@ -130,7 +131,16 @@ class TransportDeleteMonitorAction @Inject constructor( xContentRegistry, LoggingDeprecationHandler.INSTANCE, getResponse.sourceAsBytesRef, XContentType.JSON ) - return ScheduledJob.parse(xcp, getResponse.id, getResponse.version) as Monitor + val scheduledJob = ScheduledJob.parse(xcp, getResponse.id, getResponse.version) + + validateMonitorV1(scheduledJob)?.let { + actionListener.onFailure(AlertingException.wrap(it)) + return null + } + + val monitor = scheduledJob as Monitor + + return monitor } } } diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportDeleteMonitorV2Action.kt b/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportDeleteMonitorV2Action.kt new file mode 100644 index 000000000..072f92cb6 --- /dev/null +++ b/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportDeleteMonitorV2Action.kt @@ -0,0 +1,114 @@ +package org.opensearch.alerting.transport + +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.launch +import org.apache.logging.log4j.LogManager +import org.opensearch.OpenSearchStatusException +import org.opensearch.action.get.GetRequest +import org.opensearch.action.get.GetResponse +import org.opensearch.action.support.ActionFilters +import org.opensearch.action.support.HandledTransportAction +import org.opensearch.alerting.AlertingV2Utils.validateMonitorV2 +import org.opensearch.alerting.actionv2.DeleteMonitorV2Action +import org.opensearch.alerting.actionv2.DeleteMonitorV2Request +import org.opensearch.alerting.actionv2.DeleteMonitorV2Response +import org.opensearch.alerting.core.modelv2.MonitorV2 +import org.opensearch.alerting.opensearchapi.suspendUntil +import org.opensearch.alerting.service.DeleteMonitorService +import org.opensearch.alerting.settings.AlertingSettings +import org.opensearch.cluster.service.ClusterService +import org.opensearch.common.inject.Inject +import org.opensearch.common.settings.Settings +import org.opensearch.common.xcontent.LoggingDeprecationHandler +import org.opensearch.common.xcontent.XContentHelper +import org.opensearch.common.xcontent.XContentType +import org.opensearch.commons.alerting.model.ScheduledJob +import org.opensearch.commons.alerting.util.AlertingException +import org.opensearch.core.action.ActionListener +import org.opensearch.core.rest.RestStatus +import org.opensearch.core.xcontent.NamedXContentRegistry +import org.opensearch.tasks.Task +import org.opensearch.transport.TransportService +import org.opensearch.transport.client.Client + +private val scope: CoroutineScope = CoroutineScope(Dispatchers.IO) +private val log = LogManager.getLogger(TransportDeleteMonitorAction::class.java) + +class TransportDeleteMonitorV2Action @Inject constructor( + transportService: TransportService, + val client: Client, + actionFilters: ActionFilters, + val clusterService: ClusterService, + settings: Settings, + val xContentRegistry: NamedXContentRegistry +) : HandledTransportAction( + DeleteMonitorV2Action.NAME, transportService, actionFilters, ::DeleteMonitorV2Request +), + SecureTransportAction { + + @Volatile override var filterByEnabled = AlertingSettings.FILTER_BY_BACKEND_ROLES.get(settings) + + init { + listenFilterBySettingChange(clusterService) + } + + override fun doExecute(task: Task, request: DeleteMonitorV2Request, actionListener: ActionListener) { + val user = readUserFromThreadContext(client) + + if (!validateUserBackendRoles(user, actionListener)) { + return + } + + scope.launch { + try { + val monitorV2 = getMonitorV2(request.monitorV2Id, actionListener) ?: return@launch + + val canDelete = user == null || !doFilterForUser(user) || + checkUserPermissionsWithResource(user, monitorV2!!.user, actionListener, "monitor_v2", request.monitorV2Id) + + if (canDelete) { + val deleteResponse = DeleteMonitorService.deleteMonitorV2(request.monitorV2Id, request.refreshPolicy) + actionListener.onResponse(deleteResponse) + } else { + actionListener.onFailure( + AlertingException("Not allowed to delete this Monitor V2", RestStatus.FORBIDDEN, IllegalStateException()) + ) + } + } catch (e: Exception) { + actionListener.onFailure(e) + } + + // we do not expire the alerts associated with the deleted monitor, but instead let its expiration time delete it + } + } + + private suspend fun getMonitorV2(monitorV2Id: String, actionListener: ActionListener): MonitorV2? { + val getRequest = GetRequest(ScheduledJob.SCHEDULED_JOBS_INDEX, monitorV2Id) + + val getResponse: GetResponse = client.suspendUntil { get(getRequest, it) } + if (!getResponse.isExists) { + actionListener.onFailure( + AlertingException.wrap( + OpenSearchStatusException("Monitor V2 with $monitorV2Id is not found", RestStatus.NOT_FOUND) + ) + ) + return null + } + + val xcp = XContentHelper.createParser( + xContentRegistry, LoggingDeprecationHandler.INSTANCE, + getResponse.sourceAsBytesRef, XContentType.JSON + ) + val scheduledJob = ScheduledJob.parse(xcp, getResponse.id, getResponse.version) + + validateMonitorV2(scheduledJob)?.let { + actionListener.onFailure(AlertingException.wrap(it)) + return null + } + + val monitorV2 = scheduledJob as MonitorV2 + + return monitorV2 + } +} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportExecuteMonitorV2Action.kt b/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportExecuteMonitorV2Action.kt new file mode 100644 index 000000000..c7f27d029 --- /dev/null +++ b/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportExecuteMonitorV2Action.kt @@ -0,0 +1,175 @@ +package org.opensearch.alerting.transport + +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.launch +import kotlinx.coroutines.withContext +import org.apache.logging.log4j.LogManager +import org.opensearch.OpenSearchStatusException +import org.opensearch.action.get.GetRequest +import org.opensearch.action.get.GetResponse +import org.opensearch.action.support.ActionFilters +import org.opensearch.action.support.HandledTransportAction +import org.opensearch.alerting.AlertingV2Utils.validateMonitorV2 +import org.opensearch.alerting.MonitorRunnerService +import org.opensearch.alerting.actionv2.ExecuteMonitorV2Action +import org.opensearch.alerting.actionv2.ExecuteMonitorV2Request +import org.opensearch.alerting.actionv2.ExecuteMonitorV2Response +import org.opensearch.alerting.core.modelv2.MonitorV2 +import org.opensearch.alerting.core.modelv2.PPLMonitor +import org.opensearch.alerting.core.modelv2.PPLMonitor.Companion.PPL_MONITOR_TYPE +import org.opensearch.alerting.settings.AlertingSettings +import org.opensearch.cluster.service.ClusterService +import org.opensearch.common.inject.Inject +import org.opensearch.common.settings.Settings +import org.opensearch.common.xcontent.LoggingDeprecationHandler +import org.opensearch.common.xcontent.XContentHelper +import org.opensearch.common.xcontent.XContentType +import org.opensearch.commons.ConfigConstants +import org.opensearch.commons.alerting.model.ScheduledJob +import org.opensearch.commons.alerting.util.AlertingException +import org.opensearch.commons.authuser.User +import org.opensearch.core.action.ActionListener +import org.opensearch.core.rest.RestStatus +import org.opensearch.core.xcontent.NamedXContentRegistry +import org.opensearch.tasks.Task +import org.opensearch.transport.TransportService +import org.opensearch.transport.client.Client +import java.time.Instant + +private val log = LogManager.getLogger(TransportExecuteMonitorV2Action::class.java) + +class TransportExecuteMonitorV2Action @Inject constructor( + private val transportService: TransportService, + private val client: Client, + private val clusterService: ClusterService, + private val runner: MonitorRunnerService, + actionFilters: ActionFilters, + val xContentRegistry: NamedXContentRegistry, + private val settings: Settings +) : HandledTransportAction( + ExecuteMonitorV2Action.NAME, transportService, actionFilters, ::ExecuteMonitorV2Request +) { + @Volatile private var indexTimeout = AlertingSettings.INDEX_TIMEOUT.get(settings) + + override fun doExecute( + task: Task, + execMonitorV2Request: ExecuteMonitorV2Request, + actionListener: ActionListener + ) { + val userStr = client.threadPool().threadContext.getTransient(ConfigConstants.OPENSEARCH_SECURITY_USER_INFO_THREAD_CONTEXT) + log.debug("User and roles string from thread context: $userStr") + val user: User? = User.parse(userStr) + + client.threadPool().threadContext.stashContext().use { + /* first define a function that will be used later to run MonitorV2s */ + val executeMonitorV2 = fun (monitorV2: MonitorV2) { + runner.launch { + // get execution time interval + val (periodStart, periodEnd) = if (execMonitorV2Request.requestStart != null) { + Pair( + Instant.ofEpochMilli(execMonitorV2Request.requestStart.millis), + Instant.ofEpochMilli(execMonitorV2Request.requestEnd.millis) + ) + } else { + monitorV2.schedule.getPeriodEndingAt(Instant.ofEpochMilli(execMonitorV2Request.requestEnd.millis)) + } + + // call the MonitorRunnerService to execute the MonitorV2 + try { + val monitorV2Type = when (monitorV2) { + is PPLMonitor -> PPL_MONITOR_TYPE + else -> throw IllegalStateException("Unexpected MonitorV2 type: ${monitorV2.javaClass.name}") + } + log.info( + "Executing MonitorV2 from API - id: ${monitorV2.id}, type: $monitorV2Type, " + + "periodStart: $periodStart, periodEnd: $periodEnd, manual: ${execMonitorV2Request.manual}" + ) + val monitorV2RunResult = runner.runJobV2( + monitorV2, + periodStart, + periodEnd, + execMonitorV2Request.dryrun, + execMonitorV2Request.manual, + transportService + ) + withContext(Dispatchers.IO) { + actionListener.onResponse(ExecuteMonitorV2Response(monitorV2RunResult)) + } + } catch (e: Exception) { + log.error("Unexpected error running monitor", e) + withContext(Dispatchers.IO) { + actionListener.onFailure(AlertingException.wrap(e)) + } + } + } + } + + /* now execute the MonitorV2 */ + + // if both monitor_v2 id and object were passed in, ignore object and proceed with id + if (execMonitorV2Request.monitorId != null && execMonitorV2Request.monitorV2 != null) { + log.info( + "Both a monitor_v2 id and monitor_v2 object were passed in to ExecuteMonitorV2" + + "request. Proceeding to execute by monitor_v2 ID and ignoring monitor_v2 object." + ) + } + + if (execMonitorV2Request.monitorId != null) { // execute with monitor ID case + // search the alerting-config index for the MonitorV2 with this ID + val getMonitorV2Request = GetRequest(ScheduledJob.SCHEDULED_JOBS_INDEX).id(execMonitorV2Request.monitorId) + client.get( + getMonitorV2Request, + object : ActionListener { + override fun onResponse(getMonitorV2Response: GetResponse) { + if (!getMonitorV2Response.isExists) { + actionListener.onFailure( + AlertingException.wrap( + OpenSearchStatusException( + "Can't find monitorV2 with id: ${getMonitorV2Response.id}", + RestStatus.NOT_FOUND + ) + ) + ) + return + } + if (!getMonitorV2Response.isSourceEmpty) { + XContentHelper.createParser( + xContentRegistry, LoggingDeprecationHandler.INSTANCE, + getMonitorV2Response.sourceAsBytesRef, XContentType.JSON + ).use { xcp -> + val scheduledJob = ScheduledJob.parse(xcp, getMonitorV2Response.id, getMonitorV2Response.version) + validateMonitorV2(scheduledJob)?.let { + actionListener.onFailure(AlertingException.wrap(it)) + return + } + val monitorV2 = scheduledJob as MonitorV2 + try { + executeMonitorV2(monitorV2) + } catch (e: Exception) { + actionListener.onFailure(AlertingException.wrap(e)) + } + } + } + } + + override fun onFailure(t: Exception) { + actionListener.onFailure(AlertingException.wrap(t)) + } + } + ) + } else { // execute with monitor object case + try { + val monitorV2 = when (execMonitorV2Request.monitorV2) { + is PPLMonitor -> execMonitorV2Request.monitorV2.copy(user = user) + else -> throw IllegalStateException( + "unexpected MonitorV2 type: ${execMonitorV2Request.monitorV2!!.javaClass.name}" + ) + } + executeMonitorV2(monitorV2) + } catch (e: Exception) { + actionListener.onFailure(AlertingException.wrap(e)) + } + } + } + } +} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportGetAlertsV2Action.kt b/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportGetAlertsV2Action.kt new file mode 100644 index 000000000..04ee04047 --- /dev/null +++ b/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportGetAlertsV2Action.kt @@ -0,0 +1,184 @@ +package org.opensearch.alerting.transport + +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.launch +import org.apache.logging.log4j.LogManager +import org.opensearch.action.search.SearchRequest +import org.opensearch.action.search.SearchResponse +import org.opensearch.action.support.ActionFilters +import org.opensearch.action.support.HandledTransportAction +import org.opensearch.alerting.actionv2.GetAlertsV2Action +import org.opensearch.alerting.actionv2.GetAlertsV2Request +import org.opensearch.alerting.actionv2.GetAlertsV2Response +import org.opensearch.alerting.alertsv2.AlertV2Indices +import org.opensearch.alerting.core.modelv2.AlertV2 +import org.opensearch.alerting.opensearchapi.addFilter +import org.opensearch.alerting.settings.AlertingSettings +import org.opensearch.alerting.util.use +import org.opensearch.cluster.service.ClusterService +import org.opensearch.common.inject.Inject +import org.opensearch.common.settings.Settings +import org.opensearch.common.xcontent.LoggingDeprecationHandler +import org.opensearch.common.xcontent.XContentHelper +import org.opensearch.common.xcontent.XContentType +import org.opensearch.commons.alerting.util.AlertingException +import org.opensearch.commons.authuser.User +import org.opensearch.core.action.ActionListener +import org.opensearch.core.common.io.stream.NamedWriteableRegistry +import org.opensearch.core.xcontent.NamedXContentRegistry +import org.opensearch.core.xcontent.XContentParser +import org.opensearch.core.xcontent.XContentParserUtils +import org.opensearch.index.query.Operator +import org.opensearch.index.query.QueryBuilders +import org.opensearch.search.builder.SearchSourceBuilder +import org.opensearch.search.sort.SortBuilders +import org.opensearch.search.sort.SortOrder +import org.opensearch.tasks.Task +import org.opensearch.transport.TransportService +import org.opensearch.transport.client.Client +import java.io.IOException + +private val log = LogManager.getLogger(TransportGetAlertsV2Action::class.java) +private val scope: CoroutineScope = CoroutineScope(Dispatchers.IO) + +class TransportGetAlertsV2Action @Inject constructor( + transportService: TransportService, + val client: Client, + clusterService: ClusterService, + actionFilters: ActionFilters, + val settings: Settings, + val xContentRegistry: NamedXContentRegistry, + val namedWriteableRegistry: NamedWriteableRegistry +) : HandledTransportAction( + GetAlertsV2Action.NAME, + transportService, + actionFilters, + ::GetAlertsV2Request +), + SecureTransportAction { + + @Volatile + override var filterByEnabled = AlertingSettings.FILTER_BY_BACKEND_ROLES.get(settings) + + init { + listenFilterBySettingChange(clusterService) + } + + override fun doExecute( + task: Task, + getAlertsV2Request: GetAlertsV2Request, + actionListener: ActionListener, + ) { + val user = readUserFromThreadContext(client) + + val tableProp = getAlertsV2Request.table + val sortBuilder = SortBuilders + .fieldSort(tableProp.sortString) + .order(SortOrder.fromString(tableProp.sortOrder)) + if (!tableProp.missing.isNullOrBlank()) { + sortBuilder.missing(tableProp.missing) + } + + val queryBuilder = getAlertsV2Request.boolQueryBuilder ?: QueryBuilders.boolQuery() + + if (getAlertsV2Request.severityLevel != "ALL") { + queryBuilder.filter(QueryBuilders.termQuery("severity", getAlertsV2Request.severityLevel)) + } + + if (!getAlertsV2Request.alertV2Ids.isNullOrEmpty()) { + queryBuilder.filter(QueryBuilders.termsQuery("_id", getAlertsV2Request.alertV2Ids)) + } + + if (getAlertsV2Request.monitorV2Id != null) { + queryBuilder.filter(QueryBuilders.termQuery("monitor_id", getAlertsV2Request.monitorV2Id)) + } else if (!getAlertsV2Request.monitorV2Ids.isNullOrEmpty()) { + queryBuilder.filter(QueryBuilders.termsQuery("monitor_id", getAlertsV2Request.monitorV2Ids)) + } + + if (!tableProp.searchString.isNullOrBlank()) { + queryBuilder + .must( + QueryBuilders + .queryStringQuery(tableProp.searchString) + .defaultOperator(Operator.AND) + .field("monitor_name") + .field("trigger_name") + ) + } + val searchSourceBuilder = SearchSourceBuilder() + .version(true) + .seqNoAndPrimaryTerm(true) + .query(queryBuilder) + .sort(sortBuilder) + .size(tableProp.size) + .from(tableProp.startIndex) + + client.threadPool().threadContext.stashContext().use { + scope.launch { + try { + getAlerts(AlertV2Indices.ALERT_V2_INDEX, searchSourceBuilder, actionListener, user) + } catch (t: Exception) { + log.error("Failed to get alerts", t) + if (t is AlertingException) { + actionListener.onFailure(t) + } else { + actionListener.onFailure(AlertingException.wrap(t)) + } + } + } + } + } + + fun getAlerts( + alertIndex: String, + searchSourceBuilder: SearchSourceBuilder, + actionListener: ActionListener, + user: User? + ) { + try { + // if user is null, security plugin is disabled or user is super-admin + // if doFilterForUser() is false, security is enabled but filterby is disabled + if (user != null && doFilterForUser(user)) { + // if security is enabled and filterby is enabled, add search filter + log.info("Filtering result by: ${user.backendRoles}") + addFilter(user, searchSourceBuilder, "monitor.user.backend_roles.keyword") + } + + search(alertIndex, searchSourceBuilder, actionListener) + } catch (ex: IOException) { + actionListener.onFailure(AlertingException.wrap(ex)) + } + } + + fun search(alertIndex: String, searchSourceBuilder: SearchSourceBuilder, actionListener: ActionListener) { + val searchRequest = SearchRequest() + .indices(alertIndex) + .source(searchSourceBuilder) + + client.search( + searchRequest, + object : ActionListener { + override fun onResponse(response: SearchResponse) { + val totalAlertCount = response.hits.totalHits?.value?.toInt() + val alerts = response.hits.map { hit -> + val xcp = XContentHelper.createParser( + xContentRegistry, + LoggingDeprecationHandler.INSTANCE, + hit.sourceRef, + XContentType.JSON + ) + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.nextToken(), xcp) + val alertV2 = AlertV2.parse(xcp, hit.id, hit.version) + alertV2 + } + actionListener.onResponse(GetAlertsV2Response(alerts, totalAlertCount)) + } + + override fun onFailure(t: Exception) { + actionListener.onFailure(t) + } + } + ) + } +} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportGetMonitorAction.kt b/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportGetMonitorAction.kt index ce42e6157..59049da44 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportGetMonitorAction.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportGetMonitorAction.kt @@ -18,6 +18,7 @@ import org.opensearch.action.search.SearchRequest import org.opensearch.action.search.SearchResponse import org.opensearch.action.support.ActionFilters import org.opensearch.action.support.HandledTransportAction +import org.opensearch.alerting.AlertingV2Utils.validateMonitorV1 import org.opensearch.alerting.opensearchapi.suspendUntil import org.opensearch.alerting.settings.AlertingSettings import org.opensearch.alerting.util.ScheduledJobUtils.Companion.WORKFLOW_DELEGATE_PATH @@ -114,7 +115,14 @@ class TransportGetMonitorAction @Inject constructor( response.sourceAsBytesRef, XContentType.JSON ).use { xcp -> - monitor = ScheduledJob.parse(xcp, response.id, response.version) as Monitor + val scheduledJob = ScheduledJob.parse(xcp, response.id, response.version) + + validateMonitorV1(scheduledJob)?.let { + actionListener.onFailure(AlertingException.wrap(it)) + return + } + + monitor = scheduledJob as Monitor // security is enabled and filterby is enabled if (!checkUserPermissionsWithResource( diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportGetMonitorV2Action.kt b/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportGetMonitorV2Action.kt new file mode 100644 index 000000000..d74e29167 --- /dev/null +++ b/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportGetMonitorV2Action.kt @@ -0,0 +1,132 @@ +package org.opensearch.alerting.transport + +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import org.apache.logging.log4j.LogManager +import org.opensearch.OpenSearchStatusException +import org.opensearch.action.get.GetRequest +import org.opensearch.action.get.GetResponse +import org.opensearch.action.support.ActionFilters +import org.opensearch.action.support.HandledTransportAction +import org.opensearch.alerting.AlertingV2Utils.validateMonitorV2 +import org.opensearch.alerting.actionv2.GetMonitorV2Action +import org.opensearch.alerting.actionv2.GetMonitorV2Request +import org.opensearch.alerting.actionv2.GetMonitorV2Response +import org.opensearch.alerting.core.modelv2.MonitorV2 +import org.opensearch.alerting.settings.AlertingSettings +import org.opensearch.cluster.service.ClusterService +import org.opensearch.common.inject.Inject +import org.opensearch.common.settings.Settings +import org.opensearch.common.xcontent.LoggingDeprecationHandler +import org.opensearch.common.xcontent.XContentHelper +import org.opensearch.common.xcontent.XContentType +import org.opensearch.commons.alerting.model.ScheduledJob +import org.opensearch.commons.alerting.util.AlertingException +import org.opensearch.core.action.ActionListener +import org.opensearch.core.rest.RestStatus +import org.opensearch.core.xcontent.NamedXContentRegistry +import org.opensearch.tasks.Task +import org.opensearch.transport.TransportService +import org.opensearch.transport.client.Client + +private val log = LogManager.getLogger(TransportGetMonitorAction::class.java) +private val scope: CoroutineScope = CoroutineScope(Dispatchers.IO) + +class TransportGetMonitorV2Action @Inject constructor( + transportService: TransportService, + val client: Client, + actionFilters: ActionFilters, + val xContentRegistry: NamedXContentRegistry, + val clusterService: ClusterService, + settings: Settings, +) : HandledTransportAction( + GetMonitorV2Action.NAME, + transportService, + actionFilters, + ::GetMonitorV2Request +), + SecureTransportAction { + + @Volatile + override var filterByEnabled = AlertingSettings.FILTER_BY_BACKEND_ROLES.get(settings) + + init { + listenFilterBySettingChange(clusterService) + } + + override fun doExecute(task: Task, request: GetMonitorV2Request, actionListener: ActionListener) { + val getRequest = GetRequest(ScheduledJob.SCHEDULED_JOBS_INDEX, request.monitorV2Id) + .version(request.version) + .fetchSourceContext(request.srcContext) + + val user = readUserFromThreadContext(client) + + if (!validateUserBackendRoles(user, actionListener)) { + return + } + + client.threadPool().threadContext.stashContext().use { + client.get( + getRequest, + object : ActionListener { + override fun onResponse(response: GetResponse) { + if (!response.isExists) { + actionListener.onFailure( + AlertingException.wrap(OpenSearchStatusException("MonitorV2 not found.", RestStatus.NOT_FOUND)) + ) + return + } + + if (response.isSourceEmpty) { + actionListener.onFailure( + AlertingException.wrap(OpenSearchStatusException("MonitorV2 found but was empty.", RestStatus.NO_CONTENT)) + ) + return + } + + val xcp = XContentHelper.createParser( + xContentRegistry, + LoggingDeprecationHandler.INSTANCE, + response.sourceAsBytesRef, + XContentType.JSON + ) + val scheduledJob = ScheduledJob.parse(xcp, response.id, response.version) + + validateMonitorV2(scheduledJob)?.let { + actionListener.onFailure(AlertingException.wrap(it)) + return + } + + val monitorV2 = scheduledJob as MonitorV2 + + // security is enabled and filterby is enabled + if (!checkUserPermissionsWithResource( + user, + monitorV2.user, + actionListener, + "monitor", + request.monitorV2Id + ) + ) { + return + } + + actionListener.onResponse( + GetMonitorV2Response( + response.id, + response.version, + response.seqNo, + response.primaryTerm, + monitorV2 + ) + ) + } + + override fun onFailure(t: Exception) { + actionListener.onFailure(AlertingException.wrap(t)) + } + } + ) + } + } +} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportIndexMonitorAction.kt b/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportIndexMonitorAction.kt index a5b849a67..4c747e737 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportIndexMonitorAction.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportIndexMonitorAction.kt @@ -29,6 +29,7 @@ import org.opensearch.action.support.ActionFilters import org.opensearch.action.support.HandledTransportAction import org.opensearch.action.support.WriteRequest.RefreshPolicy import org.opensearch.action.support.clustermanager.AcknowledgedResponse +import org.opensearch.alerting.AlertingV2Utils.validateMonitorV1 import org.opensearch.alerting.MonitorMetadataService import org.opensearch.alerting.core.ScheduledJobIndices import org.opensearch.alerting.opensearchapi.suspendUntil @@ -614,7 +615,15 @@ class TransportIndexMonitorAction @Inject constructor( xContentRegistry, LoggingDeprecationHandler.INSTANCE, getResponse.sourceAsBytesRef, XContentType.JSON ) - val monitor = ScheduledJob.parse(xcp, getResponse.id, getResponse.version) as Monitor + val scheduledJob = ScheduledJob.parse(xcp, getResponse.id, getResponse.version) + + validateMonitorV1(scheduledJob)?.let { + actionListener.onFailure(AlertingException.wrap(it)) + return + } + + val monitor = scheduledJob as Monitor + onGetResponse(monitor) } catch (t: Exception) { actionListener.onFailure(AlertingException.wrap(t)) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportIndexMonitorV2Action.kt b/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportIndexMonitorV2Action.kt new file mode 100644 index 000000000..23681ac14 --- /dev/null +++ b/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportIndexMonitorV2Action.kt @@ -0,0 +1,682 @@ +package org.opensearch.alerting.transport + +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.launch +import org.apache.logging.log4j.LogManager +import org.opensearch.ExceptionsHelper +import org.opensearch.OpenSearchException +import org.opensearch.OpenSearchSecurityException +import org.opensearch.OpenSearchStatusException +import org.opensearch.ResourceAlreadyExistsException +import org.opensearch.action.admin.cluster.health.ClusterHealthAction +import org.opensearch.action.admin.cluster.health.ClusterHealthRequest +import org.opensearch.action.admin.cluster.health.ClusterHealthResponse +import org.opensearch.action.admin.indices.create.CreateIndexResponse +import org.opensearch.action.get.GetRequest +import org.opensearch.action.get.GetResponse +import org.opensearch.action.index.IndexRequest +import org.opensearch.action.index.IndexResponse +import org.opensearch.action.search.SearchRequest +import org.opensearch.action.search.SearchResponse +import org.opensearch.action.support.ActionFilters +import org.opensearch.action.support.HandledTransportAction +import org.opensearch.action.support.clustermanager.AcknowledgedResponse +import org.opensearch.alerting.AlertingV2Utils.validateMonitorV2 +import org.opensearch.alerting.PPLMonitorRunner.appendCustomCondition +import org.opensearch.alerting.PPLMonitorRunner.executePplQuery +import org.opensearch.alerting.PPLMonitorRunner.findEvalResultVar +import org.opensearch.alerting.PPLMonitorRunner.findEvalResultVarIdxInSchema +import org.opensearch.alerting.actionv2.IndexMonitorV2Action +import org.opensearch.alerting.actionv2.IndexMonitorV2Request +import org.opensearch.alerting.actionv2.IndexMonitorV2Response +import org.opensearch.alerting.core.ScheduledJobIndices +import org.opensearch.alerting.core.modelv2.MonitorV2 +import org.opensearch.alerting.core.modelv2.PPLMonitor +import org.opensearch.alerting.core.modelv2.PPLTrigger.ConditionType +import org.opensearch.alerting.opensearchapi.suspendUntil +import org.opensearch.alerting.settings.AlertingSettings +import org.opensearch.alerting.settings.AlertingSettings.Companion.ALERTING_MAX_MONITORS +import org.opensearch.alerting.settings.AlertingSettings.Companion.INDEX_TIMEOUT +import org.opensearch.alerting.settings.AlertingSettings.Companion.REQUEST_TIMEOUT +import org.opensearch.alerting.util.IndexUtils +import org.opensearch.alerting.util.use +import org.opensearch.cluster.service.ClusterService +import org.opensearch.common.inject.Inject +import org.opensearch.common.settings.Settings +import org.opensearch.common.xcontent.LoggingDeprecationHandler +import org.opensearch.common.xcontent.XContentFactory.jsonBuilder +import org.opensearch.common.xcontent.XContentHelper +import org.opensearch.common.xcontent.XContentType +import org.opensearch.commons.alerting.model.Monitor +import org.opensearch.commons.alerting.model.ScheduledJob +import org.opensearch.commons.alerting.model.ScheduledJob.Companion.SCHEDULED_JOBS_INDEX +import org.opensearch.commons.alerting.model.userErrorMessage +import org.opensearch.commons.alerting.util.AlertingException +import org.opensearch.commons.authuser.User +import org.opensearch.core.action.ActionListener +import org.opensearch.core.common.io.stream.NamedWriteableRegistry +import org.opensearch.core.rest.RestStatus +import org.opensearch.core.xcontent.NamedXContentRegistry +import org.opensearch.core.xcontent.ToXContent +import org.opensearch.index.query.QueryBuilders +import org.opensearch.rest.RestRequest +import org.opensearch.search.builder.SearchSourceBuilder +import org.opensearch.tasks.Task +import org.opensearch.transport.TransportService +import org.opensearch.transport.client.Client +import org.opensearch.transport.client.node.NodeClient + +private val log = LogManager.getLogger(TransportIndexMonitorV2Action::class.java) +private val scope: CoroutineScope = CoroutineScope(Dispatchers.IO) + +class TransportIndexMonitorV2Action @Inject constructor( + transportService: TransportService, + val client: Client, + actionFilters: ActionFilters, + val scheduledJobIndices: ScheduledJobIndices, + val clusterService: ClusterService, + val settings: Settings, + val xContentRegistry: NamedXContentRegistry, + val namedWriteableRegistry: NamedWriteableRegistry, +) : HandledTransportAction( + IndexMonitorV2Action.NAME, transportService, actionFilters, ::IndexMonitorV2Request +), + SecureTransportAction { + + // TODO: add monitor v2 versions of these settings + @Volatile private var maxMonitors = ALERTING_MAX_MONITORS.get(settings) + @Volatile private var requestTimeout = REQUEST_TIMEOUT.get(settings) + @Volatile private var indexTimeout = INDEX_TIMEOUT.get(settings) + @Volatile override var filterByEnabled = AlertingSettings.FILTER_BY_BACKEND_ROLES.get(settings) + + init { + clusterService.clusterSettings.addSettingsUpdateConsumer(ALERTING_MAX_MONITORS) { maxMonitors = it } + clusterService.clusterSettings.addSettingsUpdateConsumer(REQUEST_TIMEOUT) { requestTimeout = it } + clusterService.clusterSettings.addSettingsUpdateConsumer(INDEX_TIMEOUT) { indexTimeout = it } + listenFilterBySettingChange(clusterService) + } + + override fun doExecute( + task: Task, + indexMonitorV2Request: IndexMonitorV2Request, + actionListener: ActionListener + ) { + // read the user from thread context immediately, before + // downstream flows spin up new threads with fresh context + val user = readUserFromThreadContext(client) + + // validate the MonitorV2 based on its type + when (indexMonitorV2Request.monitorV2) { + is PPLMonitor -> validateMonitorPplQuery( + indexMonitorV2Request.monitorV2 as PPLMonitor, + object : ActionListener { // validationListener + override fun onResponse(response: Unit) { + checkUserAndIndicesAccess(client, actionListener, indexMonitorV2Request, user) + } + + override fun onFailure(e: Exception) { + actionListener.onFailure(e) + } + } + ) + else -> actionListener.onFailure( + AlertingException.wrap( + IllegalStateException( + "unexpected MonitorV2 type: ${indexMonitorV2Request.monitorV2.javaClass.name}" + ) + ) + ) + } + } + + // validates the PPL Monitor query by submitting it to SQL/PPL plugin + private fun validateMonitorPplQuery(pplMonitor: PPLMonitor, validationListener: ActionListener) { + scope.launch { + try { + val nodeClient = client as NodeClient + + // first attempt to run the base query + // if there are any PPL syntax errors, this will throw an exception + executePplQuery(pplMonitor.query, nodeClient) + + // now scan all the triggers with custom conditions, and ensure each query constructed + // from the base query + custom condition is valid + val allCustomTriggersValid = true + for (pplTrigger in pplMonitor.triggers) { + if (pplTrigger.conditionType != ConditionType.CUSTOM) { + continue + } + + val evalResultVar = findEvalResultVar(pplTrigger.customCondition!!) + + val queryWithCustomCondition = appendCustomCondition(pplMonitor.query, pplTrigger.customCondition!!) + + val executePplQueryResponse = executePplQuery(queryWithCustomCondition, nodeClient) + + val evalResultVarIdx = findEvalResultVarIdxInSchema(executePplQueryResponse, evalResultVar) + + val resultVarType = executePplQueryResponse + .getJSONArray("schema") + .getJSONObject(evalResultVarIdx) + .getString("type") + + // custom conditions must evaluate to a boolean result, otherwise it's invalid + if (resultVarType != "boolean") { + validationListener.onFailure( + AlertingException.wrap( + IllegalArgumentException( + "Custom condition in trigger ${pplTrigger.name} is invalid because it does not " + + "evaluate to a boolean, but instead to type: $resultVarType" + ) + ) + ) + return@launch + } + } + + validationListener.onResponse(Unit) + } catch (e: Exception) { + validationListener.onFailure( + AlertingException.wrap( + IllegalArgumentException("Invalid PPL Query in PPL Monitor: ${e.userErrorMessage()}") + ) + ) + } + } + } + + private fun checkUserAndIndicesAccess( + client: Client, + actionListener: ActionListener, + indexMonitorV2Request: IndexMonitorV2Request, + user: User? + ) { + /* check initial user permissions */ + if (!validateUserBackendRoles(user, actionListener)) { + return + } + + if ( + user != null && + !isAdmin(user) && + indexMonitorV2Request.rbacRoles != null + ) { + if (indexMonitorV2Request.rbacRoles.stream().anyMatch { !user.backendRoles.contains(it) }) { + log.debug( + "User specified backend roles, ${indexMonitorV2Request.rbacRoles}, " + + "that they don't have access to. User backend roles: ${user.backendRoles}" + ) + actionListener.onFailure( + AlertingException.wrap( + OpenSearchStatusException( + "User specified backend roles that they don't have access to. Contact administrator", RestStatus.FORBIDDEN + ) + ) + ) + return + } else if (indexMonitorV2Request.rbacRoles.isEmpty()) { + log.debug( + "Non-admin user are not allowed to specify an empty set of backend roles. " + + "Please don't pass in the parameter or pass in at least one backend role." + ) + actionListener.onFailure( + AlertingException.wrap( + OpenSearchStatusException( + "Non-admin user are not allowed to specify an empty set of backend roles.", RestStatus.FORBIDDEN + ) + ) + ) + return + } + } + + /* check user access to indices */ + when (indexMonitorV2Request.monitorV2) { + is PPLMonitor -> { + checkPplQueryIndices(indexMonitorV2Request, client, actionListener, user) + } + } + } + + private fun checkPplQueryIndices( + indexMonitorV2Request: IndexMonitorV2Request, + client: Client, + actionListener: ActionListener, + user: User? + ) { + val pplMonitor = indexMonitorV2Request.monitorV2 as PPLMonitor + val pplQuery = pplMonitor.query + val indices = getIndicesFromPplQuery(pplQuery) + + val searchRequest = SearchRequest().indices(*indices.toTypedArray()) + .source(SearchSourceBuilder.searchSource().size(1).query(QueryBuilders.matchAllQuery())) + client.search( + searchRequest, + object : ActionListener { + override fun onResponse(searchResponse: SearchResponse) { + // User has read access to configured indices in the monitor, now create monitor without user context. + client.threadPool().threadContext.stashContext().use { + if (user == null) { + // Security is disabled, add empty user to Monitor. user is null for older versions. + indexMonitorV2Request.monitorV2 = pplMonitor + .copy(user = User("", listOf(), listOf(), listOf())) + checkScheduledJobIndex(indexMonitorV2Request, actionListener, user) + } else { + indexMonitorV2Request.monitorV2 = pplMonitor + .copy(user = User(user.name, user.backendRoles, user.roles, user.customAttNames)) + checkScheduledJobIndex(indexMonitorV2Request, actionListener, user) + } + } + } + + // Due to below issue with security plugin, we get security_exception when invalid index name is mentioned. + // https://github.com/opendistro-for-elasticsearch/security/issues/718 + override fun onFailure(t: Exception) { + actionListener.onFailure( + AlertingException.wrap( + when (t is OpenSearchSecurityException) { + true -> OpenSearchStatusException( + "User doesn't have read permissions for one or more configured index " + + "$indices", + RestStatus.FORBIDDEN + ) + false -> t + } + ) + ) + } + } + ) + } + + private fun checkScheduledJobIndex( + indexMonitorRequest: IndexMonitorV2Request, + actionListener: ActionListener, + user: User? + ) { + /* check to see if alerting-config index (scheduled job index) is created and updated before indexing MonitorV2 into it */ + if (!scheduledJobIndices.scheduledJobIndexExists()) { // if alerting-config index doesn't exist, send request to create it + scheduledJobIndices.initScheduledJobIndex(object : ActionListener { + override fun onResponse(response: CreateIndexResponse) { + onCreateMappingsResponse(response.isAcknowledged, indexMonitorRequest, actionListener, user) + } + + override fun onFailure(e: Exception) { + if (ExceptionsHelper.unwrapCause(e) is ResourceAlreadyExistsException) { + scope.launch { + // Wait for the yellow status + val clusterHealthRequest = ClusterHealthRequest() + .indices(SCHEDULED_JOBS_INDEX) + .waitForYellowStatus() + val response: ClusterHealthResponse = client.suspendUntil { + execute(ClusterHealthAction.INSTANCE, clusterHealthRequest, it) + } + if (response.isTimedOut) { + actionListener.onFailure( + OpenSearchException("Cannot determine that the $SCHEDULED_JOBS_INDEX index is healthy") + ) + } + // Retry mapping of monitor + onCreateMappingsResponse(true, indexMonitorRequest, actionListener, user) + } + } else { + actionListener.onFailure(AlertingException.wrap(e)) + } + } + }) + } else if (!IndexUtils.scheduledJobIndexUpdated) { + IndexUtils.updateIndexMapping( + SCHEDULED_JOBS_INDEX, + ScheduledJobIndices.scheduledJobMappings(), clusterService.state(), client.admin().indices(), + object : ActionListener { + override fun onResponse(response: AcknowledgedResponse) { + onUpdateMappingsResponse(response, indexMonitorRequest, actionListener, user) + } + override fun onFailure(t: Exception) { + actionListener.onFailure(AlertingException.wrap(t)) + } + } + ) + } else { + prepareMonitorIndexing(indexMonitorRequest, actionListener, user) + } + } + + private fun onCreateMappingsResponse( + isAcknowledged: Boolean, + request: IndexMonitorV2Request, + actionListener: ActionListener, + user: User? + ) { + if (isAcknowledged) { + log.info("Created $SCHEDULED_JOBS_INDEX with mappings.") + prepareMonitorIndexing(request, actionListener, user) + IndexUtils.scheduledJobIndexUpdated() + } else { + log.info("Create $SCHEDULED_JOBS_INDEX mappings call not acknowledged.") + actionListener.onFailure( + AlertingException.wrap( + OpenSearchStatusException( + "Create $SCHEDULED_JOBS_INDEX mappings call not acknowledged", RestStatus.INTERNAL_SERVER_ERROR + ) + ) + ) + } + } + + private fun onUpdateMappingsResponse( + response: AcknowledgedResponse, + indexMonitorRequest: IndexMonitorV2Request, + actionListener: ActionListener, + user: User? + ) { + if (response.isAcknowledged) { + log.info("Updated $SCHEDULED_JOBS_INDEX with mappings.") + IndexUtils.scheduledJobIndexUpdated() + prepareMonitorIndexing(indexMonitorRequest, actionListener, user) + } else { + log.info("Update $SCHEDULED_JOBS_INDEX mappings call not acknowledged.") + actionListener.onFailure( + AlertingException.wrap( + OpenSearchStatusException( + "Updated $SCHEDULED_JOBS_INDEX mappings call not acknowledged.", + RestStatus.INTERNAL_SERVER_ERROR + ) + ) + ) + } + } + + /** + * This function prepares for indexing a new monitor. + * If this is an update request we can simply update the monitor. Otherwise we first check to see how many monitors already exist, + * and compare this to the [maxMonitorCount]. Requests that breach this threshold will be rejected. + */ + private fun prepareMonitorIndexing( + indexMonitorRequest: IndexMonitorV2Request, + actionListener: ActionListener, + user: User? + ) { + if (indexMonitorRequest.method == RestRequest.Method.PUT) { // update monitor case + scope.launch { + updateMonitor(indexMonitorRequest, actionListener, user) + } + } else { // create monitor case + val query = QueryBuilders.boolQuery().filter(QueryBuilders.termQuery("${Monitor.MONITOR_TYPE}.type", Monitor.MONITOR_TYPE)) + val searchSource = SearchSourceBuilder().query(query).timeout(requestTimeout) + val searchRequest = SearchRequest(SCHEDULED_JOBS_INDEX).source(searchSource) + + client.search( + searchRequest, + object : ActionListener { + override fun onResponse(searchResponse: SearchResponse) { + onMonitorCountSearchResponse(searchResponse, indexMonitorRequest, actionListener, user) + } + + override fun onFailure(t: Exception) { + actionListener.onFailure(AlertingException.wrap(t)) + } + } + ) + } + } + + /* Functions for Update Monitor flow */ + + private suspend fun updateMonitor( + indexMonitorRequest: IndexMonitorV2Request, + actionListener: ActionListener, + user: User? + ) { + val getRequest = GetRequest(SCHEDULED_JOBS_INDEX, indexMonitorRequest.monitorId) + try { + val getResponse: GetResponse = client.suspendUntil { client.get(getRequest, it) } + if (!getResponse.isExists) { + actionListener.onFailure( + AlertingException.wrap( + OpenSearchStatusException("MonitorV2 with ${indexMonitorRequest.monitorId} is not found", RestStatus.NOT_FOUND) + ) + ) + return + } + val xcp = XContentHelper.createParser( + xContentRegistry, LoggingDeprecationHandler.INSTANCE, + getResponse.sourceAsBytesRef, XContentType.JSON + ) + val scheduledJob = ScheduledJob.parse(xcp, getResponse.id, getResponse.version) + + validateMonitorV2(scheduledJob)?.let { + actionListener.onFailure(AlertingException.wrap(it)) + return + } + + val monitorV2 = scheduledJob as MonitorV2 + + onGetMonitorResponseForUpdate(monitorV2, indexMonitorRequest, actionListener, user) + } catch (e: Exception) { + actionListener.onFailure(AlertingException.wrap(e)) + } + } + + private suspend fun onGetMonitorResponseForUpdate( + existingMonitorV2: MonitorV2, + indexMonitorRequest: IndexMonitorV2Request, + actionListener: ActionListener, + user: User? + ) { + log.info("user: $user") + log.info("monitor user: ${existingMonitorV2.user}") + if ( + !checkUserPermissionsWithResource( + user, + existingMonitorV2.user, + actionListener, + "monitor_v2", + indexMonitorRequest.monitorId + ) + ) { + return + } + + var newMonitorV2: MonitorV2 + val currentMonitorV2: MonitorV2 // this is the same as existingMonitorV2, but will be cast to a specific MonitorV2 type + + when (indexMonitorRequest.monitorV2) { + is PPLMonitor -> { + newMonitorV2 = indexMonitorRequest.monitorV2 as PPLMonitor + currentMonitorV2 = existingMonitorV2 as PPLMonitor + } + else -> throw IllegalStateException("received unsupported monitor type to index: ${indexMonitorRequest.monitorV2.javaClass}") + } + + // If both are enabled, use the current existing monitor enabled time, otherwise the next execution will be + // incorrect. + if (newMonitorV2.enabled && currentMonitorV2.enabled) { + newMonitorV2 = newMonitorV2.copy(enabledTime = currentMonitorV2.enabledTime) + } + + /** + * On update monitor check which backend roles to associate to the monitor. + * Below are 2 examples of how the logic works + * + * Example 1, say we have a Monitor with backend roles [a, b, c, d] associated with it. + * If I'm User A (non-admin user) and I have backend roles [a, b, c] associated with me and I make a request to update + * the Monitor's backend roles to [a, b]. This would mean that the roles to remove are [c] and the roles to add are [a, b]. + * The Monitor's backend roles would then be [a, b, d]. + * + * Example 2, say we have a Monitor with backend roles [a, b, c, d] associated with it. + * If I'm User A (admin user) and I have backend roles [a, b, c] associated with me and I make a request to update + * the Monitor's backend roles to [a, b]. This would mean that the roles to remove are [c, d] and the roles to add are [a, b]. + * The Monitor's backend roles would then be [a, b]. + */ + if (user != null) { + if (indexMonitorRequest.rbacRoles != null) { + if (isAdmin(user)) { + newMonitorV2 = newMonitorV2.copy( + user = User(user.name, indexMonitorRequest.rbacRoles, user.roles, user.customAttNames) + ) + } else { + // rolesToRemove: these are the backend roles to remove from the monitor + val rolesToRemove = user.backendRoles - indexMonitorRequest.rbacRoles + // remove the monitor's roles with rolesToRemove and add any roles passed into the request.rbacRoles + val updatedRbac = currentMonitorV2.user?.backendRoles.orEmpty() - rolesToRemove + indexMonitorRequest.rbacRoles + newMonitorV2 = newMonitorV2.copy( + user = User(user.name, updatedRbac, user.roles, user.customAttNames) + ) + } + } else { + newMonitorV2 = newMonitorV2 + .copy(user = User(user.name, currentMonitorV2.user!!.backendRoles, user.roles, user.customAttNames)) + } + log.info("Update monitor backend roles to: ${newMonitorV2.user?.backendRoles}") + } + + newMonitorV2 = newMonitorV2.copy(schemaVersion = IndexUtils.scheduledJobIndexSchemaVersion) + val indexRequest = IndexRequest(SCHEDULED_JOBS_INDEX) + .setRefreshPolicy(indexMonitorRequest.refreshPolicy) + .source(newMonitorV2.toXContentWithUser(jsonBuilder(), ToXContent.MapParams(mapOf("with_type" to "true")))) + .id(indexMonitorRequest.monitorId) + .routing(indexMonitorRequest.monitorId) + .setIfSeqNo(indexMonitorRequest.seqNo) + .setIfPrimaryTerm(indexMonitorRequest.primaryTerm) + .timeout(indexTimeout) + + log.info( + "Updating monitor, ${currentMonitorV2.id}, from: ${currentMonitorV2.toXContentWithUser( + jsonBuilder(), + ToXContent.MapParams(mapOf("with_type" to "true")) + )} \n to: ${newMonitorV2.toXContentWithUser(jsonBuilder(), ToXContent.MapParams(mapOf("with_type" to "true")))}" + ) + + try { + val indexResponse: IndexResponse = client.suspendUntil { client.index(indexRequest, it) } + val failureReasons = IndexUtils.checkShardsFailure(indexResponse) + if (failureReasons != null) { + actionListener.onFailure( + AlertingException.wrap(OpenSearchStatusException(failureReasons.toString(), indexResponse.status())) + ) + return + } + + actionListener.onResponse( + IndexMonitorV2Response( + indexResponse.id, indexResponse.version, indexResponse.seqNo, + indexResponse.primaryTerm, newMonitorV2 + ) + ) + } catch (e: Exception) { + actionListener.onFailure(AlertingException.wrap(e)) + } + } + + /* Functions for Create Monitor flow */ + + /** + * After searching for all existing monitors we validate the system can support another monitor to be created. + */ + private fun onMonitorCountSearchResponse( + monitorCountSearchResponse: SearchResponse, + indexMonitorRequest: IndexMonitorV2Request, + actionListener: ActionListener, + user: User? + ) { + val totalHits = monitorCountSearchResponse.hits.totalHits?.value + if (totalHits != null && totalHits >= maxMonitors) { + log.info("This request would create more than the allowed monitors [$maxMonitors].") + actionListener.onFailure( + AlertingException.wrap( + IllegalArgumentException( + "This request would create more than the allowed monitors [$maxMonitors]." + ) + ) + ) + } else { + scope.launch { + indexMonitor(indexMonitorRequest, actionListener, user) + } + } + } + + private suspend fun indexMonitor( + indexMonitorRequest: IndexMonitorV2Request, + actionListener: ActionListener, + user: User? + ) { + var monitorV2 = when (indexMonitorRequest.monitorV2) { + is PPLMonitor -> indexMonitorRequest.monitorV2 as PPLMonitor + else -> throw IllegalArgumentException("received unsupported monitor type to index: ${indexMonitorRequest.monitorV2.javaClass}") + } + + if (user != null) { + // Use the backend roles which is an intersection of the requested backend roles and the user's backend roles. + // Admins can pass in any backend role. Also if no backend role is passed in, all the user's backend roles are used. + val rbacRoles = if (indexMonitorRequest.rbacRoles == null) user.backendRoles.toSet() + else if (!isAdmin(user)) indexMonitorRequest.rbacRoles.intersect(user.backendRoles).toSet() + else indexMonitorRequest.rbacRoles + + monitorV2 = when (monitorV2) { + is PPLMonitor -> monitorV2.copy( + user = User(user.name, rbacRoles.toList(), user.roles, user.customAttNames) + ) + else -> throw IllegalArgumentException( + "received unsupported monitor type when resolving backend roles: ${indexMonitorRequest.monitorV2.javaClass}" + ) + } + log.debug("Created monitor's backend roles: $rbacRoles") + } + + // TODO: only works because monitorV2 is always of type PPLMonitor, not extensible to other potential MonitorV2 types + val indexRequest = IndexRequest(SCHEDULED_JOBS_INDEX) + .setRefreshPolicy(indexMonitorRequest.refreshPolicy) + .source(monitorV2.toXContentWithUser(jsonBuilder(), ToXContent.MapParams(mapOf("with_type" to "true")))) + .routing(indexMonitorRequest.monitorId) + .setIfSeqNo(indexMonitorRequest.seqNo) + .setIfPrimaryTerm(indexMonitorRequest.primaryTerm) + .timeout(indexTimeout) + + log.info( + "Creating new monitorV2: ${monitorV2.toXContentWithUser( + jsonBuilder(), + ToXContent.MapParams(mapOf("with_type" to "true")) + )}" + ) + + try { + val indexResponse: IndexResponse = client.suspendUntil { client.index(indexRequest, it) } + val failureReasons = IndexUtils.checkShardsFailure(indexResponse) + if (failureReasons != null) { + log.info(failureReasons.toString()) + actionListener.onFailure( + AlertingException.wrap(OpenSearchStatusException(failureReasons.toString(), indexResponse.status())) + ) + return + } + + actionListener.onResponse( + IndexMonitorV2Response( + indexResponse.id, indexResponse.version, indexResponse.seqNo, + indexResponse.primaryTerm, monitorV2 + ) + ) + } catch (t: Exception) { + actionListener.onFailure(AlertingException.wrap(t)) + } + } + + /* Utils */ + private fun getIndicesFromPplQuery(pplQuery: String): List { + // captures comma-separated concrete indices, index patterns, and index aliases + val indicesRegex = """(?i)source(?:\s*)=(?:\s*)([-\w.*'+]+(?:\*)?(?:\s*,\s*[-\w.*'+]+\*?)*)\s*\|*""".toRegex() + + // use find() instead of findAll() because a PPL query only ever has one source statement + // the only capture group specified in the regex captures the comma separated list of indices/index patterns + val indices = indicesRegex.find(pplQuery)?.groupValues?.get(1)?.split(",")?.map { it.trim() } + ?: throw IllegalStateException( + "Could not find indices that PPL Monitor query searches even " + + "after validating the query through SQL/PPL plugin" + ) + + return indices + } +} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportSearchMonitorV2Action.kt b/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportSearchMonitorV2Action.kt new file mode 100644 index 000000000..3f164932c --- /dev/null +++ b/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportSearchMonitorV2Action.kt @@ -0,0 +1,86 @@ +package org.opensearch.alerting.transport + +import org.apache.logging.log4j.LogManager +import org.opensearch.action.search.SearchResponse +import org.opensearch.action.support.ActionFilters +import org.opensearch.action.support.HandledTransportAction +import org.opensearch.alerting.actionv2.SearchMonitorV2Action +import org.opensearch.alerting.actionv2.SearchMonitorV2Request +import org.opensearch.alerting.core.modelv2.MonitorV2.Companion.MONITOR_V2_TYPE +import org.opensearch.alerting.opensearchapi.addFilter +import org.opensearch.alerting.settings.AlertingSettings +import org.opensearch.cluster.service.ClusterService +import org.opensearch.common.inject.Inject +import org.opensearch.common.settings.Settings +import org.opensearch.commons.alerting.util.AlertingException +import org.opensearch.core.action.ActionListener +import org.opensearch.core.common.io.stream.NamedWriteableRegistry +import org.opensearch.index.query.BoolQueryBuilder +import org.opensearch.index.query.QueryBuilders +import org.opensearch.tasks.Task +import org.opensearch.transport.TransportService +import org.opensearch.transport.client.Client + +private val log = LogManager.getLogger(TransportSearchMonitorV2Action::class.java) + +class TransportSearchMonitorV2Action @Inject constructor( + transportService: TransportService, + val settings: Settings, + val client: Client, + clusterService: ClusterService, + actionFilters: ActionFilters, + val namedWriteableRegistry: NamedWriteableRegistry +) : HandledTransportAction( + SearchMonitorV2Action.NAME, transportService, actionFilters, ::SearchMonitorV2Request +), + SecureTransportAction { + + @Volatile + override var filterByEnabled: Boolean = AlertingSettings.FILTER_BY_BACKEND_ROLES.get(settings) + + init { + listenFilterBySettingChange(clusterService) + } + + override fun doExecute(task: Task, request: SearchMonitorV2Request, actionListener: ActionListener) { + + // TODO: if alerting-config index doesnt exist, OS error is thrown to customer saying that much, try to catch that and + // throw more explicit error like "no monitorV2 exists" + + val searchSourceBuilder = request.searchRequest.source() + + val queryBuilder = if (searchSourceBuilder.query() == null) BoolQueryBuilder() + else QueryBuilders.boolQuery().must(searchSourceBuilder.query()) + + // filter out MonitorV1s in the alerting config index + // only return MonitorV2s that match the user-given search query + queryBuilder.filter(QueryBuilders.existsQuery(MONITOR_V2_TYPE)) + + searchSourceBuilder.query(queryBuilder) + .seqNoAndPrimaryTerm(true) + .version(true) + + val user = readUserFromThreadContext(client) + client.threadPool().threadContext.stashContext().use { + // if user is null, security plugin is disabled or user is super-admin + // if doFilterForUser() is false, security is enabled but filterby is disabled + if (user != null && doFilterForUser(user)) { + log.info("Filtering result by: ${user.backendRoles}") + addFilter(user, request.searchRequest.source(), "monitor_v2.ppl_monitor.user.backend_roles.keyword") + } + + client.search( + request.searchRequest, + object : ActionListener { + override fun onResponse(response: SearchResponse) { + actionListener.onResponse(response) + } + + override fun onFailure(t: Exception) { + actionListener.onFailure(AlertingException.wrap(t)) + } + } + ) + } + } +} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/IndexUtils.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/IndexUtils.kt index 093b0bd39..b388ae757 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/IndexUtils.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/IndexUtils.kt @@ -6,9 +6,11 @@ package org.opensearch.alerting.util import org.opensearch.action.admin.indices.mapping.put.PutMappingRequest +import org.opensearch.action.index.IndexResponse import org.opensearch.action.support.IndicesOptions import org.opensearch.action.support.clustermanager.AcknowledgedResponse import org.opensearch.alerting.alerts.AlertIndices +import org.opensearch.alerting.alertsv2.AlertV2Indices import org.opensearch.alerting.comments.CommentsIndices import org.opensearch.alerting.core.ScheduledJobIndices import org.opensearch.cluster.ClusterState @@ -38,6 +40,8 @@ class IndexUtils { private set var alertingCommentIndexSchemaVersion: Int private set + var alertV2IndexSchemaVersion: Int + private set var scheduledJobIndexUpdated: Boolean = false private set @@ -47,15 +51,20 @@ class IndexUtils { private set var commentsIndexUpdated: Boolean = false private set + var alertV2IndexUpdated: Boolean = false + private set + var lastUpdatedAlertHistoryIndex: String? = null var lastUpdatedFindingHistoryIndex: String? = null var lastUpdatedCommentsHistoryIndex: String? = null + var lastUpdatedAlertV2HistoryIndex: String? = null init { scheduledJobIndexSchemaVersion = getSchemaVersion(ScheduledJobIndices.scheduledJobMappings()) alertIndexSchemaVersion = getSchemaVersion(AlertIndices.alertMapping()) findingIndexSchemaVersion = getSchemaVersion(AlertIndices.findingMapping()) alertingCommentIndexSchemaVersion = getSchemaVersion(CommentsIndices.commentsMapping()) + alertV2IndexSchemaVersion = getSchemaVersion(AlertV2Indices.alertV2Mapping()) } @JvmStatic @@ -78,6 +87,11 @@ class IndexUtils { commentsIndexUpdated = true } + @JvmStatic + fun alertV2IndexUpdated() { + commentsIndexUpdated = true + } + @JvmStatic fun getSchemaVersion(mapping: String): Int { val xcp = XContentType.JSON.xContent().createParser( @@ -205,5 +219,18 @@ class IndexUtils { fun getCreationDateForIndex(index: String, clusterState: ClusterState): Long { return clusterState.metadata.index(index).creationDate } + + @JvmStatic + fun checkShardsFailure(response: IndexResponse): String? { + val failureReasons = StringBuilder() + if (response.shardInfo.failed > 0) { + response.shardInfo.failures.forEach { + entry -> + failureReasons.append(entry.reason()) + } + return failureReasons.toString() + } + return null + } } } diff --git a/alerting/src/main/resources/org/opensearch/alerting/alertsv2/alert_v2_mapping.json b/alerting/src/main/resources/org/opensearch/alerting/alertsv2/alert_v2_mapping.json new file mode 100644 index 000000000..b658dd579 --- /dev/null +++ b/alerting/src/main/resources/org/opensearch/alerting/alertsv2/alert_v2_mapping.json @@ -0,0 +1,102 @@ +{ + "dynamic": "strict", + "_routing": { + "required": false + }, + "_meta" : { + "schema_version": 5 + }, + "properties": { + "schema_version": { + "type": "integer" + }, + "monitor_id": { + "type": "keyword" + }, + "monitor_version": { + "type": "long" + }, + "id": { + "type": "keyword" + }, + "version": { + "type": "long" + }, + "severity": { + "type": "keyword" + }, + "monitor_name": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "monitor_user": { + "properties": { + "name": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "backend_roles": { + "type" : "text", + "fields" : { + "keyword" : { + "type" : "keyword" + } + } + }, + "roles": { + "type" : "text", + "fields" : { + "keyword" : { + "type" : "keyword" + } + } + }, + "custom_attribute_names": { + "type" : "text", + "fields" : { + "keyword" : { + "type" : "keyword" + } + } + } + } + }, + "execution_id": { + "type": "keyword" + }, + "trigger_id": { + "type": "keyword" + }, + "trigger_name": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "triggered_time": { + "type": "date" + }, + "expiration_time": { + "type": "date" + }, + "error_message": { + "type": "text" + }, + "query": { + "type": "text" + } + } +} \ No newline at end of file diff --git a/core/build.gradle b/core/build.gradle index 25a943bee..31749234e 100644 --- a/core/build.gradle +++ b/core/build.gradle @@ -3,11 +3,45 @@ * SPDX-License-Identifier: Apache-2.0 */ +buildscript { + ext { + opensearch_group = "org.opensearch" + opensearch_version = System.getProperty("opensearch.version", "3.1.0-SNAPSHOT") + isSnapshot = "true" == System.getProperty("build.snapshot", "true") + buildVersionQualifier = System.getProperty("build.version_qualifier", "") + kotlin_version = System.getProperty("kotlin.version", "1.9.25") + version_tokens = opensearch_version.tokenize('-') + opensearch_build = version_tokens[0] + '.0' + if (buildVersionQualifier) { + opensearch_build += "-${buildVersionQualifier}" + } + if (isSnapshot) { + opensearch_build += "-SNAPSHOT" + } + } +} + apply plugin: 'java' apply plugin: 'opensearch.java-rest-test' apply plugin: 'org.jetbrains.kotlin.jvm' apply plugin: 'jacoco' +configurations { + zipArchive +} + +def sqlJarDirectory = "$buildDir/dependencies/opensearch-sql-plugin" + +task addJarsToClasspath(type: Copy) { + from(fileTree(dir: sqlJarDirectory)) { + include "opensearch-sql-${opensearch_build}.jar" + include "ppl-${opensearch_build}.jar" + include "protocol-${opensearch_build}.jar" + include "core-${opensearch_build}.jar" + } + into("$buildDir/classes") +} + dependencies { compileOnly "org.opensearch:opensearch:${opensearch_version}" implementation "org.jetbrains.kotlin:kotlin-stdlib:${kotlin_version}" @@ -17,8 +51,44 @@ dependencies { api "org.opensearch.client:opensearch-rest-client:${opensearch_version}" api "org.opensearch:common-utils:${common_utils_version}@jar" implementation 'commons-validator:commons-validator:1.7' + implementation 'org.json:json:20240303' + + api fileTree(dir: sqlJarDirectory, include: ["opensearch-sql-thin-${opensearch_build}.jar", "ppl-${opensearch_build}.jar", "protocol-${opensearch_build}.jar", "core-${opensearch_build}.jar"]) + + zipArchive group: 'org.opensearch.plugin', name:'opensearch-sql-plugin', version: "${opensearch_build}" testImplementation "org.opensearch.test:framework:${opensearch_version}" testImplementation "org.jetbrains.kotlin:kotlin-test:${kotlin_version}" testImplementation "org.jetbrains.kotlin:kotlin-test-junit:${kotlin_version}" } + +task extractSqlJar(type: Copy) { + mustRunAfter() + from(zipTree(configurations.zipArchive.find { it.name.startsWith("opensearch-sql-plugin") })) + into sqlJarDirectory +} + +task extractSqlClass(type: Copy, dependsOn: [extractSqlJar]) { + from zipTree("${sqlJarDirectory}/opensearch-sql-${opensearch_build}.jar") + into("$buildDir/opensearch-sql") + include 'org/opensearch/sql/**' +} + +task replaceSqlJar(type: Jar, dependsOn: [extractSqlClass]) { + from("$buildDir/opensearch-sql") + archiveFileName = "opensearch-sql-thin-${opensearch_build}.jar" + destinationDirectory = file(sqlJarDirectory) + doLast { + file("${sqlJarDirectory}/opensearch-sql-${opensearch_build}.jar").delete() + } +} + +tasks.addJarsToClasspath.dependsOn(replaceSqlJar) + +compileJava { + dependsOn addJarsToClasspath +} + +compileKotlin { + dependsOn addJarsToClasspath +} diff --git a/core/src/main/kotlin/org/opensearch/alerting/core/action/node/ScheduledJobsStatsRequest.kt b/core/src/main/kotlin/org/opensearch/alerting/core/action/node/ScheduledJobsStatsRequest.kt index 6a82e8204..79e36c3b0 100644 --- a/core/src/main/kotlin/org/opensearch/alerting/core/action/node/ScheduledJobsStatsRequest.kt +++ b/core/src/main/kotlin/org/opensearch/alerting/core/action/node/ScheduledJobsStatsRequest.kt @@ -17,18 +17,24 @@ import java.io.IOException class ScheduledJobsStatsRequest : BaseNodesRequest { var jobSchedulingMetrics: Boolean = true var jobsInfo: Boolean = true + var showAlertingV2ScheduledJobs: Boolean = false // show Alerting V2 scheduled jobs if true, Alerting V1 scheduled jobs if false constructor(si: StreamInput) : super(si) { jobSchedulingMetrics = si.readBoolean() jobsInfo = si.readBoolean() + showAlertingV2ScheduledJobs = si.readBoolean() + } + + constructor(nodeIds: Array, showAlertingV2ScheduledJobs: Boolean) : super(*nodeIds) { + this.showAlertingV2ScheduledJobs = showAlertingV2ScheduledJobs } - constructor(nodeIds: Array) : super(*nodeIds) @Throws(IOException::class) override fun writeTo(out: StreamOutput) { super.writeTo(out) out.writeBoolean(jobSchedulingMetrics) out.writeBoolean(jobsInfo) + out.writeBoolean(showAlertingV2ScheduledJobs) } fun all(): ScheduledJobsStatsRequest { diff --git a/core/src/main/kotlin/org/opensearch/alerting/core/action/node/ScheduledJobsStatsTransportAction.kt b/core/src/main/kotlin/org/opensearch/alerting/core/action/node/ScheduledJobsStatsTransportAction.kt index f2ed94623..42343e296 100644 --- a/core/src/main/kotlin/org/opensearch/alerting/core/action/node/ScheduledJobsStatsTransportAction.kt +++ b/core/src/main/kotlin/org/opensearch/alerting/core/action/node/ScheduledJobsStatsTransportAction.kt @@ -93,7 +93,12 @@ class ScheduledJobsStatsTransportAction : TransportNodesAction { + return mapOf( + ALERT_ID_FIELD to id, + ALERT_VERSION_FIELD to version, + ERROR_MESSAGE_FIELD to errorMessage, + EXECUTION_ID_FIELD to executionId, + EXPIRATION_TIME_FIELD to expirationTime?.toEpochMilli(), + SEVERITY_FIELD to severity + ) + } + + companion object { + const val TRIGGERED_TIME_FIELD = "triggered_time" + const val EXPIRATION_TIME_FIELD = "expiration_time" + const val QUERY_FIELD = "query" + + @JvmStatic + @JvmOverloads + @Throws(IOException::class) + fun parse(xcp: XContentParser, id: String = NO_ID, version: Long = NO_VERSION): AlertV2 { + var schemaVersion = NO_SCHEMA_VERSION + lateinit var monitorId: String + lateinit var monitorName: String + var monitorVersion: Long = Versions.NOT_FOUND + var monitorUser: User? = null + lateinit var triggerId: String + lateinit var triggerName: String + lateinit var query: String + lateinit var severity: Severity + var triggeredTime: Instant? = null + var expirationTime: Instant? = null + var errorMessage: String? = null + var executionId: String? = null + + ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.currentToken(), xcp) + while (xcp.nextToken() != XContentParser.Token.END_OBJECT) { + val fieldName = xcp.currentName() + xcp.nextToken() + + when (fieldName) { + MONITOR_ID_FIELD -> monitorId = xcp.text() + SCHEMA_VERSION_FIELD -> schemaVersion = xcp.intValue() + MONITOR_NAME_FIELD -> monitorName = xcp.text() + MONITOR_VERSION_FIELD -> monitorVersion = xcp.longValue() + MONITOR_USER_FIELD -> + monitorUser = if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { + null + } else { + User.parse(xcp) + } + TRIGGER_ID_FIELD -> triggerId = xcp.text() + TRIGGER_NAME_FIELD -> triggerName = xcp.text() + QUERY_FIELD -> query = xcp.text() + TRIGGERED_TIME_FIELD -> triggeredTime = xcp.instant() + EXPIRATION_TIME_FIELD -> expirationTime = xcp.instant() + ERROR_MESSAGE_FIELD -> errorMessage = xcp.textOrNull() + EXECUTION_ID_FIELD -> executionId = xcp.textOrNull() + TriggerV2.SEVERITY_FIELD -> { + val input = xcp.text() + val enumMatchResult = Severity.enumFromString(input) + ?: throw IllegalStateException( + "Invalid value for ${TriggerV2.SEVERITY_FIELD}: $input. " + + "Supported values are ${Severity.entries.map { it.value }}" + ) + severity = enumMatchResult + } + } + } + + return AlertV2( + id = id, + version = version, + schemaVersion = schemaVersion, + monitorId = requireNotNull(monitorId), + monitorName = requireNotNull(monitorName), + monitorVersion = monitorVersion, + monitorUser = monitorUser, + triggerId = requireNotNull(triggerId), + triggerName = requireNotNull(triggerName), + query = requireNotNull(query), + triggeredTime = requireNotNull(triggeredTime), + expirationTime = requireNotNull(expirationTime), + errorMessage = errorMessage, + severity = severity, + executionId = executionId + ) + } + + @JvmStatic + @Throws(IOException::class) + fun readFrom(sin: StreamInput): AlertV2 { + return AlertV2(sin) + } + } +} diff --git a/core/src/main/kotlin/org/opensearch/alerting/core/modelv2/MonitorV2.kt b/core/src/main/kotlin/org/opensearch/alerting/core/modelv2/MonitorV2.kt new file mode 100644 index 000000000..eaa3bfcce --- /dev/null +++ b/core/src/main/kotlin/org/opensearch/alerting/core/modelv2/MonitorV2.kt @@ -0,0 +1,111 @@ +package org.opensearch.alerting.core.modelv2 + +import org.opensearch.alerting.core.modelv2.PPLMonitor.Companion.PPL_MONITOR_TYPE +import org.opensearch.common.CheckedFunction +import org.opensearch.common.unit.TimeValue +import org.opensearch.commons.alerting.model.Schedule +import org.opensearch.commons.alerting.model.ScheduledJob +import org.opensearch.commons.authuser.User +import org.opensearch.core.ParseField +import org.opensearch.core.common.io.stream.StreamInput +import org.opensearch.core.common.io.stream.StreamOutput +import org.opensearch.core.xcontent.NamedXContentRegistry +import org.opensearch.core.xcontent.XContentParser +import org.opensearch.core.xcontent.XContentParserUtils +import java.io.IOException +import java.time.Instant + +interface MonitorV2 : ScheduledJob { + override val id: String + override val version: Long + override val name: String + override val enabled: Boolean + override val schedule: Schedule + override val lastUpdateTime: Instant // required for scheduled job maintenance + override val enabledTime: Instant? // required for scheduled job maintenance + val user: User? + val triggers: List + val schemaVersion: Int // for updating monitors + val lookBackWindow: TimeValue // how far back to look when querying data during monitor execution + + fun asTemplateArg(): Map + + enum class MonitorV2Type(val value: String) { + PPL_MONITOR(PPL_MONITOR_TYPE); + + override fun toString(): String { + return value + } + + companion object { + fun enumFromString(value: String): MonitorV2Type? { + return MonitorV2Type.entries.find { it.value == value } + } + } + } + + companion object { + // scheduled job field names + const val MONITOR_V2_TYPE = "monitor_v2" // scheduled job type is MonitorV2 + + // field names + const val NAME_FIELD = "name" + const val ENABLED_FIELD = "enabled" + const val SCHEDULE_FIELD = "schedule" + const val LAST_UPDATE_TIME_FIELD = "last_update_time" + const val ENABLED_TIME_FIELD = "enabled_time" + const val USER_FIELD = "user" + const val TRIGGERS_FIELD = "triggers" + const val LOOK_BACK_WINDOW_FIELD = "look_back_window" + const val SCHEMA_VERSION_FIELD = "schema_version" + + // default values + const val NO_ID = "" + const val NO_VERSION = 1L + + val XCONTENT_REGISTRY = NamedXContentRegistry.Entry( + ScheduledJob::class.java, + ParseField(MONITOR_V2_TYPE), + CheckedFunction { parse(it) } + ) + + @JvmStatic + @Throws(IOException::class) + fun parse(xcp: XContentParser): MonitorV2 { + /* parse outer object for monitorV2 type, then delegate to correct monitorV2 parser */ + + XContentParserUtils.ensureExpectedToken( // outer monitor object start + XContentParser.Token.START_OBJECT, + xcp.currentToken(), + xcp + ) + + XContentParserUtils.ensureExpectedToken(XContentParser.Token.FIELD_NAME, xcp.nextToken(), xcp) // monitor type field name + val monitorTypeText = xcp.currentName() + val monitorType = MonitorV2Type.enumFromString(monitorTypeText) + ?: throw IllegalStateException("when parsing MonitorV2, received invalid monitor type: $monitorTypeText") + + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.nextToken(), xcp) // inner monitor object start + + return when (monitorType) { + MonitorV2Type.PPL_MONITOR -> PPLMonitor.parse(xcp) + } + } + + fun readFrom(sin: StreamInput): MonitorV2 { + return when (val monitorType = sin.readEnum(MonitorV2Type::class.java)) { + MonitorV2Type.PPL_MONITOR -> PPLMonitor(sin) + else -> throw IllegalStateException("Unexpected input \"$monitorType\" when reading MonitorV2") + } + } + + fun writeTo(out: StreamOutput, monitorV2: MonitorV2) { + when (monitorV2) { + is PPLMonitor -> { + out.writeEnum(MonitorV2Type.PPL_MONITOR) + monitorV2.writeTo(out) + } + } + } + } +} diff --git a/core/src/main/kotlin/org/opensearch/alerting/core/modelv2/MonitorV2RunResult.kt b/core/src/main/kotlin/org/opensearch/alerting/core/modelv2/MonitorV2RunResult.kt new file mode 100644 index 000000000..0db8b9ce8 --- /dev/null +++ b/core/src/main/kotlin/org/opensearch/alerting/core/modelv2/MonitorV2RunResult.kt @@ -0,0 +1,44 @@ +package org.opensearch.alerting.core.modelv2 + +import org.opensearch.core.common.io.stream.StreamInput +import org.opensearch.core.common.io.stream.StreamOutput +import org.opensearch.core.common.io.stream.Writeable +import org.opensearch.core.xcontent.ToXContent +import java.time.Instant + +interface MonitorV2RunResult : Writeable, ToXContent { + val monitorName: String + val error: Exception? + val periodStart: Instant + val periodEnd: Instant + val triggerResults: Map + + enum class MonitorV2RunResultType() { + PPL_MONITOR_RUN_RESULT; + } + + companion object { + const val MONITOR_V2_NAME_FIELD = "monitor_v2_name" + const val ERROR_FIELD = "error" + const val PERIOD_START_FIELD = "period_start" + const val PERIOD_END_FIELD = "period_end" + const val TRIGGER_RESULTS_FIELD = "trigger_results" + + fun readFrom(sin: StreamInput): MonitorV2RunResult { + val monitorRunResultType = sin.readEnum(MonitorV2RunResultType::class.java) + return when (monitorRunResultType) { + MonitorV2RunResultType.PPL_MONITOR_RUN_RESULT -> PPLMonitorRunResult(sin) + else -> throw IllegalStateException("Unexpected input [$monitorRunResultType] when reading MonitorV2RunResult") + } + } + + fun writeTo(out: StreamOutput, monitorV2RunResult: MonitorV2RunResult) { + when (monitorV2RunResult) { + is PPLMonitorRunResult -> { + out.writeEnum(MonitorV2RunResultType.PPL_MONITOR_RUN_RESULT) + monitorV2RunResult.writeTo(out) + } + } + } + } +} diff --git a/core/src/main/kotlin/org/opensearch/alerting/core/modelv2/PPLMonitor.kt b/core/src/main/kotlin/org/opensearch/alerting/core/modelv2/PPLMonitor.kt new file mode 100644 index 000000000..3e3eeded5 --- /dev/null +++ b/core/src/main/kotlin/org/opensearch/alerting/core/modelv2/PPLMonitor.kt @@ -0,0 +1,360 @@ +package org.opensearch.alerting.core.modelv2 + +import org.opensearch.alerting.core.modelv2.MonitorV2.Companion.ENABLED_FIELD +import org.opensearch.alerting.core.modelv2.MonitorV2.Companion.ENABLED_TIME_FIELD +import org.opensearch.alerting.core.modelv2.MonitorV2.Companion.LAST_UPDATE_TIME_FIELD +import org.opensearch.alerting.core.modelv2.MonitorV2.Companion.LOOK_BACK_WINDOW_FIELD +import org.opensearch.alerting.core.modelv2.MonitorV2.Companion.NAME_FIELD +import org.opensearch.alerting.core.modelv2.MonitorV2.Companion.NO_ID +import org.opensearch.alerting.core.modelv2.MonitorV2.Companion.NO_VERSION +import org.opensearch.alerting.core.modelv2.MonitorV2.Companion.SCHEDULE_FIELD +import org.opensearch.alerting.core.modelv2.MonitorV2.Companion.SCHEMA_VERSION_FIELD +import org.opensearch.alerting.core.modelv2.MonitorV2.Companion.TRIGGERS_FIELD +import org.opensearch.alerting.core.modelv2.MonitorV2.Companion.USER_FIELD +import org.opensearch.alerting.core.util.nonOptionalTimeField +import org.opensearch.common.unit.TimeValue +import org.opensearch.commons.alerting.model.CronSchedule +import org.opensearch.commons.alerting.model.Schedule +import org.opensearch.commons.alerting.util.IndexUtils +import org.opensearch.commons.alerting.util.instant +import org.opensearch.commons.alerting.util.optionalTimeField +import org.opensearch.commons.alerting.util.optionalUserField +import org.opensearch.commons.authuser.User +import org.opensearch.core.common.io.stream.StreamInput +import org.opensearch.core.common.io.stream.StreamOutput +import org.opensearch.core.xcontent.ToXContent +import org.opensearch.core.xcontent.XContentBuilder +import org.opensearch.core.xcontent.XContentParser +import org.opensearch.core.xcontent.XContentParserUtils +import java.io.IOException +import java.time.Instant + +// TODO: probably change this to be called PPLSQLMonitor. A PPL Monitor and SQL Monitor +// would have the exact same functionality, except the choice of language +// when calling PPL/SQL plugin's execute API would be different. +// we dont need 2 different monitor types for that, just a simple if check +// for query language at monitor execution time +/** + * PPL (Piped Processing Language) Monitor for OpenSearch Alerting V2 + * + * @property id Monitor ID. Defaults to [NO_ID]. + * @property version Version number of the monitor. Defaults to [NO_VERSION]. + * @property name Display name of the monitor. + * @property enabled Boolean flag indicating whether the monitor is currently on or off. + * @property schedule Defines when and how often the monitor should run. Can be a CRON or interval schedule. + * @property lookBackWindow How far back each Monitor execution's query should look back when searching data. + * Only applicable if Monitor uses CRON schedule. Optional even if CRON schedule is used. + * @property lastUpdateTime Timestamp of the last update to this monitor. + * @property enabledTime Timestamp when the monitor was last enabled. Null if never enabled. + * @property triggers List of [PPLTrigger]s associated with this monitor. + * @property schemaVersion Version of the alerting-config index schema used when this Monitor was indexed. Defaults to [NO_SCHEMA_VERSION]. + * @property queryLanguage The query language used. Defaults to [QueryLanguage.PPL]. + * @property query The PPL query string to be executed by this monitor. + */ +data class PPLMonitor( + override val id: String = NO_ID, + override val version: Long = NO_VERSION, + override val name: String, + override val enabled: Boolean, + override val schedule: Schedule, + override val lookBackWindow: TimeValue, + override val lastUpdateTime: Instant, + override val enabledTime: Instant?, + override val user: User?, + override val triggers: List, + override val schemaVersion: Int = IndexUtils.NO_SCHEMA_VERSION, + val queryLanguage: QueryLanguage = QueryLanguage.PPL, // default to PPL, SQL not currently supported + val query: String +) : MonitorV2 { + + // specify scheduled job type + override val type = MonitorV2.MONITOR_V2_TYPE + + override fun fromDocument(id: String, version: Long): PPLMonitor = copy(id = id, version = version) + + init { + // SQL monitors are not yet supported + if (this.queryLanguage == QueryLanguage.SQL) { + throw IllegalStateException("Monitors with SQL queries are not supported") + } + + // for checking trigger ID uniqueness + val triggerIds = mutableSetOf() + triggers.forEach { trigger -> + require(triggerIds.add(trigger.id)) { "Duplicate trigger id: ${trigger.id}. Trigger ids must be unique." } + } + + if (enabled) { + requireNotNull(enabledTime) + } else { + require(enabledTime == null) + } + + // TODO: create setting for max triggers and check for max triggers here + } + + @Throws(IOException::class) + constructor(sin: StreamInput) : this( + id = sin.readString(), + version = sin.readLong(), + name = sin.readString(), + enabled = sin.readBoolean(), + schedule = Schedule.readFrom(sin), + lookBackWindow = TimeValue.parseTimeValue(sin.readString(), PLACEHOLDER_LOOK_BACK_WINDOW_SETTING_NAME), + lastUpdateTime = sin.readInstant(), + enabledTime = sin.readOptionalInstant(), + user = if (sin.readBoolean()) { + User(sin) + } else { + null + }, + triggers = sin.readList(PPLTrigger::readFrom), + schemaVersion = sin.readInt(), + queryLanguage = sin.readEnum(QueryLanguage::class.java), + query = sin.readString() + ) + + fun toXContentWithUser(builder: XContentBuilder, params: ToXContent.Params): XContentBuilder { + return createXContentBuilder(builder, params, true) + } + + override fun toXContent(builder: XContentBuilder, params: ToXContent.Params): XContentBuilder { + return createXContentBuilder(builder, params, false) + } + + private fun createXContentBuilder(builder: XContentBuilder, params: ToXContent.Params, withUser: Boolean): XContentBuilder { + builder.startObject() // overall start object + + // if this is being written as ScheduledJob, add extra object layer and add ScheduledJob + // related metadata, default to false + if (params.paramAsBoolean("with_type", false)) { + builder.startObject(MonitorV2.MONITOR_V2_TYPE) + } + + // wrap PPLMonitor in outer object named after its monitor type + // required for MonitorV2 XContentParser to first encounter this, + // read in monitor type, then delegate to correct parse() function + builder.startObject(PPL_MONITOR_TYPE) // monitor type start object + + builder.field(NAME_FIELD, name) + builder.field(SCHEDULE_FIELD, schedule) + builder.field(LOOK_BACK_WINDOW_FIELD, lookBackWindow?.toHumanReadableString(0)) + builder.field(ENABLED_FIELD, enabled) + builder.nonOptionalTimeField(LAST_UPDATE_TIME_FIELD, lastUpdateTime) + builder.optionalTimeField(ENABLED_TIME_FIELD, enabledTime) + builder.field(TRIGGERS_FIELD, triggers.toTypedArray()) + builder.field(SCHEMA_VERSION_FIELD, schemaVersion) + builder.field(QUERY_LANGUAGE_FIELD, queryLanguage.value) + builder.field(QUERY_FIELD, query) + + if (withUser) { + builder.optionalUserField(USER_FIELD, user) + } + + builder.endObject() // monitor type end object + + // if ScheduledJob metadata was added, end the extra object layer that was created + if (params.paramAsBoolean("with_type", false)) { + builder.endObject() + } + + builder.endObject() // overall end object + + return builder + } + + @Throws(IOException::class) + override fun writeTo(out: StreamOutput) { + out.writeString(id) + out.writeLong(version) + out.writeString(name) + out.writeBoolean(enabled) + if (schedule is CronSchedule) { + out.writeEnum(Schedule.TYPE.CRON) + } else { + out.writeEnum(Schedule.TYPE.INTERVAL) + } + + out.writeBoolean(lookBackWindow != null) + lookBackWindow?.let { out.writeString(lookBackWindow.toHumanReadableString(0)) } + + out.writeInstant(lastUpdateTime) + out.writeOptionalInstant(enabledTime) + + out.writeBoolean(user != null) + user?.writeTo(out) + + out.writeVInt(triggers.size) + triggers.forEach { it.writeTo(out) } + out.writeInt(schemaVersion) + out.writeEnum(queryLanguage) + out.writeString(query) + } + + override fun asTemplateArg(): Map { + return mapOf( + IndexUtils._ID to id, + IndexUtils._VERSION to version, + NAME_FIELD to name, + ENABLED_FIELD to enabled, + SCHEDULE_FIELD to schedule, + LOOK_BACK_WINDOW_FIELD to lookBackWindow?.toHumanReadableString(0), + LAST_UPDATE_TIME_FIELD to lastUpdateTime.toEpochMilli(), + ENABLED_TIME_FIELD to enabledTime?.toEpochMilli(), + TRIGGERS_FIELD to triggers, + QUERY_LANGUAGE_FIELD to queryLanguage.value, + QUERY_FIELD to query + ) + } + + enum class QueryLanguage(val value: String) { + PPL(PPL_QUERY_LANGUAGE), + SQL(SQL_QUERY_LANGUAGE); + + companion object { + fun enumFromString(value: String): QueryLanguage? = QueryLanguage.entries.firstOrNull { it.value == value } + } + } + + companion object { + // monitor type name + const val PPL_MONITOR_TYPE = "ppl_monitor" // TODO: eventually change to SQL_PPL_MONITOR_TYPE + + // query languages + const val PPL_QUERY_LANGUAGE = "ppl" + const val SQL_QUERY_LANGUAGE = "sql" + + // field names + const val QUERY_LANGUAGE_FIELD = "query_language" + const val QUERY_FIELD = "query" + + // mock setting name used when parsing TimeValue + // TimeValue class is usually reserved for declaring settings, but we're using it + // outside that use case here, which is why we need these placeholders + private const val PLACEHOLDER_LOOK_BACK_WINDOW_SETTING_NAME = "ppl_monitor_look_back_window" + + @JvmStatic + @JvmOverloads + @Throws(IOException::class) + fun parse(xcp: XContentParser, id: String = NO_ID, version: Long = NO_VERSION): PPLMonitor { + var name: String? = null + var enabled = true + var schedule: Schedule? = null + var lookBackWindow: TimeValue? = null // TODO: default value + var lastUpdateTime: Instant? = null + var enabledTime: Instant? = null + var user: User? = null + val triggers: MutableList = mutableListOf() + var schemaVersion = IndexUtils.NO_SCHEMA_VERSION + var queryLanguage: QueryLanguage = QueryLanguage.PPL // default to PPL + var query: String? = null + + /* parse */ + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.currentToken(), xcp) + while (xcp.nextToken() != XContentParser.Token.END_OBJECT) { + val fieldName = xcp.currentName() + xcp.nextToken() + + when (fieldName) { + NAME_FIELD -> name = xcp.text() + ENABLED_FIELD -> enabled = xcp.booleanValue() + SCHEDULE_FIELD -> schedule = Schedule.parse(xcp) + LOOK_BACK_WINDOW_FIELD -> { + lookBackWindow = if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { + null + } else { + val input = xcp.text() + // throws IllegalArgumentException if there's parsing error + TimeValue.parseTimeValue(input, PLACEHOLDER_LOOK_BACK_WINDOW_SETTING_NAME) + } + } + LAST_UPDATE_TIME_FIELD -> lastUpdateTime = xcp.instant() + ENABLED_TIME_FIELD -> enabledTime = xcp.instant() + USER_FIELD -> user = if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) null else User.parse(xcp) + TRIGGERS_FIELD -> { + XContentParserUtils.ensureExpectedToken( + XContentParser.Token.START_ARRAY, + xcp.currentToken(), + xcp + ) + while (xcp.nextToken() != XContentParser.Token.END_ARRAY) { + triggers.add(PPLTrigger.parseInner(xcp)) + } + } + SCHEMA_VERSION_FIELD -> schemaVersion = xcp.intValue() + QUERY_LANGUAGE_FIELD -> { + val input = xcp.text() + val enumMatchResult = QueryLanguage.enumFromString(input) + ?: throw IllegalArgumentException( + "Invalid value for $QUERY_LANGUAGE_FIELD: $input. " + + "Supported values are ${QueryLanguage.entries.map { it.value }}" + ) + queryLanguage = enumMatchResult + } + QUERY_FIELD -> query = xcp.text() + else -> throw IllegalArgumentException("Unexpected field when parsing PPL Monitor: $fieldName") + } + } + + /* validations */ + + // ensure there's at least 1 trigger + if (triggers.isEmpty()) { + throw IllegalArgumentException("Monitor must include at least 1 trigger") + } + + // ensure the trigger suppress durations are valid + triggers.forEach { trigger -> + trigger.suppressDuration?.let { suppressDuration -> + // TODO: these max and min values are completely arbitrary, make them settings + val minValue = TimeValue.timeValueMinutes(1) + val maxValue = TimeValue.timeValueDays(5) + + require(suppressDuration <= maxValue) { "Suppress duration must be at most $maxValue but was $suppressDuration" } + + require(suppressDuration >= minValue) { "Suppress duration must be at least $minValue but was $suppressDuration" } + } + } + + // if no lookback window was given, set a default one + lookBackWindow = lookBackWindow ?: TimeValue.timeValueHours(1L) + + // if enabled, set time of MonitorV2 creation/update is set as enable time + if (enabled && enabledTime == null) { + enabledTime = Instant.now() + } else if (!enabled) { + enabledTime = null + } + + lastUpdateTime = lastUpdateTime ?: Instant.now() + + // check for required fields + requireNotNull(name) { "Monitor name is null" } + requireNotNull(schedule) { "Schedule is null" } + requireNotNull(query) { "Query is null" } + requireNotNull(lastUpdateTime) { "Last update time is null" } + requireNotNull(lookBackWindow) { "Look back window is null" } + + if (queryLanguage == QueryLanguage.SQL) { + throw IllegalArgumentException("SQL queries are not supported. Please use a PPL query.") + } + + /* return PPLMonitor */ + return PPLMonitor( + id, + version, + name, + enabled, + schedule, + lookBackWindow, + lastUpdateTime, + enabledTime, + user, + triggers, + schemaVersion, + queryLanguage, + query + ) + } + } +} diff --git a/core/src/main/kotlin/org/opensearch/alerting/core/modelv2/PPLMonitorRunResult.kt b/core/src/main/kotlin/org/opensearch/alerting/core/modelv2/PPLMonitorRunResult.kt new file mode 100644 index 000000000..135186abb --- /dev/null +++ b/core/src/main/kotlin/org/opensearch/alerting/core/modelv2/PPLMonitorRunResult.kt @@ -0,0 +1,61 @@ +package org.opensearch.alerting.core.modelv2 + +import org.opensearch.alerting.core.modelv2.MonitorV2RunResult.Companion.ERROR_FIELD +import org.opensearch.alerting.core.modelv2.MonitorV2RunResult.Companion.MONITOR_V2_NAME_FIELD +import org.opensearch.alerting.core.modelv2.MonitorV2RunResult.Companion.PERIOD_END_FIELD +import org.opensearch.alerting.core.modelv2.MonitorV2RunResult.Companion.PERIOD_START_FIELD +import org.opensearch.alerting.core.modelv2.MonitorV2RunResult.Companion.TRIGGER_RESULTS_FIELD +import org.opensearch.alerting.core.util.nonOptionalTimeField +import org.opensearch.core.common.io.stream.StreamInput +import org.opensearch.core.common.io.stream.StreamOutput +import org.opensearch.core.xcontent.ToXContent +import org.opensearch.core.xcontent.XContentBuilder +import java.io.IOException +import java.time.Instant + +data class PPLMonitorRunResult( + override val monitorName: String, + override val error: Exception?, + override val periodStart: Instant, + override val periodEnd: Instant, + override val triggerResults: Map, + val pplQueryResults: Map> // key: trigger id, value: query results +) : MonitorV2RunResult { + + @Throws(IOException::class) + @Suppress("UNCHECKED_CAST") + constructor(sin: StreamInput) : this( + sin.readString(), // monitorName + sin.readException(), // error + sin.readInstant(), // periodStart + sin.readInstant(), // periodEnd + sin.readMap() as Map, // triggerResults + sin.readMap() as Map> // pplQueryResults + ) + + override fun toXContent(builder: XContentBuilder, params: ToXContent.Params): XContentBuilder { + builder.startObject() + builder.field(MONITOR_V2_NAME_FIELD, monitorName) + builder.nonOptionalTimeField(PERIOD_START_FIELD, periodStart) + builder.nonOptionalTimeField(PERIOD_END_FIELD, periodEnd) + builder.field(ERROR_FIELD, error?.message) + builder.field(TRIGGER_RESULTS_FIELD, triggerResults) + builder.field(PPL_QUERY_RESULTS_FIELD, pplQueryResults) + builder.endObject() + return builder + } + + @Throws(IOException::class) + override fun writeTo(out: StreamOutput) { + out.writeString(monitorName) + out.writeException(error) + out.writeInstant(periodStart) + out.writeInstant(periodEnd) + out.writeMap(triggerResults) + out.writeMap(pplQueryResults) + } + + companion object { + const val PPL_QUERY_RESULTS_FIELD = "ppl_query_results" + } +} diff --git a/core/src/main/kotlin/org/opensearch/alerting/core/modelv2/PPLTrigger.kt b/core/src/main/kotlin/org/opensearch/alerting/core/modelv2/PPLTrigger.kt new file mode 100644 index 000000000..655761870 --- /dev/null +++ b/core/src/main/kotlin/org/opensearch/alerting/core/modelv2/PPLTrigger.kt @@ -0,0 +1,392 @@ +package org.opensearch.alerting.core.modelv2 + +import org.opensearch.alerting.core.modelv2.TriggerV2.Companion.ACTIONS_FIELD +import org.opensearch.alerting.core.modelv2.TriggerV2.Companion.EXPIRE_FIELD +import org.opensearch.alerting.core.modelv2.TriggerV2.Companion.ID_FIELD +import org.opensearch.alerting.core.modelv2.TriggerV2.Companion.LAST_TRIGGERED_FIELD +import org.opensearch.alerting.core.modelv2.TriggerV2.Companion.NAME_FIELD +import org.opensearch.alerting.core.modelv2.TriggerV2.Companion.SEVERITY_FIELD +import org.opensearch.alerting.core.modelv2.TriggerV2.Companion.SUPPRESS_FIELD +import org.opensearch.alerting.core.modelv2.TriggerV2.Severity +import org.opensearch.common.CheckedFunction +import org.opensearch.common.UUIDs +import org.opensearch.common.unit.TimeValue +import org.opensearch.commons.alerting.model.action.Action +import org.opensearch.commons.alerting.util.AlertingException +import org.opensearch.commons.alerting.util.instant +import org.opensearch.commons.alerting.util.optionalTimeField +import org.opensearch.core.ParseField +import org.opensearch.core.common.io.stream.StreamInput +import org.opensearch.core.common.io.stream.StreamOutput +import org.opensearch.core.xcontent.NamedXContentRegistry +import org.opensearch.core.xcontent.ToXContent +import org.opensearch.core.xcontent.XContentBuilder +import org.opensearch.core.xcontent.XContentParser +import org.opensearch.core.xcontent.XContentParserUtils +import java.io.IOException +import java.time.Instant + +/** + * The PPL Trigger for PPL Monitors + * + * There are two types of PPLTrigger conditions: NUMBER_OF_RESULT and CUSTOM + * NUMBER_OF_RESULTS: triggers based on if the number of query results returned by the PPLMonitor + * query meets some threshold + * CUSTOM: triggers based on a custom condition that user specifies + * This trigger can operate in either result set or per-result mode and supports + * both numeric result conditions and custom conditions. + * + * PPLTriggers can run on two modes: RESULT_SET and PER_RESULT + * RESULT_SET: exactly one Alert is generated when the Trigger condition is met + * PER_RESULT: one Alert is generated per trigger condition-meeting query result row + * + * @property id Trigger ID, defaults to a base64 UUID. + * @property name Display name of the Trigger. + * @property severity The severity level of the Trigger. + * @property suppressDuration Optional duration for which alerts from this Trigger should be suppressed. + * Null indicates no suppression. + * @property expireDuration Duration after which alerts from this Trigger should be deleted permanently. + * @property lastTriggeredTime The last time this Trigger generated an Alert. Null if Trigger hasn't generated an Alert yet. + * @property actions List of notification-sending actions to run when the Trigger condition is met. + * @property mode Specifies whether the trigger evaluates the entire result set or each result individually. + * Can be either [TriggerMode.RESULT_SET] or [TriggerMode.PER_RESULT]. + * @property conditionType The type of condition to evaluate. + * Can be either [ConditionType.NUMBER_OF_RESULTS] or [ConditionType.CUSTOM]. + * @property numResultsCondition The comparison operator for NUMBER_OF_RESULTS conditions. Required if using NUMBER_OF_RESULTS conditions, + * null otherwise. + * @property numResultsValue The threshold value for NUMBER_OF_RESULTS conditions. Required if using NUMBER_OF_RESULTS conditions, + * null otherwise. + * @property customCondition A custom condition expression. Required if using CUSTOM conditions, + * null otherwise. + */ +data class PPLTrigger( + override val id: String = UUIDs.base64UUID(), + override val name: String, + override val severity: Severity, + override val suppressDuration: TimeValue?, + override val expireDuration: TimeValue, + override var lastTriggeredTime: Instant?, + override val actions: List, + val mode: TriggerMode, // result_set or per_result + val conditionType: ConditionType, + val numResultsCondition: NumResultsCondition?, + val numResultsValue: Long?, + val customCondition: String? +) : TriggerV2 { + + @Throws(IOException::class) + constructor(sin: StreamInput) : this( + sin.readString(), // id + sin.readString(), // name + sin.readEnum(Severity::class.java), // severity + // parseTimeValue() is typically used to parse OpenSearch settings + // the second param is supposed to accept a setting name, but here we're passing in our own name + TimeValue.parseTimeValue(sin.readOptionalString(), PLACEHOLDER_SUPPRESS_SETTING_NAME), // suppressDuration + TimeValue.parseTimeValue(sin.readString(), PLACEHOLDER_EXPIRE_SETTING_NAME), // expireDuration + sin.readOptionalInstant(), // lastTriggeredTime + sin.readList(::Action), // actions + sin.readEnum(TriggerMode::class.java), // trigger mode + sin.readEnum(ConditionType::class.java), // condition type + if (sin.readBoolean()) sin.readEnum(NumResultsCondition::class.java) else null, // num results condition + sin.readOptionalLong(), // num results value + sin.readOptionalString() // custom condition + ) + + @Throws(IOException::class) + override fun writeTo(out: StreamOutput) { + out.writeString(id) + out.writeString(name) + out.writeEnum(severity) + + out.writeBoolean(suppressDuration != null) + suppressDuration?.let { out.writeString(suppressDuration.toHumanReadableString(0)) } + + out.writeString(expireDuration.toHumanReadableString(0)) + out.writeOptionalInstant(lastTriggeredTime) + out.writeCollection(actions) + out.writeEnum(mode) + out.writeEnum(conditionType) + + out.writeBoolean(numResultsCondition != null) + numResultsCondition?.let { out.writeEnum(numResultsCondition) } + + out.writeOptionalLong(numResultsValue) + out.writeOptionalString(customCondition) + } + + override fun toXContent(builder: XContentBuilder, params: ToXContent.Params?): XContentBuilder { + builder.startObject() + builder.field(ID_FIELD, id) + builder.field(NAME_FIELD, name) + builder.field(SEVERITY_FIELD, severity.value) + builder.field(SUPPRESS_FIELD, suppressDuration?.toHumanReadableString(0)) + builder.field(EXPIRE_FIELD, expireDuration.toHumanReadableString(0)) + builder.optionalTimeField(LAST_TRIGGERED_FIELD, lastTriggeredTime) + builder.field(ACTIONS_FIELD, actions.toTypedArray()) + builder.field(MODE_FIELD, mode.value) + builder.field(CONDITION_TYPE_FIELD, conditionType.value) + numResultsCondition?.let { builder.field(NUM_RESULTS_CONDITION_FIELD, numResultsCondition.value) } + numResultsValue?.let { builder.field(NUM_RESULTS_VALUE_FIELD, numResultsValue) } + customCondition?.let { builder.field(CUSTOM_CONDITION_FIELD, customCondition) } + builder.endObject() + return builder + } + + fun asTemplateArg(): Map { + return mapOf( + ID_FIELD to id, + NAME_FIELD to name, + SEVERITY_FIELD to severity.value, + SUPPRESS_FIELD to suppressDuration?.toHumanReadableString(0), + EXPIRE_FIELD to expireDuration?.toHumanReadableString(0), + ACTIONS_FIELD to actions.map { it.asTemplateArg() }, + MODE_FIELD to mode.value, + CONDITION_TYPE_FIELD to conditionType.value, + NUM_RESULTS_CONDITION_FIELD to numResultsCondition?.value, + NUM_RESULTS_VALUE_FIELD to numResultsValue, + CUSTOM_CONDITION_FIELD to customCondition + ) + } + + enum class TriggerMode(val value: String) { + RESULT_SET("result_set"), + PER_RESULT("per_result"); + + companion object { + fun enumFromString(value: String): TriggerMode? = entries.firstOrNull { it.value == value } + } + } + + enum class ConditionType(val value: String) { + NUMBER_OF_RESULTS("number_of_results"), + CUSTOM("custom"); + + companion object { + fun enumFromString(value: String): ConditionType? = entries.firstOrNull { it.value == value } + } + } + + enum class NumResultsCondition(val value: String) { + GREATER_THAN(">"), + GREATER_THAN_EQUAL(">="), + LESS_THAN("<"), + LESS_THAN_EQUAL("<="), + EQUAL("=="), + NOT_EQUAL("!="); + + companion object { + fun enumFromString(value: String): NumResultsCondition? = entries.firstOrNull { it.value == value } + } + } + + companion object { + // trigger wrapper object field name + const val PPL_TRIGGER_FIELD = "ppl_trigger" + + // field names + const val MODE_FIELD = "mode" + const val CONDITION_TYPE_FIELD = "type" + const val NUM_RESULTS_CONDITION_FIELD = "num_results_condition" + const val NUM_RESULTS_VALUE_FIELD = "num_results_value" + const val CUSTOM_CONDITION_FIELD = "custom_condition" + + // mock setting name used when parsing TimeValue + // TimeValue class is usually reserved for declaring settings, but we're using it + // outside that use case here, which is why we need these placeholders + private const val PLACEHOLDER_SUPPRESS_SETTING_NAME = "ppl_trigger_suppress_duration" + private const val PLACEHOLDER_EXPIRE_SETTING_NAME = "ppl_trigger_expire_duration" + + val XCONTENT_REGISTRY = NamedXContentRegistry.Entry( + TriggerV2::class.java, + ParseField(PPL_TRIGGER_FIELD), + CheckedFunction { parseInner(it) } + ) + + @JvmStatic + @Throws(IOException::class) + fun parseInner(xcp: XContentParser): PPLTrigger { + var id = UUIDs.base64UUID() // assign a default triggerId if one is not specified + var name: String? = null + var severity: Severity? = null + var suppressDuration: TimeValue? = null + var expireDuration: TimeValue = + TimeValue.timeValueDays(7) // default to 7 days // TODO: add this as a setting + var lastTriggeredTime: Instant? = null + val actions: MutableList = mutableListOf() + var mode: TriggerMode? = null + var conditionType: ConditionType? = null + var numResultsCondition: NumResultsCondition? = null + var numResultsValue: Long? = null + var customCondition: String? = null + + /* parse */ + XContentParserUtils.ensureExpectedToken( // outer trigger object start + XContentParser.Token.START_OBJECT, + xcp.currentToken(), xcp + ) + + while (xcp.nextToken() != XContentParser.Token.END_OBJECT) { + val fieldName = xcp.currentName() + xcp.nextToken() + + when (fieldName) { + ID_FIELD -> id = xcp.text() + NAME_FIELD -> name = xcp.text() + SEVERITY_FIELD -> { + val input = xcp.text() + val enumMatchResult = Severity.enumFromString(input) + ?: throw IllegalArgumentException( + "Invalid value for $SEVERITY_FIELD: $input. " + + "Supported values are ${Severity.entries.map { it.value }}" + ) + severity = enumMatchResult + } + MODE_FIELD -> { + val input = xcp.text() + val enumMatchResult = TriggerMode.enumFromString(input) + ?: throw IllegalArgumentException( + "Invalid value for $MODE_FIELD: $input. " + + "Supported values are ${TriggerMode.entries.map { it.value }}" + ) + mode = enumMatchResult + } + CONDITION_TYPE_FIELD -> { + val input = xcp.text() + val enumMatchResult = ConditionType.enumFromString(input) + ?: throw IllegalArgumentException( + "Invalid value for $CONDITION_TYPE_FIELD: $input. " + + "Supported values are ${ConditionType.entries.map { it.value }}" + ) + conditionType = enumMatchResult + } + NUM_RESULTS_CONDITION_FIELD -> { + numResultsCondition = if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { + null + } else { + val input = xcp.text() + val enumMatchResult = NumResultsCondition.enumFromString(input) + ?: throw IllegalArgumentException( + "Invalid value for $NUM_RESULTS_CONDITION_FIELD: $input. " + + "Supported values are ${NumResultsCondition.entries.map { it.value }}" + ) + enumMatchResult + } + } + NUM_RESULTS_VALUE_FIELD -> { + numResultsValue = if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { + null + } else { + xcp.longValue() + } + } + CUSTOM_CONDITION_FIELD -> { + customCondition = if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { + null + } else { + xcp.text() + } + } + SUPPRESS_FIELD -> { + suppressDuration = if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { + // if expire field is null, skip reading it and let it retain the default value + null + } else { + val input = xcp.text() + try { + TimeValue.parseTimeValue(input, PLACEHOLDER_SUPPRESS_SETTING_NAME) + } catch (e: Exception) { + throw AlertingException.wrap( + IllegalArgumentException("Invalid value for field: $SUPPRESS_FIELD", e) + ) + } + } + } + EXPIRE_FIELD -> { + // if expire field is null, skip reading it and let it retain the default value + if (xcp.currentToken() != XContentParser.Token.VALUE_NULL) { + val input = xcp.text() + try { + expireDuration = TimeValue.parseTimeValue(input, PLACEHOLDER_EXPIRE_SETTING_NAME) + } catch (e: Exception) { + throw AlertingException.wrap( + IllegalArgumentException("Invalid value for field: $EXPIRE_FIELD", e) + ) + } + } + } + LAST_TRIGGERED_FIELD -> lastTriggeredTime = xcp.instant() + ACTIONS_FIELD -> { + XContentParserUtils.ensureExpectedToken( + XContentParser.Token.START_ARRAY, + xcp.currentToken(), + xcp + ) + while (xcp.nextToken() != XContentParser.Token.END_ARRAY) { + actions.add(Action.parse(xcp)) + } + } + else -> throw IllegalArgumentException("Unexpected field when parsing PPL Trigger: $fieldName") + } + } + + /* validations */ + requireNotNull(name) { "Trigger name must be included" } + requireNotNull(severity) { "Trigger severity must be included" } + requireNotNull(mode) { "Trigger mode must be included" } + requireNotNull(conditionType) { "Trigger condition type must be included" } + + when (conditionType) { + ConditionType.NUMBER_OF_RESULTS -> { + requireNotNull(numResultsCondition) { + "if trigger condition is of type ${ConditionType.NUMBER_OF_RESULTS.value}," + + "$NUM_RESULTS_CONDITION_FIELD must be included" + } + requireNotNull(numResultsValue) { + "if trigger condition is of type ${ConditionType.NUMBER_OF_RESULTS.value}," + + "$NUM_RESULTS_VALUE_FIELD must be included" + } + require(customCondition == null) { + "if trigger condition is of type ${ConditionType.NUMBER_OF_RESULTS.value}," + + "$CUSTOM_CONDITION_FIELD must not be included" + } + } + ConditionType.CUSTOM -> { + requireNotNull(customCondition) { + "if trigger condition is of type ${ConditionType.CUSTOM.value}," + + "$CUSTOM_CONDITION_FIELD must be included" + } + require(numResultsCondition == null) { + "if trigger condition is of type ${ConditionType.CUSTOM.value}," + + "$NUM_RESULTS_CONDITION_FIELD must not be included" + } + require(numResultsValue == null) { + "if trigger condition is of type ${ConditionType.CUSTOM.value}," + + "$NUM_RESULTS_VALUE_FIELD must not be included" + } + } + } + + // 3. prepare and return PPLTrigger object + return PPLTrigger( + id, + name, + severity, + suppressDuration, + expireDuration, + lastTriggeredTime, + actions, + mode, + conditionType, + numResultsCondition, + numResultsValue, + customCondition + ) + } + + @JvmStatic + @Throws(IOException::class) + fun readFrom(sin: StreamInput): PPLTrigger { + return PPLTrigger(sin) + } + } +} diff --git a/core/src/main/kotlin/org/opensearch/alerting/core/modelv2/PPLTriggerRunResult.kt b/core/src/main/kotlin/org/opensearch/alerting/core/modelv2/PPLTriggerRunResult.kt new file mode 100644 index 000000000..175275a97 --- /dev/null +++ b/core/src/main/kotlin/org/opensearch/alerting/core/modelv2/PPLTriggerRunResult.kt @@ -0,0 +1,51 @@ +package org.opensearch.alerting.core.modelv2 + +import org.opensearch.alerting.core.modelv2.TriggerV2RunResult.Companion.ERROR_FIELD +import org.opensearch.alerting.core.modelv2.TriggerV2RunResult.Companion.NAME_FIELD +import org.opensearch.alerting.core.modelv2.TriggerV2RunResult.Companion.TRIGGERED_FIELD +import org.opensearch.commons.alerting.model.QueryLevelTriggerRunResult +import org.opensearch.commons.alerting.model.TriggerRunResult +import org.opensearch.core.common.io.stream.StreamInput +import org.opensearch.core.common.io.stream.StreamOutput +import org.opensearch.core.xcontent.ToXContent +import org.opensearch.core.xcontent.XContentBuilder +import java.io.IOException + +data class PPLTriggerRunResult( + override var triggerName: String, + override var triggered: Boolean, + override var error: Exception?, +) : TriggerV2RunResult { + + @Throws(IOException::class) + @Suppress("UNCHECKED_CAST") + constructor(sin: StreamInput) : this( + triggerName = sin.readString(), + triggered = sin.readBoolean(), + error = sin.readException() + ) + + override fun toXContent(builder: XContentBuilder, params: ToXContent.Params): XContentBuilder { + builder.startObject() + builder.field(NAME_FIELD, triggerName) + builder.field(TRIGGERED_FIELD, triggered) + builder.field(ERROR_FIELD, error?.message) + builder.endObject() + return builder + } + + @Throws(IOException::class) + override fun writeTo(out: StreamOutput) { + out.writeString(triggerName) + out.writeBoolean(triggered) + out.writeException(error) + } + + companion object { + @JvmStatic + @Throws(IOException::class) + fun readFrom(sin: StreamInput): TriggerRunResult { + return QueryLevelTriggerRunResult(sin) + } + } +} diff --git a/core/src/main/kotlin/org/opensearch/alerting/core/modelv2/TriggerV2.kt b/core/src/main/kotlin/org/opensearch/alerting/core/modelv2/TriggerV2.kt new file mode 100644 index 000000000..c544bf275 --- /dev/null +++ b/core/src/main/kotlin/org/opensearch/alerting/core/modelv2/TriggerV2.kt @@ -0,0 +1,51 @@ +package org.opensearch.alerting.core.modelv2 + +import org.opensearch.alerting.core.modelv2.PPLTrigger.Companion.PPL_TRIGGER_FIELD +import org.opensearch.common.unit.TimeValue +import org.opensearch.commons.alerting.model.action.Action +import org.opensearch.commons.notifications.model.BaseModel +import java.time.Instant + +interface TriggerV2 : BaseModel { + + val id: String + val name: String + val severity: Severity + val suppressDuration: TimeValue? + val expireDuration: TimeValue? + var lastTriggeredTime: Instant? + val actions: List + + enum class TriggerV2Type(val value: String) { + PPL_TRIGGER(PPL_TRIGGER_FIELD); + + override fun toString(): String { + return value + } + } + + enum class Severity(val value: String) { + INFO("info"), + ERROR("error"), + LOW("low"), + MEDIUM("medium"), + HIGH("high"), + CRITICAL("critical"); + + companion object { + fun enumFromString(value: String): Severity? { + return entries.find { it.value == value } + } + } + } + + companion object { + const val ID_FIELD = "id" + const val NAME_FIELD = "name" + const val SEVERITY_FIELD = "severity" + const val SUPPRESS_FIELD = "suppress" + const val LAST_TRIGGERED_FIELD = "last_triggered_time" + const val EXPIRE_FIELD = "expires" + const val ACTIONS_FIELD = "actions" + } +} diff --git a/core/src/main/kotlin/org/opensearch/alerting/core/modelv2/TriggerV2RunResult.kt b/core/src/main/kotlin/org/opensearch/alerting/core/modelv2/TriggerV2RunResult.kt new file mode 100644 index 000000000..37ea50c90 --- /dev/null +++ b/core/src/main/kotlin/org/opensearch/alerting/core/modelv2/TriggerV2RunResult.kt @@ -0,0 +1,17 @@ +package org.opensearch.alerting.core.modelv2 + +import org.opensearch.core.common.io.stream.Writeable +import org.opensearch.core.xcontent.ToXContent + +interface TriggerV2RunResult : Writeable, ToXContent { + + val triggerName: String + val triggered: Boolean + val error: Exception? + + companion object { + const val NAME_FIELD = "name" + const val TRIGGERED_FIELD = "triggered" + const val ERROR_FIELD = "error" + } +} diff --git a/core/src/main/kotlin/org/opensearch/alerting/core/ppl/PPLPluginInterface.kt b/core/src/main/kotlin/org/opensearch/alerting/core/ppl/PPLPluginInterface.kt new file mode 100644 index 000000000..477b417ce --- /dev/null +++ b/core/src/main/kotlin/org/opensearch/alerting/core/ppl/PPLPluginInterface.kt @@ -0,0 +1,50 @@ +package org.opensearch.alerting.core.ppl + +import org.opensearch.commons.utils.recreateObject +import org.opensearch.core.action.ActionListener +import org.opensearch.core.action.ActionResponse +import org.opensearch.core.common.io.stream.Writeable +import org.opensearch.sql.plugin.transport.PPLQueryAction +import org.opensearch.sql.plugin.transport.TransportPPLQueryRequest +import org.opensearch.sql.plugin.transport.TransportPPLQueryResponse +import org.opensearch.transport.client.node.NodeClient + +/** + * Various transport action plugin interfaces for the SQL/PPL plugin + */ +object PPLPluginInterface { + fun executeQuery( + client: NodeClient, + request: TransportPPLQueryRequest, + listener: ActionListener + ) { + client.execute( + PPLQueryAction.INSTANCE, + request, + wrapActionListener(listener) { response -> recreateObject(response) { TransportPPLQueryResponse(it) } } + ) + } + + /** + * Wrap action listener on concrete response class by a new created one on ActionResponse. + * This is required because the response may be loaded by different classloader across plugins. + * The onResponse(ActionResponse) avoids type cast exception and give a chance to recreate + * the response object. + */ + @Suppress("UNCHECKED_CAST") + private fun wrapActionListener( + listener: ActionListener, + recreate: (Writeable) -> Response + ): ActionListener { + return object : ActionListener { + override fun onResponse(response: ActionResponse) { + val recreated = recreate(response) + listener.onResponse(recreated) + } + + override fun onFailure(exception: java.lang.Exception) { + listener.onFailure(exception) + } + } as ActionListener + } +} diff --git a/core/src/main/kotlin/org/opensearch/alerting/core/resthandler/RestScheduledJobStatsHandler.kt b/core/src/main/kotlin/org/opensearch/alerting/core/resthandler/RestScheduledJobStatsHandler.kt index fbe57ab19..a8fce9bf3 100644 --- a/core/src/main/kotlin/org/opensearch/alerting/core/resthandler/RestScheduledJobStatsHandler.kt +++ b/core/src/main/kotlin/org/opensearch/alerting/core/resthandler/RestScheduledJobStatsHandler.kt @@ -7,7 +7,6 @@ package org.opensearch.alerting.core.resthandler import org.opensearch.alerting.core.action.node.ScheduledJobsStatsAction import org.opensearch.alerting.core.action.node.ScheduledJobsStatsRequest -import org.opensearch.core.common.Strings import org.opensearch.rest.BaseRestHandler import org.opensearch.rest.BaseRestHandler.RestChannelConsumer import org.opensearch.rest.RestHandler @@ -16,8 +15,6 @@ import org.opensearch.rest.RestRequest import org.opensearch.rest.RestRequest.Method.GET import org.opensearch.rest.action.RestActions import org.opensearch.transport.client.node.NodeClient -import java.util.Locale -import java.util.TreeSet /** * RestScheduledJobStatsHandler is handler for getting ScheduledJob Stats. @@ -27,7 +24,7 @@ class RestScheduledJobStatsHandler(private val path: String) : BaseRestHandler() companion object { const val JOB_SCHEDULING_METRICS: String = "job_scheduling_metrics" const val JOBS_INFO: String = "jobs_info" - private val METRICS = mapOf Unit>( + val METRICS = mapOf Unit>( JOB_SCHEDULING_METRICS to { it -> it.jobSchedulingMetrics = true }, JOBS_INFO to { it -> it.jobsInfo = true } ) @@ -71,7 +68,7 @@ class RestScheduledJobStatsHandler(private val path: String) : BaseRestHandler() } override fun prepareRequest(request: RestRequest, client: NodeClient): RestChannelConsumer { - val scheduledJobNodesStatsRequest = getRequest(request) + val scheduledJobNodesStatsRequest = StatsRequestUtils.getStatsRequest(request, false, this::unrecognized) return RestChannelConsumer { channel -> client.execute( ScheduledJobsStatsAction.INSTANCE, @@ -80,43 +77,4 @@ class RestScheduledJobStatsHandler(private val path: String) : BaseRestHandler() ) } } - - private fun getRequest(request: RestRequest): ScheduledJobsStatsRequest { - val nodesIds = Strings.splitStringByCommaToArray(request.param("nodeId")) - val metrics = Strings.tokenizeByCommaToSet(request.param("metric")) - val scheduledJobsStatsRequest = ScheduledJobsStatsRequest(nodesIds) - scheduledJobsStatsRequest.timeout(request.param("timeout")) - - if (metrics.isEmpty()) { - return scheduledJobsStatsRequest - } else if (metrics.size == 1 && metrics.contains("_all")) { - scheduledJobsStatsRequest.all() - } else if (metrics.contains("_all")) { - throw IllegalArgumentException( - String.format( - Locale.ROOT, - "request [%s] contains _all and individual metrics [%s]", - request.path(), - request.param("metric") - ) - ) - } else { - // use a sorted set so the unrecognized parameters appear in a reliable sorted order - scheduledJobsStatsRequest.clear() - val invalidMetrics = TreeSet() - for (metric in metrics) { - val handler = METRICS[metric] - if (handler != null) { - handler.invoke(scheduledJobsStatsRequest) - } else { - invalidMetrics.add(metric) - } - } - - if (!invalidMetrics.isEmpty()) { - throw IllegalArgumentException(unrecognized(request, invalidMetrics, METRICS.keys, "metric")) - } - } - return scheduledJobsStatsRequest - } } diff --git a/core/src/main/kotlin/org/opensearch/alerting/core/resthandler/RestScheduledJobStatsV2Handler.kt b/core/src/main/kotlin/org/opensearch/alerting/core/resthandler/RestScheduledJobStatsV2Handler.kt new file mode 100644 index 000000000..7bfbfc47c --- /dev/null +++ b/core/src/main/kotlin/org/opensearch/alerting/core/resthandler/RestScheduledJobStatsV2Handler.kt @@ -0,0 +1,51 @@ +package org.opensearch.alerting.core.resthandler + +import org.opensearch.alerting.core.action.node.ScheduledJobsStatsAction +import org.opensearch.rest.BaseRestHandler +import org.opensearch.rest.RestHandler.Route +import org.opensearch.rest.RestRequest +import org.opensearch.rest.RestRequest.Method.GET +import org.opensearch.rest.action.RestActions +import org.opensearch.transport.client.node.NodeClient + +/** + * RestScheduledJobStatsHandler is handler for getting ScheduledJob Stats for Alerting V2 Scheduled Jobs. + */ +class RestScheduledJobStatsV2Handler : BaseRestHandler() { + + override fun getName(): String { + return "alerting_jobs_stats_v2" + } + + override fun routes(): List { + return listOf( + Route( + GET, + "/_plugins/_alerting/v2/stats/" + ), + Route( + GET, + "/_plugins/_alerting/v2/stats/{metric}" + ), + Route( + GET, + "/_plugins/_alerting/v2/{nodeId}/stats/" + ), + Route( + GET, + "/_plugins/_alerting/v2/{nodeId}/stats/{metric}" + ) + ) + } + + override fun prepareRequest(request: RestRequest, client: NodeClient): RestChannelConsumer { + val scheduledJobNodesStatsRequest = StatsRequestUtils.getStatsRequest(request, true, this::unrecognized) + return RestChannelConsumer { channel -> + client.execute( + ScheduledJobsStatsAction.INSTANCE, + scheduledJobNodesStatsRequest, + RestActions.NodesResponseRestListener(channel) + ) + } + } +} diff --git a/core/src/main/kotlin/org/opensearch/alerting/core/resthandler/StatsRequestUtils.kt b/core/src/main/kotlin/org/opensearch/alerting/core/resthandler/StatsRequestUtils.kt new file mode 100644 index 000000000..db05dd5e6 --- /dev/null +++ b/core/src/main/kotlin/org/opensearch/alerting/core/resthandler/StatsRequestUtils.kt @@ -0,0 +1,53 @@ +package org.opensearch.alerting.core.resthandler + +import org.opensearch.alerting.core.action.node.ScheduledJobsStatsRequest +import org.opensearch.alerting.core.resthandler.RestScheduledJobStatsHandler.Companion.METRICS +import org.opensearch.core.common.Strings +import org.opensearch.rest.RestRequest +import java.util.Locale +import java.util.TreeSet + +internal object StatsRequestUtils { + fun getStatsRequest( + request: RestRequest, + showAlertingV2ScheduledJobs: Boolean, + unrecognizedFn: (RestRequest, Set, Set, String) -> String + ): ScheduledJobsStatsRequest { + val nodesIds = Strings.splitStringByCommaToArray(request.param("nodeId")) + val metrics = Strings.tokenizeByCommaToSet(request.param("metric")) + val scheduledJobsStatsRequest = ScheduledJobsStatsRequest(nodeIds = nodesIds, showAlertingV2ScheduledJobs) + scheduledJobsStatsRequest.timeout(request.param("timeout")) + + if (metrics.isEmpty()) { + return scheduledJobsStatsRequest + } else if (metrics.size == 1 && metrics.contains("_all")) { + scheduledJobsStatsRequest.all() + } else if (metrics.contains("_all")) { + throw IllegalArgumentException( + String.format( + Locale.ROOT, + "request [%s] contains _all and individual metrics [%s]", + request.path(), + request.param("metric") + ) + ) + } else { + // use a sorted set so the unrecognized parameters appear in a reliable sorted order + scheduledJobsStatsRequest.clear() + val invalidMetrics = TreeSet() + for (metric in metrics) { + val handler = METRICS[metric] + if (handler != null) { + handler.invoke(scheduledJobsStatsRequest) + } else { + invalidMetrics.add(metric) + } + } + + if (!invalidMetrics.isEmpty()) { + throw IllegalArgumentException(unrecognizedFn(request, invalidMetrics, METRICS.keys, "metric")) + } + } + return scheduledJobsStatsRequest + } +} diff --git a/core/src/main/kotlin/org/opensearch/alerting/core/schedule/JobScheduler.kt b/core/src/main/kotlin/org/opensearch/alerting/core/schedule/JobScheduler.kt index a4a729121..0dddb80b1 100644 --- a/core/src/main/kotlin/org/opensearch/alerting/core/schedule/JobScheduler.kt +++ b/core/src/main/kotlin/org/opensearch/alerting/core/schedule/JobScheduler.kt @@ -7,6 +7,7 @@ package org.opensearch.alerting.core.schedule import org.apache.logging.log4j.LogManager import org.opensearch.alerting.core.JobRunner +import org.opensearch.alerting.core.modelv2.MonitorV2 import org.opensearch.common.unit.TimeValue import org.opensearch.commons.alerting.model.ScheduledJob import org.opensearch.threadpool.Scheduler @@ -192,7 +193,21 @@ class JobScheduler(private val threadPool: ThreadPool, private val jobRunner: Jo } fun getJobSchedulerMetric(): List { - return scheduledJobIdToInfo.entries.stream() + return scheduledJobIdToInfo.entries.filter { it.value.scheduledJob !is MonitorV2 } + .stream() + .map { entry -> + JobSchedulerMetrics( + entry.value.scheduledJobId, + entry.value.actualPreviousExecutionTime?.toEpochMilli(), + entry.value.scheduledJob.schedule.runningOnTime(entry.value.actualPreviousExecutionTime) + ) + } + .collect(Collectors.toList()) + } + + fun getJobSchedulerV2Metric(): List { + return scheduledJobIdToInfo.entries.filter { it.value.scheduledJob is MonitorV2 } + .stream() .map { entry -> JobSchedulerMetrics( entry.value.scheduledJobId, diff --git a/core/src/main/kotlin/org/opensearch/alerting/core/util/XContentExtensions.kt b/core/src/main/kotlin/org/opensearch/alerting/core/util/XContentExtensions.kt new file mode 100644 index 000000000..c36ed2dc5 --- /dev/null +++ b/core/src/main/kotlin/org/opensearch/alerting/core/util/XContentExtensions.kt @@ -0,0 +1,8 @@ +package org.opensearch.alerting.core.util + +import org.opensearch.core.xcontent.XContentBuilder +import java.time.Instant + +fun XContentBuilder.nonOptionalTimeField(name: String, instant: Instant): XContentBuilder { + return this.timeField(name, "${name}_in_millis", instant.toEpochMilli()) +} diff --git a/core/src/main/kotlin/org/opensearch/alerting/opensearchapi/OpenSearchExtensions.kt b/core/src/main/kotlin/org/opensearch/alerting/opensearchapi/OpenSearchExtensions.kt index 582d13fbe..fd500ef1d 100644 --- a/core/src/main/kotlin/org/opensearch/alerting/opensearchapi/OpenSearchExtensions.kt +++ b/core/src/main/kotlin/org/opensearch/alerting/opensearchapi/OpenSearchExtensions.kt @@ -14,6 +14,7 @@ import org.opensearch.OpenSearchException import org.opensearch.action.bulk.BackoffPolicy import org.opensearch.action.search.SearchResponse import org.opensearch.action.search.ShardSearchFailure +import org.opensearch.alerting.core.ppl.PPLPluginInterface import org.opensearch.common.settings.Settings import org.opensearch.common.util.concurrent.ThreadContext import org.opensearch.common.xcontent.XContentHelper @@ -170,6 +171,20 @@ suspend fun NotificationsPluginInterface.suspendUntil(block: NotificationsPl }) } +/** + * Converts [PPLPluginInterface] methods that take a callback into a kotlin suspending function. + * + * @param block - a block of code that is passed an [ActionListener] that should be passed to the PPLPluginInterface API. + */ +suspend fun PPLPluginInterface.suspendUntil(block: PPLPluginInterface.(ActionListener) -> Unit): T = + suspendCoroutine { cont -> + block(object : ActionListener { + override fun onResponse(response: T) = cont.resume(response) + + override fun onFailure(e: Exception) = cont.resumeWithException(e) + }) + } + class InjectorContextElement( id: String, settings: Settings,