Skip to content

Commit bad8fc6

Browse files
coutoPLcoutoPL
authored
[RORDEV-1554] ES 8.19.0 support (#1142)
1 parent 1896a72 commit bad8fc6

File tree

8 files changed

+124
-62
lines changed

8 files changed

+124
-62
lines changed

‎ci/supported-es-versions/es8x.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
8.19.0
12
8.18.4
23
8.18.3
34
8.18.2

‎es818x/gradle.properties‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
latestSupportedEsVersion=8.18.4
1+
latestSupportedEsVersion=8.19.0

‎es818x/src/main/scala/tech/beshu/ror/es/IndexLevelActionFilter.scala‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -162,7 +162,8 @@ class IndexLevelActionFilter(clusterService: ClusterService,
162162
request,
163163
listener,
164164
chain,
165-
JavaConverters.flattenPair(threadPool.getThreadContext.getResponseHeaders).toCovariantSet
165+
JavaConverters.flattenPair(threadPool.getThreadContext.getResponseHeaders).toCovariantSet,
166+
esEnv.esVersion
166167
)
167168
)
168169
}

‎es818x/src/main/scala/tech/beshu/ror/es/handler/AclAwareRequestFilter.scala‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -74,7 +74,7 @@ import tech.beshu.ror.es.handler.request.context.types.repositories.*
7474
import tech.beshu.ror.es.handler.request.context.types.ror.*
7575
import tech.beshu.ror.es.handler.request.context.types.snapshots.*
7676
import tech.beshu.ror.es.handler.request.context.types.templates.*
77-
import tech.beshu.ror.es.{RorClusterService, RorRestChannel}
77+
import tech.beshu.ror.es.{EsVersion, RorClusterService, RorRestChannel}
7878
import tech.beshu.ror.implicits.*
7979
import tech.beshu.ror.syntax.*
8080

@@ -274,7 +274,8 @@ object AclAwareRequestFilter {
274274
val actionRequest: ActionRequest,
275275
val listener: ActionListener[ActionResponse],
276276
val chain: EsChain,
277-
val threadContextResponseHeaders: Set[(String, String)]) {
277+
val threadContextResponseHeaders: Set[(String, String)],
278+
val esVersion: EsVersion) {
278279

279280
val timestamp: Instant = Instant.now()
280281

‎es818x/src/main/scala/tech/beshu/ror/es/handler/request/context/types/EsqlIndicesEsRequestContext.scala‎

Lines changed: 11 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ import tech.beshu.ror.es.handler.request.context.ModificationResult
3434
import tech.beshu.ror.es.handler.request.context.ModificationResult.{CannotModify, UpdateResponse}
3535
import tech.beshu.ror.es.handler.response.FLSContextHeaderHandler
3636
import tech.beshu.ror.es.utils.EsqlRequestHelper
37-
import tech.beshu.ror.es.utils.EsqlRequestHelper.{ClassificationError, EsqlRequestClassification}
37+
import tech.beshu.ror.es.utils.EsqlRequestHelper.{ClassificationError, EsqlRequestClassification, ModificationError}
3838
import tech.beshu.ror.exceptions.SecurityPermissionException
3939
import tech.beshu.ror.implicits.*
4040
import tech.beshu.ror.syntax.*
@@ -43,12 +43,13 @@ class EsqlIndicesEsRequestContext private(actionRequest: ActionRequest with Comp
4343
esContext: EsContext,
4444
aclContext: AccessControlStaticContext,
4545
clusterService: RorClusterService,
46-
override val threadPool: ThreadPool)
46+
override val threadPool: ThreadPool,
47+
esqlRequestHelper: EsqlRequestHelper)
4748
extends BaseFilterableEsRequestContext[ActionRequest with CompositeIndicesRequest](actionRequest, esContext, aclContext, clusterService, threadPool) {
4849

4950
override protected def requestFieldsUsage: RequestFieldsUsage = RequestFieldsUsage.NotUsingFields
5051

51-
private lazy val requestClassification = EsqlRequestHelper.classifyEsqlRequest(actionRequest) match {
52+
private lazy val requestClassification = esqlRequestHelper.classifyEsqlRequest(actionRequest) match {
5253
case result@Right(_) => result
5354
case result@Left(ClassificationError.ParsingException) => result
5455
case Left(ClassificationError.UnexpectedException(ex)) =>
@@ -70,7 +71,7 @@ class EsqlIndicesEsRequestContext private(actionRequest: ActionRequest with Comp
7071
filteredRequestedIndices: NonEmptyList[RequestedIndex[ClusterIndexName]],
7172
filter: Option[Filter],
7273
fieldLevelSecurity: Option[FieldLevelSecurity]): ModificationResult = {
73-
val result: Either[EsqlRequestHelper.ModificationError, UpdateResponse] = for {
74+
val result: Either[ModificationError, UpdateResponse] = for {
7475
_ <- modifyRequestIndices(request, filteredRequestedIndices)
7576
_ <- Right(applyFieldLevelSecurityTo(request, fieldLevelSecurity))
7677
_ <- Right(applyFilterTo(request, filter))
@@ -80,7 +81,7 @@ class EsqlIndicesEsRequestContext private(actionRequest: ActionRequest with Comp
8081
applyFieldLevelSecurityTo(response, fieldLevelSecurity) match {
8182
case Right(modifiedResponse) =>
8283
modifiedResponse
83-
case Left(EsqlRequestHelper.ModificationError.UnexpectedException(ex)) =>
84+
case Left(ModificationError.UnexpectedException(ex)) =>
8485
throw new SecurityPermissionException("Cannot apply field level security to the ESQL response", ex)
8586
}
8687
}
@@ -96,14 +97,14 @@ class EsqlIndicesEsRequestContext private(actionRequest: ActionRequest with Comp
9697
}
9798

9899
private def modifyRequestIndices(request: ActionRequest with CompositeIndicesRequest,
99-
filteredIndices: NonEmptyList[RequestedIndex[ClusterIndexName]]): Either[EsqlRequestHelper.ModificationError, CompositeIndicesRequest] = {
100+
filteredIndices: NonEmptyList[RequestedIndex[ClusterIndexName]]): Either[ModificationError, CompositeIndicesRequest] = {
100101
requestClassification match {
101102
case Right(EsqlRequestClassification.NonIndicesRelated) =>
102103
Right(request)
103104
case Right(r@EsqlRequestClassification.IndicesRelated(tables)) =>
104105
val filteredIndicesStrings = filteredIndices.stringify.toCovariantSet
105106
if (filteredIndicesStrings != r.indices) {
106-
EsqlRequestHelper.modifyIndicesOf(request, tables, filteredIndicesStrings)
107+
esqlRequestHelper.modifyIndicesOf(request, tables, filteredIndicesStrings)
107108
} else {
108109
Right(request)
109110
}
@@ -135,7 +136,7 @@ class EsqlIndicesEsRequestContext private(actionRequest: ActionRequest with Comp
135136
fieldLevelSecurity: Option[FieldLevelSecurity]) = {
136137
fieldLevelSecurity match {
137138
case Some(fls) =>
138-
EsqlRequestHelper.modifyResponseAccordingToFieldLevelSecurity(response, fls)
139+
esqlRequestHelper.modifyResponseAccordingToFieldLevelSecurity(response, fls)
139140
case None =>
140141
Right(response)
141142
}
@@ -159,7 +160,8 @@ object EsqlIndicesEsRequestContext {
159160
arg.esContext,
160161
arg.aclContext,
161162
arg.clusterService,
162-
arg.threadPool
163+
arg.threadPool,
164+
new EsqlRequestHelper(arg.esContext.esVersion)
163165
))
164166
} else {
165167
None

‎es818x/src/main/scala/tech/beshu/ror/es/utils/EsqlRequestHelper.scala‎

Lines changed: 98 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -23,29 +23,22 @@ import org.elasticsearch.action.{ActionResponse, CompositeIndicesRequest}
2323
import org.joor.Reflect.*
2424
import tech.beshu.ror.accesscontrol.domain.FieldLevelSecurity
2525
import tech.beshu.ror.accesscontrol.domain.FieldLevelSecurity.FieldsRestrictions
26+
import tech.beshu.ror.es.EsVersion
2627
import tech.beshu.ror.es.handler.response.FieldsFiltering
2728
import tech.beshu.ror.es.handler.response.FieldsFiltering.NonMetadataDocumentFields
28-
import tech.beshu.ror.es.utils.*
29+
import tech.beshu.ror.es.utils.EsqlRequestHelper.*
2930
import tech.beshu.ror.syntax.*
3031
import tech.beshu.ror.utils.ReflecUtils
3132
import tech.beshu.ror.utils.ScalaOps.*
3233

3334
import java.lang.reflect.Modifier
34-
import java.util.List as JList
35+
import java.time.ZoneOffset
36+
import java.util.{Locale, List as JList}
3537
import java.util.regex.Pattern
3638
import scala.jdk.CollectionConverters.*
3739
import scala.util.{Failure, Success, Try}
3840

39-
object EsqlRequestHelper {
40-
41-
final case class IndexTable(tableStringInQuery: String, indices: NonEmptyList[String])
42-
43-
sealed trait ModificationError
44-
object ModificationError {
45-
final case class UnexpectedException(ex: Throwable) extends ModificationError
46-
47-
implicit val show: Show[ModificationError] = Show.show(_.toString)
48-
}
41+
class EsqlRequestHelper(esVersion: EsVersion) {
4942

5043
def modifyIndicesOf(request: CompositeIndicesRequest,
5144
requestTables: NonEmptyList[IndexTable],
@@ -61,37 +54,21 @@ object EsqlRequestHelper {
6154
.toEither.left.map(ModificationError.UnexpectedException.apply)
6255
}
6356

64-
sealed trait EsqlRequestClassification
65-
object EsqlRequestClassification {
66-
final case class IndicesRelated(tables: NonEmptyList[IndexTable]) extends EsqlRequestClassification {
67-
lazy val indices: Set[String] = tables.toCovariantSet.flatMap(_.indices.toIterable)
68-
}
69-
case object NonIndicesRelated extends EsqlRequestClassification
70-
}
71-
72-
sealed trait ClassificationError
73-
object ClassificationError {
74-
final case class UnexpectedException(ex: Throwable) extends ClassificationError
75-
case object ParsingException extends ClassificationError
76-
}
77-
78-
import EsqlRequestClassification._
57+
import EsqlRequestClassification.*
7958

8059
def classifyEsqlRequest(request: CompositeIndicesRequest): Either[ClassificationError, EsqlRequestClassification] = {
81-
val result = Try {
82-
val query = getQuery(request)
83-
val params = ReflecUtils.invokeMethodCached(request, request.getClass, "params")
84-
85-
implicit val classLoader: ClassLoader = request.getClass.getClassLoader
86-
Try(new EsqlParser().createStatement(query, params)) match {
87-
case Success(statement: IndicesRelatedStatement) => Right(IndicesRelated(statement.indices))
88-
case Success(command: OtherCommand) => Right(NonIndicesRelated)
89-
case Failure(_) => Left(ClassificationError.ParsingException: ClassificationError)
90-
}
60+
createStatement(request) match {
61+
case Right(statement: IndicesRelatedStatement) => Right(IndicesRelated(statement.indices))
62+
case Right(command: OtherCommand) => Right(NonIndicesRelated)
63+
case Left(error) => Left(error)
9164
}
92-
result match {
93-
case Success(value) => value
94-
case Failure(exception) => Left(ClassificationError.UnexpectedException(exception))
65+
}
66+
67+
private def createStatement(request: CompositeIndicesRequest): Either[ClassificationError, Statement] = {
68+
implicit val classLoader: ClassLoader = request.getClass.getClassLoader
69+
Try(new EsqlParser(esVersion)) match {
70+
case Success(parser) => parser.createStatementBasedOn(request)
71+
case Failure(ex) => Left(ClassificationError.UnexpectedException(ex))
9572
}
9673
}
9774

@@ -106,6 +83,30 @@ object EsqlRequestHelper {
10683
request
10784
}
10885

86+
private def getParams(request: CompositeIndicesRequest): AnyRef = {
87+
ReflecUtils.invokeMethodCached(request, request.getClass, "params")
88+
}
89+
90+
private def createConfiguration(request: CompositeIndicesRequest): AnyRef = {
91+
val classLoader = request.getClass.getClassLoader
92+
onClass(classLoader.loadClass("org.elasticsearch.xpack.esql.session.Configuration"))
93+
.create(
94+
ZoneOffset.UTC,
95+
Option(on(request).call("locale").get[Locale]).getOrElse(Locale.US),
96+
null, // at the moment it's not used anywhere, so it's null here - probably to be fixed in the future
97+
"ROR", // at the moment it's not used anywhere, so it's placeholder here - probably to be fixed in the future
98+
on(request).call("pragmas").get[AnyRef],
99+
Int.MaxValue,
100+
Int.MaxValue,
101+
getQuery(request),
102+
on(request).call("profile").get[AnyRef],
103+
on(request).call("tables").get[AnyRef],
104+
System.nanoTime(),
105+
Option(on(request).call("allowPartialResults").get[Any]).getOrElse(true)
106+
)
107+
.get[AnyRef]()
108+
}
109+
109110
private def newQueryFrom(oldQuery: String, requestTables: NonEmptyList[IndexTable], finalIndices: Set[String]) = {
110111
requestTables.toList.foldLeft(oldQuery) {
111112
case (currentQuery, table) =>
@@ -123,18 +124,43 @@ object EsqlRequestHelper {
123124
currentQuery.replaceAll(Pattern.quote(originTable), finalIndices.mkString(","))
124125
}
125126

126-
private final class EsqlParser(implicit classLoader: ClassLoader) {
127+
private final class EsqlParser(esVersion: EsVersion)
128+
(implicit classLoader: ClassLoader) {
127129

128130
private val underlyingObject =
129131
onClass(classLoader.loadClass("org.elasticsearch.xpack.esql.parser.EsqlParser"))
130132
.create().get[Any]()
131133

132-
def createStatement(query: String, params: AnyRef): Statement = {
133-
val statement = on(underlyingObject).call("createStatement", query, params).get[Any]
134-
NonEmptyList.fromList(indicesFrom(statement)) match {
135-
case Some(indices) => new IndicesRelatedStatement(statement, indices)
136-
case None => OtherCommand(statement)
134+
def createStatementBasedOn(request: CompositeIndicesRequest): Either[ClassificationError, Statement] = {
135+
val statement = esVersion match {
136+
case v if v >= EsVersion(8, 19, 0) => createStatementForEsEqualOrAbove8190(request)
137+
case v => createStatementForEsBelow8190(request)
137138
}
139+
statement.map { s =>
140+
NonEmptyList.fromList(indicesFrom(s)) match {
141+
case Some(indices) => new IndicesRelatedStatement(statement, indices)
142+
case None => OtherCommand(statement)
143+
}
144+
}
145+
}
146+
147+
private def createStatementForEsBelow8190(request: CompositeIndicesRequest) = {
148+
for {
149+
query <- Try(getQuery(request)).toEither.left.map(ClassificationError.UnexpectedException.apply)
150+
params <- Try(getParams(request)).toEither.left.map(ClassificationError.UnexpectedException.apply)
151+
statement <- Try(on(underlyingObject).call("createStatement", query, params).get[Any])
152+
.toEither.left.map(_ => ClassificationError.ParsingException)
153+
} yield statement
154+
}
155+
156+
private def createStatementForEsEqualOrAbove8190(request: CompositeIndicesRequest) = {
157+
for {
158+
query <- Try(getQuery(request)).toEither.left.map(ClassificationError.UnexpectedException.apply)
159+
params <- Try(getParams(request)).toEither.left.map(ClassificationError.UnexpectedException.apply)
160+
configuration <- Try(createConfiguration(request)).toEither.left.map(ClassificationError.UnexpectedException.apply)
161+
statement <- Try(on(underlyingObject).call("createStatement", query, params, configuration).get[Any])
162+
.toEither.left.map(_ => ClassificationError.ParsingException)
163+
} yield statement
138164
}
139165

140166
private def indicesFrom(statement: Any) = {
@@ -273,3 +299,29 @@ object EsqlRequestHelper {
273299
}
274300
}
275301
}
302+
object EsqlRequestHelper {
303+
304+
final case class IndexTable(tableStringInQuery: String, indices: NonEmptyList[String])
305+
306+
sealed trait ModificationError
307+
object ModificationError {
308+
final case class UnexpectedException(ex: Throwable) extends ModificationError
309+
310+
implicit val show: Show[ModificationError] = Show.show(_.toString)
311+
}
312+
313+
sealed trait EsqlRequestClassification
314+
object EsqlRequestClassification {
315+
final case class IndicesRelated(tables: NonEmptyList[IndexTable]) extends EsqlRequestClassification {
316+
lazy val indices: Set[String] = tables.toCovariantSet.flatMap(_.indices.toIterable)
317+
}
318+
case object NonIndicesRelated extends EsqlRequestClassification
319+
}
320+
321+
sealed trait ClassificationError
322+
object ClassificationError {
323+
final case class UnexpectedException(ex: Throwable) extends ClassificationError
324+
case object ParsingException extends ClassificationError
325+
}
326+
327+
}

‎ror-tools-core/src/main/scala/tech/beshu/ror/tools/core/patches/internal/modifiers/bytecodeJars/ModifyFilesEntitlementsValidationClass.scala‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -19,12 +19,12 @@ package tech.beshu.ror.tools.core.patches.internal.modifiers.bytecodeJars
1919
import just.semver.SemVer
2020
import org.objectweb.asm.{ClassReader, ClassVisitor, ClassWriter, Label, MethodVisitor, Opcodes}
2121
import tech.beshu.ror.tools.core.patches.internal.modifiers.BytecodeJarModifier
22-
import tech.beshu.ror.tools.core.utils.EsUtil.es8182
22+
import tech.beshu.ror.tools.core.utils.EsUtil.{es8182, es8190, es900}
2323

2424
import java.io.{File, InputStream}
2525

2626
/**
27-
* Modifies the EntitlementInitialization class to bypass forbidden file path validation
27+
* Modifies the FilesEntitlementsValidation class to bypass forbidden file path validation
2828
* specifically for the ReadonlyREST plugin. This is necessary because ReadonlyREST
2929
* requires access to certain paths that would otherwise be blocked by Elasticsearch's
3030
* security entitlements system in versions 8.18.2+
@@ -35,7 +35,11 @@ private[patches] class ModifyFilesEntitlementsValidationClass(esVersion: SemVer)
3535
override def apply(jar: File): Unit = {
3636
modifyFileInJar(
3737
jar = jar,
38-
filePathString = "org/elasticsearch/entitlement/initialization/FilesEntitlementsValidation.class",
38+
filePathString = esVersion match {
39+
case v if v >= es900 => "org/elasticsearch/entitlement/initialization/FilesEntitlementsValidation.class"
40+
case v if v >= es8190 => "org/elasticsearch/entitlement/bootstrap/FilesEntitlementsValidation.class"
41+
case v => "org/elasticsearch/entitlement/initialization/FilesEntitlementsValidation.class"
42+
},
3943
processFileContent = dontValidateForbiddenPathsInCaseOfRorPlugin
4044
)
4145
}

‎ror-tools-core/src/main/scala/tech/beshu/ror/tools/core/utils/EsUtil.scala‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ object EsUtil {
2626
val es902: SemVer = SemVer.unsafeParse("9.0.2")
2727
val es901: SemVer = SemVer.unsafeParse("9.0.1")
2828
val es900: SemVer = SemVer.unsafeParse("9.0.0")
29+
val es8190: SemVer = SemVer.unsafeParse("8.19.0")
2930
val es8182: SemVer = SemVer.unsafeParse("8.18.2")
3031
val es8181: SemVer = SemVer.unsafeParse("8.18.1")
3132
val es8180: SemVer = SemVer.unsafeParse("8.18.0")

0 commit comments

Comments
 (0)