Make kyuubi.engine.security parameters public

docs/configuration/settings.md

@@ -354,6 +354,17 @@ You can configure the Kyuubi properties in `$KYUUBI_HOME/conf/kyuubi-defaults.co
 | kyuubi.ha.zookeeper.quorum                                                                                     || (deprecated) The connection string for the ZooKeeper ensemble                                                                                                                                                            | string   | 1.0.0  |
 | kyuubi.ha.zookeeper.session.timeout            | 60000                                                          | The timeout(ms) of a connected session to be idled                                                                                                                                                                       | int      | 1.0.0  |
 
+### Internal
+
+|                          Key                           |      Default      |                                                                                                                                                                                                                                          Meaning                                                                                                                                                                                                                                          |   Type   | Since  |
+|--------------------------------------------------------|-------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------|--------|
+| kyuubi.internal.security.crypto.ivLength               | 16                | Initial vector length, in bytes. Only take affects when kyuubi.internal.security.enabled is set to true.                                                                                                                                                                                                                                                                                                                                                                                  | int      | 1.12.0 |
+| kyuubi.internal.security.crypto.keyLength              | 128               | The length in bits of the encryption key to generate. Only take affects when kyuubi.internal.security.enabled is set to true. Valid values are 128, 192 and 256                                                                                                                                                                                                                                                                                                                           | int      | 1.12.0 |
+| kyuubi.internal.security.enabled                       | false             | Whether to enable secure access across all the internal communications, both<ul><li>between kyuubi server and kyuubi engine</li><li>across kyuubi server instances</li></ul>You need to also provide configure                                                                                                                                                                                                                                                                            | boolean  | 1.12.0 |
+| kyuubi.internal.security.secret.provider               | zookeeper         | The class used to manage the internal security secret. This class must be a subclass of `EngineSecuritySecretProvider`. Kyuubi provides the following built-in implementations: <ul><li>simple: Use the secret set by kyuubi.internal.security.secret.provider.simple.secret.</li> <li>zookeeper: Use the secret stored in ZooKeeper. kyuubi.ha.zookeeper.engine.secure.secret.node must be configured.</li></ul> Only take affects when kyuubi.internal.security.enabled is set to true. | string   | 1.12.0 |
+| kyuubi.internal.security.secret.provider.simple.secret | &lt;undefined&gt; | The secret key used for internal security access. Only take affects when kyuubi.internal.security.enabled is set to true and kyuubi.internal.security.secret.provider is 'simple'                                                                                                                                                                                                                                                                                                         | string   | 1.12.0 |
+| kyuubi.internal.security.token.max.lifetime            | PT10M             | The max lifetime of the token used for internal secure access. Only take affects when kyuubi.internal.security.enabled is set to true.                                                                                                                                                                                                                                                                                                                                                    | duration | 1.12.0 |
+
 ### Kinit
 
 |            Key            |      Default      |                                                       Meaning                                                       |   Type   | Since |

kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala

@@ -2747,6 +2747,7 @@ object KyuubiConf {
       .stringConf
       .createWithDefault("engine_operation_logs")
 
+  @deprecated("using kyuubi.internal.security.enabled instead", "1.12.0")
   val ENGINE_SECURITY_ENABLED: ConfigEntry[Boolean] =
     buildConf("kyuubi.engine.security.enabled")
       .internal
@@ -2757,6 +2758,7 @@ object KyuubiConf {
       .booleanConf
       .createWithDefault(false)
 
+  @deprecated("using kyuubi.internal.security.token.max.lifetime instead", "1.12.0")
   val ENGINE_SECURITY_TOKEN_MAX_LIFETIME: ConfigEntry[Long] =
     buildConf("kyuubi.engine.security.token.max.lifetime")
       .internal
@@ -2765,6 +2767,7 @@ object KyuubiConf {
       .timeConf
       .createWithDefault(Duration.ofMinutes(10).toMillis)
 
+  @deprecated("using kyuubi.internal.security.secret.provider instead", "1.12.0")
   val ENGINE_SECURITY_SECRET_PROVIDER: ConfigEntry[String] =
     buildConf("kyuubi.engine.security.secret.provider")
       .internal
@@ -2781,6 +2784,7 @@ object KyuubiConf {
       }
       .createWithDefault("zookeeper")
 
+  @deprecated("using kyuubi.internal.security.secret.provider.simple.secret instead", "1.12.0")
   val SIMPLE_SECURITY_SECRET_PROVIDER_PROVIDER_SECRET: OptionalConfigEntry[String] =
     buildConf("kyuubi.engine.security.secret.provider.simple.secret")
       .internal
@@ -2790,6 +2794,7 @@ object KyuubiConf {
       .stringConf
       .createOptional
 
+  @deprecated("using kyuubi.internal.security.crypto.keyLength instead", "1.12.0")
   val ENGINE_SECURITY_CRYPTO_KEY_LENGTH: ConfigEntry[Int] =
     buildConf("kyuubi.engine.security.crypto.keyLength")
       .internal
@@ -2800,6 +2805,7 @@ object KyuubiConf {
       .checkValues(Set(128, 192, 256))
       .createWithDefault(128)
 
+  @deprecated("using kyuubi.internal.security.crypto.ivLength instead", "1.12.0")
   val ENGINE_SECURITY_CRYPTO_IV_LENGTH: ConfigEntry[Int] =
     buildConf("kyuubi.engine.security.crypto.ivLength")
       .internal
@@ -2808,6 +2814,7 @@ object KyuubiConf {
       .intConf
       .createWithDefault(16)
 
+  @deprecated("using kyuubi.internal.security.crypto.keyAlgorithm instead", "1.12.0")
   val ENGINE_SECURITY_CRYPTO_KEY_ALGORITHM: ConfigEntry[String] =
     buildConf("kyuubi.engine.security.crypto.keyAlgorithm")
       .internal
@@ -2816,6 +2823,7 @@ object KyuubiConf {
       .stringConf
       .createWithDefault("AES")
 
+  @deprecated("using kyuubi.internal.security.crypto.cipher instead", "1.12.0")
   val ENGINE_SECURITY_CRYPTO_CIPHER_TRANSFORMATION: ConfigEntry[String] =
     buildConf("kyuubi.engine.security.crypto.cipher")
       .internal
@@ -2824,6 +2832,72 @@ object KyuubiConf {
       .stringConf
       .createWithDefault("AES/CBC/PKCS5PADDING")
 
+  val INTERNAL_SECURITY_ENABLED: ConfigEntry[Boolean] =
+    buildConf("kyuubi.internal.security.enabled")
+      .doc("Whether to enable secure access across all the internal communications, both<ul>" +
+        "<li>between kyuubi server and kyuubi engine</li>" +
+        "<li>across kyuubi server instances</li></ul>" +
+        "You need to also provide configure ")
+      .version("1.12.0")
+      .fallbackConf(ENGINE_SECURITY_ENABLED)
+
+  val INTERNAL_SECURITY_TOKEN_MAX_LIFETIME: ConfigEntry[Long] =
+    buildConf("kyuubi.internal.security.token.max.lifetime")
+      .doc("The max lifetime of the token used for internal secure access. Only take affects" +
+        s" when ${INTERNAL_SECURITY_ENABLED.key} is set to true.")
+      .version("1.12.0")
+      .fallbackConf(ENGINE_SECURITY_TOKEN_MAX_LIFETIME)
+
+  val INTERNAL_SECURITY_SECRET_PROVIDER: ConfigEntry[String] =
+    buildConf("kyuubi.internal.security.secret.provider")
+      .doc("The class used to manage the internal security secret. This class must be a " +
+        "subclass of `EngineSecuritySecretProvider`. Kyuubi provides the following " +
+        "built-in implementations: <ul><li>simple: Use the secret set by " +
+        s"kyuubi.internal.security.secret.provider.simple.secret.</li> " +
+        "<li>zookeeper: Use the secret stored in ZooKeeper. " +
+        "kyuubi.ha.zookeeper.engine.secure.secret.node must be configured.</li></ul> " +
+        s"Only take affects when ${INTERNAL_SECURITY_ENABLED.key} is set to true.")
+      .version("1.12.0")
+      .fallbackConf(ENGINE_SECURITY_SECRET_PROVIDER)
+
+  val INTERNAL_SECURITY_SECRET_PROVIDER_SIMPLE_SECRET: ConfigEntry[Option[String]] =
+    buildConf("kyuubi.internal.security.secret.provider.simple.secret")
+      .doc("The secret key used for internal security access. Only take affects when" +
+        s" ${INTERNAL_SECURITY_ENABLED.key} is set to true and" +
+        s" ${INTERNAL_SECURITY_SECRET_PROVIDER.key} is 'simple'")
+      .version("1.12.0")
+      .fallbackConf(SIMPLE_SECURITY_SECRET_PROVIDER_PROVIDER_SECRET)
+
+  val INTERNAL_SECURITY_CRYPTO_KEY_LENGTH: ConfigEntry[Int] =
+    buildConf("kyuubi.internal.security.crypto.keyLength")
+      .doc("The length in bits of the encryption key to generate. Only take affects when" +
+        s" ${INTERNAL_SECURITY_ENABLED.key} is set to true. Valid values are 128, 192 and 256")
+      .version("1.12.0")
+      .fallbackConf(ENGINE_SECURITY_CRYPTO_KEY_LENGTH)
+
+  val INTERNAL_SECURITY_CRYPTO_IV_LENGTH: ConfigEntry[Int] =
+    buildConf("kyuubi.internal.security.crypto.ivLength")
+      .doc("Initial vector length, in bytes. Only take affects when" +
+        s" ${INTERNAL_SECURITY_ENABLED.key} is set to true.")
+      .version("1.12.0")
+      .fallbackConf(ENGINE_SECURITY_CRYPTO_IV_LENGTH)
+
+  val INTERNAL_SECURITY_CRYPTO_KEY_ALGORITHM: ConfigEntry[String] =
+    buildConf("kyuubi.internal.security.crypto.keyAlgorithm")
+      .internal
+      .doc("The algorithm for generated secret keys. Only take affects when" +
+        s" ${INTERNAL_SECURITY_ENABLED.key} is set to true.")
+      .version("1.12.0")
+      .fallbackConf(ENGINE_SECURITY_CRYPTO_KEY_ALGORITHM)
+
+  val INTERNAL_SECURITY_CRYPTO_CIPHER_TRANSFORMATION: ConfigEntry[String] =
+    buildConf("kyuubi.internal.security.crypto.cipher")
+      .internal
+      .doc("The cipher transformation to use for encrypting internal access token." +
+        s" Only take affects when ${INTERNAL_SECURITY_ENABLED.key} is set to true.")
+      .version("1.12.0")
+      .fallbackConf(ENGINE_SECURITY_CRYPTO_CIPHER_TRANSFORMATION)
+
   val SESSION_NAME: OptionalConfigEntry[String] =
     buildConf("kyuubi.session.name")
       .doc("A human readable name of the session and we use empty string by default. " +

kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/AuthenticationProviderFactory.scala

@@ -57,7 +57,7 @@ object AuthenticationProviderFactory {
   }
 
   private def getAuthenticationProviderForEngine(conf: KyuubiConf): PasswdAuthenticationProvider = {
-    if (conf.get(KyuubiConf.ENGINE_SECURITY_ENABLED)) {
+    if (conf.get(KyuubiConf.INTERNAL_SECURITY_ENABLED)) {
       new EngineSecureAuthenticationProviderImpl
     } else {
       new AnonymousAuthenticationProviderImpl

kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/EngineSecuritySecretProvider.scala

@@ -41,18 +41,18 @@ class SimpleEngineSecuritySecretProviderImpl extends EngineSecuritySecretProvide
   override def initialize(conf: KyuubiConf): Unit = _conf = conf
 
   override def getSecret(): String = {
-    _conf.get(SIMPLE_SECURITY_SECRET_PROVIDER_PROVIDER_SECRET).getOrElse {
+    _conf.get(INTERNAL_SECURITY_SECRET_PROVIDER_SIMPLE_SECRET).getOrElse {
       throw new IllegalArgumentException(
-        s"${SIMPLE_SECURITY_SECRET_PROVIDER_PROVIDER_SECRET.key} must be configured " +
-          s"when ${ENGINE_SECURITY_SECRET_PROVIDER.key} is `simple`.")
+        s"${INTERNAL_SECURITY_SECRET_PROVIDER_SIMPLE_SECRET.key} must be configured " +
+          s"when ${INTERNAL_SECURITY_SECRET_PROVIDER.key} is `simple`.")
     }
   }
 }
 
 object EngineSecuritySecretProvider {
   def create(conf: KyuubiConf): EngineSecuritySecretProvider = {
     val provider = DynConstructors.builder()
-      .impl(conf.get(ENGINE_SECURITY_SECRET_PROVIDER))
+      .impl(conf.get(INTERNAL_SECURITY_SECRET_PROVIDER))
       .buildChecked[EngineSecuritySecretProvider]()
       .newInstance(conf)
     provider.initialize(conf)

kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/InternalSecurityAccessor.scala

@@ -29,12 +29,12 @@ import org.apache.kyuubi.config.KyuubiConf
 import org.apache.kyuubi.config.KyuubiConf._
 
 class InternalSecurityAccessor(conf: KyuubiConf, val isServer: Boolean) {
-  val cryptoKeyLengthBytes = conf.get(ENGINE_SECURITY_CRYPTO_KEY_LENGTH) / java.lang.Byte.SIZE
-  val cryptoIvLength = conf.get(ENGINE_SECURITY_CRYPTO_IV_LENGTH)
-  val cryptoKeyAlgorithm = conf.get(ENGINE_SECURITY_CRYPTO_KEY_ALGORITHM)
-  val cryptoCipher = conf.get(ENGINE_SECURITY_CRYPTO_CIPHER_TRANSFORMATION)
+  val cryptoKeyLengthBytes = conf.get(INTERNAL_SECURITY_CRYPTO_KEY_LENGTH) / java.lang.Byte.SIZE
+  val cryptoIvLength = conf.get(INTERNAL_SECURITY_CRYPTO_IV_LENGTH)
+  val cryptoKeyAlgorithm = conf.get(INTERNAL_SECURITY_CRYPTO_KEY_ALGORITHM)
+  val cryptoCipher = conf.get(INTERNAL_SECURITY_CRYPTO_CIPHER_TRANSFORMATION)
 
-  private val tokenMaxLifeTime: Long = conf.get(ENGINE_SECURITY_TOKEN_MAX_LIFETIME)
+  private val tokenMaxLifeTime: Long = conf.get(INTERNAL_SECURITY_TOKEN_MAX_LIFETIME)
   private val provider: EngineSecuritySecretProvider = EngineSecuritySecretProvider.create(conf)
   private val (secretKeySpec, encryptor, decryptor) =
     initializeForAuth(cryptoCipher, normalizeSecret(provider.getSecret()))

kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/KyuubiAuthenticationFactory.scala

@@ -50,7 +50,7 @@ class KyuubiAuthenticationFactory(conf: KyuubiConf, isServer: Boolean = true) ex
     }
   }
 
-  if (conf.get(ENGINE_SECURITY_ENABLED)) {
+  if (conf.get(INTERNAL_SECURITY_ENABLED)) {
     InternalSecurityAccessor.initialize(conf, isServer)
   }
 

kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/InternalSecurityAccessorSuite.scala

@@ -22,13 +22,13 @@ import org.apache.kyuubi.config.KyuubiConf
 
 class InternalSecurityAccessorSuite extends KyuubiFunSuite {
   private val conf = KyuubiConf()
-    .set(KyuubiConf.ENGINE_SECURITY_SECRET_PROVIDER, "simple")
-    .set(KyuubiConf.SIMPLE_SECURITY_SECRET_PROVIDER_PROVIDER_SECRET, "ENGINE____SECRET")
+    .set(KyuubiConf.INTERNAL_SECURITY_SECRET_PROVIDER, "simple")
+    .set(KyuubiConf.INTERNAL_SECURITY_SECRET_PROVIDER_SIMPLE_SECRET.key, "ENGINE____SECRET")
 
   test("test encrypt/decrypt, issue token/auth token") {
     Seq("AES/CBC/PKCS5PADDING", "AES/CTR/NoPadding").foreach { cipher =>
       val newConf = conf.clone
-      newConf.set(KyuubiConf.ENGINE_SECURITY_CRYPTO_CIPHER_TRANSFORMATION, cipher)
+      newConf.set(KyuubiConf.INTERNAL_SECURITY_CRYPTO_CIPHER_TRANSFORMATION, cipher)
 
       val secureAccessor = new InternalSecurityAccessor(newConf, true)
       val value = "tokenToEncrypt"