HADOOP-19546. Include cipher feature for HttpServer2 and SSLFactory (#7629)

Author: K0K0V0K

hadoop-common-project/hadoop-common/src/main/conf/ssl-server.xml.example

@@ -82,7 +82,25 @@
   SSL_RSA_EXPORT_WITH_RC4_40_MD5,SSL_RSA_EXPORT_WITH_DES40_CBC_SHA,
   SSL_RSA_WITH_RC4_128_MD5</value>
   <description>Optional. The weak security cipher suites that you want excluded
-  from SSL communication.</description>
+  from SSL communication.
+  Both ssl.server.include.cipher.list and ssl.server.exclude.cipher.list can be used simultaneously
+  to fine-tune the cipher suites utilized by Hadoop services.
+  If a cipher suite is present in both the inclusion and exclusion lists, it will be denied.
+  </description>
+</property>
+
+<property>
+  <name>ssl.server.include.cipher.list</name>
+  <value>TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,
+  TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
+  TLS_AES_128_GCM_SHA256, TLS_AES_256_GCM_SHA384, TLS_CHACHA20_POLY1305_SHA256,
+  TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256, TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256</value>
+  <description>Optional. If the inclusion list is populated,
+  any cipher not present in the list will not be allowed.
+  Both ssl.server.include.cipher.list and ssl.server.exclude.cipher.list can be used simultaneously
+  to fine-tune the cipher suites utilized by Hadoop services.
+  If a cipher suite is present in both the inclusion and exclusion lists, it will be denied.
+  </description>
 </property>
 
 </configuration>

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java

@@ -253,6 +253,7 @@ public static class Builder {
         new ArrayList<>(Collections.singletonList(
             "hadoop.http.authentication."));
     private String excludeCiphers;
+    private String includeCiphers;
 
     private boolean xFrameEnabled;
     private XFrameOption xFrameOption = XFrameOption.SAMEORIGIN;
@@ -400,6 +401,11 @@ public Builder excludeCiphers(String pExcludeCiphers) {
       return this;
     }
 
+    public Builder includeCiphers(String pIncludeCiphers) {
+      this.includeCiphers = pIncludeCiphers;
+      return this;
+    }
+
     /**
      * Adds the ability to control X_FRAME_OPTIONS on HttpServer2.
      * @param xFrameEnabled - True enables X_FRAME_OPTIONS false disables it.
@@ -481,6 +487,7 @@ private void loadSSLConfiguration() throws IOException {
       trustStoreType = sslConf.get(SSLFactory.SSL_SERVER_TRUSTSTORE_TYPE,
           SSLFactory.SSL_SERVER_TRUSTSTORE_TYPE_DEFAULT);
       excludeCiphers = sslConf.get(SSLFactory.SSL_SERVER_EXCLUDE_CIPHER_LIST);
+      includeCiphers = sslConf.get(SSLFactory.SSL_SERVER_INCLUDE_CIPHER_LIST);
     }
 
     public HttpServer2 build() throws IOException {
@@ -608,10 +615,17 @@ private ServerConnector createHttpsChannelConnector(
           sslContextFactory.setTrustStorePassword(trustStorePassword);
         }
       }
-      if(null != excludeCiphers && !excludeCiphers.isEmpty()) {
+
+      if (StringUtils.hasLength(excludeCiphers)) {
         sslContextFactory.setExcludeCipherSuites(
             StringUtils.getTrimmedStrings(excludeCiphers));
-        LOG.info("Excluded Cipher List:" + excludeCiphers);
+        LOG.info("Excluded Cipher List:{}", excludeCiphers);
+      }
+
+      if (StringUtils.hasLength(includeCiphers)) {
+        sslContextFactory.setIncludeCipherSuites(
+          StringUtils.getTrimmedStrings(includeCiphers));
+        LOG.info("Included Cipher List:{}", includeCiphers);
       }
 
       setEnabledProtocols(sslContextFactory);

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/SSLFactory.java

@@ -17,6 +17,7 @@
 */
 package org.apache.hadoop.security.ssl;
 
+import org.apache.commons.collections4.CollectionUtils;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configuration;
@@ -38,10 +39,9 @@
 import java.io.IOException;
 import java.net.HttpURLConnection;
 import java.security.GeneralSecurityException;
-import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.Iterator;
 import java.util.List;
+import java.util.stream.Stream;
 
 /**
  * Factory that creates SSLEngine and SSLSocketFactory instances using
@@ -101,6 +101,9 @@ public enum Mode { CLIENT, SERVER }
   public static final String SSL_SERVER_EXCLUDE_CIPHER_LIST =
       "ssl.server.exclude.cipher.list";
 
+  public static final String SSL_SERVER_INCLUDE_CIPHER_LIST =
+      "ssl.server.include.cipher.list";
+
   public static final String KEY_MANAGER_SSLCERTIFICATE =
       IBM_JAVA ? "ibmX509" :
           KeyManagerFactory.getDefaultAlgorithm();
@@ -125,6 +128,7 @@ public enum Mode { CLIENT, SERVER }
 
   private String[] enabledProtocols = null;
   private List<String> excludeCiphers;
+  private List<String> includeCiphers;
 
   /**
    * Creates an SSLFactory.
@@ -153,9 +157,13 @@ public SSLFactory(Mode mode, Configuration conf) {
         SSL_ENABLED_PROTOCOLS_DEFAULT);
     excludeCiphers = Arrays.asList(
         sslConf.getTrimmedStrings(SSL_SERVER_EXCLUDE_CIPHER_LIST));
+    includeCiphers = Arrays.asList(
+      sslConf.getTrimmedStrings(SSL_SERVER_INCLUDE_CIPHER_LIST));
     if (LOG.isDebugEnabled()) {
       LOG.debug("will exclude cipher suites: {}",
           StringUtils.join(",", excludeCiphers));
+      LOG.debug("will include cipher suites: {}",
+          StringUtils.join(",", includeCiphers));
     }
   }
 
@@ -261,30 +269,23 @@ public SSLEngine createSSLEngine()
     } else {
       sslEngine.setUseClientMode(false);
       sslEngine.setNeedClientAuth(requireClientCert);
-      disableExcludedCiphers(sslEngine);
+      callSetEnabledCipherSuites(sslEngine);
     }
     sslEngine.setEnabledProtocols(enabledProtocols);
     return sslEngine;
   }
 
-  private void disableExcludedCiphers(SSLEngine sslEngine) {
-    String[] cipherSuites = sslEngine.getEnabledCipherSuites();
-
-    ArrayList<String> defaultEnabledCipherSuites =
-        new ArrayList<String>(Arrays.asList(cipherSuites));
-    Iterator iterator = excludeCiphers.iterator();
-
-    while(iterator.hasNext()) {
-      String cipherName = (String)iterator.next();
-      if(defaultEnabledCipherSuites.contains(cipherName)) {
-        defaultEnabledCipherSuites.remove(cipherName);
-        LOG.debug("Disabling cipher suite {}.", cipherName);
-      }
+  private void callSetEnabledCipherSuites(SSLEngine sslEngine) {
+    Stream<String> cipherSuites = Arrays.stream(sslEngine.getSupportedCipherSuites());
+    if (CollectionUtils.isNotEmpty(includeCiphers)) {
+      cipherSuites = cipherSuites.filter(s -> includeCiphers.contains(s));
     }
-
-    cipherSuites = defaultEnabledCipherSuites.toArray(
-        new String[defaultEnabledCipherSuites.size()]);
-    sslEngine.setEnabledCipherSuites(cipherSuites);
+    if (CollectionUtils.isNotEmpty(excludeCiphers)) {
+      cipherSuites = cipherSuites.filter(s -> !excludeCiphers.contains(s));
+    }
+    String[] enabledCipherSuites = cipherSuites.toArray(String[]::new);
+    LOG.debug("Enabled cipher suites: {}", StringUtils.join(",", enabledCipherSuites));
+    sslEngine.setEnabledCipherSuites(enabledCipherSuites);
   }
 
   /**

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java

@@ -1367,4 +1367,14 @@ public static String wrap(String str, int wrapLength, String newLineStr,
       return wrappedLine.toString();
     }
   }
+
+  /**
+   * Checks whether the given string is not {@code null} and has a length greater than zero.
+   *
+   * @param str the string to check
+   * @return {@code true} if the string is not {@code null} and not empty; {@code false} otherwise
+   */
+  public static boolean hasLength(String str) {
+    return str != null && !str.isEmpty();
+  }
 }

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpCookieFlag.java

@@ -105,8 +105,8 @@ public static void setUp() throws Exception {
             .trustStore(sslConf.get("ssl.server.truststore.location"),
                     sslConf.get("ssl.server.truststore.password"),
                     sslConf.get("ssl.server.truststore.type", "jks"))
-            .excludeCiphers(
-                    sslConf.get("ssl.server.exclude.cipher.list"))
+            .excludeCiphers(sslConf.get("ssl.server.exclude.cipher.list"))
+            .includeCiphers(sslConf.get("ssl.server.include.cipher.list"))
             .build();
     server.addServlet("echo", "/echo", TestHttpServer.EchoServlet.class);
     server.start();

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestSSLHttpServer.java

@@ -149,7 +149,9 @@ private static void setupServer(Configuration conf, Configuration sslConf)
             sslConf.get(SSL_SERVER_TRUSTSTORE_PROP_PREFIX + ".location"),
             sslConf.get(SSL_SERVER_TRUSTSTORE_PROP_PREFIX + ".password"),
             sslConf.get(SSL_SERVER_TRUSTSTORE_PROP_PREFIX + ".type", "jks"))
-        .excludeCiphers(sslConf.get("ssl.server.exclude.cipher.list")).build();
+        .excludeCiphers(sslConf.get("ssl.server.exclude.cipher.list"))
+        .includeCiphers(sslConf.get("ssl.server.include.cipher.list"))
+        .build();
     server.addServlet(SERVLET_NAME_ECHO, SERVLET_PATH_ECHO, EchoServlet.class);
     server.addServlet(SERVLET_NAME_LONGHEADER, SERVLET_PATH_LONGHEADER,
         LongHeaderServlet.class);

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestSSLHttpServerConfigs.java

@@ -110,7 +110,9 @@ private HttpServer2 setupServer(String keyStoreKeyPassword,
             sslConf.get(SSL_SERVER_TRUSTSTORE_PROP_PREFIX + ".location"),
             trustStorePassword,
             sslConf.get(SSL_SERVER_TRUSTSTORE_PROP_PREFIX + ".type", "jks"))
-        .excludeCiphers(sslConf.get("ssl.server.exclude.cipher.list")).build();
+        .excludeCiphers(sslConf.get("ssl.server.exclude.cipher.list"))
+        .includeCiphers(sslConf.get("ssl.server.include.cipher.list"))
+        .build();
 
     return server;
   }

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java

@@ -1630,7 +1630,9 @@ public static HttpServer2.Builder loadSslConfToHttpServerBuilder(HttpServer2.Bui
             getPassword(sslConf, DFS_SERVER_HTTPS_TRUSTSTORE_PASSWORD_KEY),
             sslConf.get("ssl.server.truststore.type", "jks"))
         .excludeCiphers(
-            sslConf.get("ssl.server.exclude.cipher.list"));
+            sslConf.get("ssl.server.exclude.cipher.list"))
+        .includeCiphers(
+            sslConf.get("ssl.server.include.cipher.list"));
   }
 
   /**

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java

@@ -494,7 +494,9 @@ public static HttpServer2.Builder loadSslConfiguration(
             getPassword(sslConf, WEB_APP_TRUSTSTORE_PASSWORD_KEY),
             sslConf.get("ssl.server.truststore.type", "jks"))
         .excludeCiphers(
-            sslConf.get("ssl.server.exclude.cipher.list"));
+            sslConf.get("ssl.server.exclude.cipher.list"))
+        .includeCiphers(
+            sslConf.get("ssl.server.include.cipher.list"));
   }
 
   /**

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/util/TestWebServiceClient.java

@@ -85,7 +85,9 @@ void testCreateClient() throws Exception {
             sslConf.get(SSL_SERVER_TRUSTSTORE_PROP_PREFIX + ".location"),
             sslConf.get(SSL_SERVER_TRUSTSTORE_PROP_PREFIX + ".password"),
             sslConf.get(SSL_SERVER_TRUSTSTORE_PROP_PREFIX + ".type", "jks"))
-        .excludeCiphers(sslConf.get("ssl.server.exclude.cipher.list")).build();
+        .excludeCiphers(sslConf.get("ssl.server.exclude.cipher.list"))
+        .includeCiphers(sslConf.get("ssl.server.include.cipher.list"))
+        .build();
     server.addServlet(SERVLET_NAME_ECHO, SERVLET_PATH_ECHO, EchoServlet.class);
     server.start();