Migrate from BC to BCFIPS libraries

Signed-off-by: Igonin <iigonin@sap.de>

Author: Igonin

buildSrc/src/main/java/org/opensearch/gradle/OpenSearchTestBasePlugin.java

@@ -164,6 +164,12 @@ public void execute(Task t) {
                 test.systemProperty("tests.seed", BuildParams.getTestSeed());
             }
 
+            var securityFile = "java.security";
+            test.systemProperty(
+                "java.security.properties",
+                project.getRootProject().getLayout().getProjectDirectory() + "/distribution/src/config/" + securityFile
+            );
+
             // don't track these as inputs since they contain absolute paths and break cache relocatability
             File gradleHome = project.getGradle().getGradleUserHomeDir();
             String gradleVersion = project.getGradle().getGradleVersion();

client/rest/build.gradle

@@ -29,6 +29,7 @@
  */
 
 import de.thetaphi.forbiddenapis.gradle.CheckForbiddenApis
+import org.opensearch.gradle.precommit.TestingConventionsTasks
 
 apply plugin: 'opensearch.build'
 apply plugin: 'opensearch.publish'
@@ -51,6 +52,9 @@ dependencies {
   api "commons-codec:commons-codec:${versions.commonscodec}"
   api "commons-logging:commons-logging:${versions.commonslogging}"
   api "org.slf4j:slf4j-api:${versions.slf4j}"
+  runtimeOnly "org.bouncycastle:bc-fips:${versions.bouncycastle_jce}"
+  runtimeOnly "org.bouncycastle:bctls-fips:${versions.bouncycastle_tls}"
+  runtimeOnly "org.bouncycastle:bcutil-fips:${versions.bouncycastle_util}"
 
   // reactor
   api "io.projectreactor:reactor-core:${versions.reactor}"
@@ -70,6 +74,10 @@ dependencies {
   testImplementation "org.apache.logging.log4j:log4j-slf4j-impl:${versions.log4j}"
 }
 
+tasks.named("dependencyLicenses").configure {
+  mapping from: /bc.*/, to: 'bouncycastle'
+}
+
 tasks.withType(CheckForbiddenApis).configureEach {
   //client does not depend on server, so only jdk and http signatures should be checked
   replaceSignatureFiles('jdk-signatures', 'http-signatures')

client/rest/licenses/bc-fips-2.1.0.jar.sha1

@@ -0,0 +1 @@
+c8df3d47f9854f3e9ca57e9fc862da18c9381fa9

client/rest/licenses/bctls-fips-2.1.20.jar.sha1

@@ -0,0 +1 @@
+9c0632a6c5ca09a86434cf5e02e72c221e1c930f

client/rest/licenses/bcutil-fips-2.1.4.jar.sha1

@@ -0,0 +1 @@
+1d37b7a28560684f5b8e4fd65478c9130d4015d0

client/rest/licenses/bouncycastle-LICENSE.txt

@@ -0,0 +1,14 @@
+Copyright (c) 2000 - 2023 The Legion of the Bouncy Castle Inc. (https://www.bouncycastle.org)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+documentation files (the "Software"), to deal in the Software without restriction, including without limitation
+the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software,
+and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the
+Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
+WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

client/rest/licenses/bouncycastle-NOTICE.txt

@@ -0,0 +1 @@
+

client/rest/src/test/java/org/opensearch/client/RestClientBuilderIntegTests.java

@@ -38,27 +38,23 @@
 import com.sun.net.httpserver.HttpsServer;
 
 import org.apache.hc.core5.http.HttpHost;
+import org.apache.hc.core5.ssl.SSLContextBuilder;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 
 import javax.net.ssl.KeyManagerFactory;
 import javax.net.ssl.SSLContext;
-import javax.net.ssl.SSLHandshakeException;
+import javax.net.ssl.SSLException;
 import javax.net.ssl.TrustManagerFactory;
 
 import java.io.IOException;
 import java.io.InputStream;
 import java.net.InetAddress;
 import java.net.InetSocketAddress;
-import java.nio.file.Files;
-import java.nio.file.Paths;
 import java.security.AccessController;
-import java.security.KeyFactory;
 import java.security.KeyStore;
 import java.security.PrivilegedAction;
-import java.security.cert.Certificate;
-import java.security.cert.CertificateFactory;
-import java.security.spec.PKCS8EncodedKeySpec;
+import java.security.SecureRandom;
 
 import static org.hamcrest.Matchers.instanceOf;
 import static org.junit.Assert.assertEquals;
@@ -75,7 +71,7 @@ public class RestClientBuilderIntegTests extends RestClientTestCase {
     @BeforeClass
     public static void startHttpServer() throws Exception {
         httpsServer = HttpsServer.create(new InetSocketAddress(InetAddress.getLoopbackAddress(), 0), 0);
-        httpsServer.setHttpsConfigurator(new HttpsConfigurator(getSslContext()));
+        httpsServer.setHttpsConfigurator(new HttpsConfigurator(getSslContext(true)));
         httpsServer.createContext("/", new ResponseHandler());
         httpsServer.start();
     }
@@ -103,11 +99,11 @@ public void testBuilderUsesDefaultSSLContext() throws Exception {
                     client.performRequest(new Request("GET", "/"));
                     fail("connection should have been rejected due to SSL handshake");
                 } catch (Exception e) {
-                    assertThat(e, instanceOf(SSLHandshakeException.class));
+                    assertThat(e.getCause(), instanceOf(SSLException.class));
                 }
             }
 
-            SSLContext.setDefault(getSslContext());
+            SSLContext.setDefault(getSslContext(false));
             try (RestClient client = buildRestClient()) {
                 Response response = client.performRequest(new Request("GET", "/"));
                 assertEquals(200, response.getStatusLine().getStatusCode());
@@ -122,34 +118,37 @@ private RestClient buildRestClient() {
         return RestClient.builder(new HttpHost("https", address.getHostString(), address.getPort())).build();
     }
 
-    private static SSLContext getSslContext() throws Exception {
-        SSLContext sslContext = SSLContext.getInstance(getProtocol());
+    private static SSLContext getSslContext(boolean server) throws Exception {
+        SSLContext sslContext;
+        char[] password = "password".toCharArray();
+        SecureRandom secureRandom = SecureRandom.getInstance("DEFAULT", "BCFIPS");
+        String fileExtension = ".jks";
+
         try (
-            InputStream certFile = RestClientBuilderIntegTests.class.getResourceAsStream("/test.crt");
-            InputStream keyStoreFile = RestClientBuilderIntegTests.class.getResourceAsStream("/test_truststore.jks")
+            InputStream trustStoreFile = RestClientBuilderIntegTests.class.getResourceAsStream("/test_truststore" + fileExtension);
+            InputStream keyStoreFile = RestClientBuilderIntegTests.class.getResourceAsStream("/testks" + fileExtension)
         ) {
-            // Build a keystore of default type programmatically since we can't use JKS keystores to
-            // init a KeyManagerFactory in FIPS 140 JVMs.
-            KeyStore keyStore = KeyStore.getInstance(KeyStore.getDefaultType());
-            keyStore.load(null, "password".toCharArray());
-            CertificateFactory certFactory = CertificateFactory.getInstance("X.509");
-            PKCS8EncodedKeySpec privateKeySpec = new PKCS8EncodedKeySpec(
-                Files.readAllBytes(Paths.get(RestClientBuilderIntegTests.class.getResource("/test.der").toURI()))
-            );
-            KeyFactory keyFactory = KeyFactory.getInstance("RSA");
-            keyStore.setKeyEntry(
-                "mykey",
-                keyFactory.generatePrivate(privateKeySpec),
-                "password".toCharArray(),
-                new Certificate[] { certFactory.generateCertificate(certFile) }
-            );
-            KeyManagerFactory kmf = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm());
-            kmf.init(keyStore, "password".toCharArray());
+            KeyStore keyStore = KeyStore.getInstance("JKS");
+            keyStore.load(keyStoreFile, password);
+            KeyManagerFactory kmf = KeyManagerFactory.getInstance("PKIX", "BCJSSE");
+            kmf.init(keyStore, password);
+
             KeyStore trustStore = KeyStore.getInstance("JKS");
-            trustStore.load(keyStoreFile, "password".toCharArray());
-            TrustManagerFactory tmf = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm());
+            trustStore.load(trustStoreFile, password);
+            TrustManagerFactory tmf = TrustManagerFactory.getInstance("PKIX", "BCJSSE");
             tmf.init(trustStore);
-            sslContext.init(kmf.getKeyManagers(), tmf.getTrustManagers(), null);
+
+            SSLContextBuilder sslContextBuilder = SSLContextBuilder.create()
+                .setProvider("BCJSSE")
+                .setProtocol(getProtocol())
+                .setSecureRandom(secureRandom);
+
+            if (server) {
+                sslContextBuilder.loadKeyMaterial(keyStore, password);
+            }
+            sslContextBuilder.loadTrustMaterial(trustStore, null);
+            sslContext = sslContextBuilder.build();
+
         }
         return sslContext;
     }

distribution/src/config/java.security

@@ -0,0 +1,9 @@
+# Security properties for non-approved mode 'org.bouncycastle.fips.approved_only=false'. Intended to be used complementary e.g.
+
+security.provider.1=org.bouncycastle.jcajce.provider.BouncyCastleFipsProvider
+security.provider.2=org.bouncycastle.jsse.provider.BouncyCastleJsseProvider
+security.provider.3=SUN
+security.provider.4=SunJGSS
+
+ssl.KeyManagerFactory.algorithm=PKIX
+ssl.TrustManagerFactory.algorithm=PKIX

distribution/tools/keystore-cli/src/test/java/org/opensearch/tools/cli/keystore/KeyStoreWrapperTests.java

@@ -68,8 +68,13 @@
 import java.nio.file.Path;
 import java.security.GeneralSecurityException;
 import java.security.KeyStore;
+import java.security.KeyStoreException;
 import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.security.NoSuchProviderException;
 import java.security.SecureRandom;
+import java.security.cert.CertificateException;
+import java.security.spec.InvalidKeySpecException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Base64;
@@ -366,29 +371,8 @@ public void testIllegalSettingName() throws Exception {
 
     public void testBackcompatV1() throws Exception {
         assumeFalse("Can't run in a FIPS JVM as PBE is not available", inFipsJvm());
+        generateV1();
         Path configDir = env.configDir();
-        NIOFSDirectory directory = new NIOFSDirectory(configDir);
-        try (IndexOutput output = EndiannessReverserUtil.createOutput(directory, "opensearch.keystore", IOContext.DEFAULT)) {
-            CodecUtil.writeHeader(output, "opensearch.keystore", 1);
-            output.writeByte((byte) 0); // hasPassword = false
-            output.writeString("PKCS12");
-            output.writeString("PBE");
-
-            SecretKeyFactory secretFactory = SecretKeyFactory.getInstance("PBE");
-            KeyStore keystore = KeyStore.getInstance("PKCS12");
-            keystore.load(null, null);
-            SecretKey secretKey = secretFactory.generateSecret(new PBEKeySpec("stringSecretValue".toCharArray()));
-            KeyStore.ProtectionParameter protectionParameter = new KeyStore.PasswordProtection(new char[0]);
-            keystore.setEntry("string_setting", new KeyStore.SecretKeyEntry(secretKey), protectionParameter);
-
-            ByteArrayOutputStream keystoreBytesStream = new ByteArrayOutputStream();
-            keystore.store(keystoreBytesStream, new char[0]);
-            byte[] keystoreBytes = keystoreBytesStream.toByteArray();
-            output.writeInt(keystoreBytes.length);
-            output.writeBytes(keystoreBytes, keystoreBytes.length);
-            CodecUtil.writeFooter(output);
-        }
-
         KeyStoreWrapper keystore = KeyStoreWrapper.load(configDir);
         keystore.decrypt(new char[0]);
         SecureString testValue = keystore.getString("string_setting");
@@ -397,47 +381,8 @@ public void testBackcompatV1() throws Exception {
 
     public void testBackcompatV2() throws Exception {
         assumeFalse("Can't run in a FIPS JVM as PBE is not available", inFipsJvm());
+        byte[] fileBytes = generateV2();
         Path configDir = env.configDir();
-        NIOFSDirectory directory = new NIOFSDirectory(configDir);
-        byte[] fileBytes = new byte[20];
-        random().nextBytes(fileBytes);
-        try (IndexOutput output = EndiannessReverserUtil.createOutput(directory, "opensearch.keystore", IOContext.DEFAULT)) {
-
-            CodecUtil.writeHeader(output, "opensearch.keystore", 2);
-            output.writeByte((byte) 0); // hasPassword = false
-            output.writeString("PKCS12");
-            output.writeString("PBE"); // string algo
-            output.writeString("PBE"); // file algo
-
-            output.writeVInt(2); // num settings
-            output.writeString("string_setting");
-            output.writeString("STRING");
-            output.writeString("file_setting");
-            output.writeString("FILE");
-
-            SecretKeyFactory secretFactory = SecretKeyFactory.getInstance("PBE");
-            KeyStore keystore = KeyStore.getInstance("PKCS12");
-            keystore.load(null, null);
-            SecretKey secretKey = secretFactory.generateSecret(new PBEKeySpec("stringSecretValue".toCharArray()));
-            KeyStore.ProtectionParameter protectionParameter = new KeyStore.PasswordProtection(new char[0]);
-            keystore.setEntry("string_setting", new KeyStore.SecretKeyEntry(secretKey), protectionParameter);
-
-            byte[] base64Bytes = Base64.getEncoder().encode(fileBytes);
-            char[] chars = new char[base64Bytes.length];
-            for (int i = 0; i < chars.length; ++i) {
-                chars[i] = (char) base64Bytes[i]; // PBE only stores the lower 8 bits, so this narrowing is ok
-            }
-            secretKey = secretFactory.generateSecret(new PBEKeySpec(chars));
-            keystore.setEntry("file_setting", new KeyStore.SecretKeyEntry(secretKey), protectionParameter);
-
-            ByteArrayOutputStream keystoreBytesStream = new ByteArrayOutputStream();
-            keystore.store(keystoreBytesStream, new char[0]);
-            byte[] keystoreBytes = keystoreBytesStream.toByteArray();
-            output.writeInt(keystoreBytes.length);
-            output.writeBytes(keystoreBytes, keystoreBytes.length);
-            CodecUtil.writeFooter(output);
-        }
-
         KeyStoreWrapper keystore = KeyStoreWrapper.load(configDir);
         keystore.decrypt(new char[0]);
         SecureString testValue = keystore.getString("string_setting");
@@ -497,6 +442,77 @@ public void testLegacyV3() throws GeneralSecurityException, IOException {
         assertThat(toByteArray(wrapper.getFile("file_setting")), equalTo("file_value".getBytes(StandardCharsets.UTF_8)));
     }
 
+    private void generateV1() throws IOException, NoSuchAlgorithmException, NoSuchProviderException, CertificateException,
+        InvalidKeySpecException, KeyStoreException {
+        Path configDir = env.configDir();
+        NIOFSDirectory directory = new NIOFSDirectory(configDir);
+        try (IndexOutput output = EndiannessReverserUtil.createOutput(directory, "opensearch.keystore", IOContext.DEFAULT)) {
+            CodecUtil.writeHeader(output, "opensearch.keystore", 1);
+            output.writeByte((byte) 0); // hasPassword = false
+            output.writeString("PKCS12");
+            output.writeString("PBE");
+
+            SecretKeyFactory secretFactory = SecretKeyFactory.getInstance("PBE", "SunJCE");
+            KeyStore keystore = KeyStore.getInstance("PKCS12", "SUN");
+            keystore.load(null, null);
+            SecretKey secretKey = secretFactory.generateSecret(new PBEKeySpec("stringSecretValue".toCharArray()));
+            KeyStore.ProtectionParameter protectionParameter = new KeyStore.PasswordProtection(new char[0]);
+            keystore.setEntry("string_setting", new KeyStore.SecretKeyEntry(secretKey), protectionParameter);
+
+            ByteArrayOutputStream keystoreBytesStream = new ByteArrayOutputStream();
+            keystore.store(keystoreBytesStream, new char[0]);
+            byte[] keystoreBytes = keystoreBytesStream.toByteArray();
+            output.writeInt(keystoreBytes.length);
+            output.writeBytes(keystoreBytes, keystoreBytes.length);
+            CodecUtil.writeFooter(output);
+        }
+    }
+
+    private byte[] generateV2() throws Exception {
+        Path configDir = env.configDir();
+        NIOFSDirectory directory = new NIOFSDirectory(configDir);
+        byte[] fileBytes = new byte[20];
+        random().nextBytes(fileBytes);
+        try (IndexOutput output = EndiannessReverserUtil.createOutput(directory, "opensearch.keystore", IOContext.DEFAULT)) {
+
+            CodecUtil.writeHeader(output, "opensearch.keystore", 2);
+            output.writeByte((byte) 0); // hasPassword = false
+            output.writeString("PKCS12");
+            output.writeString("PBE"); // string algo
+            output.writeString("PBE"); // file algo
+
+            output.writeVInt(2); // num settings
+            output.writeString("string_setting");
+            output.writeString("STRING");
+            output.writeString("file_setting");
+            output.writeString("FILE");
+
+            SecretKeyFactory secretFactory = SecretKeyFactory.getInstance("PBE", "SunJCE");
+            KeyStore keystore = KeyStore.getInstance("PKCS12", "SUN");
+            keystore.load(null, null);
+            SecretKey secretKey = secretFactory.generateSecret(new PBEKeySpec("stringSecretValue".toCharArray()));
+            KeyStore.ProtectionParameter protectionParameter = new KeyStore.PasswordProtection(new char[0]);
+            keystore.setEntry("string_setting", new KeyStore.SecretKeyEntry(secretKey), protectionParameter);
+
+            byte[] base64Bytes = Base64.getEncoder().encode(fileBytes);
+            char[] chars = new char[base64Bytes.length];
+            for (int i = 0; i < chars.length; ++i) {
+                chars[i] = (char) base64Bytes[i]; // PBE only stores the lower 8 bits, so this narrowing is ok
+            }
+            secretKey = secretFactory.generateSecret(new PBEKeySpec(chars));
+            keystore.setEntry("file_setting", new KeyStore.SecretKeyEntry(secretKey), protectionParameter);
+
+            ByteArrayOutputStream keystoreBytesStream = new ByteArrayOutputStream();
+            keystore.store(keystoreBytesStream, new char[0]);
+            byte[] keystoreBytes = keystoreBytesStream.toByteArray();
+            output.writeInt(keystoreBytes.length);
+            output.writeBytes(keystoreBytes, keystoreBytes.length);
+            CodecUtil.writeFooter(output);
+        }
+
+        return fileBytes;
+    }
+
     private byte[] toByteArray(final InputStream is) throws IOException {
         final ByteArrayOutputStream os = new ByteArrayOutputStream();
         final byte[] buffer = new byte[1024];