Truncate a hostname longer than 64 characters for self-signed certificates (#6900)

Motivation:

Self-signed certificate generation uses the local hostname as the
certificate CN. On a machine whose hostname exceeds
64 characters — e.g. a GitHub Actions macOS runner named
`sjc20-cw714-0e535ccf-aeaf-489f-8a27-1ab749ed681c-fe6ab157f2e4.local`
(67 characters) — it fails with:

```
java.lang.IllegalArgumentException: commonName length 67 exceeds RFC 5280 ub-common-name (64)
```

Both `TlsKeyPair.ofSelfSigned()` and `ServerBuilder.tlsSelfSigned()` are
affected, so any test using them fails whenever such
a runner is assigned ([example
failure](https://github.com/line/armeria/actions/runs/30982702653/job/92230410002#step:10:1982):
`:athenz:shadedTest` failed at class initialization).

Modifications:

- Truncate an fqdn longer than 64 characters in the
`SelfSignedCertificate` constructor, which covers every self-signed
certificate generation path, and log a debug message when truncation
happens.

Result:

- Self-signed certificate generation no longer fails on a machine whose
hostname exceeds 64 characters.

Author: ikhoon

core/src/main/java/com/linecorp/armeria/common/TlsKeyPair.java

@@ -138,6 +138,9 @@ public static TlsKeyPair ofSelfSigned(String hostname) {
 
     /**
      * Generates a self-signed certificate for the local hostname.
+     *
+     * <p>Note that if the local hostname exceeds 64 characters, it is truncated to satisfy the
+     * RFC 5280 common name length limit.
      */
     public static TlsKeyPair ofSelfSigned() {
         return ofSelfSigned(SystemInfo.hostname());

core/src/main/java/com/linecorp/armeria/internal/common/util/SelfSignedCertificate.java

@@ -36,6 +36,9 @@
 import java.util.Random;
 import java.util.concurrent.ThreadLocalRandom;
 
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 import com.google.common.collect.ImmutableList;
 
 /**
@@ -53,6 +56,10 @@
  */
 public final class SelfSignedCertificate extends SignedCertificate {
 
+    private static final Logger logger = LoggerFactory.getLogger(SelfSignedCertificate.class);
+
+    private static final int MAX_COMMON_NAME_LENGTH = 64;
+
     // Forked from:
     // https://github.com/netty/netty/blob/11e6a77fba9ec7184a558d869373d0ce506d7236/handler/src/main/java/io/netty/handler/ssl/util/SelfSignedCertificate.java
     // https://github.com/netty/netty/blob/11e6a77fba9ec7184a558d869373d0ce506d7236/handler/src/main/java/io/netty/handler/ssl/util/BouncyCastleSelfSignedCertGenerator.java
@@ -235,7 +242,17 @@ public SelfSignedCertificate(String fqdn, Random random, int bits, Date notBefor
     public SelfSignedCertificate(String fqdn, Random random, int bits, Date notBefore, Date notAfter,
                                  String algorithm, Iterable<String> subjectAlternativeNames, boolean isCA)
             throws CertificateException {
-        super(new CertificateParams(fqdn, random, bits, notBefore, notAfter, algorithm,
-                                    subjectAlternativeNames, isCA));
+        super(new CertificateParams(truncateToCommonNameLength(fqdn), random, bits, notBefore, notAfter,
+                                    algorithm, subjectAlternativeNames, isCA));
+    }
+
+    private static String truncateToCommonNameLength(String fqdn) {
+        if (fqdn.length() <= MAX_COMMON_NAME_LENGTH) {
+            return fqdn;
+        }
+        final String truncated = fqdn.substring(0, MAX_COMMON_NAME_LENGTH);
+        logger.debug("Truncating the fqdn '{}' to '{}' to satisfy " +
+                     "the RFC 5280 common name length limit (64).", fqdn, truncated);
+        return truncated;
     }
 }

core/src/test/java/com/linecorp/armeria/common/TlsKeyPairTest.java

@@ -31,6 +31,7 @@
 import org.junit.jupiter.api.Test;
 
 import com.linecorp.armeria.common.annotation.Nullable;
+import com.linecorp.armeria.common.util.SystemInfo;
 import com.linecorp.armeria.internal.common.util.SelfSignedCertificate;
 
 class TlsKeyPairTest {
@@ -45,7 +46,14 @@ void selfSignedIsAValidPair() throws CertificateException {
 
     @Test
     void ofSelfSignedIsAValidPair() {
-        assertThat(TlsKeyPair.ofSelfSigned().privateKey()).isNotNull();
+        // Must not fail even on a machine whose hostname exceeds the 64-character common name limit
+        // of RFC 5280, such as a GitHub Actions macOS runner.
+        final TlsKeyPair keyPair = TlsKeyPair.ofSelfSigned();
+        assertThat(keyPair.privateKey()).isNotNull();
+        final String hostname = SystemInfo.hostname();
+        final String expectedCommonName = hostname.length() <= 64 ? hostname : hostname.substring(0, 64);
+        assertThat(keyPair.certificateChain().get(0).getSubjectX500Principal().getName())
+                .isEqualTo("CN=" + expectedCommonName);
     }
 
     @Test

core/src/test/java/com/linecorp/armeria/internal/common/util/SelfSignedCertificateTest.java

@@ -26,6 +26,7 @@
 
 import org.junit.jupiter.api.Test;
 
+import com.google.common.base.Strings;
 import com.google.common.collect.ImmutableList;
 
 class SelfSignedCertificateTest {
@@ -42,6 +43,18 @@ void fqdnAsteriskFileNameTest() throws CertificateException {
         assertThat(ssc.privateKey().getName()).doesNotContain("*");
     }
 
+    @Test
+    void fqdnLongerThanCommonNameLimitTest() throws Exception {
+        // The hostname of some machines, such as a GitHub Actions macOS runner, exceeds the
+        // 64-character common name limit of RFC 5280.
+        final String fqdn = "very-long-hostname-" + Strings.repeat("a", 42) + ".local";
+        assertThat(fqdn).hasSize(67);
+
+        final SelfSignedCertificate ssc = new SelfSignedCertificate(fqdn);
+        assertThat(ssc.cert().getSubjectX500Principal().getName())
+                .isEqualTo("CN=" + fqdn.substring(0, 64));
+    }
+
     @Test
     void subjectAlternativeNamesWithUriTest() throws Exception {
         final List<String> additionalSans = ImmutableList.of(

core/src/test/java/com/linecorp/armeria/server/ServerTlsHandshakeMetricsTest.java

@@ -97,7 +97,7 @@ void handshakeSuccess(SessionProtocol sessionProtocol, String tlsProtocol, Strin
             final Counter counter =
                     meterRegistry.find("armeria.server.tls.handshakes")
                                  .tag("cipher.suite", value -> value.startsWith("TLS_"))
-                                 .tag("hostname", server.server().defaultHostname())
+                                 .tag("hostname", certificateHostname())
                                  .tag("protocol", expectedProtocol)
                                  .tag("result", "success")
                                  .tag("tls.protocol", tlsProtocol)
@@ -134,7 +134,7 @@ void handshakeFailure() {
         final Counter counter =
                 meterRegistry.find("armeria.server.tls.handshakes")
                              .tag("cipher.suite", "")
-                             .tag("hostname", server.server().defaultHostname())
+                             .tag("hostname", certificateHostname())
                              .tag("protocol", "")
                              .tag("result", "failure")
                              .tag("tls.protocol", "")
@@ -145,4 +145,11 @@ void handshakeFailure() {
                 .isNotNull();
         assertThat(counter.count()).isOne();
     }
+
+    private static String certificateHostname() {
+        // The hostname tag is derived from the self-signed certificate, whose common name is
+        // truncated to the 64-character limit of RFC 5280.
+        final String hostname = server.server().defaultHostname();
+        return hostname.length() <= 64 ? hostname : hostname.substring(0, 64);
+    }
 }

core/src/test/java/com/linecorp/armeria/server/VirtualHostBuilderTest.java

@@ -32,6 +32,7 @@
 import org.junit.jupiter.params.provider.CsvSource;
 import org.slf4j.LoggerFactory;
 
+import com.google.common.base.Strings;
 import com.google.common.collect.ImmutableSet;
 
 import com.linecorp.armeria.common.Flags;
@@ -89,6 +90,19 @@ void defaultVirtualHostSetDefaultHostname() {
         assertThat(virtualHost.defaultHostname()).isEqualTo("foo");
     }
 
+    @Test
+    void tlsSelfSignedWithLongDefaultHostname() {
+        // The hostname of some machines exceeds the 64-character common name limit of RFC 5280.
+        final String hostname = "very-long-hostname-" + Strings.repeat("a", 42) + ".local";
+        final Server server = Server.builder()
+                                    .defaultHostname(hostname)
+                                    .tlsSelfSigned()
+                                    .service("/test", (ctx, req) -> HttpResponse.of(OK))
+                                    .build();
+
+        assertThat(server.config().defaultVirtualHost().defaultHostname()).isEqualTo(hostname);
+    }
+
     @Test
     void defaultVirtualHostWithImplicitStyle() {
         final ServerBuilder sb = Server.builder();