8366261: Provide utility methods for sun.security.util.Password

src/java.base/share/classes/java/io/Console.java

@@ -25,6 +25,7 @@
 
 package java.io;
 
+import java.lang.annotation.Native;
 import java.security.AccessController;
 import java.security.PrivilegedAction;
 import java.util.*;
@@ -349,11 +350,17 @@ private static UnsupportedOperationException newUnsupportedOperationException()
                 "Console class itself does not provide implementation");
     }
 
+    @Native static final int TTY_STDIN_MASK = 0x00000001;
+    @Native static final int TTY_STDOUT_MASK = 0x00000002;
+    @Native static final int TTY_STDERR_MASK = 0x00000004;
+    // ttyStatus() returns bit patterns above, a bit is set if the corresponding file
+    // descriptor is a character device
+    private static final int ttyStatus = ttyStatus();
     private static native String encoding();
     static final Charset CHARSET;
     static {
         Charset cs = null;
-        boolean istty = istty();
+        boolean istty = isStdinTty() && isStdoutTty();
 
         if (istty) {
             String csname = encoding();
@@ -378,6 +385,9 @@ private static UnsupportedOperationException newUnsupportedOperationException()
             public Console console() {
                 return cons;
             }
+            public boolean isStdinTty() {
+                return Console.isStdinTty();
+            }
         });
     }
 
@@ -419,5 +429,14 @@ private static Console instantiateConsole(boolean istty) {
     }
 
     private static final Console cons;
-    private static native boolean istty();
+    private static boolean isStdinTty() {
+        return (ttyStatus & TTY_STDIN_MASK) != 0;
+    }
+    private static boolean isStdoutTty() {
+        return (ttyStatus & TTY_STDOUT_MASK) != 0;
+    }
+    private static boolean isStderrTty() {
+        return (ttyStatus & TTY_STDERR_MASK) != 0;
+    }
+    private static native int ttyStatus();
 }

src/java.base/share/classes/jdk/internal/access/JavaIOAccess.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2005, 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2005, 2025, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -29,4 +29,5 @@
 
 public interface JavaIOAccess {
     Console console();
+    boolean isStdinTty();
 }

src/java.base/share/classes/jdk/internal/io/JdkConsoleImpl.java

@@ -37,10 +37,14 @@
 import java.util.Arrays;
 import java.util.Formatter;
 import java.util.Objects;
+import java.util.Optional;
+import java.util.concurrent.atomic.AtomicReference;
 
 import jdk.internal.access.SharedSecrets;
+import jdk.internal.util.StaticProperty;
 import sun.nio.cs.StreamDecoder;
 import sun.nio.cs.StreamEncoder;
+import sun.nio.cs.UTF_8;
 
 /**
  * JdkConsole implementation based on the platform's TTY.
@@ -92,7 +96,55 @@ public String readLine() {
     }
 
     @Override
-    public char[] readPassword(String fmt, Object ... args) {
+    public char[] readPassword(String format, Object ... args) {
+        return readPassword0(false, format, args);
+    }
+
+    // These two methods are intended for sun.security.util.Password, so tools like keytool can
+    // use JdkConsoleImpl even when standard output is redirected. The Password class should first
+    // check if `System.console()` returns a Console instance and use it if available. Otherwise,
+    // it should call this method to obtain a JdkConsoleImpl. This ensures only one Console
+    // instance exists in the Java runtime.
+    private static final AtomicReference<Optional<JdkConsoleImpl>> INSTANCE = new AtomicReference<>();
+    public static Optional<JdkConsoleImpl> passwordConsole() {
+         Optional<JdkConsoleImpl> result = INSTANCE.get();
+         if (result != null) {
+             return result;
+         }
+
+         synchronized (JdkConsoleImpl.class) {
+             result = INSTANCE.get();
+             if (result != null) {
+                 return result;
+             }
+
+            // If there's already a proper console, throw an exception
+            if (System.console() != null) {
+                throw new IllegalStateException("Can't create a dedicated password " +
+                    "console since a real console already exists");
+            }
+
+            // If stdin is NOT redirected, return an Optional containing a JdkConsoleImpl
+            // instance, otherwise an empty Optional.
+            result = SharedSecrets.getJavaIOAccess().isStdinTty() ?
+                Optional.of(
+                    new JdkConsoleImpl(
+                        UTF_8.INSTANCE)) :
+                Optional.empty();
+
+            INSTANCE.set(result);
+            return result;
+        }
+    }
+
+    // Dedicated entry for sun.security.util.Password when stdout is redirected.
+    // This method strictly avoids producing any output by using noNewLine = true
+    // and an empty format string.
+    public char[] readPasswordNoNewLine() {
+        return readPassword0(true, "");
+    }
+
+    private char[] readPassword0(boolean noNewLine, String fmt, Object ... args) {
         char[] passwd = null;
         synchronized (writeLock) {
             synchronized(readLock) {
@@ -131,7 +183,9 @@ public char[] readPassword(String fmt, Object ... args) {
                         throw ioe;
                     }
                 }
-                pw.println();
+                if (!noNewLine) {
+                    pw.println();
+                }
             }
         }
         return passwd;

src/java.base/unix/native/libjava/Console_md.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2005, 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2005, 2025, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -31,10 +31,21 @@
 #include <stdlib.h>
 #include <unistd.h>
 
-JNIEXPORT jboolean JNICALL
-Java_java_io_Console_istty(JNIEnv *env, jclass cls)
+JNIEXPORT jint JNICALL
+Java_java_io_Console_ttyStatus(JNIEnv *env, jclass cls)
 {
-    return isatty(fileno(stdin)) && isatty(fileno(stdout));
+    jint ret = 0;
+
+    if (isatty(fileno(stdin))) {
+        ret |= java_io_Console_TTY_STDIN_MASK;
+    }
+    if (isatty(fileno(stdout))) {
+        ret |= java_io_Console_TTY_STDOUT_MASK;
+    }
+    if (isatty(fileno(stderr))) {
+        ret |= java_io_Console_TTY_STDERR_MASK;
+    }
+    return ret;
 }
 
 JNIEXPORT jstring JNICALL

src/java.base/windows/native/libjava/Console_md.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2005, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2005, 2025, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -31,23 +31,30 @@
 #include <stdlib.h>
 #include <Wincon.h>
 
-JNIEXPORT jboolean JNICALL
-Java_java_io_Console_istty(JNIEnv *env, jclass cls)
+JNIEXPORT jint JNICALL
+Java_java_io_Console_ttyStatus(JNIEnv *env, jclass cls)
 {
+    jint ret = 0;
     HANDLE hStdOut = GetStdHandle(STD_OUTPUT_HANDLE);
     HANDLE hStdIn = GetStdHandle(STD_INPUT_HANDLE);
+    HANDLE hStdErr = GetStdHandle(STD_ERROR_HANDLE);
 
-    if (hStdIn == INVALID_HANDLE_VALUE ||
-        hStdOut == INVALID_HANDLE_VALUE) {
-        return JNI_FALSE;
+    if (hStdIn != INVALID_HANDLE_VALUE &&
+        GetFileType(hStdIn) == FILE_TYPE_CHAR) {
+        ret |= java_io_Console_TTY_STDIN_MASK;
     }
 
-    if (GetFileType(hStdIn) != FILE_TYPE_CHAR ||
-        GetFileType(hStdOut) != FILE_TYPE_CHAR) {
-        return JNI_FALSE;
+    if (hStdOut != INVALID_HANDLE_VALUE &&
+        GetFileType(hStdOut) == FILE_TYPE_CHAR) {
+        ret |= java_io_Console_TTY_STDOUT_MASK;
     }
 
-    return JNI_TRUE;
+    if (hStdErr != INVALID_HANDLE_VALUE &&
+        GetFileType(hStdErr) == FILE_TYPE_CHAR) {
+        ret |= java_io_Console_TTY_STDERR_MASK;
+    }
+
+    return ret;
 }
 
 JNIEXPORT jstring JNICALL

test/jdk/java/io/Console/ModuleSelectionTest.java

@@ -21,9 +21,9 @@
  * questions.
  */
 
-/**
+/*
  * @test
- * @bug 8295803 8299689 8361613
+ * @bug 8295803 8299689 8361613 8366261
  * @summary Tests System.console() returns correct Console (or null) from the expected
  *          module.
  * @library /test/lib
@@ -92,9 +92,12 @@ public static void main(String... args) throws Throwable {
         var con = System.console();
         var pc = Class.forName("java.io.ProxyingConsole");
         var jdkc = Class.forName("jdk.internal.io.JdkConsole");
-        var istty = (boolean)MethodHandles.privateLookupIn(Console.class, MethodHandles.lookup())
-                .findStatic(Console.class, "istty", MethodType.methodType(boolean.class))
-                .invoke();
+        var lookup = MethodHandles.privateLookupIn(Console.class, MethodHandles.lookup());
+        var istty = (boolean)lookup.findStatic(Console.class, "isStdinTty", MethodType.methodType(boolean.class))
+                        .invoke() &&
+                   (boolean)lookup.findStatic(Console.class, "isStdoutTty", MethodType.methodType(boolean.class))
+                        .invoke();
+
         var impl = con != null ? MethodHandles.privateLookupIn(pc, MethodHandles.lookup())
                 .findGetter(pc, "delegate", jdkc)
                 .invoke(con) : null;