8381519: Enhance Der Value Handling

Reviewed-by: tpushkin, andrew
Backport-of: 96239a84aaa93b1e1ffd704799fd7cef1bf98a5e

Author: Aleksei Voitylov

src/java.base/share/classes/sun/security/util/DerValue.java

@@ -1,5 +1,5 @@
 /**
- * Copyright (c) 1996, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1996, 2026, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -156,6 +156,9 @@ public class DerValue {
      */
     public static final byte    tag_SetOf = 0x31;
 
+    // Max nested depth for constructed data
+    private static final int MAX_CONSTRUCTED_NEST = 30;
+
     /*
      * These values are the high order bits for the other kinds of tags.
      */
@@ -503,6 +506,14 @@ private byte[] append(byte[] a, byte[] b) {
      * @return the octet string held in this DER value
      */
     public byte[] getOctetString() throws IOException {
+        return getOctetString(0);
+    }
+
+    private byte[] getOctetString(int limit) throws IOException {
+        if (++limit > MAX_CONSTRUCTED_NEST) {
+            throw new IOException("Nested OctetString limit reached ("
+                + MAX_CONSTRUCTED_NEST + ").");
+        }
 
         if (tag != tag_OctetString && !isConstructed(tag_OctetString)) {
             throw new IOException(
@@ -529,7 +540,7 @@ public byte[] getOctetString() throws IOException {
                 buffer.allowBER);
             bytes = null;
             while (in.available() != 0) {
-                bytes = append(bytes, in.getOctetString());
+                bytes = append(bytes, in.getDerValue().getOctetString(limit));
             }
         }
         return bytes;