Add maximum_index_characters_per_message setting from 3.3.2

src/main/java/org/opensearch/security/OpenSearchSecurityPlugin.java

@@ -1998,6 +1998,16 @@ public List<Setting<?>> getSettings() {
                     Property.Filtered
                 )
             );
+            settings.add(
+                Setting.intSetting(
+                    ConfigConstants.SECURITY_AUDIT_CONFIG_DEFAULT_PREFIX
+                        + ConfigConstants.SECURITY_AUDIT_LOG4J_MAXIMUM_INDEX_CHARACTERS_PER_MESSAGE,
+                    Integer.MAX_VALUE,
+                    255,
+                    Property.NodeScope,
+                    Property.Filtered
+                )
+            );
 
             // Kerberos
             settings.add(Setting.simpleString(ConfigConstants.SECURITY_KERBEROS_KRB5_FILEPATH, Property.NodeScope, Property.Filtered));

src/main/java/org/opensearch/security/auditlog/impl/AuditMessage.java

@@ -17,7 +17,10 @@
 import java.nio.file.Path;
 import java.nio.file.attribute.FileTime;
 import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
 import java.util.HashMap;
+import java.util.IntSummaryStatistics;
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
@@ -39,6 +42,7 @@
 import org.opensearch.core.common.Strings;
 import org.opensearch.core.common.bytes.BytesReference;
 import org.opensearch.core.common.transport.TransportAddress;
+import org.opensearch.core.common.util.CollectionUtils;
 import org.opensearch.core.index.shard.ShardId;
 import org.opensearch.core.xcontent.MediaType;
 import org.opensearch.rest.RestRequest;
@@ -510,6 +514,98 @@ public String toString() {
         }
     }
 
+    public List<String> toJsonSplitIndices(final int maximumIndexCharsPerMessage) {
+        final List<String> indices = Arrays.asList((String[]) auditInfo.getOrDefault(INDICES, new String[0]));
+        final List<String> resolvedIndices = Arrays.asList((String[]) auditInfo.getOrDefault(RESOLVED_INDICES, new String[0]));
+
+        // Calculates sum and max at the same time
+        final IntSummaryStatistics indicesCharsStats = indices.stream().mapToInt(String::length).summaryStatistics();
+        final IntSummaryStatistics resolvedIndicesCharsStats = resolvedIndices.stream().mapToInt(String::length).summaryStatistics();
+
+        final long totalIndexChars = indicesCharsStats.getSum() + resolvedIndicesCharsStats.getSum();
+
+        // Only split if there are too many characters
+        if (totalIndexChars < maximumIndexCharsPerMessage) {
+            return List.of(toJson());
+        }
+
+        final int longestIndexName = Math.max(indicesCharsStats.getMax(), resolvedIndicesCharsStats.getMax());
+
+        // How many index names we can safely include without exceeding maximumIndexCharsPerMessage.
+        // This may cause some messages to be smaller than they need to be, but simplifies processing logic compared to
+        // inspecting the length of each index name individually.
+        final int maximumIndicesPerMessage = maximumIndexCharsPerMessage / longestIndexName;
+
+        final List<String> splitMessages = new ArrayList<>();
+
+        int indicesRemaining = indices.size();
+        int resolvedIndicesRemaining = resolvedIndices.size();
+
+        while (indicesRemaining + resolvedIndicesRemaining > 0) {
+            List<String> indicesPartition;
+            List<String> resolvedIndicesPartition;
+
+            // Process all indices first before starting on resolvedIndices.
+            if (indicesRemaining > 0) {
+                // Grab the next sublist of up to maximumIndicesPerMessage length
+                final int fromIndex = indices.size() - indicesRemaining;
+                indicesPartition = indices.subList(fromIndex, Math.min(indices.size(), fromIndex + maximumIndicesPerMessage));
+
+                // If there weren't enough indices to reach the maximum, add resolved indices up to the maximum
+                if (indicesPartition.size() < maximumIndicesPerMessage) {
+                    resolvedIndicesPartition = resolvedIndices.subList(
+                        resolvedIndices.size() - resolvedIndicesRemaining,
+                        Math.min(resolvedIndices.size(), maximumIndicesPerMessage - indicesPartition.size())
+                    );
+                } else { // Otherwise, don't include any resolvedIndices in this split message
+                    resolvedIndicesPartition = Collections.emptyList();
+                }
+            } else { // Only resolvedIndices remain
+                indicesPartition = Collections.emptyList();
+
+                // Grab the next sublist of up to maximumIndicesPerMessage length
+                final int fromIndex = resolvedIndices.size() - resolvedIndicesRemaining;
+                resolvedIndicesPartition = resolvedIndices.subList(
+                    fromIndex,
+                    Math.min(resolvedIndices.size(), fromIndex + maximumIndicesPerMessage)
+                );
+            }
+
+            indicesRemaining -= indicesPartition.size();
+            resolvedIndicesRemaining -= resolvedIndicesPartition.size();
+
+            // Create and add new split message with the indices and resolvedIndices partitions
+            splitMessages.add(getSplitMessage(indicesPartition, resolvedIndicesPartition));
+        }
+
+        return splitMessages;
+    }
+
+    private String getSplitMessage(final List<String> indices, final List<String> resolvedIndices) {
+        // Create a shallow copy of the audit message information, which will have indices information overwritten
+        final HashMap<String, Object> splitAuditInfo = new HashMap<>(auditInfo);
+
+        // If either indices or resolvedIndices is empty, remove the corresponding field from the split message.
+        // Otherwise, overwrite the shallow copy with the split lists.
+        if (CollectionUtils.isEmpty(indices)) {
+            splitAuditInfo.remove(INDICES);
+        } else {
+            splitAuditInfo.put(INDICES, indices);
+        }
+
+        if (CollectionUtils.isEmpty(resolvedIndices)) {
+            splitAuditInfo.remove(RESOLVED_INDICES);
+        } else {
+            splitAuditInfo.put(RESOLVED_INDICES, resolvedIndices);
+        }
+
+        try {
+            return JsonXContent.contentBuilder().map(splitAuditInfo).toString();
+        } catch (IOException e) {
+            throw ExceptionsHelper.convertToOpenSearchException(e);
+        }
+    }
+
     public String toPrettyString() {
         try {
             return JsonXContent.contentBuilder().prettyPrint().map(getAsMap()).toString();

src/main/java/org/opensearch/security/auditlog/sink/Log4JSink.java

@@ -24,12 +24,17 @@ public final class Log4JSink extends AuditLogSink {
     final String loggerName;
     final Level logLevel;
     final boolean enabled;
+    final Integer maximumIndexCharactersPerMessage;
 
     public Log4JSink(final String name, final Settings settings, final String settingsPrefix, AuditLogSink fallbackSink) {
         super(name, settings, settingsPrefix, fallbackSink);
         loggerName = settings.get(settingsPrefix + ".log4j.logger_name", "audit");
         auditLogger = LogManager.getLogger(loggerName);
         logLevel = Level.toLevel(settings.get(settingsPrefix + ".log4j.level", "INFO").toUpperCase());
+        maximumIndexCharactersPerMessage = settings.getAsInt(
+            settingsPrefix + ".log4j.maximum_index_characters_per_message",
+            Integer.MAX_VALUE
+        );
         enabled = auditLogger.isEnabled(logLevel);
     }
 
@@ -39,7 +44,7 @@ public boolean isHandlingBackpressure() {
 
     public boolean doStore(final AuditMessage msg) {
         if (enabled) {
-            auditLogger.log(logLevel, msg.toJson());
+            msg.toJsonSplitIndices(maximumIndexCharactersPerMessage).forEach(message -> auditLogger.log(logLevel, message));
         }
         return true;
     }

src/main/java/org/opensearch/security/support/ConfigConstants.java

@@ -269,6 +269,7 @@ public class ConfigConstants {
     // Log4j
     public static final String SECURITY_AUDIT_LOG4J_LOGGER_NAME = "log4j.logger_name";
     public static final String SECURITY_AUDIT_LOG4J_LEVEL = "log4j.level";
+    public static final String SECURITY_AUDIT_LOG4J_MAXIMUM_INDEX_CHARACTERS_PER_MESSAGE = "log4j.maximum_index_characters_per_message";
 
     // retry
     public static final String SECURITY_AUDIT_RETRY_COUNT = SECURITY_SETTINGS_PREFIX + "audit.config.retry_count";

src/test/java/org/opensearch/security/auditlog/impl/AuditMessageTest.java

@@ -12,6 +12,7 @@
 package org.opensearch.security.auditlog.impl;
 
 import java.nio.ByteBuffer;
+import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
@@ -68,17 +69,19 @@ public class AuditMessageTest {
         "test-4"
     );
 
+    private final ClusterService clusterServiceMock = mock(ClusterService.class);
+    private final DiscoveryNode discoveryNodeMock = mock(DiscoveryNode.class);
+    private final ClusterName clusterNameMock = mock(ClusterName.class);
+    private final AuditConfig auditConfig = mock(AuditConfig.class);
+    private final AuditConfig.Filter auditFilterMock = mock(AuditConfig.Filter.class);
+
     private AuditMessage message;
-    private AuditConfig auditConfig;
 
     @Before
     public void setUp() {
-        final ClusterService clusterServiceMock = mock(ClusterService.class);
-        when(clusterServiceMock.localNode()).thenReturn(mock(DiscoveryNode.class));
-        when(clusterServiceMock.getClusterName()).thenReturn(mock(ClusterName.class));
-        auditConfig = mock(AuditConfig.class);
-        final AuditConfig.Filter auditFilter = mock(AuditConfig.Filter.class);
-        when(auditConfig.getFilter()).thenReturn(auditFilter);
+        when(clusterServiceMock.localNode()).thenReturn(discoveryNodeMock);
+        when(clusterServiceMock.getClusterName()).thenReturn(clusterNameMock);
+        when(auditConfig.getFilter()).thenReturn(auditFilterMock);
         message = new AuditMessage(AuditCategory.AUTHENTICATED, clusterServiceMock, AuditLog.Origin.REST, AuditLog.Origin.REST);
     }
 
@@ -200,4 +203,53 @@ public void testRequestBodyLoggingWithInvalidSourceOrContentTypeParam() {
         message.addRestRequestInfo(request, auditConfig.getFilter());
         assertThat(message.getAsMap().get(AuditMessage.REQUEST_BODY), is("ERROR: Unable to generate request body"));
     }
+
+    private AuditMessage dummyAuditMessage(final String[] indices, String[] resolvedIndices) {
+        final AuditMessage auditMessage = new AuditMessage(
+            AuditCategory.AUTHENTICATED,
+            clusterServiceMock,
+            AuditLog.Origin.REST,
+            AuditLog.Origin.REST
+        );
+
+        if (indices != null) {
+            auditMessage.addIndices(indices);
+        }
+        if (resolvedIndices != null) {
+            auditMessage.addResolvedIndices(resolvedIndices);
+        }
+        return auditMessage;
+    }
+
+    private String[] getTestIndices(final int indexNameLength, final int numberOfIndices) {
+        ArrayList<String> indices = new ArrayList<>();
+        for (int i = 0; i < numberOfIndices; i++) {
+            indices.add("a".repeat(indexNameLength));
+        }
+        return indices.toArray(new String[0]);
+    }
+
+    @Test
+    public void testToJsonSplitIndices() {
+        // test standard case, should be split into 4 messages
+        AuditMessage auditMessage = dummyAuditMessage(new String[] { "*" }, getTestIndices(255, 3));
+        List<String> splitMessages = auditMessage.toJsonSplitIndices(255);
+        assertThat(splitMessages.size(), is(4));
+
+        // test when audit_trace_indices is not present, should be split into 3 messages
+        auditMessage = dummyAuditMessage(null, getTestIndices(255, 3));
+        splitMessages = auditMessage.toJsonSplitIndices(255);
+        assertThat(splitMessages.size(), is(3));
+
+        // test when splitting isn't required, should return a single message
+        auditMessage = dummyAuditMessage(new String[] { "*" }, getTestIndices(255, 2));
+        splitMessages = auditMessage.toJsonSplitIndices(700);
+        assertThat(splitMessages.size(), is(1));
+
+        // test when there aren't enough indices to fill a whole message so some resolved indices are added too.
+        // Should be split into 2 messages. First with "*" and one resolved index, second with the remaining resolved indices
+        auditMessage = dummyAuditMessage(new String[] { "*" }, getTestIndices(255, 3));
+        splitMessages = auditMessage.toJsonSplitIndices(700);
+        assertThat(splitMessages.size(), is(2));
+    }
 }