From 51256d6177be6ab5c3747d154fa1d848310b57f5 Mon Sep 17 00:00:00 2001 From: Paris Larkins Date: Wed, 3 Sep 2025 11:31:54 +1000 Subject: [PATCH] Add maximum_index_characters_per_message setting from 3.3.2 --- .../security/OpenSearchSecurityPlugin.java | 10 ++ .../security/auditlog/impl/AuditMessage.java | 96 +++++++++++++++++++ .../security/auditlog/sink/Log4JSink.java | 7 +- .../security/support/ConfigConstants.java | 1 + .../auditlog/impl/AuditMessageTest.java | 66 +++++++++++-- 5 files changed, 172 insertions(+), 8 deletions(-) diff --git a/src/main/java/org/opensearch/security/OpenSearchSecurityPlugin.java b/src/main/java/org/opensearch/security/OpenSearchSecurityPlugin.java index 23e12f567e..3e6a3a8d4d 100644 --- a/src/main/java/org/opensearch/security/OpenSearchSecurityPlugin.java +++ b/src/main/java/org/opensearch/security/OpenSearchSecurityPlugin.java @@ -1998,6 +1998,16 @@ public List> getSettings() { Property.Filtered ) ); + settings.add( + Setting.intSetting( + ConfigConstants.SECURITY_AUDIT_CONFIG_DEFAULT_PREFIX + + ConfigConstants.SECURITY_AUDIT_LOG4J_MAXIMUM_INDEX_CHARACTERS_PER_MESSAGE, + Integer.MAX_VALUE, + 255, + Property.NodeScope, + Property.Filtered + ) + ); // Kerberos settings.add(Setting.simpleString(ConfigConstants.SECURITY_KERBEROS_KRB5_FILEPATH, Property.NodeScope, Property.Filtered)); diff --git a/src/main/java/org/opensearch/security/auditlog/impl/AuditMessage.java b/src/main/java/org/opensearch/security/auditlog/impl/AuditMessage.java index 81b701bd7f..3883b90eb1 100644 --- a/src/main/java/org/opensearch/security/auditlog/impl/AuditMessage.java +++ b/src/main/java/org/opensearch/security/auditlog/impl/AuditMessage.java @@ -17,7 +17,10 @@ import java.nio.file.Path; import java.nio.file.attribute.FileTime; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; +import java.util.IntSummaryStatistics; import java.util.List; import java.util.Map; import java.util.Map.Entry; @@ -39,6 +42,7 @@ import org.opensearch.core.common.Strings; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.common.transport.TransportAddress; +import org.opensearch.core.common.util.CollectionUtils; import org.opensearch.core.index.shard.ShardId; import org.opensearch.core.xcontent.MediaType; import org.opensearch.rest.RestRequest; @@ -510,6 +514,98 @@ public String toString() { } } + public List toJsonSplitIndices(final int maximumIndexCharsPerMessage) { + final List indices = Arrays.asList((String[]) auditInfo.getOrDefault(INDICES, new String[0])); + final List resolvedIndices = Arrays.asList((String[]) auditInfo.getOrDefault(RESOLVED_INDICES, new String[0])); + + // Calculates sum and max at the same time + final IntSummaryStatistics indicesCharsStats = indices.stream().mapToInt(String::length).summaryStatistics(); + final IntSummaryStatistics resolvedIndicesCharsStats = resolvedIndices.stream().mapToInt(String::length).summaryStatistics(); + + final long totalIndexChars = indicesCharsStats.getSum() + resolvedIndicesCharsStats.getSum(); + + // Only split if there are too many characters + if (totalIndexChars < maximumIndexCharsPerMessage) { + return List.of(toJson()); + } + + final int longestIndexName = Math.max(indicesCharsStats.getMax(), resolvedIndicesCharsStats.getMax()); + + // How many index names we can safely include without exceeding maximumIndexCharsPerMessage. + // This may cause some messages to be smaller than they need to be, but simplifies processing logic compared to + // inspecting the length of each index name individually. + final int maximumIndicesPerMessage = maximumIndexCharsPerMessage / longestIndexName; + + final List splitMessages = new ArrayList<>(); + + int indicesRemaining = indices.size(); + int resolvedIndicesRemaining = resolvedIndices.size(); + + while (indicesRemaining + resolvedIndicesRemaining > 0) { + List indicesPartition; + List resolvedIndicesPartition; + + // Process all indices first before starting on resolvedIndices. + if (indicesRemaining > 0) { + // Grab the next sublist of up to maximumIndicesPerMessage length + final int fromIndex = indices.size() - indicesRemaining; + indicesPartition = indices.subList(fromIndex, Math.min(indices.size(), fromIndex + maximumIndicesPerMessage)); + + // If there weren't enough indices to reach the maximum, add resolved indices up to the maximum + if (indicesPartition.size() < maximumIndicesPerMessage) { + resolvedIndicesPartition = resolvedIndices.subList( + resolvedIndices.size() - resolvedIndicesRemaining, + Math.min(resolvedIndices.size(), maximumIndicesPerMessage - indicesPartition.size()) + ); + } else { // Otherwise, don't include any resolvedIndices in this split message + resolvedIndicesPartition = Collections.emptyList(); + } + } else { // Only resolvedIndices remain + indicesPartition = Collections.emptyList(); + + // Grab the next sublist of up to maximumIndicesPerMessage length + final int fromIndex = resolvedIndices.size() - resolvedIndicesRemaining; + resolvedIndicesPartition = resolvedIndices.subList( + fromIndex, + Math.min(resolvedIndices.size(), fromIndex + maximumIndicesPerMessage) + ); + } + + indicesRemaining -= indicesPartition.size(); + resolvedIndicesRemaining -= resolvedIndicesPartition.size(); + + // Create and add new split message with the indices and resolvedIndices partitions + splitMessages.add(getSplitMessage(indicesPartition, resolvedIndicesPartition)); + } + + return splitMessages; + } + + private String getSplitMessage(final List indices, final List resolvedIndices) { + // Create a shallow copy of the audit message information, which will have indices information overwritten + final HashMap splitAuditInfo = new HashMap<>(auditInfo); + + // If either indices or resolvedIndices is empty, remove the corresponding field from the split message. + // Otherwise, overwrite the shallow copy with the split lists. + if (CollectionUtils.isEmpty(indices)) { + splitAuditInfo.remove(INDICES); + } else { + splitAuditInfo.put(INDICES, indices); + } + + if (CollectionUtils.isEmpty(resolvedIndices)) { + splitAuditInfo.remove(RESOLVED_INDICES); + } else { + splitAuditInfo.put(RESOLVED_INDICES, resolvedIndices); + } + + try { + return JsonXContent.contentBuilder().map(splitAuditInfo).toString(); + } catch (IOException e) { + throw ExceptionsHelper.convertToOpenSearchException(e); + } + } + public String toPrettyString() { try { return JsonXContent.contentBuilder().prettyPrint().map(getAsMap()).toString(); diff --git a/src/main/java/org/opensearch/security/auditlog/sink/Log4JSink.java b/src/main/java/org/opensearch/security/auditlog/sink/Log4JSink.java index cf535e48b1..2bf344ab31 100644 --- a/src/main/java/org/opensearch/security/auditlog/sink/Log4JSink.java +++ b/src/main/java/org/opensearch/security/auditlog/sink/Log4JSink.java @@ -24,12 +24,17 @@ public final class Log4JSink extends AuditLogSink { final String loggerName; final Level logLevel; final boolean enabled; + final Integer maximumIndexCharactersPerMessage; public Log4JSink(final String name, final Settings settings, final String settingsPrefix, AuditLogSink fallbackSink) { super(name, settings, settingsPrefix, fallbackSink); loggerName = settings.get(settingsPrefix + ".log4j.logger_name", "audit"); auditLogger = LogManager.getLogger(loggerName); logLevel = Level.toLevel(settings.get(settingsPrefix + ".log4j.level", "INFO").toUpperCase()); + maximumIndexCharactersPerMessage = settings.getAsInt( + settingsPrefix + ".log4j.maximum_index_characters_per_message", + Integer.MAX_VALUE + ); enabled = auditLogger.isEnabled(logLevel); } @@ -39,7 +44,7 @@ public boolean isHandlingBackpressure() { public boolean doStore(final AuditMessage msg) { if (enabled) { - auditLogger.log(logLevel, msg.toJson()); + msg.toJsonSplitIndices(maximumIndexCharactersPerMessage).forEach(message -> auditLogger.log(logLevel, message)); } return true; } diff --git a/src/main/java/org/opensearch/security/support/ConfigConstants.java b/src/main/java/org/opensearch/security/support/ConfigConstants.java index 8549b0a004..5d82a64c24 100644 --- a/src/main/java/org/opensearch/security/support/ConfigConstants.java +++ b/src/main/java/org/opensearch/security/support/ConfigConstants.java @@ -269,6 +269,7 @@ public class ConfigConstants { // Log4j public static final String SECURITY_AUDIT_LOG4J_LOGGER_NAME = "log4j.logger_name"; public static final String SECURITY_AUDIT_LOG4J_LEVEL = "log4j.level"; + public static final String SECURITY_AUDIT_LOG4J_MAXIMUM_INDEX_CHARACTERS_PER_MESSAGE = "log4j.maximum_index_characters_per_message"; // retry public static final String SECURITY_AUDIT_RETRY_COUNT = SECURITY_SETTINGS_PREFIX + "audit.config.retry_count"; diff --git a/src/test/java/org/opensearch/security/auditlog/impl/AuditMessageTest.java b/src/test/java/org/opensearch/security/auditlog/impl/AuditMessageTest.java index 21cd9ce968..2ea419936d 100644 --- a/src/test/java/org/opensearch/security/auditlog/impl/AuditMessageTest.java +++ b/src/test/java/org/opensearch/security/auditlog/impl/AuditMessageTest.java @@ -12,6 +12,7 @@ package org.opensearch.security.auditlog.impl; import java.nio.ByteBuffer; +import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Map; @@ -68,17 +69,19 @@ public class AuditMessageTest { "test-4" ); + private final ClusterService clusterServiceMock = mock(ClusterService.class); + private final DiscoveryNode discoveryNodeMock = mock(DiscoveryNode.class); + private final ClusterName clusterNameMock = mock(ClusterName.class); + private final AuditConfig auditConfig = mock(AuditConfig.class); + private final AuditConfig.Filter auditFilterMock = mock(AuditConfig.Filter.class); + private AuditMessage message; - private AuditConfig auditConfig; @Before public void setUp() { - final ClusterService clusterServiceMock = mock(ClusterService.class); - when(clusterServiceMock.localNode()).thenReturn(mock(DiscoveryNode.class)); - when(clusterServiceMock.getClusterName()).thenReturn(mock(ClusterName.class)); - auditConfig = mock(AuditConfig.class); - final AuditConfig.Filter auditFilter = mock(AuditConfig.Filter.class); - when(auditConfig.getFilter()).thenReturn(auditFilter); + when(clusterServiceMock.localNode()).thenReturn(discoveryNodeMock); + when(clusterServiceMock.getClusterName()).thenReturn(clusterNameMock); + when(auditConfig.getFilter()).thenReturn(auditFilterMock); message = new AuditMessage(AuditCategory.AUTHENTICATED, clusterServiceMock, AuditLog.Origin.REST, AuditLog.Origin.REST); } @@ -200,4 +203,53 @@ public void testRequestBodyLoggingWithInvalidSourceOrContentTypeParam() { message.addRestRequestInfo(request, auditConfig.getFilter()); assertThat(message.getAsMap().get(AuditMessage.REQUEST_BODY), is("ERROR: Unable to generate request body")); } + + private AuditMessage dummyAuditMessage(final String[] indices, String[] resolvedIndices) { + final AuditMessage auditMessage = new AuditMessage( + AuditCategory.AUTHENTICATED, + clusterServiceMock, + AuditLog.Origin.REST, + AuditLog.Origin.REST + ); + + if (indices != null) { + auditMessage.addIndices(indices); + } + if (resolvedIndices != null) { + auditMessage.addResolvedIndices(resolvedIndices); + } + return auditMessage; + } + + private String[] getTestIndices(final int indexNameLength, final int numberOfIndices) { + ArrayList indices = new ArrayList<>(); + for (int i = 0; i < numberOfIndices; i++) { + indices.add("a".repeat(indexNameLength)); + } + return indices.toArray(new String[0]); + } + + @Test + public void testToJsonSplitIndices() { + // test standard case, should be split into 4 messages + AuditMessage auditMessage = dummyAuditMessage(new String[] { "*" }, getTestIndices(255, 3)); + List splitMessages = auditMessage.toJsonSplitIndices(255); + assertThat(splitMessages.size(), is(4)); + + // test when audit_trace_indices is not present, should be split into 3 messages + auditMessage = dummyAuditMessage(null, getTestIndices(255, 3)); + splitMessages = auditMessage.toJsonSplitIndices(255); + assertThat(splitMessages.size(), is(3)); + + // test when splitting isn't required, should return a single message + auditMessage = dummyAuditMessage(new String[] { "*" }, getTestIndices(255, 2)); + splitMessages = auditMessage.toJsonSplitIndices(700); + assertThat(splitMessages.size(), is(1)); + + // test when there aren't enough indices to fill a whole message so some resolved indices are added too. + // Should be split into 2 messages. First with "*" and one resolved index, second with the remaining resolved indices + auditMessage = dummyAuditMessage(new String[] { "*" }, getTestIndices(255, 3)); + splitMessages = auditMessage.toJsonSplitIndices(700); + assertThat(splitMessages.size(), is(2)); + } }