From 54f64efe3da00622cbf821a0b3a4e3c5046a1604 Mon Sep 17 00:00:00 2001 From: Ramesh Mani Date: Wed, 11 Feb 2026 11:24:15 -0800 Subject: [PATCH 01/12] RANGER-5482:Create Ranger Audit Server with SOLR and HDFS as audit consumer --- .gitignore | 6 + .../audit/provider/AuditProviderFactory.java | 2 + .../audit/provider/BaseAuditHandler.java | 96 +-- .../utils/AbstractRangerAuditWriter.java | 18 +- .../audit/utils/RangerJSONAuditWriter.java | 4 + agents-audit/dest-auditserver/pom.xml | 63 ++ .../RangerAuditServerDestination.java | 749 +++++++++++++++++ .../destination/SolrAuditDestination.java | 4 +- agents-audit/pom.xml | 1 + .../audit/RangerDefaultAuditHandler.java | 29 +- .../policyengine/RangerAccessResult.java | 10 + .../audit/TestRangerDefaultAuditHandler.java | 6 +- dev-support/ranger-docker/.dockerignore | 3 + .../Dockerfile.ranger-audit-consumer-hdfs | 85 ++ .../Dockerfile.ranger-audit-consumer-solr | 86 ++ .../Dockerfile.ranger-audit-server | 86 ++ dev-support/ranger-docker/README.md | 5 + .../docker-compose.ranger-audit-server.yml | 168 ++++ .../docker-compose.ranger-kafka.yml | 2 +- .../scripts/admin/create-ranger-services.py | 23 +- .../ranger-audit-consumer-hdfs.sh | 138 ++++ .../ranger-audit-consumer-solr.sh | 137 +++ .../audit-server/ranger-audit-server.sh | 137 +++ .../audit-server/service-check-functions.sh | 76 ++ .../scripts/hadoop/hdfs-site.xml | 12 + .../ranger-hdfs-plugin-install.properties | 10 +- .../ranger-hive-plugin-install.properties | 10 +- .../scripts/kafka/ranger-kafka-setup.sh | 61 +- .../scripts/kafka/ranger-kafka.sh | 11 +- .../ranger-docker/scripts/kdc/entrypoint.sh | 15 +- distro/pom.xml | 30 + distro/src/main/assembly/hdfs-agent.xml | 1 + distro/src/main/assembly/hive-agent.xml | 1 + .../assembly/ranger-audit-consumer-hdfs.xml | 95 +++ .../assembly/ranger-audit-consumer-solr.xml | 93 +++ .../assembly/ranger-audit-server-service.xml | 93 +++ hdfs-agent/conf/ranger-hdfs-audit-changes.cfg | 5 + hdfs-agent/conf/ranger-hdfs-audit.xml | 19 +- hdfs-agent/pom.xml | 5 + hdfs-agent/scripts/install.properties | 4 + .../hadoop/RangerHdfsAuditHandler.java | 6 +- hive-agent/conf/ranger-hive-audit-changes.cfg | 5 + hive-agent/conf/ranger-hive-audit.xml | 19 +- hive-agent/pom.xml | 5 + hive-agent/scripts/install.properties | 4 + pom.xml | 27 + ranger-audit-server/pom.xml | 460 +++++++++++ .../ranger-audit-common/pom.xml | 127 +++ .../audit/consumer/kafka/AuditConsumer.java | 78 ++ .../consumer/kafka/AuditConsumerBase.java | 101 +++ .../consumer/kafka/AuditConsumerFactory.java | 38 + .../kafka/AuditConsumerRebalanceListener.java | 119 +++ .../consumer/kafka/AuditConsumerRegistry.java | 143 ++++ .../ranger/audit/server/AuditConfig.java | 90 ++ .../audit/server/AuditServerConstants.java | 99 +++ .../ranger/audit/server/EmbeddedServer.java | 780 ++++++++++++++++++ .../audit/utils/AuditMessageQueueUtils.java | 348 ++++++++ .../audit/utils/AuditServerLogFormatter.java | 148 ++++ .../ranger/audit/utils/AuditServerUtils.java | 286 +++++++ .../ranger-audit-consumer-hdfs/pom.xml | 345 ++++++++ .../scripts/start-consumer-hdfs.sh | 164 ++++ .../scripts/stop-consumer-hdfs.sh | 72 ++ .../consumer/HdfsConsumerApplication.java | 59 ++ .../audit/consumer/HdfsConsumerManager.java | 205 +++++ .../consumer/kafka/AuditHDFSConsumer.java | 497 +++++++++++ .../audit/consumer/kafka/AuditRouterHDFS.java | 354 ++++++++ .../ranger/audit/rest/HealthCheckREST.java | 112 +++ .../audit/server/HdfsConsumerConfig.java | 146 ++++ .../src/main/resources/conf/core-site.xml | 39 + .../src/main/resources/conf/hdfs-site.xml | 52 ++ .../src/main/resources/conf/logback.xml | 71 ++ .../conf/ranger-audit-consumer-hdfs-site.xml | 156 ++++ .../webapp/WEB-INF/applicationContext.xml | 39 + .../src/main/webapp/WEB-INF/web.xml | 60 ++ .../ranger-audit-consumer-solr/pom.xml | 336 ++++++++ .../scripts/start-consumer-solr.sh | 163 ++++ .../scripts/stop-consumer-solr.sh | 72 ++ .../consumer/SolrConsumerApplication.java | 65 ++ .../audit/consumer/SolrConsumerManager.java | 210 +++++ .../consumer/kafka/AuditSolrConsumer.java | 457 ++++++++++ .../ranger/audit/rest/HealthCheckREST.java | 112 +++ .../audit/server/SolrConsumerConfig.java | 71 ++ .../src/main/resources/conf/logback.xml | 76 ++ .../conf/ranger-audit-consumer-solr-site.xml | 204 +++++ .../webapp/WEB-INF/applicationContext.xml | 39 + .../src/main/webapp/WEB-INF/web.xml | 60 ++ .../ranger-audit-server-service/pom.xml | 383 +++++++++ .../scripts/start-audit-server.sh | 162 ++++ .../scripts/stop-audit-server.sh | 72 ++ .../audit/producer/AuditDestinationMgr.java | 102 +++ .../producer/kafka/AuditMessageQueue.java | 277 +++++++ .../audit/producer/kafka/AuditProducer.java | 132 +++ .../producer/kafka/AuditRecoveryManager.java | 222 +++++ .../producer/kafka/AuditRecoveryRetry.java | 362 ++++++++ .../producer/kafka/AuditRecoveryWriter.java | 319 +++++++ .../apache/ranger/audit/rest/AuditREST.java | 219 +++++ .../audit/security/AuditAuthEntryPoint.java | 52 ++ .../security/AuditDelegationTokenFilter.java | 316 +++++++ .../audit/security/AuditJwtAuthFilter.java | 196 +++++ .../audit/security/FilterChainWrapper.java | 159 ++++ .../audit/security/NullServletContext.java | 240 ++++++ .../audit/server/AuditServerApplication.java | 61 ++ .../audit/server/AuditServerConfig.java | 71 ++ .../src/main/resources/conf/logback.xml | 71 ++ .../conf/ranger-audit-server-site.xml | 359 ++++++++ .../webapp/WEB-INF/applicationContext.xml | 54 ++ .../WEB-INF/security-applicationContext.xml | 63 ++ .../src/main/webapp/WEB-INF/web.xml | 73 ++ ranger-audit-server/scripts/README.md | 549 ++++++++++++ .../scripts/start-all-services.sh | 78 ++ .../scripts/stop-all-services.sh | 66 ++ 111 files changed, 14074 insertions(+), 102 deletions(-) create mode 100644 agents-audit/dest-auditserver/pom.xml create mode 100644 agents-audit/dest-auditserver/src/main/java/org/apache/ranger/audit/destination/RangerAuditServerDestination.java create mode 100644 dev-support/ranger-docker/Dockerfile.ranger-audit-consumer-hdfs create mode 100644 dev-support/ranger-docker/Dockerfile.ranger-audit-consumer-solr create mode 100644 dev-support/ranger-docker/Dockerfile.ranger-audit-server create mode 100644 dev-support/ranger-docker/docker-compose.ranger-audit-server.yml create mode 100755 dev-support/ranger-docker/scripts/audit-server/ranger-audit-consumer-hdfs.sh create mode 100755 dev-support/ranger-docker/scripts/audit-server/ranger-audit-consumer-solr.sh create mode 100755 dev-support/ranger-docker/scripts/audit-server/ranger-audit-server.sh create mode 100755 dev-support/ranger-docker/scripts/audit-server/service-check-functions.sh create mode 100644 distro/src/main/assembly/ranger-audit-consumer-hdfs.xml create mode 100644 distro/src/main/assembly/ranger-audit-consumer-solr.xml create mode 100644 distro/src/main/assembly/ranger-audit-server-service.xml create mode 100644 ranger-audit-server/pom.xml create mode 100644 ranger-audit-server/ranger-audit-common/pom.xml create mode 100644 ranger-audit-server/ranger-audit-common/src/main/java/org/apache/ranger/audit/consumer/kafka/AuditConsumer.java create mode 100644 ranger-audit-server/ranger-audit-common/src/main/java/org/apache/ranger/audit/consumer/kafka/AuditConsumerBase.java create mode 100644 ranger-audit-server/ranger-audit-common/src/main/java/org/apache/ranger/audit/consumer/kafka/AuditConsumerFactory.java create mode 100644 ranger-audit-server/ranger-audit-common/src/main/java/org/apache/ranger/audit/consumer/kafka/AuditConsumerRebalanceListener.java create mode 100644 ranger-audit-server/ranger-audit-common/src/main/java/org/apache/ranger/audit/consumer/kafka/AuditConsumerRegistry.java create mode 100644 ranger-audit-server/ranger-audit-common/src/main/java/org/apache/ranger/audit/server/AuditConfig.java create mode 100644 ranger-audit-server/ranger-audit-common/src/main/java/org/apache/ranger/audit/server/AuditServerConstants.java create mode 100644 ranger-audit-server/ranger-audit-common/src/main/java/org/apache/ranger/audit/server/EmbeddedServer.java create mode 100644 ranger-audit-server/ranger-audit-common/src/main/java/org/apache/ranger/audit/utils/AuditMessageQueueUtils.java create mode 100644 ranger-audit-server/ranger-audit-common/src/main/java/org/apache/ranger/audit/utils/AuditServerLogFormatter.java create mode 100644 ranger-audit-server/ranger-audit-common/src/main/java/org/apache/ranger/audit/utils/AuditServerUtils.java create mode 100644 ranger-audit-server/ranger-audit-consumer-hdfs/pom.xml create mode 100755 ranger-audit-server/ranger-audit-consumer-hdfs/scripts/start-consumer-hdfs.sh create mode 100755 ranger-audit-server/ranger-audit-consumer-hdfs/scripts/stop-consumer-hdfs.sh create mode 100644 ranger-audit-server/ranger-audit-consumer-hdfs/src/main/java/org/apache/ranger/audit/consumer/HdfsConsumerApplication.java create mode 100644 ranger-audit-server/ranger-audit-consumer-hdfs/src/main/java/org/apache/ranger/audit/consumer/HdfsConsumerManager.java create mode 100644 ranger-audit-server/ranger-audit-consumer-hdfs/src/main/java/org/apache/ranger/audit/consumer/kafka/AuditHDFSConsumer.java create mode 100644 ranger-audit-server/ranger-audit-consumer-hdfs/src/main/java/org/apache/ranger/audit/consumer/kafka/AuditRouterHDFS.java create mode 100644 ranger-audit-server/ranger-audit-consumer-hdfs/src/main/java/org/apache/ranger/audit/rest/HealthCheckREST.java create mode 100644 ranger-audit-server/ranger-audit-consumer-hdfs/src/main/java/org/apache/ranger/audit/server/HdfsConsumerConfig.java create mode 100644 ranger-audit-server/ranger-audit-consumer-hdfs/src/main/resources/conf/core-site.xml create mode 100644 ranger-audit-server/ranger-audit-consumer-hdfs/src/main/resources/conf/hdfs-site.xml create mode 100644 ranger-audit-server/ranger-audit-consumer-hdfs/src/main/resources/conf/logback.xml create mode 100644 ranger-audit-server/ranger-audit-consumer-hdfs/src/main/resources/conf/ranger-audit-consumer-hdfs-site.xml create mode 100644 ranger-audit-server/ranger-audit-consumer-hdfs/src/main/webapp/WEB-INF/applicationContext.xml create mode 100644 ranger-audit-server/ranger-audit-consumer-hdfs/src/main/webapp/WEB-INF/web.xml create mode 100644 ranger-audit-server/ranger-audit-consumer-solr/pom.xml create mode 100755 ranger-audit-server/ranger-audit-consumer-solr/scripts/start-consumer-solr.sh create mode 100755 ranger-audit-server/ranger-audit-consumer-solr/scripts/stop-consumer-solr.sh create mode 100644 ranger-audit-server/ranger-audit-consumer-solr/src/main/java/org/apache/ranger/audit/consumer/SolrConsumerApplication.java create mode 100644 ranger-audit-server/ranger-audit-consumer-solr/src/main/java/org/apache/ranger/audit/consumer/SolrConsumerManager.java create mode 100644 ranger-audit-server/ranger-audit-consumer-solr/src/main/java/org/apache/ranger/audit/consumer/kafka/AuditSolrConsumer.java create mode 100644 ranger-audit-server/ranger-audit-consumer-solr/src/main/java/org/apache/ranger/audit/rest/HealthCheckREST.java create mode 100644 ranger-audit-server/ranger-audit-consumer-solr/src/main/java/org/apache/ranger/audit/server/SolrConsumerConfig.java create mode 100644 ranger-audit-server/ranger-audit-consumer-solr/src/main/resources/conf/logback.xml create mode 100644 ranger-audit-server/ranger-audit-consumer-solr/src/main/resources/conf/ranger-audit-consumer-solr-site.xml create mode 100644 ranger-audit-server/ranger-audit-consumer-solr/src/main/webapp/WEB-INF/applicationContext.xml create mode 100644 ranger-audit-server/ranger-audit-consumer-solr/src/main/webapp/WEB-INF/web.xml create mode 100644 ranger-audit-server/ranger-audit-server-service/pom.xml create mode 100755 ranger-audit-server/ranger-audit-server-service/scripts/start-audit-server.sh create mode 100755 ranger-audit-server/ranger-audit-server-service/scripts/stop-audit-server.sh create mode 100644 ranger-audit-server/ranger-audit-server-service/src/main/java/org/apache/ranger/audit/producer/AuditDestinationMgr.java create mode 100644 ranger-audit-server/ranger-audit-server-service/src/main/java/org/apache/ranger/audit/producer/kafka/AuditMessageQueue.java create mode 100644 ranger-audit-server/ranger-audit-server-service/src/main/java/org/apache/ranger/audit/producer/kafka/AuditProducer.java create mode 100644 ranger-audit-server/ranger-audit-server-service/src/main/java/org/apache/ranger/audit/producer/kafka/AuditRecoveryManager.java create mode 100644 ranger-audit-server/ranger-audit-server-service/src/main/java/org/apache/ranger/audit/producer/kafka/AuditRecoveryRetry.java create mode 100644 ranger-audit-server/ranger-audit-server-service/src/main/java/org/apache/ranger/audit/producer/kafka/AuditRecoveryWriter.java create mode 100644 ranger-audit-server/ranger-audit-server-service/src/main/java/org/apache/ranger/audit/rest/AuditREST.java create mode 100644 ranger-audit-server/ranger-audit-server-service/src/main/java/org/apache/ranger/audit/security/AuditAuthEntryPoint.java create mode 100644 ranger-audit-server/ranger-audit-server-service/src/main/java/org/apache/ranger/audit/security/AuditDelegationTokenFilter.java create mode 100644 ranger-audit-server/ranger-audit-server-service/src/main/java/org/apache/ranger/audit/security/AuditJwtAuthFilter.java create mode 100644 ranger-audit-server/ranger-audit-server-service/src/main/java/org/apache/ranger/audit/security/FilterChainWrapper.java create mode 100644 ranger-audit-server/ranger-audit-server-service/src/main/java/org/apache/ranger/audit/security/NullServletContext.java create mode 100644 ranger-audit-server/ranger-audit-server-service/src/main/java/org/apache/ranger/audit/server/AuditServerApplication.java create mode 100644 ranger-audit-server/ranger-audit-server-service/src/main/java/org/apache/ranger/audit/server/AuditServerConfig.java create mode 100644 ranger-audit-server/ranger-audit-server-service/src/main/resources/conf/logback.xml create mode 100644 ranger-audit-server/ranger-audit-server-service/src/main/resources/conf/ranger-audit-server-site.xml create mode 100644 ranger-audit-server/ranger-audit-server-service/src/main/webapp/WEB-INF/applicationContext.xml create mode 100644 ranger-audit-server/ranger-audit-server-service/src/main/webapp/WEB-INF/security-applicationContext.xml create mode 100644 ranger-audit-server/ranger-audit-server-service/src/main/webapp/WEB-INF/web.xml create mode 100644 ranger-audit-server/scripts/README.md create mode 100755 ranger-audit-server/scripts/start-all-services.sh create mode 100755 ranger-audit-server/scripts/stop-all-services.sh diff --git a/.gitignore b/.gitignore index 5bd45411b9..92e70d43fb 100644 --- a/.gitignore +++ b/.gitignore @@ -19,3 +19,9 @@ winpkg/target .python-version /security-admin/src/main/webapp/react-webapp/node_modules **/target + +# Runtime logs and process files +logs/ +*.log +*.pid +catalina.out diff --git a/agents-audit/core/src/main/java/org/apache/ranger/audit/provider/AuditProviderFactory.java b/agents-audit/core/src/main/java/org/apache/ranger/audit/provider/AuditProviderFactory.java index f6e63c23dd..2fca686273 100644 --- a/agents-audit/core/src/main/java/org/apache/ranger/audit/provider/AuditProviderFactory.java +++ b/agents-audit/core/src/main/java/org/apache/ranger/audit/provider/AuditProviderFactory.java @@ -439,6 +439,8 @@ private AuditHandler getProviderFromConfig(Properties props, String propPrefix, provider = createDestination("org.apache.ranger.audit.provider.kafka.KafkaAuditProvider"); } else if (providerName.equalsIgnoreCase("log4j")) { provider = createDestination("org.apache.ranger.audit.destination.Log4JAuditDestination"); + } else if (providerName.equalsIgnoreCase("auditserver")) { + provider = createDestination("org.apache.ranger.audit.destination.RangerAuditServerDestination"); } else if (providerName.equalsIgnoreCase("batch")) { provider = getAuditProvider(props, propPrefix, consumer); } else if (providerName.equalsIgnoreCase("async")) { diff --git a/agents-audit/core/src/main/java/org/apache/ranger/audit/provider/BaseAuditHandler.java b/agents-audit/core/src/main/java/org/apache/ranger/audit/provider/BaseAuditHandler.java index 32ea8b9c3a..5a23b37e5c 100644 --- a/agents-audit/core/src/main/java/org/apache/ranger/audit/provider/BaseAuditHandler.java +++ b/agents-audit/core/src/main/java/org/apache/ranger/audit/provider/BaseAuditHandler.java @@ -74,16 +74,16 @@ public abstract class BaseAuditHandler implements AuditHandler { int errorLogIntervalMS = 30 * 1000; // Every 30 seconds long lastErrorLogMS; - long totalCount; - long totalSuccessCount; - long totalFailedCount; - long totalStashedCount; - long totalDeferredCount; - long lastIntervalCount; - long lastIntervalSuccessCount; - long lastIntervalFailedCount; - long lastStashedCount; - long lastDeferredCount; + AtomicLong totalCount = new AtomicLong(0); + AtomicLong totalSuccessCount = new AtomicLong(0); + AtomicLong totalFailedCount = new AtomicLong(0); + AtomicLong totalStashedCount = new AtomicLong(0); + AtomicLong totalDeferredCount = new AtomicLong(0); + AtomicLong lastIntervalCount = new AtomicLong(0); + AtomicLong lastIntervalSuccessCount = new AtomicLong(0); + AtomicLong lastIntervalFailedCount = new AtomicLong(0); + AtomicLong lastStashedCount = new AtomicLong(0); + AtomicLong lastDeferredCount = new AtomicLong(0); boolean statusLogEnabled = DEFAULT_AUDIT_LOG_STATUS_LOG_ENABLED; long statusLogIntervalMS = DEFAULT_AUDIT_LOG_STATUS_LOG_INTERVAL_SEC * 1000; long lastStatusLogTime = System.currentTimeMillis(); @@ -237,61 +237,51 @@ public String getFinalPath() { } public long addTotalCount(int count) { - totalCount += count; - - return totalCount; + return totalCount.addAndGet(count); } public long addSuccessCount(int count) { - totalSuccessCount += count; - - return totalSuccessCount; + return totalSuccessCount.addAndGet(count); } public long addFailedCount(int count) { - totalFailedCount += count; - - return totalFailedCount; + return totalFailedCount.addAndGet(count); } public long addStashedCount(int count) { - totalStashedCount += count; - - return totalStashedCount; + return totalStashedCount.addAndGet(count); } public long addDeferredCount(int count) { - totalDeferredCount += count; - - return totalDeferredCount; + return totalDeferredCount.addAndGet(count); } public long getTotalCount() { - return totalCount; + return totalCount.get(); } public long getTotalSuccessCount() { - return totalSuccessCount; + return totalSuccessCount.get(); } public long getTotalFailedCount() { - return totalFailedCount; + return totalFailedCount.get(); } public long getTotalStashedCount() { - return totalStashedCount; + return totalStashedCount.get(); } public long getLastStashedCount() { - return lastStashedCount; + return lastStashedCount.get(); } public long getTotalDeferredCount() { - return totalDeferredCount; + return totalDeferredCount.get(); } public long getLastDeferredCount() { - return lastDeferredCount; + return lastDeferredCount.get(); } public boolean isStatusLogEnabled() { @@ -312,21 +302,27 @@ public void logStatus() { lastStatusLogTime = currTime; nextStatusLogTime = currTime + statusLogIntervalMS; - long diffCount = totalCount - lastIntervalCount; - long diffSuccess = totalSuccessCount - lastIntervalSuccessCount; - long diffFailed = totalFailedCount - lastIntervalFailedCount; - long diffStashed = totalStashedCount - lastStashedCount; - long diffDeferred = totalDeferredCount - lastDeferredCount; + long currentTotalCount = totalCount.get(); + long currentSuccessCount = totalSuccessCount.get(); + long currentFailedCount = totalFailedCount.get(); + long currentStashedCount = totalStashedCount.get(); + long currentDeferredCount = totalDeferredCount.get(); + + long diffCount = currentTotalCount - lastIntervalCount.get(); + long diffSuccess = currentSuccessCount - lastIntervalSuccessCount.get(); + long diffFailed = currentFailedCount - lastIntervalFailedCount.get(); + long diffStashed = currentStashedCount - lastStashedCount.get(); + long diffDeferred = currentDeferredCount - lastDeferredCount.get(); if (diffCount == 0 && diffSuccess == 0 && diffFailed == 0 && diffStashed == 0 && diffDeferred == 0) { return; } - lastIntervalCount = totalCount; - lastIntervalSuccessCount = totalSuccessCount; - lastIntervalFailedCount = totalFailedCount; - lastStashedCount = totalStashedCount; - lastDeferredCount = totalDeferredCount; + lastIntervalCount.set(currentTotalCount); + lastIntervalSuccessCount.set(currentSuccessCount); + lastIntervalFailedCount.set(currentFailedCount); + lastStashedCount.set(currentStashedCount); + lastDeferredCount.set(currentDeferredCount); if (statusLogEnabled) { String finalPath = ""; @@ -475,6 +471,12 @@ public void logFailedEventJSON(Collection events, Throwable excp) { } private void logAuditStatus(long diffTime, long diffCount, long diffSuccess, long diffFailed, long diffStashed, long diffDeferred, String finalPath) { + long currentTotalCount = totalCount.get(); + long currentTotalSuccessCount = totalSuccessCount.get(); + long currentTotalFailedCount = totalFailedCount.get(); + long currentTotalStashedCount = totalStashedCount.get(); + long currentTotalDeferredCount = totalDeferredCount.get(); + String msg = "Audit Status Log: name=" + getName() + finalPath @@ -489,14 +491,14 @@ private void logAuditStatus(long diffTime, long diffCount, long diffSuccess, lon + (diffDeferred > 0 ? (", deferredCount=" + diffDeferred) : "") + ", totalEvents=" - + totalCount - + (totalSuccessCount > 0 ? (", totalSuccessCount=" + totalSuccessCount) + + currentTotalCount + + (currentTotalSuccessCount > 0 ? (", totalSuccessCount=" + currentTotalSuccessCount) : "") - + (totalFailedCount > 0 ? (", totalFailedCount=" + totalFailedCount) + + (currentTotalFailedCount > 0 ? (", totalFailedCount=" + currentTotalFailedCount) : "") - + (totalStashedCount > 0 ? (", totalStashedCount=" + totalStashedCount) + + (currentTotalStashedCount > 0 ? (", totalStashedCount=" + currentTotalStashedCount) : "") - + (totalDeferredCount > 0 ? (", totalDeferredCount=" + totalDeferredCount) + + (currentTotalDeferredCount > 0 ? (", totalDeferredCount=" + currentTotalDeferredCount) : ""); LOG.info(msg); } diff --git a/agents-audit/core/src/main/java/org/apache/ranger/audit/utils/AbstractRangerAuditWriter.java b/agents-audit/core/src/main/java/org/apache/ranger/audit/utils/AbstractRangerAuditWriter.java index df707d1017..d1ac210512 100644 --- a/agents-audit/core/src/main/java/org/apache/ranger/audit/utils/AbstractRangerAuditWriter.java +++ b/agents-audit/core/src/main/java/org/apache/ranger/audit/utils/AbstractRangerAuditWriter.java @@ -133,13 +133,15 @@ public void createFileSystemFolders() throws Exception { String defaultPath = fullPath; + fileSystemScheme = getFileSystemScheme(); + conf = createConfiguration(); URI uri = URI.create(fullPath); - fileSystem = FileSystem.get(uri, conf); - auditPath = new Path(fullPath); - fileSystemScheme = getFileSystemScheme(); + fileSystem = FileSystem.get(uri, conf); + + auditPath = new Path(fullPath); logger.info("Checking whether log file exists. {} Path={}, UGI={}", fileSystemScheme, fullPath, MiscUtil.getUGILoginUser()); @@ -195,6 +197,9 @@ public void createParents(Path pathLogfile, FileSystem fileSystem) throws Except if (parentPath != null && fileSystem != null && !fileSystem.exists(parentPath)) { fileSystem.mkdirs(parentPath); + logger.info("Successfully created parent folder: {}", parentPath); + } else { + logger.info("Parent folder already exists or not required: {}", parentPath); } } @@ -308,14 +313,17 @@ public PrintWriter createWriter() throws Exception { if (!appendMode) { // Create the file to write - logger.info("Creating new log file. auditPath = {}", fullPath); - createFileSystemFolders(); + logger.info("Creating new log file. fullPath = {}", fullPath); + ostream = fileSystem.create(auditPath); + logger.info("Successfully created {} output stream for file: {}", fileSystemScheme, fullPath); } logWriter = new PrintWriter(ostream); isHFlushCapableStream = ostream.hasCapability(StreamCapabilities.HFLUSH); + + logger.info("{} audit writer initialized successfully. File: {}, HFlush capable: {}", fileSystemScheme, fullPath, isHFlushCapableStream); } logger.debug("<== AbstractRangerAuditWriter.createWriter()"); diff --git a/agents-audit/core/src/main/java/org/apache/ranger/audit/utils/RangerJSONAuditWriter.java b/agents-audit/core/src/main/java/org/apache/ranger/audit/utils/RangerJSONAuditWriter.java index ab796b81d2..c8b98fa759 100644 --- a/agents-audit/core/src/main/java/org/apache/ranger/audit/utils/RangerJSONAuditWriter.java +++ b/agents-audit/core/src/main/java/org/apache/ranger/audit/utils/RangerJSONAuditWriter.java @@ -102,9 +102,13 @@ public synchronized boolean logJSON(final Collection events) throws Exce } else { out1 = getLogFileStream(); + logger.debug("Writing {} audit events to HDFS file: {}", events.size(), currentFileName); + for (String event : events) { out1.println(event); } + + logger.debug("Successfully wrote {} audit events to HDFS", events.size()); } return out1; diff --git a/agents-audit/dest-auditserver/pom.xml b/agents-audit/dest-auditserver/pom.xml new file mode 100644 index 0000000000..c489e1da9c --- /dev/null +++ b/agents-audit/dest-auditserver/pom.xml @@ -0,0 +1,63 @@ + + + + 4.0.0 + + org.apache.ranger + ranger + 3.0.0-SNAPSHOT + ../.. + + ranger-audit-dest-auditserver + jar + Ranger Audit Destination - auditserver + Ranger Audit Destination - auditserver + + UTF-8 + 1.2 + + + + com.google.code.gson + gson + + + org.apache.ranger + ranger-audit-core + ${project.version} + + + org.slf4j + slf4j-api + ${slf4j.version} + + + + + org.slf4j + log4j-over-slf4j + ${slf4j.version} + test + + + org.testng + testng + test + + + \ No newline at end of file diff --git a/agents-audit/dest-auditserver/src/main/java/org/apache/ranger/audit/destination/RangerAuditServerDestination.java b/agents-audit/dest-auditserver/src/main/java/org/apache/ranger/audit/destination/RangerAuditServerDestination.java new file mode 100644 index 0000000000..572b3c430d --- /dev/null +++ b/agents-audit/dest-auditserver/src/main/java/org/apache/ranger/audit/destination/RangerAuditServerDestination.java @@ -0,0 +1,749 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.audit.destination; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.http.HttpResponse; +import org.apache.http.HttpStatus; +import org.apache.http.auth.AuthSchemeProvider; +import org.apache.http.auth.AuthScope; +import org.apache.http.auth.KerberosCredentials; +import org.apache.http.auth.UsernamePasswordCredentials; +import org.apache.http.client.config.AuthSchemes; +import org.apache.http.client.config.CookieSpecs; +import org.apache.http.client.config.RequestConfig; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.client.methods.HttpRequestBase; +import org.apache.http.client.protocol.HttpClientContext; +import org.apache.http.client.utils.URIBuilder; +import org.apache.http.config.Lookup; +import org.apache.http.config.Registry; +import org.apache.http.config.RegistryBuilder; +import org.apache.http.conn.socket.ConnectionSocketFactory; +import org.apache.http.conn.ssl.NoopHostnameVerifier; +import org.apache.http.conn.ssl.SSLConnectionSocketFactory; +import org.apache.http.entity.StringEntity; +import org.apache.http.impl.auth.SPNegoSchemeFactory; +import org.apache.http.impl.client.BasicCredentialsProvider; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClientBuilder; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.impl.client.StandardHttpRequestRetryHandler; +import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; +import org.apache.http.protocol.HttpContext; +import org.apache.ranger.audit.model.AuditEventBase; +import org.apache.ranger.audit.provider.MiscUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.net.ssl.KeyManager; +import javax.net.ssl.KeyManagerFactory; +import javax.net.ssl.SSLContext; +import javax.net.ssl.TrustManager; +import javax.net.ssl.TrustManagerFactory; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.charset.Charset; +import java.security.KeyManagementException; +import java.security.KeyStore; +import java.security.KeyStoreException; +import java.security.NoSuchAlgorithmException; +import java.security.PrivilegedExceptionAction; +import java.security.SecureRandom; +import java.security.UnrecoverableKeyException; +import java.security.cert.CertificateException; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; + +public class RangerAuditServerDestination extends AuditDestination { + public static final String PROP_AUDITSERVER_URL = "xasecure.audit.destination.auditserver.url"; + public static final String PROP_AUDITSERVER_USER_NAME = "xasecure.audit.destination.auditserver.username"; + public static final String PROP_AUDITSERVER_USER_PASSWORD = "xasecure.audit.destination.auditserver.password"; + public static final String PROP_AUDITSERVER_AUTH_TYPE = "xasecure.audit.destination.auditserver.authentication.type"; + public static final String PROP_AUDITSERVER_JWT_TOKEN = "xasecure.audit.destination.auditserver.jwt.token"; + public static final String PROP_AUDITSERVER_JWT_TOKEN_FILE = "xasecure.audit.destination.auditserver.jwt.token.file"; + public static final String PROP_AUDITSERVER_CLIENT_CONN_TIMEOUT_MS = "xasecure.audit.destination.auditserver.connection.timeout.ms"; + public static final String PROP_AUDITSERVER_CLIENT_READ_TIMEOUT_MS = "xasecure.audit.destination.auditserver.read.timeout.ms"; + public static final String PROP_AUDITSERVER_MAX_CONNECTION = "xasecure.audit.destination.auditserver.max.connections"; + public static final String PROP_AUDITSERVER_MAX_CONNECTION_PER_HOST = "xasecure.audit.destination.auditserver.max.connections.per.host"; + public static final String PROP_AUDITSERVER_VALIDATE_INACTIVE_MS = "xasecure.audit.destination.auditserver.validate.inactivity.ms"; + public static final String PROP_AUDITSERVER_POOL_RETRY_COUNT = "xasecure.audit.destination.auditserver.pool.retry.count"; + public static final String GSON_DATE_FORMAT = "yyyy-MM-dd'T'HH:mm:ss.SSSZ"; + public static final String REST_ACCEPTED_MIME_TYPE_JSON = "application/json"; + public static final String REST_CONTENT_TYPE_MIME_TYPE_JSON = "application/json"; + public static final String REST_HEADER_ACCEPT = "Accept"; + public static final String REST_HEADER_CONTENT_TYPE = "Content-type"; + public static final String REST_HEADER_AUTHORIZATION = "Authorization"; + public static final String REST_RELATIVE_PATH_POST = "/api/audit/post"; + + // Authentication types + public static final String AUTH_TYPE_KERBEROS = "kerberos"; + public static final String AUTH_TYPE_BASIC = "basic"; + public static final String AUTH_TYPE_JWT = "jwt"; + + private static final Logger LOG = LoggerFactory.getLogger(RangerAuditServerDestination.class); + private static final String PROTOCOL_HTTPS = "https"; + private volatile CloseableHttpClient httpClient; + private volatile Gson gsonBuilder; + private String httpURL; + private String authType; + private String jwtToken; + + @Override + public void init(Properties props, String propPrefix) { + LOG.info("==> RangerAuditServerDestination:init()"); + super.init(props, propPrefix); + + this.authType = MiscUtil.getStringProperty(props, PROP_AUDITSERVER_AUTH_TYPE); + if (AUTH_TYPE_JWT.equalsIgnoreCase(authType)) { + LOG.info("JWT authentication configured...."); + initJwtToken(); + } else if (StringUtils.isEmpty(authType)) { + // Authentication priority: JWT → Kerberos → Basic + try { + if (StringUtils.isNotEmpty(MiscUtil.getStringProperty(props, PROP_AUDITSERVER_JWT_TOKEN)) || + StringUtils.isNotEmpty(MiscUtil.getStringProperty(props, PROP_AUDITSERVER_JWT_TOKEN_FILE))) { + this.authType = AUTH_TYPE_JWT; + initJwtToken(); + } else if (isKerberosAuthenticated()) { + this.authType = AUTH_TYPE_KERBEROS; + } else if (StringUtils.isNotEmpty(MiscUtil.getStringProperty(props, PROP_AUDITSERVER_USER_NAME))) { + this.authType = AUTH_TYPE_BASIC; + } + } catch (Exception e) { + LOG.warn("Failed to auto-detect authentication type", e); + } + } + + LOG.info("Audit destination authentication type: {}", authType); + + if (AUTH_TYPE_KERBEROS.equalsIgnoreCase(authType)) { + preAuthenticateKerberos(); + } + + this.httpClient = buildHTTPClient(); + this.gsonBuilder = new GsonBuilder().setDateFormat(GSON_DATE_FORMAT).create(); + LOG.info("<== RangerAuditServerDestination:init()"); + } + + private void initJwtToken() { + LOG.info("==> RangerAuditServerDestination:initJwtToken()"); + + this.jwtToken = MiscUtil.getStringProperty(props, PROP_AUDITSERVER_JWT_TOKEN); + if (StringUtils.isEmpty(jwtToken)) { + String jwtTokenFile = MiscUtil.getStringProperty(props, PROP_AUDITSERVER_JWT_TOKEN_FILE); + if (StringUtils.isNotEmpty(jwtTokenFile)) { + try { + this.jwtToken = readJwtTokenFromFile(jwtTokenFile); + LOG.info("JWT token loaded from file: {}", jwtTokenFile); + } catch (Exception e) { + LOG.error("Failed to read JWT token from file: {}", jwtTokenFile, e); + } + } + } + + if (StringUtils.isEmpty(jwtToken)) { + LOG.warn("JWT authentication configured but no token found. Configure {} or {}", PROP_AUDITSERVER_JWT_TOKEN, PROP_AUDITSERVER_JWT_TOKEN_FILE); + } else { + LOG.info("JWT authentication initialized successfully"); + } + + LOG.info("<== RangerAuditServerDestination:initJwtToken()"); + } + + private String readJwtTokenFromFile(String tokenFile) throws IOException { + InputStream in = null; + try { + in = getFileInputStream(tokenFile); + if (in != null) { + String token = IOUtils.toString(in, Charset.defaultCharset()).trim(); + return token; + } else { + throw new IOException("Unable to read JWT token file: " + tokenFile); + } + } finally { + close(in, tokenFile); + } + } + + /** + * This method proactively obtains the TGT and service ticket for the audit server + * during initialization, so they are cached and ready when the first audit event arrives. + */ + private void preAuthenticateKerberos() { + LOG.info("==> RangerAuditServerDestination:preAuthenticateKerberos()"); + + try { + UserGroupInformation ugi = UserGroupInformation.getLoginUser(); + + if (ugi == null) { + LOG.warn("No UserGroupInformation available for Kerberos pre-authentication"); + return; + } + + if (!ugi.hasKerberosCredentials()) { + LOG.warn("User {} does not have Kerberos credentials for pre-authentication", ugi.getUserName()); + return; + } + + LOG.info("Pre-authenticating Kerberos for user: {}, authMethod: {}", ugi.getUserName(), ugi.getAuthenticationMethod()); + + ugi.checkTGTAndReloginFromKeytab(); + LOG.debug("TGT verified and refreshed if needed for user: {}", ugi.getUserName()); + + // Get the audit server URL to determine the target hostname for service ticket + String auditServerUrl = MiscUtil.getStringProperty(props, PROP_AUDITSERVER_URL); + + if (StringUtils.isNotEmpty(auditServerUrl)) { + try { + URI uri = new URI(auditServerUrl); + String hostname = uri.getHost(); + + LOG.info("Pre-fetching Kerberos service ticket for HTTP/{}", hostname); + + ugi.doAs((PrivilegedExceptionAction) () -> { + LOG.debug("Kerberos security context initialized for audit server: {}", hostname); + return null; + }); + + LOG.info("Kerberos pre-authentication completed successfully. Service ticket cached for HTTP/{}", hostname); + } catch (URISyntaxException e) { + LOG.warn("Invalid audit server URL format: {}. Skipping service ticket pre-fetch", auditServerUrl, e); + } catch (Exception e) { + LOG.warn("Failed to pre-fetch service ticket for audit server: {}. First request may need to obtain ticket", auditServerUrl, e); + } + } else { + LOG.warn("Audit server URL not configured. Cannot pre-fetch service ticket"); + } + } catch (Exception e) { + LOG.warn("Kerberos pre-authentication failed. First request will retry authentication", e); + } + + LOG.info("<== RangerAuditServerDestination:preAuthenticateKerberos()"); + } + + @Override + public void stop() { + LOG.info("==> RangerAuditServerDestination.stop() called.."); + logStatus(); + if (httpClient != null) { + try { + httpClient.close(); + } catch (IOException ioe) { + LOG.error("Error while closing httpclient in RangerAuditServerDestination!", ioe); + } finally { + httpClient = null; + } + } + } + + /* + * (non-Javadoc) + * + * @see org.apache.ranger.audit.provider.AuditProvider#flush() + */ + @Override + public void flush() { + } + + @Override + public boolean log(Collection events) { + boolean ret = false; + try { + logStatusIfRequired(); + addTotalCount(events.size()); + + if (httpClient == null) { + httpClient = buildHTTPClient(); + if (httpClient == null) { + // HTTP Server is still not initialized. So need return error + addDeferredCount(events.size()); + return ret; + } + } + ret = logAsBatch(events); + } catch (Throwable t) { + addDeferredCount(events.size()); + logError("Error sending audit to HTTP Server", t); + } + return ret; + } + + public boolean isAsync() { + return true; + } + + private boolean logAsBatch(Collection events) { + boolean batchSuccess = false; + int totalEvents = events.size(); + + LOG.debug("==> logAsBatch() Sending batch of {} events to Audit Server....", totalEvents); + + batchSuccess = sendBatch(events); + if (batchSuccess) { + addSuccessCount(totalEvents); + } else { + LOG.error("Failed to send batch of {} events", totalEvents); + addFailedCount(totalEvents); + } + + LOG.debug("<== logAsBatch() Batch processing complete: {}/{} events sent successfully", batchSuccess ? totalEvents : 0, totalEvents); + + return batchSuccess; + } + + private boolean sendBatch(Collection events) { + boolean ret = false; + Map queryParams = new HashMap<>(); + try { + UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); + LOG.debug("Sending audit batch of {} events as user: {}", events.size(), ugi.getUserName()); + + PrivilegedExceptionAction> action = + () -> executeHttpBatchRequest(REST_RELATIVE_PATH_POST, queryParams, events, Map.class); + Map response = executeAction(action, ugi); + + if (response != null) { + LOG.info("Audit batch sent successfully. {} events delivered. Response: {}", events.size(), response); + ret = true; + } else { + LOG.error("Received null response from audit server for batch of {} events", events.size()); + ret = false; + } + } catch (Exception e) { + LOG.error("Failed to send audit batch of {} events to {}. Error: {}", events.size(), httpURL, e.getMessage(), e); + + // Log additional context for authentication errors + if (e.getMessage() != null && e.getMessage().contains("401")) { + LOG.error("Authentication failure detected. Verify Kerberos credentials are valid and audit server is reachable."); + } + ret = false; + } + + return ret; + } + + public T executeHttpBatchRequest(String relativeUrl, Map params, + Collection events, Class clazz) throws Exception { + T finalResponse = postAndParse(httpURL + relativeUrl, params, events, clazz); + return finalResponse; + } + + public T executeHttpRequestPOST(String relativeUrl, Map params, Object obj, Class clazz) throws Exception { + LOG.debug("==> RangerAuditServerDestination().executeHttpRequestPOST()"); + + T finalResponse = postAndParse(httpURL + relativeUrl, params, obj, clazz); + + LOG.debug("<== RangerAuditServerDestination().executeHttpRequestPOST()"); + return finalResponse; + } + + public T postAndParse(String url, Map queryParams, Object obj, Class clazz) throws Exception { + HttpPost httpPost = new HttpPost(buildURI(url, queryParams)); + StringEntity entity = new StringEntity(gsonBuilder.toJson(obj)); + httpPost.setEntity(entity); + return executeAndParseResponse(httpPost, clazz, queryParams); + } + + synchronized CloseableHttpClient buildHTTPClient() { + Lookup authRegistry = RegistryBuilder.create().register(AuthSchemes.SPNEGO, new SPNegoSchemeFactory(true, true)).build(); + HttpClientBuilder clientBuilder = HttpClients.custom().setDefaultAuthSchemeRegistry(authRegistry); + String userName = MiscUtil.getStringProperty(props, PROP_AUDITSERVER_USER_NAME); + String passWord = MiscUtil.getStringProperty(props, PROP_AUDITSERVER_USER_PASSWORD); + int clientConnTimeOutMs = MiscUtil.getIntProperty(props, PROP_AUDITSERVER_CLIENT_CONN_TIMEOUT_MS, 1000); + int clientReadTimeOutMs = MiscUtil.getIntProperty(props, PROP_AUDITSERVER_CLIENT_READ_TIMEOUT_MS, 1000); + int maxConnections = MiscUtil.getIntProperty(props, PROP_AUDITSERVER_MAX_CONNECTION, 10); + int maxConnectionsPerHost = MiscUtil.getIntProperty(props, PROP_AUDITSERVER_MAX_CONNECTION_PER_HOST, 10); + int validateAfterInactivityMs = MiscUtil.getIntProperty(props, PROP_AUDITSERVER_VALIDATE_INACTIVE_MS, 1000); + int poolRetryCount = MiscUtil.getIntProperty(props, PROP_AUDITSERVER_POOL_RETRY_COUNT, 5); + httpURL = MiscUtil.getStringProperty(props, PROP_AUDITSERVER_URL); + + LOG.info("Building HTTP client for audit destination: url={}, connTimeout={}ms, readTimeout={}ms, maxConn={}", httpURL, clientConnTimeOutMs, clientReadTimeOutMs, maxConnections); + + try { + if (AUTH_TYPE_JWT.equalsIgnoreCase(authType)) { + // JWT authentication - token will be added to request headers directly + LOG.info("HTTP client configured for JWT Bearer token authentication"); + } else { + // Kerberos or Basic authentication - use credentials provider + BasicCredentialsProvider credentialsProvider = new BasicCredentialsProvider(); + + if (AUTH_TYPE_KERBEROS.equalsIgnoreCase(authType) || isKerberosAuthenticated()) { + credentialsProvider.setCredentials(AuthScope.ANY, new KerberosCredentials(null)); + LOG.info("HTTP client configured for Kerberos authentication (SPNEGO)...."); + try { + UserGroupInformation ugi = UserGroupInformation.getLoginUser(); + LOG.info("Current Kerberos principal: {}, authMethod: {}, hasKerberosCredentials: {}", ugi.getUserName(), ugi.getAuthenticationMethod(), ugi.hasKerberosCredentials()); + } catch (Exception e) { + LOG.warn("Failed to get current UGI details", e); + } + } else if (AUTH_TYPE_BASIC.equalsIgnoreCase(authType) || (StringUtils.isNotEmpty(userName) && StringUtils.isNotEmpty(passWord))) { + credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(userName, passWord)); + LOG.info("HTTP client configured for basic authentication with username: {}", userName); + } else { + LOG.warn("No authentication credentials configured for HTTP audit destination"); + } + + clientBuilder.setDefaultCredentialsProvider(credentialsProvider); + } + } catch (Exception excp) { + LOG.error("Exception while configuring authentication credentials. Audits may fail to send!", excp); + } + + PoolingHttpClientConnectionManager connectionManager; + + KeyManager[] kmList = getKeyManagers(); + TrustManager[] tmList = getTrustManagers(); + SSLContext sslContext = getSSLContext(kmList, tmList); + + if (sslContext != null) { + SSLContext.setDefault(sslContext); + } + + boolean isSSL = (httpURL != null && httpURL.contains("https://")); + if (isSSL) { + SSLConnectionSocketFactory sslsf = new SSLConnectionSocketFactory(sslContext, NoopHostnameVerifier.INSTANCE); + Registry socketFactoryRegistry = RegistryBuilder.create().register(PROTOCOL_HTTPS, sslsf).build(); + + connectionManager = new PoolingHttpClientConnectionManager(socketFactoryRegistry); + + clientBuilder.setSSLSocketFactory(sslsf); + } else { + connectionManager = new PoolingHttpClientConnectionManager(); + } + + connectionManager.setMaxTotal(maxConnections); + connectionManager.setDefaultMaxPerRoute(maxConnectionsPerHost); + connectionManager.setValidateAfterInactivity(validateAfterInactivityMs); + + RequestConfig.Builder requestConfigBuilder = RequestConfig.custom() + .setCookieSpec(CookieSpecs.DEFAULT) + .setConnectTimeout(clientConnTimeOutMs) + .setSocketTimeout(clientReadTimeOutMs); + + // Configure authentication based on auth type + if (AUTH_TYPE_JWT.equalsIgnoreCase(authType)) { + // JWT doesn't use HttpClient's authentication mechanism - token is added to headers + LOG.info("RequestConfig configured for JWT authentication...."); + } else { + // For Kerberos and Basic auth, enable HttpClient's authentication mechanism + requestConfigBuilder.setAuthenticationEnabled(true); + + try { + // For Kerberos authentication, specify SPNEGO as target auth scheme + if (isKerberosAuthenticated() || (AUTH_TYPE_KERBEROS.equalsIgnoreCase(authType))) { + requestConfigBuilder.setTargetPreferredAuthSchemes(Arrays.asList(AuthSchemes.SPNEGO)); + LOG.info("Configured SPNEGO as target authentication scheme for audit HTTP client"); + } + } catch (Exception e) { + LOG.warn("Failed to check Kerberos authentication status for request config", e); + } + } + + RequestConfig customizedRequestConfig = requestConfigBuilder.build(); + + CloseableHttpClient httpClient = clientBuilder.setConnectionManager(connectionManager).setRetryHandler(new AuditHTTPRetryHandler(poolRetryCount, true)).setDefaultRequestConfig(customizedRequestConfig).build(); + + return httpClient; + } + + boolean isKerberosAuthenticated() throws Exception { + boolean status = false; + try { + UserGroupInformation loggedInUser = UserGroupInformation.getLoginUser(); + boolean isSecurityEnabled = UserGroupInformation.isSecurityEnabled(); + boolean hasKerberosCredentials = loggedInUser.hasKerberosCredentials(); + UserGroupInformation.AuthenticationMethod loggedInUserAuthMethod = loggedInUser.getAuthenticationMethod(); + + status = isSecurityEnabled && hasKerberosCredentials && loggedInUserAuthMethod.equals(UserGroupInformation.AuthenticationMethod.KERBEROS); + } catch (IOException e) { + throw new Exception("Failed to get authentication details.", e); + } + return status; + } + + private void close(InputStream str, String filename) { + if (str != null) { + try { + str.close(); + } catch (IOException excp) { + LOG.error("Error while closing file: [{}]", filename, excp); + } + } + } + + private PrivilegedExceptionAction getPrivilegedAction(String relativeUrl, Map queryParams, Object postParam, Class clazz) { + return () -> executeHttpRequestPOST(relativeUrl, queryParams, postParam, clazz); + } + + private R executeAndParseResponse(T request, Class responseClazz, Map queryParams) throws Exception { + R ret = null; + CloseableHttpClient client = getCloseableHttpClient(); + + if (client != null) { + addCommonHeaders(request, queryParams); + // Create an HttpClientContext to maintain authentication state across potential retries + HttpClientContext context = HttpClientContext.create(); + try (CloseableHttpResponse response = client.execute(request, context)) { + ret = parseResponse(response, responseClazz); + } + } else { + LOG.error("Cannot process request as Audit HTTPClient is null..."); + } + return ret; + } + + private T addCommonHeaders(T t, Map queryParams) { + t.addHeader(REST_HEADER_ACCEPT, REST_ACCEPTED_MIME_TYPE_JSON); + t.setHeader(REST_HEADER_CONTENT_TYPE, REST_CONTENT_TYPE_MIME_TYPE_JSON); + + // Add JWT Bearer token if JWT authentication is configured + if (AUTH_TYPE_JWT.equalsIgnoreCase(authType) && StringUtils.isNotEmpty(jwtToken)) { + t.setHeader(REST_HEADER_AUTHORIZATION, "Bearer " + jwtToken); + LOG.debug("Added JWT Bearer token to request Authorization header"); + } + + return t; + } + + @SuppressWarnings("unchecked") + private R parseResponse(HttpResponse response, Class clazz) throws Exception { + R type = null; + + if (response == null) { + String responseError = "Received NULL response from server"; + LOG.error(responseError); + throw new Exception(responseError); + } + + int httpStatus = response.getStatusLine().getStatusCode(); + + if (httpStatus == HttpStatus.SC_OK) { + InputStream responseInputStream = response.getEntity().getContent(); + if (clazz.equals(String.class)) { + type = (R) IOUtils.toString(responseInputStream, Charset.defaultCharset()); + } else { + type = gsonBuilder.fromJson(new InputStreamReader(responseInputStream), clazz); + } + responseInputStream.close(); + } else { + String responseBody = ""; + try { + if (response.getEntity() != null && response.getEntity().getContent() != null) { + responseBody = IOUtils.toString(response.getEntity().getContent(), Charset.defaultCharset()); + } + } catch (Exception e) { + LOG.debug("Failed to read error response body", e); + } + + String error = String.format("Request failed with HTTP status %d. Response body: %s", httpStatus, responseBody); + if (httpStatus == HttpStatus.SC_UNAUTHORIZED) { + LOG.error("{} - Authentication failed. Verify Kerberos credentials are valid and properly configured.", error); + } else { + LOG.error(error); + } + throw new Exception(error); + } + return type; + } + + private T executeAction(PrivilegedExceptionAction action, UserGroupInformation owner) throws Exception { + T ret = null; + if (owner != null) { + ret = owner.doAs(action); + } else { + ret = action.run(); + } + return ret; + } + + private URI buildURI(String url, Map queryParams) throws URISyntaxException { + URIBuilder builder = new URIBuilder(url); + if (queryParams != null) { + for (Map.Entry param : queryParams.entrySet()) { + builder.addParameter(param.getKey(), param.getValue()); + } + } + return builder.build(); + } + + private CloseableHttpClient getCloseableHttpClient() { + CloseableHttpClient client = httpClient; + if (client == null) { + synchronized (this) { + client = httpClient; + + if (client == null) { + client = buildHTTPClient(); + httpClient = client; + } + } + } + return client; + } + + private KeyManager[] getKeyManagers() { + KeyManager[] kmList = null; + String credentialProviderPath = MiscUtil.getStringProperty(props, RANGER_POLICYMGR_CLIENT_KEY_FILE_CREDENTIAL); + String keyStoreAlias = RANGER_POLICYMGR_CLIENT_KEY_FILE_CREDENTIAL_ALIAS; + String keyStoreFile = MiscUtil.getStringProperty(props, RANGER_POLICYMGR_CLIENT_KEY_FILE); + String keyStoreFilepwd = MiscUtil.getCredentialString(credentialProviderPath, keyStoreAlias); + if (StringUtils.isNotEmpty(keyStoreFile) && StringUtils.isNotEmpty(keyStoreFilepwd)) { + InputStream in = null; + try { + in = getFileInputStream(keyStoreFile); + + if (in != null) { + String keyStoreType = MiscUtil.getStringProperty(props, RANGER_POLICYMGR_CLIENT_KEY_FILE_TYPE); + keyStoreType = StringUtils.isNotEmpty(keyStoreType) ? keyStoreType : RANGER_POLICYMGR_CLIENT_KEY_FILE_TYPE_DEFAULT; + KeyStore keyStore = KeyStore.getInstance(keyStoreType); + + keyStore.load(in, keyStoreFilepwd.toCharArray()); + + KeyManagerFactory keyManagerFactory = KeyManagerFactory.getInstance(RANGER_SSL_KEYMANAGER_ALGO_TYPE); + + keyManagerFactory.init(keyStore, keyStoreFilepwd.toCharArray()); + + kmList = keyManagerFactory.getKeyManagers(); + } else { + LOG.error("Unable to obtain keystore from file [{}]", keyStoreFile); + } + } catch (KeyStoreException e) { + LOG.error("Unable to obtain from KeyStore :{}", e.getMessage(), e); + } catch (NoSuchAlgorithmException e) { + LOG.error("SSL algorithm is NOT available in the environment", e); + } catch (CertificateException e) { + LOG.error("Unable to obtain the requested certification ", e); + } catch (FileNotFoundException e) { + LOG.error("Unable to find the necessary SSL Keystore Files", e); + } catch (IOException e) { + LOG.error("Unable to read the necessary SSL Keystore Files", e); + } catch (UnrecoverableKeyException e) { + LOG.error("Unable to recover the key from keystore", e); + } finally { + close(in, keyStoreFile); + } + } + return kmList; + } + + private TrustManager[] getTrustManagers() { + TrustManager[] tmList = null; + String credentialProviderPath = MiscUtil.getStringProperty(props, RANGER_POLICYMGR_TRUSTSTORE_FILE_CREDENTIAL); + String trustStoreAlias = RANGER_POLICYMGR_TRUSTSTORE_FILE_CREDENTIAL_ALIAS; + String trustStoreFile = MiscUtil.getStringProperty(props, RANGER_POLICYMGR_TRUSTSTORE_FILE); + String trustStoreFilepwd = MiscUtil.getCredentialString(credentialProviderPath, trustStoreAlias); + if (StringUtils.isNotEmpty(trustStoreFile) && StringUtils.isNotEmpty(trustStoreFilepwd)) { + InputStream in = null; + try { + in = getFileInputStream(trustStoreFile); + + if (in != null) { + String trustStoreType = MiscUtil.getStringProperty(props, RANGER_POLICYMGR_TRUSTSTORE_FILE_TYPE); + trustStoreType = StringUtils.isNotEmpty(trustStoreType) ? trustStoreType : RANGER_POLICYMGR_TRUSTSTORE_FILE_TYPE_DEFAULT; + KeyStore trustStore = KeyStore.getInstance(trustStoreType); + + trustStore.load(in, trustStoreFilepwd.toCharArray()); + + TrustManagerFactory trustManagerFactory = TrustManagerFactory.getInstance(RANGER_SSL_TRUSTMANAGER_ALGO_TYPE); + + trustManagerFactory.init(trustStore); + + tmList = trustManagerFactory.getTrustManagers(); + } else { + LOG.error("Unable to obtain truststore from file [{}]", trustStoreFile); + } + } catch (KeyStoreException e) { + LOG.error("Unable to obtain from KeyStore", e); + } catch (NoSuchAlgorithmException e) { + LOG.error("SSL algorithm is NOT available in the environment :{}", e.getMessage(), e); + } catch (CertificateException e) { + LOG.error("Unable to obtain the requested certification :{}", e.getMessage(), e); + } catch (FileNotFoundException e) { + LOG.error("Unable to find the necessary SSL TrustStore File:{}", trustStoreFile, e); + } catch (IOException e) { + LOG.error("Unable to read the necessary SSL TrustStore Files :{}", trustStoreFile, e); + } finally { + close(in, trustStoreFile); + } + } + return tmList; + } + + private SSLContext getSSLContext(KeyManager[] kmList, TrustManager[] tmList) { + SSLContext sslContext = null; + try { + sslContext = SSLContext.getInstance(RANGER_SSL_CONTEXT_ALGO_TYPE); + if (sslContext != null) { + sslContext.init(kmList, tmList, new SecureRandom()); + } + } catch (NoSuchAlgorithmException e) { + LOG.error("SSL algorithm is not available in the environment", e); + } catch (KeyManagementException e) { + LOG.error("Unable to initialise the SSLContext", e); + } + return sslContext; + } + + private InputStream getFileInputStream(String fileName) throws IOException { + InputStream in = null; + if (StringUtils.isNotEmpty(fileName)) { + File file = new File(fileName); + if (file != null && file.exists()) { + in = new FileInputStream(file); + } else { + in = ClassLoader.getSystemResourceAsStream(fileName); + } + } + return in; + } + + private class AuditHTTPRetryHandler extends StandardHttpRequestRetryHandler { + public AuditHTTPRetryHandler(Integer poolRetryCount, boolean isIdempotentRequest) { + super(poolRetryCount, isIdempotentRequest); + } + + @Override + public boolean retryRequest(IOException exception, int executionCount, HttpContext context) { + LOG.debug("==> AuditHTTPRetryHandler.retryRequest {} Execution Count = {}", exception.getMessage(), executionCount); + + boolean ret = super.retryRequest(exception, executionCount, context); + + LOG.debug("<== AuditHTTPRetryHandler.retryRequest(): ret= {}", ret); + + return ret; + } + } +} diff --git a/agents-audit/dest-solr/src/main/java/org/apache/ranger/audit/destination/SolrAuditDestination.java b/agents-audit/dest-solr/src/main/java/org/apache/ranger/audit/destination/SolrAuditDestination.java index 81a576a222..61ec236921 100644 --- a/agents-audit/dest-solr/src/main/java/org/apache/ranger/audit/destination/SolrAuditDestination.java +++ b/agents-audit/dest-solr/src/main/java/org/apache/ranger/audit/destination/SolrAuditDestination.java @@ -487,9 +487,9 @@ private UpdateResponse addDocsToSolr(final SolrClient solrClient, final Collecti if (kerberosUser != null) { // execute the privileged action as the given keytab user - final KerberosAction kerberosAction = new KerberosAction<>(kerberosUser, action, LOG); + final KerberosAction kerberosAction = new KerberosAction<>(kerberosUser, action, LOG); - ret = (UpdateResponse) kerberosAction.execute(); + ret = kerberosAction.execute(); } else { ret = action.run(); } diff --git a/agents-audit/pom.xml b/agents-audit/pom.xml index 250bb582e0..a25edde925 100644 --- a/agents-audit/pom.xml +++ b/agents-audit/pom.xml @@ -31,6 +31,7 @@ core + dest-auditserver dest-cloudwatch dest-es dest-hdfs diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/audit/RangerDefaultAuditHandler.java b/agents-common/src/main/java/org/apache/ranger/plugin/audit/RangerDefaultAuditHandler.java index a9fb1863af..6e20ebe7ab 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/audit/RangerDefaultAuditHandler.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/audit/RangerDefaultAuditHandler.java @@ -20,6 +20,7 @@ package org.apache.ranger.plugin.audit; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.MapUtils; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.ranger.audit.model.AuthzAuditEvent; @@ -138,7 +139,7 @@ public AuthzAuditEvent getAuthzEvents(RangerAccessResult result) { ret.setDatasets(getDatasets(request)); ret.setProjects(getProjects(request)); ret.setDatasetIds(getDatasetIds(request)); - ret.setAdditionalInfo(getAdditionalInfo(request)); + ret.setAdditionalInfo(getAdditionalInfo(request, result)); ret.setClusterName(request.getClusterName()); ret.setZoneName(result.getZoneName()); ret.setAgentHostname(restUtils.getAgentHostname()); @@ -230,16 +231,28 @@ public final Set getDatasetIds(RangerAccessRequest request) { return gdsResult != null ? gdsResult.getDatasetIds() : null; } - public String getAdditionalInfo(RangerAccessRequest request) { - if (StringUtils.isBlank(request.getRemoteIPAddress()) && CollectionUtils.isEmpty(request.getForwardedAddresses())) { - return null; + public String getAdditionalInfo(RangerAccessRequest request, RangerAccessResult result) { + String ret = org.apache.commons.lang3.StringUtils.EMPTY; + Map addInfomap = new HashMap<>(); + + if (!CollectionUtils.isEmpty(request.getForwardedAddresses())) { + addInfomap.put("forwarded-ip-addresses", "[" + org.apache.commons.lang3.StringUtils.join(request.getForwardedAddresses(), ", ") + "]"); } - Map addInfomap = new HashMap<>(); - addInfomap.put("forwarded-ip-addresses", "[" + StringUtils.join(request.getForwardedAddresses(), ", ") + "]"); - addInfomap.put("remote-ip-address", request.getRemoteIPAddress()); + if (org.apache.commons.lang3.StringUtils.isNotEmpty(request.getRemoteIPAddress())) { + addInfomap.put("remote-ip-address", request.getRemoteIPAddress()); + } - return JsonUtils.mapToJson(addInfomap); + String serviceType = result.getServiceTypeName(); + if (org.apache.commons.lang3.StringUtils.isNotEmpty(serviceType)) { + addInfomap.put("serviceType", serviceType); + } + + if (MapUtils.isNotEmpty(addInfomap)) { + ret = JsonUtils.mapToJson(addInfomap); + } + + return ret; } protected final Set getTags(RangerAccessRequest request) { diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerAccessResult.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerAccessResult.java index 45441dac07..32a0aaab7a 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerAccessResult.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerAccessResult.java @@ -259,6 +259,16 @@ public int getServiceType() { return ret; } + public String getServiceTypeName() { + String ret = null; + + if (serviceDef != null && serviceDef.getName() != null) { + ret = serviceDef.getName(); + } + + return ret; + } + public Map getAdditionalInfo() { return additionalInfo; } diff --git a/agents-common/src/test/java/org/apache/ranger/plugin/audit/TestRangerDefaultAuditHandler.java b/agents-common/src/test/java/org/apache/ranger/plugin/audit/TestRangerDefaultAuditHandler.java index acdad81e02..54a63c42d7 100644 --- a/agents-common/src/test/java/org/apache/ranger/plugin/audit/TestRangerDefaultAuditHandler.java +++ b/agents-common/src/test/java/org/apache/ranger/plugin/audit/TestRangerDefaultAuditHandler.java @@ -286,15 +286,17 @@ public void test07_getDatasetsAndProjects_fromContext() { public void test08_getAdditionalInfo_nullAndNonNull() { RangerDefaultAuditHandler handler = new RangerDefaultAuditHandler(); RangerAccessRequestImpl req = new RangerAccessRequestImpl(); + RangerAccessResult res = new RangerAccessResult(0, "svc", null, req); req.setRemoteIPAddress(null); req.setForwardedAddresses(new ArrayList<>()); - Assertions.assertNull(handler.getAdditionalInfo(req)); + Assertions.assertNull(handler.getAdditionalInfo(req, res)); RangerAccessRequestImpl req2 = new RangerAccessRequestImpl(); + RangerAccessResult res2 = new RangerAccessResult(0, "svc", null, req); req2.setRemoteIPAddress("10.1.1.1"); List fwd = new ArrayList<>(Arrays.asList("1.1.1.1")); req2.setForwardedAddresses(fwd); - String info = handler.getAdditionalInfo(req2); + String info = handler.getAdditionalInfo(req2, res2); Assertions.assertNotNull(info); Assertions.assertTrue(info.contains("10.1.1.1")); } diff --git a/dev-support/ranger-docker/.dockerignore b/dev-support/ranger-docker/.dockerignore index 8453ea111d..b27010823a 100644 --- a/dev-support/ranger-docker/.dockerignore +++ b/dev-support/ranger-docker/.dockerignore @@ -2,6 +2,9 @@ !config !dist/version !dist/ranger-*-admin.tar.gz +!dist/ranger-*-ranger-audit-server-service.tar.gz +!dist/ranger-*-ranger-audit-consumer-solr.tar.gz +!dist/ranger-*-ranger-audit-consumer-hdfs.tar.gz !dist/ranger-*-kms.tar.gz !dist/ranger-*-usersync.tar.gz !dist/ranger-*-tagsync.tar.gz diff --git a/dev-support/ranger-docker/Dockerfile.ranger-audit-consumer-hdfs b/dev-support/ranger-docker/Dockerfile.ranger-audit-consumer-hdfs new file mode 100644 index 0000000000..e143c3247a --- /dev/null +++ b/dev-support/ranger-docker/Dockerfile.ranger-audit-consumer-hdfs @@ -0,0 +1,85 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ARG RANGER_BASE_IMAGE +ARG RANGER_BASE_VERSION +FROM ${RANGER_BASE_IMAGE}:${RANGER_BASE_VERSION} + +# Declare ARG for use in this build stage +ARG RANGER_VERSION +ARG KERBEROS_ENABLED + +# Install required packages including Kerberos client +RUN apt-get update && apt-get install -y \ + python3 \ + python3-pip \ + curl \ + vim \ + net-tools \ + krb5-user \ + && rm -rf /var/lib/apt/lists/* + +# Volume for keytabs +VOLUME /etc/keytabs + +# Set up working directory +WORKDIR /opt/ranger-audit-consumer-hdfs + +# Copy ranger-audit-consumer-hdfs distribution +COPY ./dist/ranger-${RANGER_VERSION}-ranger-audit-consumer-hdfs.tar.gz /tmp/ + +# Copy scripts, Hadoop configs, and Kerberos configuration +COPY ./scripts/audit-server/service-check-functions.sh ${RANGER_SCRIPTS}/ +COPY ./scripts/audit-server/ranger-audit-consumer-hdfs.sh ${RANGER_SCRIPTS}/ +COPY ./scripts/kdc/krb5.conf /etc/krb5.conf + +# Extract and setup +RUN tar xzf /tmp/ranger-${RANGER_VERSION}-ranger-audit-consumer-hdfs.tar.gz -C /opt/ranger-audit-consumer-hdfs --strip-components=1 && \ + rm -f /tmp/ranger-${RANGER_VERSION}-ranger-audit-consumer-hdfs.tar.gz && \ + mkdir -p /var/log/ranger/ranger-audit-consumer-hdfs && \ + mkdir -p /home/ranger/scripts && \ + rm -rf /opt/ranger-audit-consumer-hdfs/logs && \ + ln -sf /var/log/ranger/ranger-audit-consumer-hdfs /opt/ranger-audit-consumer-hdfs/logs && \ + chown -R ranger:ranger /opt/ranger-audit-consumer-hdfs/ /var/log/ranger/ ${RANGER_SCRIPTS} && \ + chmod -R 755 /opt/ranger-audit-consumer-hdfs && \ + chmod 755 ${RANGER_SCRIPTS}/ranger-audit-consumer-hdfs.sh + +# Set Java home +ENV JAVA_HOME=/opt/java/openjdk +ENV PATH=$JAVA_HOME/bin:$PATH + +# Environment variables for HDFS consumer +ENV AUDIT_CONSUMER_HOME_DIR=/opt/ranger-audit-consumer-hdfs +ENV AUDIT_CONSUMER_CONF_DIR=/opt/ranger-audit-consumer-hdfs/conf +ENV AUDIT_CONSUMER_LOG_DIR=/var/log/ranger/ranger-audit-consumer-hdfs +ENV RANGER_USER=ranger + +# Expose health check port +EXPOSE 7092 + +# Health check - test the health endpoint +HEALTHCHECK --interval=30s --timeout=10s --start-period=60s --retries=3 \ + CMD curl -f -o /dev/null -s -w "%{http_code}" http://localhost:7092/api/health | grep -q "200" || exit 1 + +# Environment variable for Kerberos support +ENV KERBEROS_ENABLED=${KERBEROS_ENABLED:-false} + +# Switch to ranger user +USER ranger + +# Start the HDFS consumer using the custom startup script +WORKDIR /opt/ranger-audit-consumer-hdfs +ENTRYPOINT ["/home/ranger/scripts/ranger-audit-consumer-hdfs.sh"] diff --git a/dev-support/ranger-docker/Dockerfile.ranger-audit-consumer-solr b/dev-support/ranger-docker/Dockerfile.ranger-audit-consumer-solr new file mode 100644 index 0000000000..abcb6240ea --- /dev/null +++ b/dev-support/ranger-docker/Dockerfile.ranger-audit-consumer-solr @@ -0,0 +1,86 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ARG RANGER_BASE_IMAGE +ARG RANGER_BASE_VERSION +FROM ${RANGER_BASE_IMAGE}:${RANGER_BASE_VERSION} + +# Declare ARG for use in this build stage +ARG RANGER_VERSION +ARG KERBEROS_ENABLED + +# Install required packages including Kerberos client +RUN apt-get update && apt-get install -y \ + python3 \ + python3-pip \ + curl \ + vim \ + net-tools \ + krb5-user \ + && rm -rf /var/lib/apt/lists/* + +# Volume for keytabs +VOLUME /etc/keytabs + +# Set up working directory +WORKDIR /opt/ranger-audit-consumer-solr + +# Copy ranger-audit-consumer-solr distribution +COPY ./dist/ranger-${RANGER_VERSION}-ranger-audit-consumer-solr.tar.gz /tmp/ + +# Copy scripts and Kerberos configuration +COPY ./scripts/audit-server/service-check-functions.sh ${RANGER_SCRIPTS}/ +COPY ./scripts/audit-server/ranger-audit-consumer-solr.sh ${RANGER_SCRIPTS}/ +COPY ./scripts/kdc/krb5.conf /etc/krb5.conf + +# Extract and setup +RUN tar xzf /tmp/ranger-${RANGER_VERSION}-ranger-audit-consumer-solr.tar.gz -C /opt/ranger-audit-consumer-solr --strip-components=1 && \ + rm -f /tmp/ranger-${RANGER_VERSION}-ranger-audit-consumer-solr.tar.gz && \ + mkdir -p /var/log/ranger/ranger-audit-consumer-solr && \ + mkdir -p /home/ranger/scripts && \ + rm -rf /opt/ranger-audit-consumer-solr/logs && \ + ln -sf /var/log/ranger/ranger-audit-consumer-solr /opt/ranger-audit-consumer-solr/logs && \ + chown -R ranger:ranger /opt/ranger-audit-consumer-solr/ /var/log/ranger/ ${RANGER_SCRIPTS} && \ + chmod -R 755 /opt/ranger-audit-consumer-solr && \ + chmod 755 ${RANGER_SCRIPTS}/ranger-audit-consumer-solr.sh && \ + chmod 755 ${RANGER_SCRIPTS}/wait_for_keytab.sh + +# Set Java home +ENV JAVA_HOME=/opt/java/openjdk +ENV PATH=$JAVA_HOME/bin:$PATH + +# Environment variables for Solr consumer +ENV AUDIT_CONSUMER_HOME_DIR=/opt/ranger-audit-consumer-solr +ENV AUDIT_CONSUMER_CONF_DIR=/opt/ranger-audit-consumer-solr/conf +ENV AUDIT_CONSUMER_LOG_DIR=/var/log/ranger/ranger-audit-consumer-solr +ENV RANGER_USER=ranger + +# Expose health check port +EXPOSE 7091 + +# Health check - test the health endpoint +HEALTHCHECK --interval=30s --timeout=10s --start-period=60s --retries=3 \ + CMD curl -f -o /dev/null -s -w "%{http_code}" http://localhost:7091/api/health | grep -q "200" || exit 1 + +# Environment variable for Kerberos support +ENV KERBEROS_ENABLED=${KERBEROS_ENABLED:-false} + +# Switch to ranger user +USER ranger + +# Start the Solr consumer using the custom startup script +WORKDIR /opt/ranger-audit-consumer-solr +ENTRYPOINT ["/home/ranger/scripts/ranger-audit-consumer-solr.sh"] diff --git a/dev-support/ranger-docker/Dockerfile.ranger-audit-server b/dev-support/ranger-docker/Dockerfile.ranger-audit-server new file mode 100644 index 0000000000..e95dc5c7c5 --- /dev/null +++ b/dev-support/ranger-docker/Dockerfile.ranger-audit-server @@ -0,0 +1,86 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ARG RANGER_BASE_IMAGE +ARG RANGER_BASE_VERSION +FROM ${RANGER_BASE_IMAGE}:${RANGER_BASE_VERSION} + +# Declare ARG for use in this build stage +ARG RANGER_VERSION +ARG KERBEROS_ENABLED + +# Install required packages including Kerberos client +RUN apt-get update && apt-get install -y \ + python3 \ + python3-pip \ + curl \ + vim \ + net-tools \ + krb5-user \ + && rm -rf /var/lib/apt/lists/* + +# Volume for keytabs +VOLUME /etc/keytabs + +# Set up working directory +WORKDIR /opt/ranger-audit-server + +# Copy ranger-audit-server-service distribution +COPY ./dist/ranger-${RANGER_VERSION}-ranger-audit-server-service.tar.gz /tmp/ + +# Copy scripts and Kerberos configuration +COPY ./scripts/audit-server/service-check-functions.sh ${RANGER_SCRIPTS}/ +COPY ./scripts/audit-server/ranger-audit-server.sh ${RANGER_SCRIPTS}/ +COPY ./scripts/kdc/krb5.conf /etc/krb5.conf + +# Extract and setup +RUN tar xzf /tmp/ranger-${RANGER_VERSION}-ranger-audit-server-service.tar.gz -C /opt/ranger-audit-server --strip-components=1 && \ + rm -f /tmp/ranger-${RANGER_VERSION}-ranger-audit-server-service.tar.gz && \ + mkdir -p /var/log/ranger/ranger-audit-server/audit/spool && \ + mkdir -p /var/log/ranger/ranger-audit-server/audit/archive && \ + mkdir -p /home/ranger/scripts && \ + rm -rf /opt/ranger-audit-server/logs && \ + ln -sf /var/log/ranger/ranger-audit-server /opt/ranger-audit-server/logs && \ + chown -R ranger:ranger /opt/ranger-audit-server/ /var/log/ranger/ ${RANGER_SCRIPTS} && \ + chmod -R 755 /opt/ranger-audit-server && \ + chmod 755 ${RANGER_SCRIPTS}/ranger-audit-server.sh + +# Set Java home +ENV JAVA_HOME=/opt/java/openjdk +ENV PATH=$JAVA_HOME/bin:$PATH + +# Environment variables for audit server +ENV AUDIT_SERVER_HOME_DIR=/opt/ranger-audit-server +ENV AUDIT_SERVER_CONF_DIR=/opt/ranger-audit-server/conf +ENV AUDIT_SERVER_LOG_DIR=/var/log/ranger/ranger-audit-server +ENV RANGER_USER=ranger + +# Expose ports (HTTP and HTTPS) +EXPOSE 7081 7182 + +# Health check - test the REST API endpoint +HEALTHCHECK --interval=30s --timeout=10s --start-period=60s --retries=3 \ + CMD curl -f -o /dev/null -s -w "%{http_code}" http://localhost:7081/api/audit/health | grep -q -E "200|401" || exit 1 + +# Environment variable for Kerberos support +ENV KERBEROS_ENABLED=${KERBEROS_ENABLED:-false} + +# Switch to ranger user +USER ranger + +# Start the audit server using the custom startup script +WORKDIR /opt/ranger-audit-server +ENTRYPOINT ["/home/ranger/scripts/ranger-audit-server.sh"] diff --git a/dev-support/ranger-docker/README.md b/dev-support/ranger-docker/README.md index 39a89440f1..bd0528909a 100644 --- a/dev-support/ranger-docker/README.md +++ b/dev-support/ranger-docker/README.md @@ -118,3 +118,8 @@ docker compose -f docker-compose.ranger.yml -f docker-compose.ranger-usersync.ym ~~~ docker compose -f docker-compose.ranger.yml -f docker-compose.ranger-usersync.yml -f docker-compose.ranger-tagsync.yml -f docker-compose.ranger-kms.yml -f docker-compose.ranger-hadoop.yml -f docker-compose.ranger-hbase.yml -f docker-compose.ranger-kafka.yml -f docker-compose.ranger-hive.yml -f docker-compose.ranger-trino.yml -f docker-compose.ranger-knox.yml up -d --no-deps --force-recreate --build ~~~ + +#### To bring up audit server, solr and hdfs consumer. Make sure kafka,solr and hdfs containers are running before bring up audit server. +~~~ +docker compose -f docker-compose.ranger.yml -f docker-compose.ranger-hadoop.yml -f docker-compose.ranger-kafka.yml -f docker-compose.ranger-audit-server.yml up -d +~~~ \ No newline at end of file diff --git a/dev-support/ranger-docker/docker-compose.ranger-audit-server.yml b/dev-support/ranger-docker/docker-compose.ranger-audit-server.yml new file mode 100644 index 0000000000..1a18d3a211 --- /dev/null +++ b/dev-support/ranger-docker/docker-compose.ranger-audit-server.yml @@ -0,0 +1,168 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +services: + # Core Audit Server Service - Receives audits from plugins and produces to Kafka + ranger-audit-server: + build: + context: . + dockerfile: Dockerfile.ranger-audit-server + args: + - RANGER_BASE_IMAGE=${RANGER_BASE_IMAGE} + - RANGER_BASE_VERSION=${RANGER_BASE_VERSION} + - RANGER_VERSION=${RANGER_VERSION} + - KERBEROS_ENABLED=${KERBEROS_ENABLED} + image: ranger-audit-server:latest + container_name: ranger-audit-server + hostname: ranger-audit-server.rangernw + stdin_open: true + tty: true + depends_on: + ranger-kdc: + condition: service_started + ranger-kafka: + condition: service_started + ports: + - "7081:7081" + - "7182:7182" + environment: + JAVA_HOME: /opt/java/openjdk + AUDIT_SERVER_HOME_DIR: /opt/ranger-audit-server + AUDIT_SERVER_CONF_DIR: /opt/ranger-audit-server/conf + AUDIT_SERVER_LOG_DIR: /var/log/ranger/ranger-audit-server + AUDIT_SERVER_HEAP: "-Xms512m -Xmx2g" + RANGER_VERSION: ${RANGER_VERSION} + KERBEROS_ENABLED: ${KERBEROS_ENABLED} + KAFKA_BOOTSTRAP_SERVERS: ranger-kafka.rangernw:9092 + networks: + - ranger + volumes: + - ./dist/keytabs/ranger-audit-server:/etc/keytabs + - ranger-audit-server-logs:/opt/ranger-audit-server/logs + - ranger-audit-server-spool:/var/log/ranger/ranger-audit-server/audit + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:7081/api/audit/health"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 60s + + # Solr Consumer Service - Consumes from Kafka and indexes to Solr + ranger-audit-consumer-solr: + build: + context: . + dockerfile: Dockerfile.ranger-audit-consumer-solr + args: + - RANGER_BASE_IMAGE=${RANGER_BASE_IMAGE} + - RANGER_BASE_VERSION=${RANGER_BASE_VERSION} + - RANGER_VERSION=${RANGER_VERSION} + - KERBEROS_ENABLED=${KERBEROS_ENABLED} + image: ranger-audit-consumer-solr:latest + container_name: ranger-audit-consumer-solr + hostname: ranger-audit-consumer-solr.rangernw + stdin_open: true + tty: true + depends_on: + ranger-kdc: + condition: service_started + ranger-kafka: + condition: service_started + ranger-solr: + condition: service_started + ranger-audit-server: + condition: service_healthy + ports: + - "7091:7091" + environment: + JAVA_HOME: /opt/java/openjdk + AUDIT_CONSUMER_HOME_DIR: /opt/ranger-audit-consumer-solr + AUDIT_CONSUMER_CONF_DIR: /opt/ranger-audit-consumer-solr/conf + AUDIT_CONSUMER_LOG_DIR: /var/log/ranger/ranger-audit-consumer-solr + AUDIT_CONSUMER_HEAP: "-Xms512m -Xmx2g" + RANGER_VERSION: ${RANGER_VERSION} + KERBEROS_ENABLED: ${KERBEROS_ENABLED} + KAFKA_BOOTSTRAP_SERVERS: ranger-kafka.rangernw:9092 + SOLR_URL: http://ranger-solr.rangernw:8983/solr + networks: + - ranger + volumes: + - ./dist/keytabs/ranger-audit-consumer-solr:/etc/keytabs + - ranger-audit-consumer-solr-logs:/opt/ranger-audit-consumer-solr/logs + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:7091/api/health"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 60s + + # HDFS Consumer Service - Consumes from Kafka and writes to HDFS + ranger-audit-consumer-hdfs: + build: + context: . + dockerfile: Dockerfile.ranger-audit-consumer-hdfs + args: + - RANGER_BASE_IMAGE=${RANGER_BASE_IMAGE} + - RANGER_BASE_VERSION=${RANGER_BASE_VERSION} + - RANGER_VERSION=${RANGER_VERSION} + - KERBEROS_ENABLED=${KERBEROS_ENABLED} + image: ranger-audit-consumer-hdfs:latest + container_name: ranger-audit-consumer-hdfs + hostname: ranger-audit-consumer-hdfs.rangernw + stdin_open: true + tty: true + depends_on: + ranger-kdc: + condition: service_started + ranger-kafka: + condition: service_started + ranger-hadoop: + condition: service_healthy + ranger-audit-server: + condition: service_healthy + ports: + - "7092:7092" + environment: + JAVA_HOME: /opt/java/openjdk + AUDIT_CONSUMER_HOME_DIR: /opt/ranger-audit-consumer-hdfs + AUDIT_CONSUMER_CONF_DIR: /opt/ranger-audit-consumer-hdfs/conf + AUDIT_CONSUMER_LOG_DIR: /var/log/ranger/ranger-audit-consumer-hdfs + AUDIT_CONSUMER_HEAP: "-Xms512m -Xmx2g" + RANGER_VERSION: ${RANGER_VERSION} + KERBEROS_ENABLED: ${KERBEROS_ENABLED} + KAFKA_BOOTSTRAP_SERVERS: ranger-kafka.rangernw:9092 + HDFS_NAMENODE: ranger-hadoop.rangernw:9000 + networks: + - ranger + volumes: + - ./dist/keytabs/ranger-audit-consumer-hdfs:/etc/keytabs + - ranger-audit-consumer-hdfs-logs:/opt/ranger-audit-consumer-hdfs/logs + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:7092/api/health"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 60s + +volumes: + ranger-audit-server-logs: + ranger-audit-server-spool: + ranger-audit-consumer-solr-logs: + ranger-audit-consumer-hdfs-logs: + +networks: + ranger: + name: rangernw + external: true diff --git a/dev-support/ranger-docker/docker-compose.ranger-kafka.yml b/dev-support/ranger-docker/docker-compose.ranger-kafka.yml index 80e386a500..b2713dead3 100644 --- a/dev-support/ranger-docker/docker-compose.ranger-kafka.yml +++ b/dev-support/ranger-docker/docker-compose.ranger-kafka.yml @@ -25,7 +25,7 @@ services: networks: - ranger ports: - - "6667:6667" + - "9092:9092" depends_on: ranger: condition: service_started diff --git a/dev-support/ranger-docker/scripts/admin/create-ranger-services.py b/dev-support/ranger-docker/scripts/admin/create-ranger-services.py index 5153d5c479..e3dd0aa213 100644 --- a/dev-support/ranger-docker/scripts/admin/create-ranger-services.py +++ b/dev-support/ranger-docker/scripts/admin/create-ranger-services.py @@ -27,6 +27,10 @@ def service_not_exists(service): 'default-policy.1.resource.path.is-recursive': 'true', 'default-policy.1.policyItem.1.users': 'hive', 'default-policy.1.policyItem.1.accessTypes': 'read,write,execute', + 'default-policy.2.name': 'ranger-audit-path', + 'default-policy.2.resource.path': '/ranger/audit/*', + 'default-policy.2.policyItem.1.users': 'rangerauditserver', + 'default-policy.2.policyItem.1.accessTypes': 'read,write,execute', 'ranger.plugin.hdfs.policy.refresh.synchronous':'true'}}) hive = RangerService({'name': 'dev_hive', 'type': 'hive', @@ -53,7 +57,19 @@ def service_not_exists(service): 'default-policy.2.resource.consumergroup': 'ranger_entities_consumer', 'default-policy.2.policyItem.1.users': 'rangertagsync', 'default-policy.2.policyItem.1.accessTypes': 'consume,describe', - 'ranger.plugin.audit.filters': "[{'accessResult': 'DENIED', 'isAudited': true},{'resources':{'topic':{'values':['ATLAS_ENTITIES']}},'users':['rangertagsync'],'actions':['create','consume','describe'],'isAudited':false},{'resources':{'consumergroup':{'values':['ranger_entities_consumer']}},'users':['rangertagsync'],'actions':['consume'],'isAudited':false}]", + 'default-policy.3.name': 'topic: ranger_audit', + 'default-policy.3.resource.topic': 'ranger_audit*,__consumer_offsets', + 'default-policy.3.policyItem.1.users': 'rangerauditserver', + 'default-policy.3.policyItem.1.accessTypes': 'create,configure,consume,publish,describe,create,delete,alter,describe_configs,alter_configs', + 'default-policy.4.name': 'consumergroup: ranger_audit_consumer_groups', + 'default-policy.4.resource.consumergroup': 'ranger_audit*', + 'default-policy.4.policyItem.1.users': 'rangerauditserver', + 'default-policy.4.policyItem.1.accessTypes': 'consume,describe,delete', + 'default-policy.5.name': 'cluster: audit server cluster permissions', + 'default-policy.5.resource.cluster': '*,dummy', + 'default-policy.5.policyItem.1.users': 'rangerauditserver', + 'default-policy.5.policyItem.1.accessTypes': 'configure,describe,alter,create,idempotent_write,describe_configs,alter_configs', + 'ranger.plugin.audit.filters': "[{'accessResult': 'DENIED', 'isAudited': true},{'resources':{'topic':{'values':['ATLAS_ENTITIES']}},'users':['rangertagsync'],'actions':['create','consume','describe'],'isAudited':false},{'resources':{'consumergroup':{'values':['ranger_entities_consumer']}},'users':['rangertagsync'],'actions':['consume'],'isAudited':false},{'users':['rangerauditserver'],'isAudited':false}]", 'userstore.download.auth.users': 'kafka', 'ranger.plugin.kafka.policy.refresh.synchronous':'true'}}) @@ -120,6 +136,11 @@ def service_not_exists(service): solr = RangerService({'name': 'dev_solr', 'type': 'solr', 'configs': {'username': 'solr', 'password': 'rangerR0cks!', 'solr.url': 'http://ranger-solr.rangernw:8983', + 'setup.additional.default.policies': 'true', + 'default-policy.1.name': 'collection: ranger_audits', + 'default-policy.1.resource.collection': 'ranger_audits', + 'default-policy.1.policyItem.1.users': 'rangerauditserver', + 'default-policy.1.policyItem.1.accessTypes': 'query,update', 'policy.download.auth.users': 'solr', 'tag.download.auth.users': 'solr', 'userstore.download.auth.users': 'solr', diff --git a/dev-support/ranger-docker/scripts/audit-server/ranger-audit-consumer-hdfs.sh b/dev-support/ranger-docker/scripts/audit-server/ranger-audit-consumer-hdfs.sh new file mode 100755 index 0000000000..7ea7cfc15d --- /dev/null +++ b/dev-support/ranger-docker/scripts/audit-server/ranger-audit-consumer-hdfs.sh @@ -0,0 +1,138 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +AUDIT_CONSUMER_HOME_DIR="${AUDIT_CONSUMER_HOME_DIR:-/opt/ranger-audit-consumer-hdfs}" +AUDIT_CONSUMER_CONF_DIR="${AUDIT_CONSUMER_CONF_DIR:-/opt/ranger-audit-consumer-hdfs/conf}" +AUDIT_CONSUMER_LOG_DIR="${AUDIT_CONSUMER_LOG_DIR:-/var/log/ranger/ranger-audit-consumer-hdfs}" + +# Create log directory if it doesn't exist +mkdir -p ${AUDIT_CONSUMER_LOG_DIR} +chown -R ranger:ranger /var/log/ranger 2>/dev/null || true + +echo "==========================================" +echo "Starting Ranger Audit Consumer - HDFS" +echo "==========================================" +echo "AUDIT_CONSUMER_HOME_DIR: ${AUDIT_CONSUMER_HOME_DIR}" +echo "AUDIT_CONSUMER_CONF_DIR: ${AUDIT_CONSUMER_CONF_DIR}" +echo "AUDIT_CONSUMER_LOG_DIR: ${AUDIT_CONSUMER_LOG_DIR}" +echo "==========================================" + +# Source service check functions +source /home/ranger/scripts/service-check-functions.sh + +# Wait for Kafka to be available +KAFKA_BOOTSTRAP_SERVERS="${KAFKA_BOOTSTRAP_SERVERS:-ranger-kafka.rangernw:9092}" +check_tcp_port "Kafka" "${KAFKA_BOOTSTRAP_SERVERS}" 60 + +# Wait for HDFS to be available +HDFS_NAMENODE="${HDFS_NAMENODE:-ranger-hadoop.rangernw:9000}" +check_tcp_port "HDFS NameNode" "${HDFS_NAMENODE}" 60 + +# Start the HDFS consumer +echo "[INFO] Starting Ranger Audit Consumer - HDFS..." +cd ${AUDIT_CONSUMER_HOME_DIR} + +# Export environment variables for Java +export JAVA_HOME=${JAVA_HOME:-/opt/java/openjdk} +export PATH=$JAVA_HOME/bin:$PATH +export AUDIT_CONSUMER_LOG_DIR=${AUDIT_CONSUMER_LOG_DIR} + +# Set heap size +AUDIT_CONSUMER_HEAP="${AUDIT_CONSUMER_HEAP:--Xms512m -Xmx2g}" +export AUDIT_CONSUMER_HEAP + +# Set JVM options including logback configuration +if [ -z "$AUDIT_CONSUMER_OPTS" ]; then + AUDIT_CONSUMER_OPTS="-Dlogback.configurationFile=${AUDIT_CONSUMER_CONF_DIR}/logback.xml" + AUDIT_CONSUMER_OPTS="${AUDIT_CONSUMER_OPTS} -Daudit.consumer.hdfs.log.dir=${AUDIT_CONSUMER_LOG_DIR}" + AUDIT_CONSUMER_OPTS="${AUDIT_CONSUMER_OPTS} -Daudit.consumer.hdfs.log.file=ranger-audit-consumer-hdfs.log" + AUDIT_CONSUMER_OPTS="${AUDIT_CONSUMER_OPTS} -Djava.net.preferIPv4Stack=true -server" + AUDIT_CONSUMER_OPTS="${AUDIT_CONSUMER_OPTS} -XX:+UseG1GC -XX:MaxGCPauseMillis=200" + AUDIT_CONSUMER_OPTS="${AUDIT_CONSUMER_OPTS} -XX:InitiatingHeapOccupancyPercent=35" + AUDIT_CONSUMER_OPTS="${AUDIT_CONSUMER_OPTS} -XX:ConcGCThreads=4 -XX:ParallelGCThreads=8" +fi + +# Point to krb5.conf and Hadoop configs for Kerberos +if [ "${KERBEROS_ENABLED}" == "true" ]; then + export AUDIT_CONSUMER_OPTS="${AUDIT_CONSUMER_OPTS} -Djava.security.krb5.conf=/etc/krb5.conf" + + # Wait for keytabs if Kerberos is enabled + if [ -f /home/ranger/scripts/wait_for_keytab.sh ]; then + echo "[INFO] Waiting for Kerberos keytabs..." + bash /home/ranger/scripts/wait_for_keytab.sh HTTP.keytab + bash /home/ranger/scripts/wait_for_keytab.sh rangerauditserver.keytab + echo "[INFO] ✓ Keytabs are available" + fi +fi + +export AUDIT_CONSUMER_OPTS + +echo "[INFO] JAVA_HOME: ${JAVA_HOME}" +echo "[INFO] AUDIT_CONSUMER_HEAP: ${AUDIT_CONSUMER_HEAP}" +echo "[INFO] AUDIT_CONSUMER_OPTS: ${AUDIT_CONSUMER_OPTS}" + +# Build classpath from WAR file +WEBAPP_ROOT="${AUDIT_CONSUMER_HOME_DIR}/webapp" +WAR_FILE="${WEBAPP_ROOT}/ranger-audit-consumer-hdfs.war" +WEBAPP_DIR="${WEBAPP_ROOT}/ranger-audit-consumer-hdfs" + +# Extract WAR if not already extracted +if [ -f "${WAR_FILE}" ] && [ ! -d "${WEBAPP_DIR}" ]; then + echo "[INFO] Extracting WAR file..." + mkdir -p "${WEBAPP_DIR}" + cd "${WEBAPP_DIR}" + jar xf "${WAR_FILE}" + cd - +fi + +# Build classpath +RANGER_CLASSPATH="${WEBAPP_DIR}/WEB-INF/classes" +for jar in "${WEBAPP_DIR}"/WEB-INF/lib/*.jar; do + RANGER_CLASSPATH="${RANGER_CLASSPATH}:${jar}" +done + +# Add libext directory if it exists +if [ -d "${AUDIT_CONSUMER_HOME_DIR}/libext" ]; then + for jar in "${AUDIT_CONSUMER_HOME_DIR}"/libext/*.jar; do + if [ -f "${jar}" ]; then + RANGER_CLASSPATH="${RANGER_CLASSPATH}:${jar}" + fi + done +fi + +export RANGER_CLASSPATH + +echo "[INFO] Starting Ranger Audit Consumer - HDFS..." +echo "[INFO] Webapp dir: ${WEBAPP_DIR}" +java ${AUDIT_CONSUMER_HEAP} ${AUDIT_CONSUMER_OPTS} \ + -Daudit.config=${AUDIT_CONSUMER_CONF_DIR}/ranger-audit-consumer-hdfs-site.xml \ + -Dhadoop.config.dir=${AUDIT_CONSUMER_CONF_DIR} \ + -Dranger.audit.consumer.webapp.dir="${WAR_FILE}" \ + -cp "${RANGER_CLASSPATH}" \ + org.apache.ranger.audit.consumer.HdfsConsumerApplication \ + >> ${AUDIT_CONSUMER_LOG_DIR}/catalina.out 2>&1 & + +PID=$! +echo $PID > ${AUDIT_CONSUMER_LOG_DIR}/ranger-audit-consumer-hdfs.pid + +echo "[INFO] Ranger Audit Consumer - HDFS started with PID: $PID" + +# Keep the container running by tailing logs +tail -f ${AUDIT_CONSUMER_LOG_DIR}/catalina.out 2>/dev/null diff --git a/dev-support/ranger-docker/scripts/audit-server/ranger-audit-consumer-solr.sh b/dev-support/ranger-docker/scripts/audit-server/ranger-audit-consumer-solr.sh new file mode 100755 index 0000000000..cb59efe095 --- /dev/null +++ b/dev-support/ranger-docker/scripts/audit-server/ranger-audit-consumer-solr.sh @@ -0,0 +1,137 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +AUDIT_CONSUMER_HOME_DIR="${AUDIT_CONSUMER_HOME_DIR:-/opt/ranger-audit-consumer-solr}" +AUDIT_CONSUMER_CONF_DIR="${AUDIT_CONSUMER_CONF_DIR:-/opt/ranger-audit-consumer-solr/conf}" +AUDIT_CONSUMER_LOG_DIR="${AUDIT_CONSUMER_LOG_DIR:-/var/log/ranger/ranger-audit-consumer-solr}" + +# Create log directory if it doesn't exist +mkdir -p ${AUDIT_CONSUMER_LOG_DIR} +chown -R ranger:ranger /var/log/ranger 2>/dev/null || true + +echo "==========================================" +echo "Starting Ranger Audit Consumer - Solr" +echo "==========================================" +echo "AUDIT_CONSUMER_HOME_DIR: ${AUDIT_CONSUMER_HOME_DIR}" +echo "AUDIT_CONSUMER_CONF_DIR: ${AUDIT_CONSUMER_CONF_DIR}" +echo "AUDIT_CONSUMER_LOG_DIR: ${AUDIT_CONSUMER_LOG_DIR}" +echo "==========================================" + +# Source service check functions +source /home/ranger/scripts/service-check-functions.sh + +# Wait for Kafka to be available +KAFKA_BOOTSTRAP_SERVERS="${KAFKA_BOOTSTRAP_SERVERS:-ranger-kafka.rangernw:9092}" +check_tcp_port "Kafka" "${KAFKA_BOOTSTRAP_SERVERS}" 60 + +# Wait for Solr to be available +SOLR_URL="${SOLR_URL:-http://ranger-solr.rangernw:8983/solr}" +check_http_service "Solr" "${SOLR_URL}/" 60 + +# Start the Solr consumer +echo "[INFO] Starting Ranger Audit Consumer - Solr..." +cd ${AUDIT_CONSUMER_HOME_DIR} + +# Export environment variables for Java +export JAVA_HOME=${JAVA_HOME:-/opt/java/openjdk} +export PATH=$JAVA_HOME/bin:$PATH +export AUDIT_CONSUMER_LOG_DIR=${AUDIT_CONSUMER_LOG_DIR} + +# Set heap size +AUDIT_CONSUMER_HEAP="${AUDIT_CONSUMER_HEAP:--Xms512m -Xmx2g}" +export AUDIT_CONSUMER_HEAP + +# Set JVM options including logback configuration +if [ -z "$AUDIT_CONSUMER_OPTS" ]; then + AUDIT_CONSUMER_OPTS="-Dlogback.configurationFile=${AUDIT_CONSUMER_CONF_DIR}/logback.xml" + AUDIT_CONSUMER_OPTS="${AUDIT_CONSUMER_OPTS} -Daudit.consumer.solr.log.dir=${AUDIT_CONSUMER_LOG_DIR}" + AUDIT_CONSUMER_OPTS="${AUDIT_CONSUMER_OPTS} -Daudit.consumer.solr.log.file=ranger-audit-consumer-solr.log" + AUDIT_CONSUMER_OPTS="${AUDIT_CONSUMER_OPTS} -Djava.net.preferIPv4Stack=true -server" + AUDIT_CONSUMER_OPTS="${AUDIT_CONSUMER_OPTS} -XX:+UseG1GC -XX:MaxGCPauseMillis=200" + AUDIT_CONSUMER_OPTS="${AUDIT_CONSUMER_OPTS} -XX:InitiatingHeapOccupancyPercent=35" + AUDIT_CONSUMER_OPTS="${AUDIT_CONSUMER_OPTS} -XX:ConcGCThreads=4 -XX:ParallelGCThreads=8" +fi + +# Point to krb5.conf for Kerberos +if [ "${KERBEROS_ENABLED}" == "true" ]; then + export AUDIT_CONSUMER_OPTS="${AUDIT_CONSUMER_OPTS} -Djava.security.krb5.conf=/etc/krb5.conf" + + # Wait for keytabs if Kerberos is enabled + if [ -f /home/ranger/scripts/wait_for_keytab.sh ]; then + echo "[INFO] Waiting for Kerberos keytabs..." + bash /home/ranger/scripts/wait_for_keytab.sh HTTP.keytab + bash /home/ranger/scripts/wait_for_keytab.sh rangerauditserver.keytab + echo "[INFO] ✓ Keytabs are available" + fi +fi + +export AUDIT_CONSUMER_OPTS + +echo "[INFO] JAVA_HOME: ${JAVA_HOME}" +echo "[INFO] AUDIT_CONSUMER_HEAP: ${AUDIT_CONSUMER_HEAP}" +echo "[INFO] AUDIT_CONSUMER_OPTS: ${AUDIT_CONSUMER_OPTS}" + +# Build classpath from WAR file +WEBAPP_ROOT="${AUDIT_CONSUMER_HOME_DIR}/webapp" +WAR_FILE="${WEBAPP_ROOT}/ranger-audit-consumer-solr.war" +WEBAPP_DIR="${WEBAPP_ROOT}/ranger-audit-consumer-solr" + +# Extract WAR if not already extracted +if [ -f "${WAR_FILE}" ] && [ ! -d "${WEBAPP_DIR}" ]; then + echo "[INFO] Extracting WAR file..." + mkdir -p "${WEBAPP_DIR}" + cd "${WEBAPP_DIR}" + jar xf "${WAR_FILE}" + cd - +fi + +# Build classpath +RANGER_CLASSPATH="${WEBAPP_DIR}/WEB-INF/classes" +for jar in "${WEBAPP_DIR}"/WEB-INF/lib/*.jar; do + RANGER_CLASSPATH="${RANGER_CLASSPATH}:${jar}" +done + +# Add libext directory if it exists +if [ -d "${AUDIT_CONSUMER_HOME_DIR}/libext" ]; then + for jar in "${AUDIT_CONSUMER_HOME_DIR}"/libext/*.jar; do + if [ -f "${jar}" ]; then + RANGER_CLASSPATH="${RANGER_CLASSPATH}:${jar}" + fi + done +fi + +export RANGER_CLASSPATH + +echo "[INFO] Starting Ranger Audit Consumer - Solr..." +echo "[INFO] Webapp dir: ${WEBAPP_DIR}" +java ${AUDIT_CONSUMER_HEAP} ${AUDIT_CONSUMER_OPTS} \ + -Daudit.config=${AUDIT_CONSUMER_CONF_DIR}/ranger-audit-consumer-solr-site.xml \ + -Dranger.audit.consumer.webapp.dir="${WAR_FILE}" \ + -cp "${RANGER_CLASSPATH}" \ + org.apache.ranger.audit.consumer.SolrConsumerApplication \ + >> ${AUDIT_CONSUMER_LOG_DIR}/catalina.out 2>&1 & + +PID=$! +echo $PID > ${AUDIT_CONSUMER_LOG_DIR}/ranger-audit-consumer-solr.pid + +echo "[INFO] Ranger Audit Consumer - Solr started with PID: $PID" + +# Keep the container running by tailing logs +tail -f ${AUDIT_CONSUMER_LOG_DIR}/catalina.out 2>/dev/null diff --git a/dev-support/ranger-docker/scripts/audit-server/ranger-audit-server.sh b/dev-support/ranger-docker/scripts/audit-server/ranger-audit-server.sh new file mode 100755 index 0000000000..827e5d4ece --- /dev/null +++ b/dev-support/ranger-docker/scripts/audit-server/ranger-audit-server.sh @@ -0,0 +1,137 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +AUDIT_SERVER_HOME_DIR="${AUDIT_SERVER_HOME_DIR:-/opt/ranger-audit-server}" +AUDIT_SERVER_CONF_DIR="${AUDIT_SERVER_CONF_DIR:-/opt/ranger-audit-server/conf}" +AUDIT_SERVER_LOG_DIR="${AUDIT_SERVER_LOG_DIR:-/var/log/ranger/ranger-audit-server}" + +# Create log directory if it doesn't exist +mkdir -p ${AUDIT_SERVER_LOG_DIR} +chown -R ranger:ranger /var/log/ranger 2>/dev/null || true + +echo "==========================================" +echo "Starting Ranger Audit Server Service..." +echo "==========================================" +echo "AUDIT_SERVER_HOME_DIR: ${AUDIT_SERVER_HOME_DIR}" +echo "AUDIT_SERVER_CONF_DIR: ${AUDIT_SERVER_CONF_DIR}" +echo "AUDIT_SERVER_LOG_DIR: ${AUDIT_SERVER_LOG_DIR}" +echo "==========================================" + +# Source service check functions +source /home/ranger/scripts/service-check-functions.sh + +# Quick check for Kafka availability +# The audit server has a built-in recovery/spool mechanism for when Kafka is unavailable +KAFKA_BOOTSTRAP_SERVERS="${KAFKA_BOOTSTRAP_SERVERS:-ranger-kafka.rangernw:9092}" +if check_tcp_port "Kafka" "${KAFKA_BOOTSTRAP_SERVERS}" 30; then + echo "[INFO] Kafka is available at startup" +else + echo "[INFO] Kafka not immediately available - audit server will use recovery/spool mechanism" +fi + +# Start the audit server +echo "[INFO] Starting Ranger Audit Server Service..." +cd ${AUDIT_SERVER_HOME_DIR} + +# Export environment variables for Java and audit server +export JAVA_HOME=${JAVA_HOME:-/opt/java/openjdk} +export PATH=$JAVA_HOME/bin:$PATH +export AUDIT_SERVER_LOG_DIR=${AUDIT_SERVER_LOG_DIR} + +# Set heap size +AUDIT_SERVER_HEAP="${AUDIT_SERVER_HEAP:--Xms512m -Xmx2g}" +export AUDIT_SERVER_HEAP + +# Set JVM options including logback configuration +if [ -z "$AUDIT_SERVER_OPTS" ]; then + AUDIT_SERVER_OPTS="-Dlogback.configurationFile=${AUDIT_SERVER_CONF_DIR}/logback.xml" + AUDIT_SERVER_OPTS="${AUDIT_SERVER_OPTS} -Daudit.server.log.dir=${AUDIT_SERVER_LOG_DIR}" + AUDIT_SERVER_OPTS="${AUDIT_SERVER_OPTS} -Daudit.server.log.file=ranger-audit-server.log" + AUDIT_SERVER_OPTS="${AUDIT_SERVER_OPTS} -Djava.net.preferIPv4Stack=true -server" + AUDIT_SERVER_OPTS="${AUDIT_SERVER_OPTS} -XX:+UseG1GC -XX:MaxGCPauseMillis=200" + AUDIT_SERVER_OPTS="${AUDIT_SERVER_OPTS} -XX:InitiatingHeapOccupancyPercent=35" + AUDIT_SERVER_OPTS="${AUDIT_SERVER_OPTS} -XX:ConcGCThreads=4 -XX:ParallelGCThreads=8" +fi + +# Point to krb5.conf for Kerberos +if [ "${KERBEROS_ENABLED}" == "true" ]; then + export AUDIT_SERVER_OPTS="${AUDIT_SERVER_OPTS} -Djava.security.krb5.conf=/etc/krb5.conf" + + # Wait for keytabs if Kerberos is enabled + if [ -f /home/ranger/scripts/wait_for_keytab.sh ]; then + echo "[INFO] Waiting for Kerberos keytabs..." + bash /home/ranger/scripts/wait_for_keytab.sh HTTP.keytab + bash /home/ranger/scripts/wait_for_keytab.sh rangerauditserver.keytab + echo "[INFO] ✓ Keytabs are available" + fi +fi + +export AUDIT_SERVER_OPTS + +echo "[INFO] JAVA_HOME: ${JAVA_HOME}" +echo "[INFO] AUDIT_SERVER_HEAP: ${AUDIT_SERVER_HEAP}" +echo "[INFO] AUDIT_SERVER_OPTS: ${AUDIT_SERVER_OPTS}" + +# Build classpath from WAR file +WEBAPP_ROOT="${AUDIT_SERVER_HOME_DIR}/webapp" +WAR_FILE="${WEBAPP_ROOT}/ranger-audit-server-service.war" +WEBAPP_DIR="${WEBAPP_ROOT}/ranger-audit-server-service" + +# Extract WAR if not already extracted +if [ -f "${WAR_FILE}" ] && [ ! -d "${WEBAPP_DIR}" ]; then + echo "[INFO] Extracting WAR file..." + mkdir -p "${WEBAPP_DIR}" + cd "${WEBAPP_DIR}" + jar xf "${WAR_FILE}" + cd - +fi + +# Build classpath +RANGER_CLASSPATH="${WEBAPP_DIR}/WEB-INF/classes" +for jar in "${WEBAPP_DIR}"/WEB-INF/lib/*.jar; do + RANGER_CLASSPATH="${RANGER_CLASSPATH}:${jar}" +done + +# Add libext directory if it exists +if [ -d "${AUDIT_SERVER_HOME_DIR}/libext" ]; then + for jar in "${AUDIT_SERVER_HOME_DIR}"/libext/*.jar; do + if [ -f "${jar}" ]; then + RANGER_CLASSPATH="${RANGER_CLASSPATH}:${jar}" + fi + done +fi + +export RANGER_CLASSPATH + +echo "[INFO] Starting Ranger Audit Server Service..." +echo "[INFO] Webapp dir: ${WEBAPP_DIR}" +java ${AUDIT_SERVER_HEAP} ${AUDIT_SERVER_OPTS} \ + -Daudit.config=${AUDIT_SERVER_CONF_DIR}/ranger-audit-server-site.xml \ + -cp "${RANGER_CLASSPATH}" \ + org.apache.ranger.audit.server.AuditServerApplication \ + >> ${AUDIT_SERVER_LOG_DIR}/catalina.out 2>&1 & + +PID=$! +echo $PID > ${AUDIT_SERVER_LOG_DIR}/ranger-audit-server.pid + +echo "[INFO] Ranger Audit Server started with PID: $PID" + +# Keep the container running by tailing logs +tail -f ${AUDIT_SERVER_LOG_DIR}/catalina.out 2>/dev/null diff --git a/dev-support/ranger-docker/scripts/audit-server/service-check-functions.sh b/dev-support/ranger-docker/scripts/audit-server/service-check-functions.sh new file mode 100755 index 0000000000..6a106d164a --- /dev/null +++ b/dev-support/ranger-docker/scripts/audit-server/service-check-functions.sh @@ -0,0 +1,76 @@ +#!/bin/bash + +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# +# Reusable service availability check functions +# + +# Check if a TCP port is available +# Usage: check_tcp_port "ServiceName" "host:port" [max_wait_seconds] +check_tcp_port() { + local SERVICE_NAME=$1 + local HOST_PORT=$2 + local MAX_WAIT=${3:-60} + + # Parse host and port + local HOST="${HOST_PORT%%:*}" + local PORT="${HOST_PORT##*:}" + + echo "[INFO] Waiting for ${SERVICE_NAME}..." + local WAIT_COUNT=0 + + while ! timeout 1 bash -c "echo > /dev/tcp/${HOST}/${PORT}" 2>/dev/null; do + WAIT_COUNT=$((WAIT_COUNT+1)) + if [ $WAIT_COUNT -ge $MAX_WAIT ]; then + echo "[WARN] ${SERVICE_NAME} not available after ${MAX_WAIT} seconds, continuing anyway..." + return 1 + fi + echo "[INFO] Waiting for ${SERVICE_NAME}... ($WAIT_COUNT/$MAX_WAIT)" + sleep 2 + done + + echo "[INFO] ✓ ${SERVICE_NAME} is available at ${HOST}:${PORT}" + return 0 +} + +# Check if an HTTP service is available +# Usage: check_http_service "ServiceName" "http://host:port/path" [max_wait_seconds] +check_http_service() { + local SERVICE_NAME=$1 + local URL=$2 + local MAX_WAIT=${3:-60} + + echo "[INFO] Waiting for ${SERVICE_NAME}..." + local WAIT_COUNT=0 + + while ! curl -f -s -m 2 "${URL}" > /dev/null 2>&1; do + WAIT_COUNT=$((WAIT_COUNT+1)) + if [ $WAIT_COUNT -ge $MAX_WAIT ]; then + echo "[WARN] ${SERVICE_NAME} not available after ${MAX_WAIT} seconds, continuing anyway..." + return 1 + fi + echo "[INFO] Waiting for ${SERVICE_NAME}... ($WAIT_COUNT/$MAX_WAIT)" + sleep 2 + done + + echo "[INFO] ✓ ${SERVICE_NAME} is available at ${URL}" + return 0 +} diff --git a/dev-support/ranger-docker/scripts/hadoop/hdfs-site.xml b/dev-support/ranger-docker/scripts/hadoop/hdfs-site.xml index ab1cc9aee8..58525a1bc9 100644 --- a/dev-support/ranger-docker/scripts/hadoop/hdfs-site.xml +++ b/dev-support/ranger-docker/scripts/hadoop/hdfs-site.xml @@ -60,4 +60,16 @@ dfs.web.authentication.kerberos.keytab /etc/keytabs/HTTP.keytab + + dfs.secondary.namenode.kerberos.principal + nn/ranger-hadoop.rangernw@EXAMPLE.COM + + + dfs.secondary.namenode.keytab.file + /etc/keytabs/nn.keytab + + + dfs.secondary.namenode.kerberos.internal.spnego.principal + HTTP/ranger-hadoop.rangernw@EXAMPLE.COM + diff --git a/dev-support/ranger-docker/scripts/hadoop/ranger-hdfs-plugin-install.properties b/dev-support/ranger-docker/scripts/hadoop/ranger-hdfs-plugin-install.properties index fb876ee274..eae056a9de 100644 --- a/dev-support/ranger-docker/scripts/hadoop/ranger-hdfs-plugin-install.properties +++ b/dev-support/ranger-docker/scripts/hadoop/ranger-hdfs-plugin-install.properties @@ -20,7 +20,7 @@ COMPONENT_INSTALL_DIR_NAME=/opt/hadoop CUSTOM_USER=hdfs CUSTOM_GROUP=hadoop -XAAUDIT.SOLR.IS_ENABLED=true +XAAUDIT.SOLR.IS_ENABLED=false XAAUDIT.SOLR.MAX_QUEUE_SIZE=1 XAAUDIT.SOLR.MAX_FLUSH_INTERVAL_MS=1000 XAAUDIT.SOLR.SOLR_URL=http://ranger-solr:8983/solr/ranger_audits @@ -39,7 +39,7 @@ XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS=60 XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS=600 XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT=10 -XAAUDIT.SOLR.ENABLE=true +XAAUDIT.SOLR.ENABLE=false XAAUDIT.SOLR.URL=http://ranger-solr:8983/solr/ranger_audits XAAUDIT.SOLR.USER=NONE XAAUDIT.SOLR.PASSWORD=NONE @@ -64,7 +64,7 @@ XAAUDIT.ELASTICSEARCH.INDEX=NONE XAAUDIT.ELASTICSEARCH.PORT=NONE XAAUDIT.ELASTICSEARCH.PROTOCOL=NONE -XAAUDIT.HDFS.ENABLE=true +XAAUDIT.HDFS.ENABLE=false XAAUDIT.HDFS.HDFS_DIR=hdfs://ranger-hadoop:9000/ranger/audit XAAUDIT.HDFS.FILE_SPOOL_DIR=/var/log/hadoop/hdfs/audit/hdfs/spool @@ -90,3 +90,7 @@ SSL_KEYSTORE_FILE_PATH=/etc/hadoop/conf/ranger-plugin-keystore.jks SSL_KEYSTORE_PASSWORD=myKeyFilePassword SSL_TRUSTSTORE_FILE_PATH=/etc/hadoop/conf/ranger-plugin-truststore.jks SSL_TRUSTSTORE_PASSWORD=changeit + +XAAUDIT.AUDITSERVER.ENABLE=true +XAAUDIT.AUDITSERVER.URL=http://ranger-audit-server.rangernw:7081 +XAAUDIT.AUDITSERVER.FILE_SPOOL_DIR=/var/log/hadoop/hdfs/audit/http/spool diff --git a/dev-support/ranger-docker/scripts/hive/ranger-hive-plugin-install.properties b/dev-support/ranger-docker/scripts/hive/ranger-hive-plugin-install.properties index 0dae26f433..f7308683b2 100644 --- a/dev-support/ranger-docker/scripts/hive/ranger-hive-plugin-install.properties +++ b/dev-support/ranger-docker/scripts/hive/ranger-hive-plugin-install.properties @@ -21,7 +21,7 @@ UPDATE_XAPOLICIES_ON_GRANT_REVOKE=true CUSTOM_USER=hive CUSTOM_GROUP=hadoop -XAAUDIT.SOLR.IS_ENABLED=true +XAAUDIT.SOLR.IS_ENABLED=false XAAUDIT.SOLR.MAX_QUEUE_SIZE=1 XAAUDIT.SOLR.MAX_FLUSH_INTERVAL_MS=1000 XAAUDIT.SOLR.SOLR_URL=http://ranger-solr:8983/solr/ranger_audits @@ -40,7 +40,7 @@ XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS=60 XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS=600 XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT=10 -XAAUDIT.SOLR.ENABLE=true +XAAUDIT.SOLR.ENABLE=false XAAUDIT.SOLR.URL=http://ranger-solr:8983/solr/ranger_audits XAAUDIT.SOLR.USER=NONE XAAUDIT.SOLR.PASSWORD=NONE @@ -65,7 +65,7 @@ XAAUDIT.ELASTICSEARCH.INDEX=NONE XAAUDIT.ELASTICSEARCH.PORT=NONE XAAUDIT.ELASTICSEARCH.PROTOCOL=NONE -XAAUDIT.HDFS.ENABLE=true +XAAUDIT.HDFS.ENABLE=false XAAUDIT.HDFS.HDFS_DIR=hdfs://ranger-hadoop:9000/ranger/audit XAAUDIT.HDFS.FILE_SPOOL_DIR=/var/log/hive/audit/hdfs/spool @@ -91,3 +91,7 @@ SSL_KEYSTORE_FILE_PATH=/etc/hive/conf/ranger-plugin-keystore.jks SSL_KEYSTORE_PASSWORD=myKeyFilePassword SSL_TRUSTSTORE_FILE_PATH=/etc/hive/conf/ranger-plugin-truststore.jks SSL_TRUSTSTORE_PASSWORD=changeit + +XAAUDIT.AUDITSERVER.ENABLE=true +XAAUDIT.AUDITSERVER.URL=http://ranger-audit-server.rangernw:7081 +XAAUDIT.AUDITSERVER.FILE_SPOOL_DIR=/var/log/hive/audit/http/spool diff --git a/dev-support/ranger-docker/scripts/kafka/ranger-kafka-setup.sh b/dev-support/ranger-docker/scripts/kafka/ranger-kafka-setup.sh index f41f7322a8..067042ada8 100755 --- a/dev-support/ranger-docker/scripts/kafka/ranger-kafka-setup.sh +++ b/dev-support/ranger-docker/scripts/kafka/ranger-kafka-setup.sh @@ -39,25 +39,44 @@ cd ${RANGER_HOME}/ranger-kafka-plugin sed -i 's/localhost:2181/ranger-zk.rangernw:2181/' ${KAFKA_HOME}/config/server.properties -cat <> ${KAFKA_HOME}/config/server.properties -# Enable SASL/GSSAPI mechanism -sasl.enabled.mechanisms=GSSAPI -sasl.mechanism.inter.broker.protocol=GSSAPI -security.inter.broker.protocol=SASL_PLAINTEXT - -# Listener configuration -listeners=SASL_PLAINTEXT://:9092 -advertised.listeners=SASL_PLAINTEXT://ranger-kafka.rangernw:9092 - -# JAAS configuration for Kerberos -listener.name.sasl_plaintext.gssapi.sasl.jaas.config=com.sun.security.auth.module.Krb5LoginModule required \ -useKeyTab=true \ -storeKey=true \ -keyTab="/etc/keytabs/kafka.keytab" \ -principal="kafka/ranger-kafka.rangernw@EXAMPLE.COM"; - -# Kerberos service name -sasl.kerberos.service.name=kafka - -authorizer.class.name=org.apache.ranger.authorization.kafka.authorizer.RangerKafkaAuthorizer +if [ "${KERBEROS_ENABLED}" == "true" ]; then + echo "Configuring Kafka with Kerberos (SASL_PLAINTEXT)" + cat <> ${KAFKA_HOME}/config/server.properties + + # Enable SASL/GSSAPI mechanism + sasl.enabled.mechanisms=GSSAPI + sasl.mechanism.inter.broker.protocol=GSSAPI + security.inter.broker.protocol=SASL_PLAINTEXT + + # Listener configuration + listeners=SASL_PLAINTEXT://:9092 + advertised.listeners=SASL_PLAINTEXT://ranger-kafka.rangernw:9092 + + # JAAS configuration for Kerberos + listener.name.sasl_plaintext.gssapi.sasl.jaas.config=com.sun.security.auth.module.Krb5LoginModule required \ + useKeyTab=true \ + storeKey=true \ + keyTab="/etc/keytabs/kafka.keytab" \ + principal="kafka/ranger-kafka.rangernw@EXAMPLE.COM"; + + # Kerberos service name + sasl.kerberos.service.name=kafka + + # Ranger authorization + authorizer.class.name=org.apache.ranger.authorization.kafka.authorizer.RangerKafkaAuthorizer + + # Super users bypass Ranger authorization for admin operations + super.users=User:kafka +EOF +else + echo "Configuring Kafka with PLAINTEXT (no Kerberos)" + cat <> ${KAFKA_HOME}/config/server.properties + # Listener configuration (PLAINTEXT - no authentication) + listeners=PLAINTEXT://:9092 + advertised.listeners=PLAINTEXT://ranger-kafka.rangernw:9092 + + # Ranger authorization + # disabling Ranger Plugin for now. + # authorizer.class.name=org.apache.ranger.authorization.kafka.authorizer.RangerKafkaAuthorizer EOF +fi diff --git a/dev-support/ranger-docker/scripts/kafka/ranger-kafka.sh b/dev-support/ranger-docker/scripts/kafka/ranger-kafka.sh index cd5e93eff4..1e22101c38 100755 --- a/dev-support/ranger-docker/scripts/kafka/ranger-kafka.sh +++ b/dev-support/ranger-docker/scripts/kafka/ranger-kafka.sh @@ -42,4 +42,13 @@ then fi fi -su -c "cd ${KAFKA_HOME} && CLASSPATH=${KAFKA_HOME}/config KAFKA_OPTS='-Djava.security.krb5.conf=/etc/krb5.conf -Djava.security.auth.login.config=/opt/kafka/config/kafka-server-jaas.conf' ./bin/kafka-server-start.sh config/server.properties" kafka +# Configure KAFKA_OPTS based on KERBEROS_ENABLED +if [ "${KERBEROS_ENABLED}" == "true" ]; then + echo "Starting Kafka with Kerberos authentication (SASL_PLAINTEXT)" + KAFKA_OPTS='-Djava.security.krb5.conf=/etc/krb5.conf -Djava.security.auth.login.config=/opt/kafka/config/kafka-server-jaas.conf' +else + echo "Starting Kafka with PLAINTEXT (no Kerberos authentication)" + KAFKA_OPTS='' +fi + +su -c "cd ${KAFKA_HOME} && CLASSPATH=${KAFKA_HOME}/config KAFKA_OPTS='${KAFKA_OPTS}' ./bin/kafka-server-start.sh config/server.properties" kafka diff --git a/dev-support/ranger-docker/scripts/kdc/entrypoint.sh b/dev-support/ranger-docker/scripts/kdc/entrypoint.sh index 7ee36e23ff..08ca178ec0 100644 --- a/dev-support/ranger-docker/scripts/kdc/entrypoint.sh +++ b/dev-support/ranger-docker/scripts/kdc/entrypoint.sh @@ -74,6 +74,19 @@ function create_keytabs() { create_principal_and_keytab rangeradmin ranger create_principal_and_keytab rangerlookup ranger + create_principal_and_keytab HTTP ranger-audit + create_principal_and_keytab rangerauditserver ranger-audit + + # Ranger Audit Server Microservices + create_principal_and_keytab HTTP ranger-audit-server + create_principal_and_keytab rangerauditserver ranger-audit-server + + create_principal_and_keytab HTTP ranger-audit-consumer-solr + create_principal_and_keytab rangerauditserver ranger-audit-consumer-solr + + create_principal_and_keytab HTTP ranger-audit-consumer-hdfs + create_principal_and_keytab rangerauditserver ranger-audit-consumer-hdfs + create_principal_and_keytab rangertagsync ranger-tagsync create_principal_and_keytab rangerusersync ranger-usersync @@ -132,7 +145,7 @@ if [ ! -f $DB_DIR/principal ]; then echo "Database initialized" create_keytabs - create_testusers ranger ranger-usersync ranger-tagsync ranger-audit ranger-hadoop ranger-hive ranger-hbase ranger-kafka ranger-solr ranger-knox ranger-kms ranger-ozone ranger-trino ranger-opensearch + create_testusers ranger ranger-usersync ranger-tagsync ranger-audit ranger-audit-server ranger-audit-consumer-solr ranger-audit-consumer-hdfs ranger-hadoop ranger-hive ranger-hbase ranger-kafka ranger-solr ranger-knox ranger-kms ranger-ozone ranger-trino ranger-opensearch else echo "KDC DB already exists; skipping create" fi diff --git a/distro/pom.xml b/distro/pom.xml index 709cf9f838..cb8277021a 100644 --- a/distro/pom.xml +++ b/distro/pom.xml @@ -66,6 +66,26 @@ ${project.version} provided + + org.apache.ranger + ranger-audit-common + ${project.version} + provided + + + org.apache.ranger + ranger-audit-consumer-hdfs + ${project.version} + war + provided + + + org.apache.ranger + ranger-audit-consumer-solr + ${project.version} + war + provided + org.apache.ranger ranger-audit-dest-cloudwatch @@ -102,6 +122,13 @@ ${project.version} provided + + org.apache.ranger + ranger-audit-server-service + ${project.version} + war + provided + org.apache.ranger ranger-authn @@ -472,6 +499,9 @@ src/main/assembly/plugin-presto.xml src/main/assembly/plugin-trino.xml src/main/assembly/sample-client.xml + src/main/assembly/ranger-audit-server-service.xml + src/main/assembly/ranger-audit-consumer-solr.xml + src/main/assembly/ranger-audit-consumer-hdfs.xml diff --git a/distro/src/main/assembly/hdfs-agent.xml b/distro/src/main/assembly/hdfs-agent.xml index bfbe915c70..56b3b14e95 100644 --- a/distro/src/main/assembly/hdfs-agent.xml +++ b/distro/src/main/assembly/hdfs-agent.xml @@ -72,6 +72,7 @@ true org.apache.ranger:ranger-audit-core + org.apache.ranger:ranger-audit-dest-auditserver org.apache.ranger:ranger-audit-dest-hdfs org.apache.ranger:ranger-audit-dest-solr org.apache.ranger:ranger-authz-api diff --git a/distro/src/main/assembly/hive-agent.xml b/distro/src/main/assembly/hive-agent.xml index 7a72129c51..d20d3f7cfd 100644 --- a/distro/src/main/assembly/hive-agent.xml +++ b/distro/src/main/assembly/hive-agent.xml @@ -42,6 +42,7 @@ true org.apache.ranger:ranger-audit-core + org.apache.ranger:ranger-audit-dest-auditserver org.apache.ranger:ranger-audit-dest-hdfs org.apache.ranger:ranger-audit-dest-solr org.apache.ranger:ranger-authz-api diff --git a/distro/src/main/assembly/ranger-audit-consumer-hdfs.xml b/distro/src/main/assembly/ranger-audit-consumer-hdfs.xml new file mode 100644 index 0000000000..0130321a67 --- /dev/null +++ b/distro/src/main/assembly/ranger-audit-consumer-hdfs.xml @@ -0,0 +1,95 @@ + + + + ranger-audit-consumer-hdfs + + tar.gz + + ${project.parent.name}-${project.version}-ranger-audit-consumer-hdfs + true + + + + bin + ${project.parent.basedir}/ranger-audit-server/ranger-audit-consumer-hdfs/scripts + + **/*.sh + **/*.py + + 755 + + + + + conf + ${project.parent.basedir}/ranger-audit-server/ranger-audit-consumer-hdfs/src/main/resources/conf + + ranger-audit-consumer-hdfs-site.xml + core-site.xml + hdfs-site.xml + logback.xml + + 644 + + + + + + ${project.build.directory} + + version + + 444 + + + + + libext + + **/* + + + + + + logs + + **/* + + + + + + + ${project.parent.basedir}/ranger-audit-server/ranger-audit-consumer-hdfs/ + + NOTICE.txt + + 644 + + + + + + + webapp + ${project.parent.basedir}/ranger-audit-server/ranger-audit-consumer-hdfs/target/ranger-audit-consumer-hdfs-${project.version}.war + ranger-audit-consumer-hdfs.war + + + diff --git a/distro/src/main/assembly/ranger-audit-consumer-solr.xml b/distro/src/main/assembly/ranger-audit-consumer-solr.xml new file mode 100644 index 0000000000..edd06938a5 --- /dev/null +++ b/distro/src/main/assembly/ranger-audit-consumer-solr.xml @@ -0,0 +1,93 @@ + + + + ranger-audit-consumer-solr + + tar.gz + + ${project.parent.name}-${project.version}-ranger-audit-consumer-solr + true + + + + bin + ${project.parent.basedir}/ranger-audit-server/ranger-audit-consumer-solr/scripts + + **/*.sh + **/*.py + + 755 + + + + + conf + ${project.parent.basedir}/ranger-audit-server/ranger-audit-consumer-solr/src/main/resources/conf + + ranger-audit-consumer-solr-site.xml + logback.xml + + 644 + + + + + + ${project.build.directory} + + version + + 444 + + + + + libext + + **/* + + + + + + logs + + **/* + + + + + + + ${project.parent.basedir}/ranger-audit-server/ranger-audit-consumer-solr/ + + NOTICE.txt + + 644 + + + + + + + webapp + ${project.parent.basedir}/ranger-audit-server/ranger-audit-consumer-solr/target/ranger-audit-consumer-solr-${project.version}.war + ranger-audit-consumer-solr.war + + + diff --git a/distro/src/main/assembly/ranger-audit-server-service.xml b/distro/src/main/assembly/ranger-audit-server-service.xml new file mode 100644 index 0000000000..53540f40f5 --- /dev/null +++ b/distro/src/main/assembly/ranger-audit-server-service.xml @@ -0,0 +1,93 @@ + + + + ranger-audit-server-service + + tar.gz + + ${project.parent.name}-${project.version}-ranger-audit-server-service + true + + + + bin + ${project.parent.basedir}/ranger-audit-server/ranger-audit-server-service/scripts + + **/*.sh + **/*.py + + 755 + + + + + conf + ${project.parent.basedir}/ranger-audit-server/ranger-audit-server-service/src/main/resources/conf + + ranger-audit-server-site.xml + logback.xml + + 644 + + + + + + ${project.build.directory} + + version + + 444 + + + + + libext + + **/* + + + + + + logs + + **/* + + + + + + + ${project.parent.basedir}/ranger-audit-server/ranger-audit-server-service/ + + NOTICE.txt + + 644 + + + + + + + webapp + ${project.parent.basedir}/ranger-audit-server/ranger-audit-server-service/target/ranger-audit-server-service-${project.version}.war + ranger-audit-server-service.war + + + diff --git a/hdfs-agent/conf/ranger-hdfs-audit-changes.cfg b/hdfs-agent/conf/ranger-hdfs-audit-changes.cfg index 49244c66bd..3be456cd4f 100644 --- a/hdfs-agent/conf/ranger-hdfs-audit-changes.cfg +++ b/hdfs-agent/conf/ranger-hdfs-audit-changes.cfg @@ -88,3 +88,8 @@ xasecure.audit.log4j.async.max.queue.size %XAAUDIT.LOG4J.ASYNC.MA xasecure.audit.log4j.async.max.flush.interval.ms %XAAUDIT.LOG4J.ASYNC.MAX.FLUSH.INTERVAL.MS% mod create-if-not-exists xasecure.audit.destination.log4j %XAAUDIT.LOG4J.DESTINATION.LOG4J% mod create-if-not-exists xasecure.audit.destination.log4j.logger %XAAUDIT.LOG4J.DESTINATION.LOG4J.LOGGER% mod create-if-not-exists + +#Audit Service Destination +xasecure.audit.destination.auditserver %XAAUDIT.AUDITSERVER.ENABLE% mod create-if-not-exists +xasecure.audit.destination.auditserver.url %XAAUDIT.AUDITSERVER.URL% mod create-if-not-exists +xasecure.audit.destination.auditserver.batch.filespool.dir %XAAUDIT.AUDITSERVER.FILE_SPOOL_DIR% mod create-if-not-exists diff --git a/hdfs-agent/conf/ranger-hdfs-audit.xml b/hdfs-agent/conf/ranger-hdfs-audit.xml index e16ea590dc..2b8d8556c7 100644 --- a/hdfs-agent/conf/ranger-hdfs-audit.xml +++ b/hdfs-agent/conf/ranger-hdfs-audit.xml @@ -211,6 +211,21 @@ xasecure.audit.solr.solr_url http://localhost:6083/solr/ranger_audits - - + + + + + xasecure.audit.destination.auditserver + false + + + + xasecure.audit.destination.auditserver.url + http://localhost:7081 + + + + xasecure.audit.destination.auditserver.batch.filespool.dir + /var/log/hive/audit/http/spool + diff --git a/hdfs-agent/pom.xml b/hdfs-agent/pom.xml index 910f837ece..1affa6c4c2 100644 --- a/hdfs-agent/pom.xml +++ b/hdfs-agent/pom.xml @@ -114,6 +114,11 @@ httpcore ${httpcomponents.httpcore.version} + + org.apache.ranger + ranger-audit-dest-auditserver + ${project.version} + org.apache.ranger ranger-audit-dest-hdfs diff --git a/hdfs-agent/scripts/install.properties b/hdfs-agent/scripts/install.properties index fedecc1d25..20deb540c9 100644 --- a/hdfs-agent/scripts/install.properties +++ b/hdfs-agent/scripts/install.properties @@ -116,6 +116,10 @@ XAAUDIT.AMAZON_CLOUDWATCH.LOG_STREAM_PREFIX=NONE XAAUDIT.AMAZON_CLOUDWATCH.FILE_SPOOL_DIR=NONE XAAUDIT.AMAZON_CLOUDWATCH.REGION=NONE +#Audit Server Provider +XAAUDIT.AUDITSERVER.ENABLE=false +XAAUDIT.AUDITSERVER.URL=http://ranger-audit:7081 +XAAUDIT.AUDITSERVER.FILE_SPOOL_DIR=/var/log/hive/audit/http/spool # End of V3 properties # diff --git a/hdfs-agent/src/main/java/org/apache/ranger/authorization/hadoop/RangerHdfsAuditHandler.java b/hdfs-agent/src/main/java/org/apache/ranger/authorization/hadoop/RangerHdfsAuditHandler.java index 4f7e74cd3a..5ce5b56315 100644 --- a/hdfs-agent/src/main/java/org/apache/ranger/authorization/hadoop/RangerHdfsAuditHandler.java +++ b/hdfs-agent/src/main/java/org/apache/ranger/authorization/hadoop/RangerHdfsAuditHandler.java @@ -94,7 +94,7 @@ public void processResult(RangerAccessResult result) { setRequestData(); auditEvent.setAction(getAccessType(request.getAccessType())); - auditEvent.setAdditionalInfo(getAdditionalInfo(request)); + auditEvent.setAdditionalInfo(getAdditionalInfo(request, result)); Set tags = getTags(request); @@ -108,8 +108,8 @@ public void processResult(RangerAccessResult result) { } @Override - public String getAdditionalInfo(RangerAccessRequest request) { - String additionalInfo = super.getAdditionalInfo(request); + public String getAdditionalInfo(RangerAccessRequest request, RangerAccessResult result) { + String additionalInfo = super.getAdditionalInfo(request, result); Map addInfoMap = JsonUtils.jsonToMapStringString(additionalInfo); if (addInfoMap == null || addInfoMap.isEmpty()) { diff --git a/hive-agent/conf/ranger-hive-audit-changes.cfg b/hive-agent/conf/ranger-hive-audit-changes.cfg index 651372d3ee..c81a52746f 100644 --- a/hive-agent/conf/ranger-hive-audit-changes.cfg +++ b/hive-agent/conf/ranger-hive-audit-changes.cfg @@ -89,3 +89,8 @@ xasecure.audit.log4j.async.max.queue.size %XAAUDIT.LOG4J.ASYNC.MA xasecure.audit.log4j.async.max.flush.interval.ms %XAAUDIT.LOG4J.ASYNC.MAX.FLUSH.INTERVAL.MS% mod create-if-not-exists xasecure.audit.destination.log4j %XAAUDIT.LOG4J.DESTINATION.LOG4J% mod create-if-not-exists xasecure.audit.destination.log4j.logger %XAAUDIT.LOG4J.DESTINATION.LOG4J.LOGGER% mod create-if-not-exists + +#Audit Service Destination +xasecure.audit.destination.auditserver %XAAUDIT.AUDITSERVER.ENABLE% mod create-if-not-exists +xasecure.audit.destination.auditserver.url %XAAUDIT.AUDITSERVER.URL% mod create-if-not-exists +xasecure.audit.destination.auditserver.batch.filespool.dir %XAAUDIT.AUDITSERVER.FILE_SPOOL_DIR% mod create-if-not-exists diff --git a/hive-agent/conf/ranger-hive-audit.xml b/hive-agent/conf/ranger-hive-audit.xml index d42e58935d..5693a1be1c 100644 --- a/hive-agent/conf/ranger-hive-audit.xml +++ b/hive-agent/conf/ranger-hive-audit.xml @@ -211,6 +211,21 @@ xasecure.audit.solr.solr_url http://localhost:6083/solr/ranger_audits - - + + + + + xasecure.audit.destination.auditserver + true + + + + xasecure.audit.destination.auditserver.url + http://localhost:7081 + + + + xasecure.audit.destination.auditserver.batch.filespool.dir + /var/log/hive/audit/http/spool + diff --git a/hive-agent/pom.xml b/hive-agent/pom.xml index 825f0f1e54..8a800e1e08 100644 --- a/hive-agent/pom.xml +++ b/hive-agent/pom.xml @@ -182,6 +182,11 @@ httpcore ${httpcomponents.httpcore.version} + + org.apache.ranger + ranger-audit-dest-auditserver + ${project.version} + org.apache.ranger ranger-audit-dest-hdfs diff --git a/hive-agent/scripts/install.properties b/hive-agent/scripts/install.properties index 4d8976b6d8..8ac29281d5 100644 --- a/hive-agent/scripts/install.properties +++ b/hive-agent/scripts/install.properties @@ -119,6 +119,10 @@ XAAUDIT.AMAZON_CLOUDWATCH.LOG_STREAM_PREFIX=NONE XAAUDIT.AMAZON_CLOUDWATCH.FILE_SPOOL_DIR=NONE XAAUDIT.AMAZON_CLOUDWATCH.REGION=NONE +#Audit Server Provider +XAAUDIT.AUDITSERVER.ENABLE=false +XAAUDIT.AUDITSERVER.URL=http://ranger-audit:7081 +XAAUDIT.AUDITSERVER.FILE_SPOOL_DIR=/var/log/hive/audit/http/spool # End of V3 properties diff --git a/pom.xml b/pom.xml index e703f964ac..1136edebb9 100755 --- a/pom.xml +++ b/pom.xml @@ -99,6 +99,8 @@ 2.3.0 + 33.4.8-jre + 1.2 1.2 22.3.0 2.9.0 @@ -165,6 +167,7 @@ 1.3.14 8.11.3 3.0.1 + 3.4.0 1.6.0 3.14.0 3.0.0-M6 @@ -824,6 +827,7 @@ plugin-trino plugin-yarn ranger-atlas-plugin-shim + ranger-audit-server ranger-authn ranger-common-ha ranger-elasticsearch-plugin-shim @@ -1152,6 +1156,7 @@ plugin-trino plugin-yarn ranger-atlas-plugin-shim + ranger-audit-server ranger-authn ranger-common-ha ranger-elasticsearch-plugin-shim @@ -1236,6 +1241,7 @@ plugin-trino plugin-yarn ranger-atlas-plugin-shim + ranger-audit-server ranger-authn ranger-common-ha ranger-elasticsearch-plugin-shim @@ -1302,6 +1308,27 @@ + + + com.github.ekryd.sortpom + sortpom-maven-plugin + 4.0.0 + + + + sort + + + recommended_2008_06 + groupId,artifactId + true + + + + diff --git a/ranger-audit-server/pom.xml b/ranger-audit-server/pom.xml new file mode 100644 index 0000000000..1652dc11a7 --- /dev/null +++ b/ranger-audit-server/pom.xml @@ -0,0 +1,460 @@ + + + + 4.0.0 + + + org.apache.ranger + ranger + 3.0.0-SNAPSHOT + .. + + + ranger-audit-server + pom + Ranger Audit Server Parent + Parent project for Ranger Audit Server microservices - Producer and Consumer services + + + + ranger-audit-common + ranger-audit-consumer-hdfs + ranger-audit-consumer-solr + ranger-audit-server-service + + + + 1.8 + 1.8 + UTF-8 + + + + + + + ch.qos.logback + logback-classic + ${logback.version} + + + + com.fasterxml.jackson.core + jackson-annotations + ${fasterxml.jackson.version} + + + com.fasterxml.jackson.core + jackson-core + ${fasterxml.jackson.version} + + + com.fasterxml.jackson.core + jackson-databind + ${fasterxml.jackson.version} + + + + com.google.guava + guava + ${google.guava.version} + + + com.google.inject + guice + ${guice.version} + + + + com.sun.jersey + jersey-bundle + ${jersey-bundle.version} + + + com.sun.jersey + jersey-core + ${jersey-bundle.version} + + + com.sun.jersey.contribs + jersey-spring + ${jersey-spring.version} + + + + com.sun.xml.bind + jaxb-impl + ${jaxb-impl.version} + + + commons-codec + commons-codec + ${commons.codec.version} + + + commons-io + commons-io + ${commons.io.version} + + + + javax.servlet + javax.servlet-api + ${javax.servlet.version} + + + + org.apache.commons + commons-lang3 + ${commons.lang3.version} + + + org.apache.hadoop + hadoop-aws + ${hadoop.version} + + + org.apache.hadoop + hadoop-azure + ${hadoop.version} + + + + org.apache.hadoop + hadoop-common + ${hadoop.version} + + + org.apache.hadoop + hadoop-hdfs-client + ${hadoop.version} + + + org.apache.hadoop.thirdparty + hadoop-shaded-guava + ${hadoop-shaded-guava.version} + + + + org.apache.hive + hive-exec + ${hive.version} + + + org.apache.hive + hive-storage-api + ${hive.version} + + + + org.apache.httpcomponents + httpasyncclient + ${httpcomponents.httpasyncclient.version} + + + org.apache.httpcomponents + httpclient + ${httpcomponents.httpclient.version} + + + org.apache.httpcomponents + httpcore + ${httpcomponents.httpcore.version} + + + org.apache.httpcomponents + httpcore-nio + ${httpcomponents.httpcore.version} + + + + org.apache.kafka + kafka-clients + ${kafka.version} + + + + org.apache.lucene + lucene-analyzers-common + ${lucene.version} + + + org.apache.lucene + lucene-backward-codecs + ${lucene.version} + + + + org.apache.lucene + lucene-core + ${lucene.version} + + + org.apache.lucene + lucene-grouping + ${lucene.version} + + + org.apache.lucene + lucene-highlighter + ${lucene.version} + + + org.apache.lucene + lucene-join + ${lucene.version} + + + org.apache.lucene + lucene-memory + ${lucene.version} + + + org.apache.lucene + lucene-misc + ${lucene.version} + + + org.apache.lucene + lucene-queries + ${lucene.version} + + + org.apache.lucene + lucene-queryparser + ${lucene.version} + + + org.apache.lucene + lucene-sandbox + ${lucene.version} + + + org.apache.lucene + lucene-spatial-extras + ${lucene.version} + + + org.apache.lucene + lucene-spatial3d + ${lucene.version} + + + org.apache.lucene + lucene-suggest + ${lucene.version} + + + + org.apache.orc + orc-core + ${orc.version} + + + + org.apache.orc + orc-shims + ${orc.version} + + + + org.apache.ranger + ranger-audit-common + ${project.version} + + + + org.apache.ranger + ranger-audit-core + ${project.version} + + + + org.apache.ranger + ranger-audit-dest-hdfs + ${project.version} + + + + org.apache.ranger + ranger-audit-dest-solr + ${project.version} + + + + org.apache.ranger + ranger-authn + ${project.version} + + + + org.apache.ranger + ranger-plugins-common + ${project.version} + + + + org.apache.ranger + ranger-plugins-cred + ${project.version} + + + + org.apache.tomcat + tomcat-annotations-api + ${tomcat.embed.version} + + + + org.apache.tomcat.embed + tomcat-embed-core + ${tomcat.embed.version} + + + + org.apache.tomcat.embed + tomcat-embed-el + ${tomcat.embed.version} + + + + org.apache.tomcat.embed + tomcat-embed-jasper + ${tomcat.embed.version} + + + + org.apache.tomcat.embed + tomcat-embed-websocket + ${tomcat.embed.version} + + + + org.slf4j + log4j-over-slf4j + ${slf4j.version} + + + + org.slf4j + log4j-over-slf4j + ${slf4j.version} + + + + org.slf4j + slf4j-api + ${slf4j.version} + + + + org.springframework + spring-beans + ${springframework.version} + + + + org.springframework + spring-context + ${springframework.version} + + + + org.springframework + spring-context-support + ${springframework.version} + + + + org.springframework + spring-tx + ${springframework.version} + + + + org.springframework + spring-web + ${springframework.version} + + + + org.springframework.security + spring-security-config + ${springframework.security.version} + + + + org.springframework.security + spring-security-web + ${springframework.security.version} + + + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.8.1 + + 1.8 + 1.8 + + + + + org.apache.maven.plugins + maven-war-plugin + ${maven-war-plugin.version} + + + + + + + org.apache.maven.plugins + maven-pmd-plugin + ${maven.pmd.plugin.version} + + + ${project.parent.parent.basedir}/dev-support/ranger-pmd-ruleset.xml + + UTF-8 + true + false + true + true + + ${basedir}/src/main/generated + + + + + + check + + verify + + + + + + diff --git a/ranger-audit-server/ranger-audit-common/pom.xml b/ranger-audit-server/ranger-audit-common/pom.xml new file mode 100644 index 0000000000..9c6c6de54b --- /dev/null +++ b/ranger-audit-server/ranger-audit-common/pom.xml @@ -0,0 +1,127 @@ + + + + 4.0.0 + + + org.apache.ranger + ranger-audit-server + 3.0.0-SNAPSHOT + .. + + + ranger-audit-common + jar + Ranger Audit Common + Shared utilities and base classes for Ranger Audit Server services + + + + com.google.guava + guava + ${google.guava.version} + + + + commons-io + commons-io + ${commons.io.version} + + + + org.apache.commons + commons-lang3 + ${commons.lang3.version} + + + + org.apache.hadoop + hadoop-common + ${hadoop.version} + + + log4j + * + + + org.slf4j + * + + + + + + org.apache.kafka + kafka-clients + ${kafka.version} + + + log4j + * + + + org.slf4j + * + + + + + + org.apache.ranger + ranger-audit-core + ${project.version} + + + + org.apache.ranger + ranger-plugins-common + ${project.version} + + + javax.servlet + javax.servlet-api + + + + + + org.apache.tomcat + tomcat-annotations-api + ${tomcat.embed.version} + + + + org.apache.tomcat.embed + tomcat-embed-core + ${tomcat.embed.version} + + + + org.slf4j + slf4j-api + ${slf4j.version} + + + + org.springframework + spring-context + ${springframework.version} + + + diff --git a/ranger-audit-server/ranger-audit-common/src/main/java/org/apache/ranger/audit/consumer/kafka/AuditConsumer.java b/ranger-audit-server/ranger-audit-common/src/main/java/org/apache/ranger/audit/consumer/kafka/AuditConsumer.java new file mode 100644 index 0000000000..3d97536481 --- /dev/null +++ b/ranger-audit-server/ranger-audit-common/src/main/java/org/apache/ranger/audit/consumer/kafka/AuditConsumer.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.audit.consumer.kafka; + +import org.apache.kafka.clients.consumer.KafkaConsumer; + +import java.util.Properties; + +/** + * Interface for Ranger Kafka consumers that consume audit events from Kafka + * and forward them to various destinations like Solr, HDFS, etc. + */ +public interface AuditConsumer extends Runnable { + /** + * Initialize the consumer with the given properties and property prefix. + * This method should set up the destination handler and configure the Kafka consumer. + * + * @param props Configuration properties + * @param propPrefix Property prefix for consumer-specific configuration + * @throws Exception if initialization fails + */ + void init(Properties props, String propPrefix) throws Exception; + + /** + * Start consuming messages from Kafka topic and process them. + * This method should implement the main consumption loop. + * Implementation should handle subscription to topics and polling for messages. + */ + @Override + void run(); + + /** + * Get the underlying Kafka consumer instance. + * + * @return KafkaConsumer instance used by this consumer + */ + KafkaConsumer getKafkaConsumer(); + + /** + * Process a single audit message received from Kafka. + * Implementation should handle the specific logic for forwarding + * the message to the appropriate destination (Solr, HDFS, etc.). + * + * @param audit The audit message in JSON format + * @throws Exception if message processing fails + */ + void processMessage(String audit) throws Exception; + + /** + * Get the topic name this consumer is subscribed to. + * + * @return Kafka topic name + */ + String getTopicName(); + + /** + * Shutdown the consumer gracefully. + * This method should clean up resources and close connections. + */ + void shutdown(); +} diff --git a/ranger-audit-server/ranger-audit-common/src/main/java/org/apache/ranger/audit/consumer/kafka/AuditConsumerBase.java b/ranger-audit-server/ranger-audit-common/src/main/java/org/apache/ranger/audit/consumer/kafka/AuditConsumerBase.java new file mode 100644 index 0000000000..0964da0501 --- /dev/null +++ b/ranger-audit-server/ranger-audit-common/src/main/java/org/apache/ranger/audit/consumer/kafka/AuditConsumerBase.java @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ranger.audit.consumer.kafka; + +import org.apache.kafka.clients.admin.AdminClientConfig; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.consumer.KafkaConsumer; +import org.apache.ranger.audit.provider.MiscUtil; +import org.apache.ranger.audit.server.AuditServerConstants; +import org.apache.ranger.audit.utils.AuditMessageQueueUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Properties; + +public class AuditConsumerBase { + private static final Logger LOG = LoggerFactory.getLogger(AuditConsumerBase.class); + + public Properties consumerProps = new Properties(); + public KafkaConsumer consumer; + public String topicName; + public String consumerGroupId; + + public AuditConsumerBase(Properties props, String propPrefix, String consumerGroupId) throws Exception { + AuditMessageQueueUtils auditMessageQueueUtils = new AuditMessageQueueUtils(props); + + this.consumerGroupId = getConsumerGroupId(props, propPrefix, consumerGroupId); + + consumerProps.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, MiscUtil.getStringProperty(props, propPrefix + "." + AuditServerConstants.PROP_BOOTSTRAP_SERVERS)); + consumerProps.put(ConsumerConfig.GROUP_ID_CONFIG, this.consumerGroupId); + consumerProps.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, Class.forName("org.apache.kafka.common.serialization.StringDeserializer")); + consumerProps.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, Class.forName("org.apache.kafka.common.serialization.StringDeserializer")); + + String securityProtocol = MiscUtil.getStringProperty(props, propPrefix + "." + AuditServerConstants.PROP_SECURITY_PROTOCOL, AuditServerConstants.DEFAULT_SECURITY_PROTOCOL); + consumerProps.put(AdminClientConfig.SECURITY_PROTOCOL_CONFIG, securityProtocol); + + consumerProps.put(AuditServerConstants.PROP_SASL_MECHANISM, MiscUtil.getStringProperty(props, propPrefix + "." + AuditServerConstants.PROP_SASL_MECHANISM, AuditServerConstants.DEFAULT_SASL_MECHANISM)); + consumerProps.put(AuditServerConstants.PROP_SASL_KERBEROS_SERVICE_NAME, AuditServerConstants.DEFAULT_SERVICE_NAME); + + if (securityProtocol.toUpperCase().contains(AuditServerConstants.PROP_SECURITY_PROTOCOL_VALUE)) { + consumerProps.put(AuditServerConstants.PROP_SASL_JAAS_CONFIG, auditMessageQueueUtils.getJAASConfig(props, propPrefix)); + } + + consumerProps.put(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, MiscUtil.getIntProperty(props, propPrefix + "." + AuditServerConstants.PROP_CONSUMER_MAX_POLL_RECORDS, AuditServerConstants.DEFAULT_MAX_POLL_RECORDS)); + consumerProps.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); + + // Configure re-balancing parameters for subscribe mode + // These ensure stable consumer group behavior during horizontal scaling + int sessionTimeoutMs = MiscUtil.getIntProperty(props, propPrefix + "." + AuditServerConstants.PROP_CONSUMER_SESSION_TIMEOUT_MS, AuditServerConstants.DEFAULT_SESSION_TIMEOUT_MS); + int maxPollIntervalMs = MiscUtil.getIntProperty(props, propPrefix + "." + AuditServerConstants.PROP_CONSUMER_MAX_POLL_INTERVAL_MS, AuditServerConstants.DEFAULT_MAX_POLL_INTERVAL_MS); + int heartbeatIntervalMs = MiscUtil.getIntProperty(props, propPrefix + "." + AuditServerConstants.PROP_CONSUMER_HEARTBEAT_INTERVAL_MS, AuditServerConstants.DEFAULT_HEARTBEAT_INTERVAL_MS); + + consumerProps.put(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG, sessionTimeoutMs); + consumerProps.put(ConsumerConfig.MAX_POLL_INTERVAL_MS_CONFIG, maxPollIntervalMs); + consumerProps.put(ConsumerConfig.HEARTBEAT_INTERVAL_MS_CONFIG, heartbeatIntervalMs); + + // Configure partition assignment strategy + String partitionAssignmentStrategy = MiscUtil.getStringProperty(props, propPrefix + "." + AuditServerConstants.PROP_CONSUMER_PARTITION_ASSIGNMENT_STRATEGY, AuditServerConstants.DEFAULT_PARTITION_ASSIGNMENT_STRATEGY); + consumerProps.put(ConsumerConfig.PARTITION_ASSIGNMENT_STRATEGY_CONFIG, partitionAssignmentStrategy); + + LOG.info("Consumer '{}' configured for subscription-based partition assignment with re-balancing support", this.consumerGroupId); + LOG.info("Re-balancing config - session.timeout.ms: {}, max.poll.interval.ms: {}, heartbeat.interval.ms: {}", sessionTimeoutMs, maxPollIntervalMs, heartbeatIntervalMs); + LOG.info("Partition assignment strategy: {}", partitionAssignmentStrategy); + + consumer = new KafkaConsumer<>(consumerProps); + + topicName = MiscUtil.getStringProperty(props, propPrefix + "." + AuditServerConstants.PROP_TOPIC_NAME, AuditServerConstants.DEFAULT_TOPIC); + } + + private String getConsumerGroupId(Properties props, String propPrefix, String defaultConsumerGroupId) { + String configuredGroupId = MiscUtil.getStringProperty(props, propPrefix + ".consumer.group.id"); + if (configuredGroupId != null && !configuredGroupId.trim().isEmpty()) { + return configuredGroupId.trim(); + } + return defaultConsumerGroupId; + } + + public KafkaConsumer getConsumer() { + return consumer; + } + + public String getConsumerGroupId() { + return consumerGroupId; + } +} diff --git a/ranger-audit-server/ranger-audit-common/src/main/java/org/apache/ranger/audit/consumer/kafka/AuditConsumerFactory.java b/ranger-audit-server/ranger-audit-common/src/main/java/org/apache/ranger/audit/consumer/kafka/AuditConsumerFactory.java new file mode 100644 index 0000000000..e986756127 --- /dev/null +++ b/ranger-audit-server/ranger-audit-common/src/main/java/org/apache/ranger/audit/consumer/kafka/AuditConsumerFactory.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.audit.consumer.kafka; + +import java.util.Properties; + +/** + * Factory interface for creating audit consumer instances. + */ +@FunctionalInterface +public interface AuditConsumerFactory { + /** + * Create a consumer instance with the given configuration. + * + * @param props Configuration properties + * @param propPrefix Property prefix for consumer configuration + * @return Initialized consumer instance ready to run + * @throws Exception if consumer creation or initialization fails + */ + AuditConsumer createConsumer(Properties props, String propPrefix) throws Exception; +} diff --git a/ranger-audit-server/ranger-audit-common/src/main/java/org/apache/ranger/audit/consumer/kafka/AuditConsumerRebalanceListener.java b/ranger-audit-server/ranger-audit-common/src/main/java/org/apache/ranger/audit/consumer/kafka/AuditConsumerRebalanceListener.java new file mode 100644 index 0000000000..1b002d629c --- /dev/null +++ b/ranger-audit-server/ranger-audit-common/src/main/java/org/apache/ranger/audit/consumer/kafka/AuditConsumerRebalanceListener.java @@ -0,0 +1,119 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.audit.consumer.kafka; + +import org.apache.kafka.clients.consumer.ConsumerRebalanceListener; +import org.apache.kafka.clients.consumer.KafkaConsumer; +import org.apache.kafka.clients.consumer.OffsetAndMetadata; +import org.apache.kafka.common.TopicPartition; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; + +/** + * Reusable ConsumerRebalanceListener for Kafka consumer group re-balancing. + * + * This listener handles graceful partition re-balancing by: + * - Committing pending offsets before partitions are revoked + * - Updating partition assignments when partitions are assigned + * - Logging re-balancing events with customizable log prefixes + * + * Used by both AuditSolrConsumer, AuditHDFSConsumer and any new consumers added to the audit serve, ensuring + * no message duplication during scaling operations. + */ +public class AuditConsumerRebalanceListener implements ConsumerRebalanceListener { + private static final Logger LOG = LoggerFactory.getLogger(AuditConsumerRebalanceListener.class); + + private final String workerId; + private final String destinationType; + private final String topicName; + private final String offsetCommitStrategy; + private final String consumerGroupId; + private final KafkaConsumer workerConsumer; + private final Map pendingOffsets; + private final AtomicInteger messagesProcessedSinceLastCommit; + private final AtomicLong lastCommitTime; + private final List assignedPartitions; + + public AuditConsumerRebalanceListener( + String workerId, + String destinationType, + String topicName, + String offsetCommitStrategy, + String consumerGroupId, + KafkaConsumer workerConsumer, + Map pendingOffsets, + AtomicInteger messagesProcessedSinceLastCommit, + AtomicLong lastCommitTime, + List assignedPartitions) { + this.workerId = workerId; + this.destinationType = destinationType; + this.topicName = topicName; + this.offsetCommitStrategy = offsetCommitStrategy; + this.consumerGroupId = consumerGroupId; + this.workerConsumer = workerConsumer; + this.pendingOffsets = pendingOffsets; + this.messagesProcessedSinceLastCommit = messagesProcessedSinceLastCommit; + this.lastCommitTime = lastCommitTime; + this.assignedPartitions = assignedPartitions; + } + + @Override + public void onPartitionsRevoked(Collection partitions) { + LOG.info("[{}-REBALANCE] Worker '{}': Partitions REVOKED: {} (count: {})", destinationType, workerId, partitions, partitions.size()); + + // Commit pending offsets before partitions are revoked + if (!pendingOffsets.isEmpty()) { + try { + workerConsumer.commitSync(pendingOffsets); + LOG.info("[{}-REBALANCE] Worker '{}': Successfully committed {} pending offsets before rebalance", + destinationType, workerId, pendingOffsets.size()); + pendingOffsets.clear(); + } catch (Exception e) { + LOG.error("[{}-REBALANCE] Worker '{}': Failed to commit offsets during rebalance", + destinationType, workerId, e); + } + } + + // Reset counters + messagesProcessedSinceLastCommit.set(0); + lastCommitTime.set(System.currentTimeMillis()); + } + + @Override + public void onPartitionsAssigned(Collection partitions) { + LOG.info("[{}-REBALANCE] Worker '{}': Partitions ASSIGNED: {} (count: {})", destinationType, workerId, partitions, partitions.size()); + + // Update assigned partitions list for tracking + assignedPartitions.clear(); + for (TopicPartition tp : partitions) { + assignedPartitions.add(tp.partition()); + } + + // Log assignment details + LOG.info("[{}-CONSUMER-ASSIGNED] Worker '{}' | Topic: '{}' | Partitions: {} | Offset-Strategy: {} | Consumer-Group: {}", + destinationType, workerId, topicName, assignedPartitions, offsetCommitStrategy, consumerGroupId); + } +} diff --git a/ranger-audit-server/ranger-audit-common/src/main/java/org/apache/ranger/audit/consumer/kafka/AuditConsumerRegistry.java b/ranger-audit-server/ranger-audit-common/src/main/java/org/apache/ranger/audit/consumer/kafka/AuditConsumerRegistry.java new file mode 100644 index 0000000000..d1cf560741 --- /dev/null +++ b/ranger-audit-server/ranger-audit-common/src/main/java/org/apache/ranger/audit/consumer/kafka/AuditConsumerRegistry.java @@ -0,0 +1,143 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.audit.consumer.kafka; + +import org.apache.ranger.audit.provider.AuditProviderFactory; +import org.apache.ranger.audit.provider.MiscUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.concurrent.ConcurrentHashMap; + +/** + * Registry for managing audit consumer factories and instances. + * Supports dynamic consumer registration and creation based on configuration. + */ +public class AuditConsumerRegistry { + private static final Logger LOG = LoggerFactory.getLogger(AuditConsumerRegistry.class); + + private static final AuditConsumerRegistry auditConsumerRegistry = new AuditConsumerRegistry(); + private final Map consumerFactories = new ConcurrentHashMap<>(); + private final Map activeConsumers = new ConcurrentHashMap<>(); + + private AuditConsumerRegistry() { + LOG.debug("AuditConsumerRegistry instance created"); + } + + public static AuditConsumerRegistry getInstance() { + return auditConsumerRegistry; + } + + /** + * Register a consumer factory for a specific destination type. + * This method can be called early in the application lifecycle (static initializers) + * to register consumers before the AuditMessageQueue is initialized. + * + * @param destinationType The destination type identifier (e.g., "solr", "hdfs", "elasticsearch") + * @param factory The factory that creates consumer instances + */ + public void registerFactory(String destinationType, AuditConsumerFactory factory) { + if (destinationType == null || destinationType.trim().isEmpty()) { + LOG.warn("Attempted to register factory with null or empty destination type"); + return; + } + + if (factory == null) { + LOG.warn("Attempted to register null factory for destination type: {}", destinationType); + return; + } + + AuditConsumerFactory existing = consumerFactories.put(destinationType, factory); + if (existing != null) { + LOG.warn("Replaced existing factory for destination type: {}", destinationType); + } else { + LOG.info("Registered consumer factory for destination type: {}", destinationType); + } + } + + /** + * Create consumer instances for all enabled destinations based on configuration. + * + * @param props Configuration properties + * @param propPrefix Property prefix for Kafka configuration + * @return List of created consumer instances + */ + public List createConsumers(Properties props, String propPrefix) { + LOG.info("==> AuditConsumerRegistry.createConsumers()"); + + List consumers = new ArrayList<>(); + + for (Map.Entry entry : consumerFactories.entrySet()) { + String destinationType = entry.getKey(); + AuditConsumerFactory factory = entry.getValue(); + String destPropPrefix = AuditProviderFactory.AUDIT_DEST_BASE + "." + destinationType; + boolean isEnabled = MiscUtil.getBooleanProperty(props, destPropPrefix, false); + + if (isEnabled) { + try { + LOG.info("Creating consumer for enabled destination: {}", destinationType); + AuditConsumer consumer = factory.createConsumer(props, propPrefix); + + if (consumer != null) { + consumers.add(consumer); + activeConsumers.put(destinationType, consumer); + LOG.info("Successfully created consumer for destination: {}", destinationType); + } else { + LOG.warn("Factory returned null consumer for destination: {}", destinationType); + } + } catch (Exception e) { + LOG.error("Failed to create consumer for destination: {}", destinationType, e); + } + } else { + LOG.debug("Destination '{}' is disabled (property: {} = false)", destinationType, destPropPrefix); + } + } + + LOG.info("<== AuditConsumerRegistry.createConsumers(): Created {} consumers out of {} registered factories", consumers.size(), consumerFactories.size()); + + return consumers; + } + + public Collection getActiveConsumers() { + return activeConsumers.values(); + } + + public Collection getRegisteredDestinationTypes() { + return consumerFactories.keySet(); + } + + /** + * Clear all active consumer references. + * Called during shutdown after consumers have been stopped. + */ + public void clearActiveConsumers() { + LOG.debug("Clearing {} active consumer references", activeConsumers.size()); + activeConsumers.clear(); + } + + public int getFactoryCount() { + return consumerFactories.size(); + } +} diff --git a/ranger-audit-server/ranger-audit-common/src/main/java/org/apache/ranger/audit/server/AuditConfig.java b/ranger-audit-server/ranger-audit-common/src/main/java/org/apache/ranger/audit/server/AuditConfig.java new file mode 100644 index 0000000000..68b8eda6cb --- /dev/null +++ b/ranger-audit-server/ranger-audit-common/src/main/java/org/apache/ranger/audit/server/AuditConfig.java @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.audit.server; + +import org.apache.ranger.authorization.hadoop.config.RangerConfiguration; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Properties; + +/** + * Base configuration class for Ranger Audit Server services. + * Can be extended by specific services to load their custom configuration files. + */ +public class AuditConfig extends RangerConfiguration { + private static final Logger LOG = LoggerFactory.getLogger(AuditConfig.class); + private static volatile AuditConfig sInstance; + + protected AuditConfig() { + super(); + } + + /** + * Get the singleton instance of AuditConfig. + * Subclasses should override this method to return their specific instance. + */ + public static AuditConfig getInstance() { + AuditConfig ret = AuditConfig.sInstance; + + if (ret == null) { + synchronized (AuditConfig.class) { + ret = AuditConfig.sInstance; + + if (ret == null) { + ret = new AuditConfig(); + AuditConfig.sInstance = ret; + } + } + } + + return ret; + } + + public Properties getProperties() { + return this.getProps(); + } + + /** + * Add a resource file to the configuration. + * Subclasses can override to load their specific config files. + * + * @param resourcePath Path to the resource file (e.g., "conf/ranger-audit-server-site.xml") + * @param required Whether this resource is required + * @return true if resource was loaded successfully or is optional, false otherwise + */ + protected boolean addAuditResource(String resourcePath, boolean required) { + LOG.debug("==> addAuditResource(path={}, required={})", resourcePath, required); + + boolean success = addResourceIfReadable(resourcePath); + + if (success) { + LOG.info("Successfully loaded configuration: {}", resourcePath); + } else if (required) { + LOG.error("Failed to load required configuration: {}", resourcePath); + } else { + LOG.warn("Failed to load optional configuration: {}", resourcePath); + } + + LOG.debug("<== addAuditResource(path={}, required={}), result={}", resourcePath, required, success); + + return success || !required; + } +} diff --git a/ranger-audit-server/ranger-audit-common/src/main/java/org/apache/ranger/audit/server/AuditServerConstants.java b/ranger-audit-server/ranger-audit-common/src/main/java/org/apache/ranger/audit/server/AuditServerConstants.java new file mode 100644 index 0000000000..9b3141bc71 --- /dev/null +++ b/ranger-audit-server/ranger-audit-common/src/main/java/org/apache/ranger/audit/server/AuditServerConstants.java @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.audit.server; + +public class AuditServerConstants { + private AuditServerConstants() {} + + public static final String AUDIT_SERVER_APPNAME = "ranger-audit"; + public static final String AUDIT_SERVER_PROP_PREFIX = "ranger.audit.service."; + public static final String JAAS_KRB5_MODULE = "com.sun.security.auth.module.Krb5LoginModule required"; + public static final String JAAS_USE_KEYTAB = "useKeyTab=true"; + public static final String JAAS_KEYTAB = "keyTab=\""; + public static final String JAAS_STOKE_KEY = "storeKey=true"; + public static final String JAAS_SERVICE_NAME = "serviceName=kafka"; + public static final String JAAS_USER_TICKET_CACHE = "useTicketCache=false"; + public static final String JAAS_PRINCIPAL = "principal=\""; + public static final String PROP_BOOTSTRAP_SERVERS = "bootstrap.servers"; + public static final String PROP_TOPIC_NAME = "topic.name"; + public static final String PROP_SECURITY_PROTOCOL = "security.protocol"; + public static final String PROP_SASL_MECHANISM = "sasl.mechanism"; + public static final String PROP_SASL_JAAS_CONFIG = "sasl.jaas.config"; + public static final String PROP_SASL_KERBEROS_SERVICE_NAME = "sasl.kerberos.service.name"; + public static final String PROP_REQ_TIMEOUT_MS = "request.timeout.ms"; + public static final String PROP_CONN_MAX_IDEAL_MS = "connections.max.idle.ms"; + public static final String PROP_SOLR_DEST_PREFIX = "solr"; + public static final String PROP_HDFS_DEST_PREFIX = "hdfs"; + public static final String PROP_CONSUMER_THREAD_COUNT = "consumer.thread.count"; + public static final String PROP_CONSUMER_OFFSET_COMMIT_STRATEGY = "consumer.offset.commit.strategy"; + public static final String PROP_CONSUMER_OFFSET_COMMIT_INTERVAL = "consumer.offset.commit.interval.ms"; + public static final String PROP_CONSUMER_MAX_POLL_RECORDS = "consumer.max.poll.records"; + public static final String PROP_CONSUMER_SESSION_TIMEOUT_MS = "consumer.session.timeout.ms"; + public static final String PROP_CONSUMER_MAX_POLL_INTERVAL_MS = "consumer.max.poll.interval.ms"; + public static final String PROP_CONSUMER_HEARTBEAT_INTERVAL_MS = "consumer.heartbeat.interval.ms"; + public static final String PROP_CONSUMER_PARTITION_ASSIGNMENT_STRATEGY = "consumer.partition.assignment.strategy"; + public static final String PROP_AUDIT_SERVICE_PRINCIPAL = "kerberos.principal"; + public static final String PROP_AUDIT_SERVICE_KEYTAB = "kerberos.keytab"; + public static final String PROP_KAFKA_PROP_PREFIX = "xasecure.audit.destination.kafka"; + public static final String PROP_KAFKA_STARTUP_MAX_RETRIES = "kafka.startup.max.retries"; + public static final String PROP_KAFKA_STARTUP_RETRY_DELAY_MS = "kafka.startup.retry.delay.ms"; + + // ranger_audits topic configuration + public static final String PROP_TOPIC_PARTITIONS = "topic.partitions"; + public static final String PROP_REPLICATION_FACTOR = "replication.factor"; + public static final short DEFAULT_REPLICATION_FACTOR = 3; + + // Hadoop security configuration for UGI + public static final String PROP_HADOOP_AUTHENTICATION_TYPE = "hadoop.security.authentication"; + public static final String PROP_HADOOP_AUTH_TYPE_KERBEROS = "kerberos"; + public static final String PROP_HADOOP_KERBEROS_NAME_RULES = "hadoop.security.auth_to_local"; + + // Kafka Topic and defaults + public static final String DEFAULT_TOPIC = "ranger_audits"; + public static final String DEFAULT_SASL_MECHANISM = "PLAIN"; + public static final String DEFAULT_SECURITY_PROTOCOL = "PLAINTEXT"; + public static final String DEFAULT_SERVICE_NAME = "kafka"; + public static final String DEFAULT_RANGER_AUDIT_HDFS_CONSUMER_GROUP = "ranger_audit_hdfs_consumer_group"; + public static final String DEFAULT_RANGER_AUDIT_SOLR_CONSUMER_GROUP = "ranger_audit_solr_consumer_group"; + public static final String PROP_SECURITY_PROTOCOL_VALUE = "SASL"; + + // Offset commit strategies + public static final String PROP_OFFSET_COMMIT_STRATEGY_MANUAL = "manual"; + public static final String PROP_OFFSET_COMMIT_STRATEGY_BATCH = "batch"; + public static final String DEFAULT_OFFSET_COMMIT_STRATEGY = PROP_OFFSET_COMMIT_STRATEGY_BATCH; + public static final long DEFAULT_OFFSET_COMMIT_INTERVAL_MS = 30000; // 30 seconds + public static final int DEFAULT_MAX_POLL_RECORDS = 500; // Kafka default batch size + + // Kafka consumer rebalancing timeouts (for subscribe mode) + public static final int DEFAULT_SESSION_TIMEOUT_MS = 60000; // 60 seconds - failure detection + public static final int DEFAULT_MAX_POLL_INTERVAL_MS = 300000; // 5 minutes - max processing time + public static final int DEFAULT_HEARTBEAT_INTERVAL_MS = 10000; // 10 seconds - heartbeat frequency + + // Kafka consumer partition assignment strategy + public static final String DEFAULT_PARTITION_ASSIGNMENT_STRATEGY = "org.apache.kafka.clients.consumer.RangeAssignor"; + + // Destination + public static final String DESTINATION_HDFS = "HDFS"; + public static final String DESTINATION_SOLR = "SOLR"; + + // Consumer Registry Configuration + public static final String PROP_CONSUMER_CLASSES = "consumer.classes"; + public static final String DEFAULT_CONSUMER_CLASSES = "org.apache.ranger.audit.consumer.kafka.AuditSolrConsumer, org.apache.ranger.audit.consumer.kafka.AuditHDFSConsumer"; +} diff --git a/ranger-audit-server/ranger-audit-common/src/main/java/org/apache/ranger/audit/server/EmbeddedServer.java b/ranger-audit-server/ranger-audit-common/src/main/java/org/apache/ranger/audit/server/EmbeddedServer.java new file mode 100644 index 0000000000..1826e5759e --- /dev/null +++ b/ranger-audit-server/ranger-audit-common/src/main/java/org/apache/ranger/audit/server/EmbeddedServer.java @@ -0,0 +1,780 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.audit.server; + +import org.apache.catalina.Context; +import org.apache.catalina.LifecycleException; +import org.apache.catalina.WebResourceRoot; +import org.apache.catalina.connector.Connector; +import org.apache.catalina.core.StandardContext; +import org.apache.catalina.startup.Tomcat; +import org.apache.catalina.valves.AccessLogValve; +import org.apache.catalina.webresources.StandardRoot; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.SecureClientLogin; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.alias.BouncyCastleFipsKeyStoreProvider; +import org.apache.hadoop.security.alias.CredentialProvider; +import org.apache.hadoop.security.alias.CredentialProviderFactory; +import org.apache.hadoop.security.alias.JavaKeyStoreProvider; +import org.apache.hadoop.security.alias.LocalBouncyCastleFipsKeyStoreProvider; +import org.apache.ranger.audit.provider.MiscUtil; +import org.apache.ranger.audit.utils.AuditServerLogFormatter; +import org.apache.ranger.authorization.hadoop.utils.RangerCredentialProvider; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.net.ssl.KeyManager; +import javax.net.ssl.KeyManagerFactory; +import javax.net.ssl.SSLContext; +import javax.net.ssl.TrustManager; +import javax.net.ssl.TrustManagerFactory; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.security.KeyManagementException; +import java.security.KeyStore; +import java.security.KeyStoreException; +import java.security.NoSuchAlgorithmException; +import java.security.SecureRandom; +import java.security.UnrecoverableKeyException; +import java.security.cert.CertificateException; +import java.util.Date; +import java.util.List; +import java.util.Map; + +public class EmbeddedServer { + private static final Logger LOG = LoggerFactory.getLogger(EmbeddedServer.class); + + private static final String ACCESS_LOG_PREFIX = "accesslog.prefix"; + private static final String ACCESS_LOG_DATE_FORMAT = "accesslog.dateformat"; + private static final String ACCESS_LOG_PATTERN = "accesslog.pattern"; + private static final String ACCESS_LOG_ROTATE_MAX_DAYS = "accesslog.rotate.max.days"; + private static final String DEFAULT_LOG_DIR = "/tmp"; + private static final int DEFAULT_HTTP_PORT = 6081; + private static final int DEFAULT_HTTPS_PORT = -1; + private static final int DEFAULT_SHUTDOWN_PORT = 6185; + private static final String DEFAULT_SHUTDOWN_COMMAND = "SHUTDOWN"; + private static final String DEFAULT_WEBAPPS_ROOT_FOLDER = "webapps"; + private static final String DEFAULT_ENABLED_PROTOCOLS = "TLSv1.2"; + public static final String DEFAULT_NAME_RULE = "DEFAULT"; + private static final String RANGER_KEYSTORE_FILE_TYPE_DEFAULT = "jks"; + private static final String RANGER_TRUSTSTORE_FILE_TYPE_DEFAULT = "jks"; + private static final String RANGER_SSL_CONTEXT_ALGO_TYPE = "TLSv1.2"; + private static final String RANGER_SSL_KEYMANAGER_ALGO_TYPE = KeyManagerFactory.getDefaultAlgorithm(); + private static final String RANGER_SSL_TRUSTMANAGER_ALGO_TYPE = TrustManagerFactory.getDefaultAlgorithm(); + public static final String KEYSTORE_FILE_TYPE_DEFAULT = KeyStore.getDefaultType(); + public static final String TRUSTSTORE_FILE_TYPE_DEFAULT = KeyStore.getDefaultType(); + + private final Configuration configuration; + private final String appName; + private final String configPrefix; + private volatile Tomcat server; + private volatile Context webappContext; + + public EmbeddedServer(Configuration configuration, String appName, String configPrefix) { + LOG.info("==> EmbeddedServer(appName={}, configPrefix={})", appName, configPrefix); + + this.configuration = new Configuration(configuration); + this.appName = appName; + this.configPrefix = configPrefix; + + LOG.info("<== EmbeddedServer(appName={}, configPrefix={}", appName, configPrefix); + } + + public static void main(String[] args) { + Configuration config = AuditConfig.getInstance(); + String appName = AuditServerConstants.AUDIT_SERVER_APPNAME; + String configPrefix = AuditServerConstants.AUDIT_SERVER_PROP_PREFIX; + + new EmbeddedServer(config, appName, configPrefix).start(); + } + + @SuppressWarnings("deprecation") + public void start() { + LOG.info("==> EmbeddedServer.start(appName={})", appName); + + SSLContext sslContext = getSSLContext(); + + if (sslContext != null) { + SSLContext.setDefault(sslContext); + } + + this.server = new Tomcat(); + + // Add shutdown hook for graceful cleanup + addShutdownHook(); + + String logDir = getConfig("log.dir", DEFAULT_LOG_DIR); + String hostName = getConfig("host"); + int serverPort = getIntConfig("http.port", DEFAULT_HTTP_PORT); + int sslPort = getIntConfig("https.port", DEFAULT_HTTPS_PORT); + int shutdownPort = getIntConfig("shutdown.port", DEFAULT_SHUTDOWN_PORT); + String shutdownCommand = getConfig("shutdown.command", DEFAULT_SHUTDOWN_COMMAND); + boolean isHttpsEnabled = getBooleanConfig("https.attrib.ssl.enabled", false); + boolean ajpEnabled = getBooleanConfig("ajp.enabled", false); + + server.setHostname(hostName); + server.setPort(serverPort); + server.getServer().setPort(shutdownPort); + server.getServer().setShutdown(shutdownCommand); + + if (ajpEnabled) { + Connector ajpConnector = new Connector("org.apache.coyote.ajp.AjpNioProtocol"); + + ajpConnector.setPort(serverPort); + ajpConnector.setProperty("protocol", "AJP/1.3"); + + server.getService().addConnector(ajpConnector); + + // Making this as a default connector + server.setConnector(ajpConnector); + + LOG.info("Created AJP Connector"); + } else if (isHttpsEnabled && sslPort > 0) { + String clientAuth = getConfig("https.attrib.clientAuth", "false"); + String providerPath = getConfig("credential.provider.path"); + String keyAlias = getConfig("https.attrib.keystore.credential.alias", "keyStoreCredentialAlias"); + String keystorePass = null; + String enabledProtocols = getConfig("https.attrib.ssl.enabled.protocols", DEFAULT_ENABLED_PROTOCOLS); + String ciphers = getConfig("tomcat.ciphers"); + + if (StringUtils.equalsIgnoreCase(clientAuth, "false")) { + clientAuth = getConfig("https.attrib.client.auth", "want"); + } + + if (providerPath != null && keyAlias != null) { + keystorePass = getDecryptedString(providerPath.trim(), keyAlias.trim(), getConfig("keystore.file.type", KEYSTORE_FILE_TYPE_DEFAULT)); + + if (StringUtils.isBlank(keystorePass) || StringUtils.equalsIgnoreCase(keystorePass.trim(), "none")) { + keystorePass = getConfig("https.attrib.keystore.pass"); + } + } + + Connector ssl = new Connector(); + String sslKeystoreKeyAlias = getConfig("https.attrib.keystore.keyalias", ""); + ssl.setPort(sslPort); + ssl.setSecure(true); + ssl.setScheme("https"); + ssl.setAttribute("SSLEnabled", "true"); + ssl.setAttribute("sslProtocol", getConfig("https.attrib.ssl.protocol", "TLSv1.2")); + ssl.setAttribute("clientAuth", clientAuth); + if (StringUtils.isNotBlank(sslKeystoreKeyAlias)) { + ssl.setAttribute("keyAlias", sslKeystoreKeyAlias); + } + ssl.setAttribute("keystorePass", keystorePass); + ssl.setAttribute("keystoreFile", getKeystoreFile()); + ssl.setAttribute("sslEnabledProtocols", enabledProtocols); + ssl.setAttribute("keystoreType", getConfig("audit.keystore.file.type", KEYSTORE_FILE_TYPE_DEFAULT)); + ssl.setAttribute("truststoreType", getConfig("audit.truststore.file.type", TRUSTSTORE_FILE_TYPE_DEFAULT)); + + if (StringUtils.isNotBlank(ciphers)) { + ssl.setAttribute("ciphers", ciphers); + } + + server.getService().addConnector(ssl); + + // + // Making this as a default connector + // + server.setConnector(ssl); + } + + updateHttpConnectorAttribConfig(server); + + File logDirectory = new File(logDir); + + if (!logDirectory.exists()) { + boolean created = logDirectory.mkdirs(); + if (!created) { + LOG.error("Failed to create log directory: {}", logDir); + throw new RuntimeException("Failed to create log directory: " + logDir); + } + } + + String logPattern = getConfig(ACCESS_LOG_PATTERN, "%h %l %u %t \"%r\" %s %b"); + + AccessLogValve valve = new AccessLogValve(); + + valve.setRotatable(true); + valve.setAsyncSupported(true); + valve.setBuffered(false); + valve.setEnabled(true); + valve.setPrefix(getConfig(ACCESS_LOG_PREFIX, "access_log-" + hostName + "-")); + valve.setFileDateFormat(getConfig(ACCESS_LOG_DATE_FORMAT, "yyyy-MM-dd.HH")); + valve.setDirectory(logDirectory.getAbsolutePath()); + valve.setSuffix(".log"); + valve.setPattern(logPattern); + valve.setMaxDays(getIntConfig(ACCESS_LOG_ROTATE_MAX_DAYS, 15)); + + server.getHost().getPipeline().addValve(valve); + + try { + String webappDir = getConfig("webapp.dir"); + + if (StringUtils.isBlank(webappDir)) { + LOG.error("Tomcat Server failed to start: {}.webapp.dir is not set", configPrefix); + + System.exit(1); + } + + String webContextName = getConfig("contextName", ""); + + if (StringUtils.isBlank(webContextName)) { + webContextName = ""; + } else if (!webContextName.startsWith("/")) { + LOG.info("Context Name [{}] is being loaded as [ /{}]", webContextName, webContextName); + + webContextName = "/" + webContextName; + } + + File wad = new File(webappDir); + + if (wad.isDirectory()) { + LOG.info("Webapp dir={}, webAppName={}", webappDir, webContextName); + } else if (wad.isFile()) { + File webAppDir = new File(DEFAULT_WEBAPPS_ROOT_FOLDER); + + if (!webAppDir.exists()) { + boolean created = webAppDir.mkdirs(); + if (!created) { + LOG.error("Failed to create webapp directory: {}", DEFAULT_WEBAPPS_ROOT_FOLDER); + throw new RuntimeException("Failed to create webapp directory: " + DEFAULT_WEBAPPS_ROOT_FOLDER); + } + } + + LOG.info("Webapp file={}, webAppName={}", webappDir, webContextName); + } + + LOG.info("Adding webapp [{}] = path [{}] .....", webContextName, webappDir); + + this.webappContext = server.addWebapp(webContextName, new File(webappDir).getAbsolutePath()); + if (webappContext instanceof StandardContext) { + boolean allowLinking = getBooleanConfig("allow.linking", true); + StandardContext standardContext = (StandardContext) webappContext; + String workDirPath = getConfig("tomcat.work.dir", ""); + if (!workDirPath.isEmpty() && new File(workDirPath).exists()) { + standardContext.setWorkDir(workDirPath); + } else { + LOG.debug("Skipping to set tomcat server work directory {} as it is blank or directory does not exist.", workDirPath); + } + WebResourceRoot resRoot = new StandardRoot(standardContext); + webappContext.setResources(resRoot); + webappContext.getResources().setAllowLinking(allowLinking); + LOG.debug("Tomcat Configuration - allowLinking=[{}]", allowLinking); + } else { + LOG.error("Tomcat Context [{}] is either NULL OR it's NOT instanceof StandardContext", webappContext); + } + + webappContext.init(); + + LOG.info("Finished init of webapp [{}] = path [{}].", webContextName, webappDir); + } catch (LifecycleException lce) { + LOG.error("Tomcat Server failed to start webapp", lce); + + lce.printStackTrace(); + } + + String authenticationType = getConfig(AuditServerConstants.PROP_HADOOP_AUTHENTICATION_TYPE); + + if (StringUtils.equalsIgnoreCase(authenticationType, AuditServerConstants.PROP_HADOOP_AUTH_TYPE_KERBEROS)) { + String keyTab = getConfig(AuditServerConstants.PROP_AUDIT_SERVICE_KEYTAB); + String principal = null; + String nameRules = getConfig(AuditServerConstants.PROP_HADOOP_KERBEROS_NAME_RULES, DEFAULT_NAME_RULE); + + try { + principal = SecureClientLogin.getPrincipal(getConfig(AuditServerConstants.PROP_AUDIT_SERVICE_PRINCIPAL), hostName); + } catch (IOException excp) { + LOG.warn("Failed to get principal" + excp); + } + + if (SecureClientLogin.isKerberosCredentialExists(principal, keyTab)) { + try { + AuditServerLogFormatter.builder("Kerberos Login Attempt") + .add("Principal", principal) + .add("Keytab", keyTab) + .add("Name rules", nameRules) + .logInfo(LOG); + + Configuration hadoopConf = new Configuration(); + hadoopConf.set(AuditServerConstants.PROP_HADOOP_AUTHENTICATION_TYPE, AuditServerConstants.PROP_HADOOP_AUTH_TYPE_KERBEROS); + hadoopConf.set(AuditServerConstants.PROP_HADOOP_KERBEROS_NAME_RULES, nameRules); + UserGroupInformation.setConfiguration(hadoopConf); + UserGroupInformation.loginUserFromKeytab(principal, keyTab); + UserGroupInformation currentUGI = UserGroupInformation.getLoginUser(); + + AuditServerLogFormatter.builder("Kerberos Login Successful") + .add("UGI Username", currentUGI.getUserName()) + .add("UGI Real User", currentUGI.getUserName()) + .add("Authentication Method", currentUGI.getAuthenticationMethod().toString()) + .add("Has Kerberos Credentials", currentUGI.hasKerberosCredentials()) + .logInfo(LOG); + + MiscUtil.setUGILoginUser(currentUGI, null); + + LOG.info("Starting Server using kerberos credential"); + startServer(server); + } catch (Exception excp) { + LOG.error("Tomcat Server failed to start", excp); + } + } else { + LOG.warn("Kerberos principal={} not found in keytab={}. Starting server in non-kerberos mode", principal, keyTab); + + startServer(server); + } + } else { + LOG.info("Starting server in non-kerberos mode"); + + startServer(server); + } + + LOG.info("<== EmbeddedServer.start(appName={})", appName); + } + + public String getDecryptedString(String credentialProviderPath, String alias, String storeType) { + String ret = null; + + if (StringUtils.isNotBlank(credentialProviderPath) && StringUtils.isNotBlank(alias)) { + try { + Configuration conf = new Configuration(); + String prefixJceks = JavaKeyStoreProvider.SCHEME_NAME + "://file"; + String prefixLocalJceks = "localjceks://file"; + String prefixBcfks = BouncyCastleFipsKeyStoreProvider.SCHEME_NAME + "://file"; + String prefixLocalBcfks = LocalBouncyCastleFipsKeyStoreProvider.SCHEME_NAME + "://file"; + + String lowerPath = credentialProviderPath.toLowerCase(); + if (lowerPath.startsWith(prefixJceks.toLowerCase()) || lowerPath.startsWith(prefixLocalJceks.toLowerCase()) || lowerPath.startsWith(prefixBcfks.toLowerCase()) || lowerPath.startsWith(prefixLocalBcfks.toLowerCase())) { + conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, credentialProviderPath); + } else { + if (credentialProviderPath.startsWith("/")) { + if ("bcfks".equalsIgnoreCase(storeType)) { + conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, credentialProviderPath); + } else { + conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, prefixJceks + credentialProviderPath); + } + } else { + conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, prefixJceks + "/" + credentialProviderPath); + } + } + + List providers = CredentialProviderFactory.getProviders(conf); + + LOG.debug("CredentialProviderPath={} alias={} storeType={}", credentialProviderPath, alias, storeType); + LOG.debug("List of CredentialProvider = {}", providers.toString()); + + for (CredentialProvider provider : providers) { + List aliasesList = provider.getAliases(); + + if (CollectionUtils.isNotEmpty(aliasesList) && aliasesList.contains(alias)) { + CredentialProvider.CredentialEntry credEntry = provider.getCredentialEntry(alias); + char[] pass = credEntry.getCredential(); + + if (pass != null && pass.length > 0) { + ret = String.valueOf(pass); + + break; + } + } + } + } catch (Exception ex) { + LOG.error("CredentialReader failed while decrypting provided string. Reason: {}", ex.toString()); + + ret = null; + } + } + + LOG.debug("getDecryptedString() : ret = {}", ret); + + return ret; + } + + protected long getLongConfig(String key, long defaultValue) { + long ret = defaultValue; + String retStr = getConfig(key); + + try { + if (retStr != null) { + ret = Long.parseLong(retStr); + } + } catch (Exception err) { + LOG.warn(retStr + " can't be parsed to long. Reason: {}", err.toString()); + } + + return ret; + } + + private void startServer(final Tomcat server) { + LOG.info("==> EmbeddedServer.startServer(appName={})", appName); + + try { + server.start(); + + server.getServer().await(); + + shutdownServer(); + } catch (LifecycleException e) { + LOG.error("Tomcat Server failed to start", e); + + e.printStackTrace(); + } catch (Exception e) { + LOG.error("Tomcat Server failed to start", e); + + e.printStackTrace(); + } + + LOG.info("<== EmbeddedServer.startServer(appName={})", appName); + } + + private void addShutdownHook() { + Runtime.getRuntime().addShutdownHook(new Thread(() -> { + LOG.info("==> EmbeddedServer shutdown hook triggered"); + try { + gracefulShutdown(); + } catch (Exception e) { + LOG.error("Error during graceful shutdown", e); + } + LOG.info("<== EmbeddedServer shutdown hook completed"); + }, "EmbeddedServer-ShutdownHook")); + } + + private void gracefulShutdown() { + LOG.info("==> EmbeddedServer.gracefulShutdown()"); + + // Instead of trying to manually shutdown Spring context, let's rely on Tomcat's + // normal webapp lifecycle which will trigger ContextLoaderListener.contextDestroyed() + // and call @PreDestroy methods automatically + + if (server != null) { + try { + LOG.info("Initiating Tomcat server stop to trigger webapp lifecycle shutdown"); + server.stop(); + LOG.info("Tomcat server stop completed - Spring @PreDestroy methods should have been called"); + } catch (Exception e) { + LOG.error("Error stopping Tomcat server during graceful shutdown", e); + + // Fallback: try to stop the webapp context directly + if (webappContext != null) { + try { + LOG.info("Fallback: stopping webapp context directly"); + webappContext.stop(); + LOG.info("Webapp context stopped"); + } catch (Exception contextE) { + LOG.error("Error stopping webapp context", contextE); + } + } + } + } + + // Give some time for Spring components to shutdown through normal lifecycle + try { + Thread.sleep(3000); // Increased to 3 seconds to allow for proper cleanup + } catch (InterruptedException e) { + LOG.warn("Interrupted while waiting for Spring shutdown", e); + Thread.currentThread().interrupt(); + } + + LOG.info("<== EmbeddedServer.gracefulShutdown()"); + } + + private void shutdownServer() { + LOG.info("==> EmbeddedServer.shutdownServer(appName={})", appName); + + int timeWaitForShutdownInSeconds = getIntConfig("waitTimeForForceShutdownInSeconds", 0); + + if (timeWaitForShutdownInSeconds > 0) { + long endTime = System.currentTimeMillis() + (timeWaitForShutdownInSeconds * 1000L); + + LOG.info("Will wait for all threads to shutdown gracefully. Final shutdown Time: {}", new Date(endTime)); + + while (System.currentTimeMillis() < endTime) { + int activeCount = Thread.activeCount(); + + if (activeCount == 0) { + LOG.info("Number of active threads = {}", activeCount); + + break; + } else { + LOG.info("Number of active threads = {}. Waiting for all threads to shutdown ...", activeCount); + + try { + Thread.sleep(5000L); + } catch (InterruptedException e) { + LOG.warn("shutdownServer process is interrupted with exception", e); + + break; + } + } + } + } + + LOG.info("Shutting down the Server."); + + LOG.info("<== EmbeddedServer.shutdownServer(appName={})", appName); + + System.exit(0); + } + + @SuppressWarnings("deprecation") + private void updateHttpConnectorAttribConfig(Tomcat server) { + server.getConnector().setAllowTrace(getBooleanConfig("http.connector.attrib.allowTrace", false)); + server.getConnector().setAsyncTimeout(getLongConfig("http.connector.attrib.asyncTimeout", 10000)); + server.getConnector().setEnableLookups(getBooleanConfig("http.connector.attrib.enableLookups", false)); + server.getConnector().setMaxParameterCount(getIntConfig("http.connector.attrib.maxParameterCount", 10000)); + server.getConnector().setMaxPostSize(getIntConfig("http.connector.attrib.maxPostSize", 2097152)); + server.getConnector().setMaxSavePostSize(getIntConfig("http.connector.attrib.maxSavePostSize", 4096)); + server.getConnector().setParseBodyMethods(getConfig("http.connector.attrib.methods", "POST")); + server.getConnector().setURIEncoding(getConfig("http.connector.attrib.URIEncoding", "UTF-8")); + server.getConnector().setProperty("acceptCount", getConfig("http.connector.attrib.acceptCount", "1024")); + server.getConnector().setXpoweredBy(false); + server.getConnector().setAttribute("server", getConfig("servername", "Audit Server")); + + int maxThreads = getIntConfig("http.connector.attrib.maxThreads", 2000); + int maxKeepAliveRequests = getIntConfig("http.connector.attrib.maxKeepAliveRequests", 1000); + int minSpareThreads = getIntConfig("http.connector.attrib.minSpareThreads", (int) (maxThreads * 0.8)); // (default 80% of maxThreads) + boolean prestartminSpareThreads = getBooleanConfig("http.connector.attrib.prestartminSpareThreads", true); + + server.getConnector().setAttribute("maxThreads", maxThreads); + server.getConnector().setAttribute("maxKeepAliveRequests", maxKeepAliveRequests); + server.getConnector().setAttribute("minSpareThreads", minSpareThreads); + server.getConnector().setAttribute("prestartminSpareThreads", prestartminSpareThreads); + server.getConnector().setProperty("sendReasonPhrase", getConfig(configPrefix + "http.connector.property.sendReasonPhrase", "true")); + + for (Map.Entry entry : configuration) { + String key = entry.getKey(); + String value = entry.getValue(); + + if (key != null && key.startsWith(configPrefix + "http.connector.property.")) { + String property = key.replace(configPrefix + "http.connector.property.", ""); + + server.getConnector().setProperty(property, value); + + LOG.info("{}:{}", property, server.getConnector().getProperty(property)); + } + } + } + + private SSLContext getSSLContext() { + SSLContext ret = null; + KeyManager[] kmList = getKeyManagers(); + TrustManager[] tmList = getTrustManagers(); + + if (tmList != null) { + try { + ret = SSLContext.getInstance(RANGER_SSL_CONTEXT_ALGO_TYPE); + + ret.init(kmList, tmList, new SecureRandom()); + } catch (NoSuchAlgorithmException e) { + LOG.error("SSL algorithm is not available in the environment", e); + } catch (KeyManagementException e) { + LOG.error("Unable to initials the SSLContext", e); + } + } + + return ret; + } + + private KeyManager[] getKeyManagers() { + KeyManager[] ret = null; + String keyStoreFile = getConfig("keystore.file"); + String keyStoreAlias = getConfig("keystore.alias"); + String credentialProviderPath = getConfig("credential.provider.path"); + String keyStoreFilepwd = getCredential(credentialProviderPath, keyStoreAlias); + + if (StringUtils.isNotBlank(keyStoreFile) && StringUtils.isNotBlank(keyStoreFilepwd)) { + InputStream in = null; + + try { + in = getFileInputStream(keyStoreFile); + + if (in != null) { + KeyStore keyStore = KeyStore.getInstance(RANGER_KEYSTORE_FILE_TYPE_DEFAULT); + + keyStore.load(in, keyStoreFilepwd.toCharArray()); + + KeyManagerFactory keyManagerFactory = KeyManagerFactory.getInstance(RANGER_SSL_KEYMANAGER_ALGO_TYPE); + + keyManagerFactory.init(keyStore, keyStoreFilepwd.toCharArray()); + + ret = keyManagerFactory.getKeyManagers(); + } else { + LOG.error("Unable to obtain keystore from file [{}]", keyStoreFile); + } + } catch (KeyStoreException e) { + LOG.error("Unable to obtain from KeyStore", e); + } catch (NoSuchAlgorithmException e) { + LOG.error("SSL algorithm is NOT available in the environment", e); + } catch (CertificateException e) { + LOG.error("Unable to obtain the requested certification", e); + } catch (FileNotFoundException e) { + LOG.error("Unable to find the necessary SSL Keystore Files", e); + } catch (IOException e) { + LOG.error("Unable to read the necessary SSL Keystore Files", e); + } catch (UnrecoverableKeyException e) { + LOG.error("Unable to recover the key from keystore", e); + } finally { + close(in, keyStoreFile); + } + } + + return ret; + } + + private TrustManager[] getTrustManagers() { + TrustManager[] ret = null; + String truststoreFile = getConfig("truststore.file"); + String truststoreAlias = getConfig("truststore.alias"); + String credentialProviderPath = getConfig("credential.provider.path"); + String trustStoreFilepwd = getCredential(credentialProviderPath, truststoreAlias); + + if (StringUtils.isNotBlank(truststoreFile) && StringUtils.isNotBlank(trustStoreFilepwd)) { + InputStream in = null; + + try { + in = getFileInputStream(truststoreFile); + + if (in != null) { + KeyStore trustStore = KeyStore.getInstance(RANGER_TRUSTSTORE_FILE_TYPE_DEFAULT); + + trustStore.load(in, trustStoreFilepwd.toCharArray()); + + TrustManagerFactory trustManagerFactory = TrustManagerFactory.getInstance(RANGER_SSL_TRUSTMANAGER_ALGO_TYPE); + + trustManagerFactory.init(trustStore); + + ret = trustManagerFactory.getTrustManagers(); + } else { + LOG.error("Unable to obtain truststore from file [{}]", truststoreFile); + } + } catch (KeyStoreException e) { + LOG.error("Unable to obtain from KeyStore", e); + } catch (NoSuchAlgorithmException e) { + LOG.error("SSL algorithm is NOT available in the environment", e); + } catch (CertificateException e) { + LOG.error("Unable to obtain the requested certification", e); + } catch (FileNotFoundException e) { + LOG.error("Unable to find the necessary SSL TrustStore File:{}", truststoreFile, e); + } catch (IOException e) { + LOG.error("Unable to read the necessary SSL TrustStore Files:{}", truststoreFile, e); + } finally { + close(in, truststoreFile); + } + } + + return ret; + } + + private String getKeystoreFile() { + String ret = getConfig("https.attrib.keystore.file"); + + if (StringUtils.isBlank(ret)) { + // new property not configured, lets use the old property + ret = getConfig("https.attrib.keystore.file"); + } + + return ret; + } + + private String getCredential(String url, String alias) { + return RangerCredentialProvider.getInstance().getCredentialString(url, alias); + } + + private String getConfig(String key) { + String propertyWithPrefix = configPrefix + key; + String value = configuration.get(propertyWithPrefix); + if (value == null) { // config-with-prefix not found; look at System properties + value = System.getProperty(propertyWithPrefix); + if (value == null) { // config-with-prefix not found in System properties; look for config-without-prefix + value = configuration.get(key); + if (value == null) { // config-without-prefix not found; look at System properties + value = System.getProperty(key); + } + } + } + + return value; + } + + private String getConfig(String key, String defaultValue) { + String ret = getConfig(key); + + if (ret == null) { + ret = defaultValue; + } + + return ret; + } + + private int getIntConfig(String key, int defaultValue) { + int ret = defaultValue; + String strValue = getConfig(key); + + try { + if (strValue != null) { + ret = Integer.parseInt(strValue); + } + } catch (Exception err) { + LOG.warn(strValue + " can't be parsed to int. Reason: {}", err.toString()); + } + + return ret; + } + + private boolean getBooleanConfig(String key, boolean defaultValue) { + boolean ret = defaultValue; + String strValue = getConfig(key); + + if (StringUtils.isNotBlank(strValue)) { + ret = Boolean.valueOf(strValue); + } + + return ret; + } + + private InputStream getFileInputStream(String fileName) throws IOException { + InputStream ret = null; + + if (StringUtils.isNotEmpty(fileName)) { + File f = new File(fileName); + + if (f.exists()) { + ret = new FileInputStream(f); + } else { + ret = ClassLoader.getSystemResourceAsStream(fileName); + } + } + + return ret; + } + + private void close(InputStream str, String filename) { + if (str != null) { + try { + str.close(); + } catch (IOException excp) { + LOG.error("Error while closing file: [{}]", filename, excp); + } + } + } +} diff --git a/ranger-audit-server/ranger-audit-common/src/main/java/org/apache/ranger/audit/utils/AuditMessageQueueUtils.java b/ranger-audit-server/ranger-audit-common/src/main/java/org/apache/ranger/audit/utils/AuditMessageQueueUtils.java new file mode 100644 index 0000000000..de8a5a7244 --- /dev/null +++ b/ranger-audit-server/ranger-audit-common/src/main/java/org/apache/ranger/audit/utils/AuditMessageQueueUtils.java @@ -0,0 +1,348 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.audit.utils; + +import org.apache.hadoop.security.SecureClientLogin; +import org.apache.kafka.clients.admin.AdminClient; +import org.apache.kafka.clients.admin.AdminClientConfig; +import org.apache.kafka.clients.admin.CreatePartitionsResult; +import org.apache.kafka.clients.admin.DescribeTopicsResult; +import org.apache.kafka.clients.admin.NewPartitions; +import org.apache.kafka.clients.admin.NewTopic; +import org.apache.kafka.clients.admin.TopicDescription; +import org.apache.ranger.audit.provider.AuditProviderFactory; +import org.apache.ranger.audit.provider.MiscUtil; +import org.apache.ranger.audit.server.AuditServerConstants; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.time.Duration; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; +import java.util.Set; + +public class AuditMessageQueueUtils { + private static final Logger LOG = LoggerFactory.getLogger(AuditMessageQueueUtils.class); + + AuditServerUtils auditServerUtils = new AuditServerUtils(); + boolean isTopicReady; + boolean isSolrConsumerEnabled; + boolean isHDFSConsumerEnabled; + + public AuditMessageQueueUtils(Properties props) { + isHDFSConsumerEnabled = MiscUtil.getBooleanProperty(props, AuditProviderFactory.AUDIT_DEST_BASE + "." + AuditServerConstants.PROP_HDFS_DEST_PREFIX, false); + isSolrConsumerEnabled = MiscUtil.getBooleanProperty(props, AuditProviderFactory.AUDIT_DEST_BASE + "." + AuditServerConstants.PROP_SOLR_DEST_PREFIX, false); + } + + public String createAuditsTopicIfNotExists(Properties props, String propPrefix) { + LOG.info("==> AuditMessageQueueUtils:createAuditsTopicIfNotExists()"); + + String ret = null; + String topicName = MiscUtil.getStringProperty(props, propPrefix + "." + AuditServerConstants.PROP_TOPIC_NAME, AuditServerConstants.DEFAULT_TOPIC); + String bootstrapServers = MiscUtil.getStringProperty(props, propPrefix + "." + AuditServerConstants.PROP_BOOTSTRAP_SERVERS); + String securityProtocol = MiscUtil.getStringProperty(props, propPrefix + "." + AuditServerConstants.PROP_SECURITY_PROTOCOL, AuditServerConstants.DEFAULT_SECURITY_PROTOCOL); + String saslMechanism = MiscUtil.getStringProperty(props, propPrefix + "." + AuditServerConstants.PROP_SASL_MECHANISM, AuditServerConstants.DEFAULT_SASL_MECHANISM); + int connMaxIdleTimeoutMS = MiscUtil.getIntProperty(props, propPrefix + "." + AuditServerConstants.PROP_CONN_MAX_IDEAL_MS, 10000); + int partitions = getPartitions(props, propPrefix); + short replicationFactor = (short) MiscUtil.getIntProperty(props, propPrefix + "." + AuditServerConstants.PROP_REPLICATION_FACTOR, AuditServerConstants.DEFAULT_REPLICATION_FACTOR); + int reqTimeoutMS = MiscUtil.getIntProperty(props, propPrefix + "." + AuditServerConstants.PROP_REQ_TIMEOUT_MS, 5000); + + // Retry configuration for Kafka connection during startup + int maxRetries = MiscUtil.getIntProperty(props, propPrefix + "." + AuditServerConstants.PROP_KAFKA_STARTUP_MAX_RETRIES, 10); + int retryDelayMs = MiscUtil.getIntProperty(props, propPrefix + "." + AuditServerConstants.PROP_KAFKA_STARTUP_RETRY_DELAY_MS, 3000); + int currentAttempt = 0; + + Map kafkaProp = new HashMap<>(); + kafkaProp.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers); + kafkaProp.put(AuditServerConstants.PROP_SASL_MECHANISM, saslMechanism); + kafkaProp.put(AdminClientConfig.SECURITY_PROTOCOL_CONFIG, securityProtocol); + + if (securityProtocol != null && securityProtocol.toUpperCase().contains("SASL")) { + kafkaProp.put(AuditServerConstants.PROP_SASL_JAAS_CONFIG, getJAASConfig(props, propPrefix)); + } + + kafkaProp.put(AdminClientConfig.REQUEST_TIMEOUT_MS_CONFIG, reqTimeoutMS); + kafkaProp.put(AdminClientConfig.CONNECTIONS_MAX_IDLE_MS_CONFIG, connMaxIdleTimeoutMS); + + while (currentAttempt <= maxRetries && ret == null) { + currentAttempt++; + + try (AdminClient admin = AdminClient.create(kafkaProp)) { + if (currentAttempt > 1) { + LOG.info("Attempting to connect to Kafka (attempt {}/{})", currentAttempt, maxRetries + 1); + } + + Set names = admin.listTopics().names().get(); + if (!names.contains(topicName)) { + NewTopic topic = new NewTopic(topicName, partitions, replicationFactor); + admin.createTopics(Collections.singletonList(topic)).all().get(); + ret = topic.name(); + LOG.info("Creating topic '{}' with {} partitions and replication factor {}", topicName, partitions, replicationFactor); + // Wait for metadata to propagate across the cluster + boolean isTopicReady = auditServerUtils.waitUntilTopicReady(admin, topicName, Duration.ofSeconds(60)); + if (isTopicReady) { + try { + DescribeTopicsResult result = admin.describeTopics(Collections.singletonList(topicName)); + TopicDescription topicDescription = result.values().get(topicName).get(); + ret = topicDescription.name(); + + int partitionCount = topicDescription.partitions().size(); + setTopicReady(isTopicReady); + + LOG.info("Topic: {} successfully created with {} partitions", ret, partitionCount); + } catch (Exception e) { + throw new RuntimeException("Failed to fetch metadata for topic:" + topicName, e); + } + } + } else { + /*** + * Topic already existing. Check and update number of partitions for audit server. This is for upgrade + * from existing audit mechanism to audit server + */ + ret = updateExistingTopicPartitions(admin, topicName, partitions, replicationFactor); + } + } catch (Exception ex) { + if (currentAttempt <= maxRetries) { + LOG.warn("AuditMessageQueueUtils:createAuditsTopicIfNotExists(): Failed to connect to Kafka on attempt {}/{}. Retrying in {} ms. Error: {}", + currentAttempt, maxRetries + 1, retryDelayMs, ex.getMessage()); + try { + Thread.sleep(retryDelayMs); + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + LOG.error("Interrupted while waiting to retry Kafka connection", ie); + break; + } + } else { + LOG.error("AuditMessageQueueUtils:createAuditsTopicIfNotExists(): Error creating topic after {} attempts", currentAttempt, ex); + } + } + } + + LOG.info("<== AuditMessageQueueUtils:createAuditsTopicIfNotExists() ret: {}", ret); + + return ret; + } + + public boolean isSolrConsumerEnabled() { + return isSolrConsumerEnabled; + } + + public boolean isHDFSConsumerEnabled() { + return isHDFSConsumerEnabled; + } + + public boolean isTopicReady() { + return isTopicReady; + } + + public void setTopicReady(boolean topicReady) { + isTopicReady = topicReady; + } + + public String getJAASConfig(Properties props, String propPrefix) { + // Use ranger service principal and keytab for Kafka authentication + // This ensures consistent identity across all Ranger services and destination writes + String hostName = props.getProperty(AuditServerConstants.AUDIT_SERVER_PROP_PREFIX + "host"); + String principal = props.getProperty(AuditServerConstants.AUDIT_SERVER_PROP_PREFIX + AuditServerConstants.PROP_AUDIT_SERVICE_PRINCIPAL); + String keytab = props.getProperty(AuditServerConstants.AUDIT_SERVER_PROP_PREFIX + AuditServerConstants.PROP_AUDIT_SERVICE_KEYTAB); + + AuditServerLogFormatter.builder("Kerberos Configuration") + .add("Principal (raw)", principal) + .add("Hostname", hostName) + .add("Keytab path", keytab) + .logInfo(LOG); + + // Validate keytab file exists and is readable + if (keytab != null) { + java.io.File keytabFile = new java.io.File(keytab); + if (!keytabFile.exists()) { + LOG.error("ERROR: Keytab file does not exist: {}", keytab); + throw new IllegalStateException("Keytab file not found: " + keytab); + } + if (!keytabFile.canRead()) { + LOG.error("ERROR: Keytab file is not readable: {}", keytab); + throw new IllegalStateException("Keytab file not readable: " + keytab); + } + + AuditServerLogFormatter.builder("Keytab File Validation") + .add("Exists", keytabFile.exists()) + .add("Readable", keytabFile.canRead()) + .add("Size (bytes)", keytabFile.length()) + .logInfo(LOG); + } + + try { + principal = SecureClientLogin.getPrincipal(props.getProperty(AuditServerConstants.AUDIT_SERVER_PROP_PREFIX + AuditServerConstants.PROP_AUDIT_SERVICE_PRINCIPAL), hostName); + LOG.info("Principal (resolved): {}", principal); + } catch (Exception e) { + principal = null; + LOG.error("ERROR: Failed to resolve principal from _HOST pattern!", e); + } + + if (keytab == null || principal == null) { + AuditServerLogFormatter.builder("Please configure the following properties in ranger-audit-server-site.xml:") + .add(AuditServerConstants.AUDIT_SERVER_PROP_PREFIX + AuditServerConstants.PROP_AUDIT_SERVICE_PRINCIPAL, "ranger/_HOST@YOUR-REALM") + .add(AuditServerConstants.AUDIT_SERVER_PROP_PREFIX + AuditServerConstants.PROP_AUDIT_SERVICE_KEYTAB, "/path/to/ranger.keytab") + .logError(LOG); + throw new IllegalStateException("Ranger service principal and keytab must be configured for Kafka authentication. "); + } + + String jaasConfig = new StringBuilder() + .append(AuditServerConstants.JAAS_KRB5_MODULE).append(" ") + .append(AuditServerConstants.JAAS_USE_KEYTAB).append(" ") + .append(AuditServerConstants.JAAS_KEYTAB).append(keytab).append("\"").append(" ") + .append(AuditServerConstants.JAAS_STOKE_KEY).append(" ") + .append(AuditServerConstants.JAAS_USER_TICKET_CACHE).append(" ") + .append(AuditServerConstants.JAAS_SERVICE_NAME).append(" ") + .append(AuditServerConstants.JAAS_PRINCIPAL).append(principal).append("\";") + .toString(); + + AuditServerLogFormatter.builder("JAAS Configuration Generated") + .add("Principal", principal) + .add("Keytab", keytab) + .add("Full JAAS Config", jaasConfig) + .logInfo(LOG); + + return jaasConfig; + } + + public String updateExistingTopicPartitions(AdminClient admin, String topicName, int partitions, short replicationFactor) { + LOG.info("==> AuditMessageQueueUtils:updateExistingTopicPartitions() topic: {}, desired partitions: {}", topicName, partitions); + + String ret = null; + int maxRetries = 3; + int retryDelayMs = 1000; // Start with 1 second + + try { + // Describe the existing topic to get current partition count + DescribeTopicsResult describeTopicsResult = admin.describeTopics(Collections.singletonList(topicName)); + TopicDescription topicDescription = describeTopicsResult.values().get(topicName).get(); + int currentPartitions = topicDescription.partitions().size(); + + ret = topicDescription.name(); + LOG.info("Topic '{}' already exists with {} partitions", ret, currentPartitions); + + // Check if we need to increase partitions + if (partitions > currentPartitions) { + LOG.info("Upgrading topic '{}' from {} to {} partitions for audit server mechanism", topicName, currentPartitions, partitions); + + boolean updateSuccess = false; + Exception lastException = null; + + // Retry logic while updating partitions + for (int attempt = 1; attempt <= maxRetries; attempt++) { + try { + LOG.info("Partition update attempt {}/{} for topic '{}'", attempt, maxRetries, topicName); + + // Create partition increase request + Map newPartitionsMap = new HashMap<>(); + newPartitionsMap.put(topicName, NewPartitions.increaseTo(partitions)); + + // Execute partition increase + CreatePartitionsResult createPartitionsResult = admin.createPartitions(newPartitionsMap); + createPartitionsResult.all().get(); // Wait for operation to complete + + LOG.info("Successfully initiated partition increase for topic '{}' on attempt {}", topicName, attempt); + + // Wait for metadata to propagate across the cluster + boolean isTopicReady = auditServerUtils.waitUntilTopicReady(admin, topicName, Duration.ofSeconds(60)); + + if (isTopicReady) { + // Verify the partition count after update + DescribeTopicsResult verifyResult = admin.describeTopics(Collections.singletonList(topicName)); + TopicDescription updatedDescription = verifyResult.values().get(topicName).get(); + int updatedPartitions = updatedDescription.partitions().size(); + + if (updatedPartitions >= partitions) { + setTopicReady(true); + LOG.info("Topic '{}' successfully upgraded from {} to {} partitions on attempt {}", topicName, currentPartitions, updatedPartitions, attempt); + updateSuccess = true; + break; + } else { + LOG.warn("Topic '{}' partition update completed but verification shows only {} partitions (expected {})", topicName, updatedPartitions, partitions); + throw new IllegalStateException("Partition count verification failed"); + } + } else { + LOG.warn("Topic '{}' partition update completed but topic not ready within timeout on attempt {}", topicName, attempt); + throw new IllegalStateException("Topic not ready after partition update"); + } + } catch (Exception e) { + lastException = e; + LOG.warn("Partition update attempt {}/{} failed for topic '{}': {}", attempt, maxRetries, topicName, e.getMessage()); + if (attempt < maxRetries) { + int currentDelay = retryDelayMs * (1 << (attempt - 1)); + LOG.info("Retrying partition update in {} ms...", currentDelay); + Thread.sleep(currentDelay); + } + } + } + if (!updateSuccess) { + String errorMsg = String.format( + "FATAL: Failed to update partitions for topic '%s' after %d attempts. " + + "Required: %d partitions, Current: %d partitions. " + + "Cannot proceed without sufficient partitions.", topicName, maxRetries, partitions, currentPartitions); + LOG.error(errorMsg, lastException); + throw new RuntimeException(errorMsg, lastException); + } + } else if (partitions < currentPartitions) { + LOG.warn("Topic '{}' has {} partitions, which is more than the configured {} partitions. " + + "Kafka does not support reducing partition count. Using existing partition count.", topicName, currentPartitions, partitions); + setTopicReady(true); + } else { + LOG.info("Topic '{}' already has the correct number of partitions: {}", topicName, currentPartitions); + setTopicReady(true); + } + } catch (Exception e) { + String errorMsg = String.format("FATAL: Error updating partitions for topic '%s'", topicName); + LOG.error(errorMsg, e); + throw new RuntimeException(errorMsg, e); + } + + LOG.info("<== AuditMessageQueueUtils:updateExistingTopicPartitions() ret: {}", ret); + + return ret; + } + + /** + * Get the number of partitions for the Kafka topic + * + * Configurable via xasecure.audit.destination.kafka.topic.partitions property. + * Default: 30 partitions for balanced distribution with Kafka's default partitioner. + * + * With default partitioner (murmur2 hash), messages with same key (appId) always + * go to the same partition, providing ordering per appId while distributing load + * evenly across all partitions. + * + * @return Number of partitions for the topic + */ + private int getPartitions(Properties prop, String propPrefix) { + int defaultPartitions = 30; + int partitions = MiscUtil.getIntProperty(prop, + propPrefix + "." + AuditServerConstants.PROP_TOPIC_PARTITIONS, + defaultPartitions); + + LOG.info("Kafka topic partition count: {} (configured: {})", + partitions, prop.getProperty(propPrefix + "." + AuditServerConstants.PROP_TOPIC_PARTITIONS, "default")); + + return partitions; + } +} diff --git a/ranger-audit-server/ranger-audit-common/src/main/java/org/apache/ranger/audit/utils/AuditServerLogFormatter.java b/ranger-audit-server/ranger-audit-common/src/main/java/org/apache/ranger/audit/utils/AuditServerLogFormatter.java new file mode 100644 index 0000000000..1fca5554b3 --- /dev/null +++ b/ranger-audit-server/ranger-audit-common/src/main/java/org/apache/ranger/audit/utils/AuditServerLogFormatter.java @@ -0,0 +1,148 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.audit.utils; + +import org.slf4j.Logger; + +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * Utility for logging in the ranger-audit-server + */ +public class AuditServerLogFormatter { + private AuditServerLogFormatter() {} + + /** + * Log a map of logDetails at INFO level with a title + * @param logger The logger to use + * @param title The title/header for this log section + * @param logDetails Map of key-value pairs to log + */ + public static void logInfo(Logger logger, String title, Map logDetails) { + if (logDetails != null && !logDetails.isEmpty()) { + logger.info("{}:", title); + logDetails.forEach((key, value) -> logger.info(" {} = [{}]", key, value)); + } + } + + /** + * Log a map of logDetails at DEBUG level with a title + * @param logger The logger to use + * @param title The title/header for this log section + * @param logDetails Map of key-value pairs to log + */ + public static void logDebug(Logger logger, String title, Map logDetails) { + if (logDetails != null && !logDetails.isEmpty()) { + logger.debug("{}:", title); + logDetails.forEach((key, value) -> logger.debug(" {} = [{}]", key, value)); + } + } + + /** + * Log a map of logDetails at DEBUG level with a title + * @param logger The logger to use + * @param title The title/header for this log section + * @param logDetails Map of key-value pairs to log + */ + public static void logError(Logger logger, String title, Map logDetails) { + if (logDetails != null && !logDetails.isEmpty()) { + logger.error("{}:", title); + logDetails.forEach((key, value) -> logger.error(" {} = [{}]", key, value)); + } + } + + /** + * Create a builder for constructing LogDetails maps to log + * @param title The title for this log section + * @return A new LogBuilder + */ + public static LogBuilder builder(String title) { + return new LogBuilder(title); + } + + /** + * Builder class for constructing structured log messages + */ + public static class LogBuilder { + private final String title; + private final Map logDetails = new LinkedHashMap<>(); + + private LogBuilder(String title) { + this.title = title; + } + + /** + * Add a log key value pair + * @param key + * @param value + * @return This builder for chaining + */ + public LogBuilder add(String key, Object value) { + logDetails.put(key, value); + return this; + } + + /** + * Add a log only if the value is not null + * @param key + * @param value + * @return This builder for chaining + */ + public LogBuilder addIfNotNull(String key, Object value) { + if (value != null) { + logDetails.put(key, value); + } + return this; + } + + /** + * Log the accumulated LogDetails at INFO level + * @param logger The logger to use + */ + public void logInfo(Logger logger) { + AuditServerLogFormatter.logInfo(logger, title, logDetails); + } + + /** + * Log the accumulated LogDetails at DEBUG level + * @param logger The logger to use + */ + public void logDebug(Logger logger) { + AuditServerLogFormatter.logDebug(logger, title, logDetails); + } + + /** + * Log the accumulated LogDetails at DEBUG level + * @param logger The logger to use + */ + public void logError(Logger logger) { + AuditServerLogFormatter.logError(logger, title, logDetails); + } + + /** + * Get the LogDetails map + * @return The LogDetails map + */ + public Map getLogDetails() { + return new LinkedHashMap<>(logDetails); + } + } +} diff --git a/ranger-audit-server/ranger-audit-common/src/main/java/org/apache/ranger/audit/utils/AuditServerUtils.java b/ranger-audit-server/ranger-audit-common/src/main/java/org/apache/ranger/audit/utils/AuditServerUtils.java new file mode 100644 index 0000000000..1536134a30 --- /dev/null +++ b/ranger-audit-server/ranger-audit-common/src/main/java/org/apache/ranger/audit/utils/AuditServerUtils.java @@ -0,0 +1,286 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.audit.utils; + +import org.apache.commons.collections.MapUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.kafka.clients.admin.Admin; +import org.apache.kafka.clients.admin.DescribeTopicsResult; +import org.apache.kafka.clients.admin.TopicDescription; +import org.apache.kafka.common.errors.UnknownTopicOrPartitionException; +import org.apache.ranger.audit.model.AuthzAuditEvent; +import org.apache.ranger.audit.provider.AuditHandler; +import org.apache.ranger.audit.provider.AuditProviderFactory; +import org.apache.ranger.authorization.utils.JsonUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ThreadLocalRandom; + +import static org.apache.ranger.audit.provider.MiscUtil.TOKEN_APP_TYPE; +import static org.apache.ranger.audit.provider.MiscUtil.TOKEN_END; +import static org.apache.ranger.audit.provider.MiscUtil.TOKEN_START; +import static org.apache.ranger.audit.provider.MiscUtil.toArray; + +public class AuditServerUtils { + private static final Logger LOG = LoggerFactory.getLogger(AuditServerUtils.class); + + private Properties auditConfig; + private ConcurrentHashMap> auditProviderMap = new ConcurrentHashMap<>(); + + public Properties getAuditConfig() { + if (auditConfig == null) { + return null; + } + Properties copy = new Properties(); + copy.putAll(auditConfig); + return copy; + } + + public void setAuditConfig(Properties auditConfig) { + if (auditConfig != null) { + this.auditConfig = new Properties(); + this.auditConfig.putAll(auditConfig); + } else { + this.auditConfig = null; + } + } + + public AuditHandler getAuditProvider(AuthzAuditEvent auditEvent) throws Exception { + AuditHandler ret = null; + + AuditProviderFactory auditProviderFactory = getAuditProvideFactory(auditEvent); + if (auditProviderFactory != null) { + ret = auditProviderFactory.getAuditProvider(); + } + + return ret; + } + + private AuditProviderFactory getAuditProvideFactory(AuthzAuditEvent auditEvent) throws Exception { + LOG.debug("==> AuditServerUtils.getAuditProviderFactory()"); + + AuditProviderFactory ret = null; + String hostName = auditEvent.getAgentHostname(); + String appId = auditEvent.getAgentId(); + String serviceType = getServiceType(auditEvent); + + Map auditProviderFactoryMap = auditProviderMap.get(hostName); + if (MapUtils.isNotEmpty(auditProviderFactoryMap)) { + ret = auditProviderFactoryMap.get(appId); + } + + if (ret == null) { + ret = createAndCacheAuditProvider(hostName, serviceType, appId); + if (MapUtils.isEmpty(auditProviderFactoryMap)) { + auditProviderFactoryMap = new HashMap<>(); + } + auditProviderFactoryMap.put(appId, ret); + auditProviderMap.put(hostName, auditProviderFactoryMap); + } + + if (LOG.isDebugEnabled()) { + logAuditProviderCreated(); + LOG.debug("<== AuditServerUtils.getAuditProviderFactory(): {}", ret); + } + + return ret; + } + + private AuditProviderFactory createAndCacheAuditProvider(String hostName, String serviceType, String appId) throws Exception { + LOG.debug("==> AuditServerUtils.createAndCacheAuditProvider(hostname: {}, serviceType: {}, appId: {})", hostName, serviceType, appId); + + AuditProviderFactory ret = initAuditProvider(serviceType, appId); + if (!ret.isInitDone()) { + ret = initAuditProvider(serviceType, appId); + } + if (!ret.isInitDone()) { + String msg = String.format("AuditHandler for appId={%s}, hostname={%s} not initialized correctly. Please check audit configuration...", appId, hostName); + LOG.error(msg); + throw new Exception(msg); + } + + LOG.debug("<== AuditServerUtils.createAndCacheAuditProvider(hostname: {}, serviceType: {}, appId: {})", hostName, serviceType, appId); + return ret; + } + + private AuditProviderFactory initAuditProvider(String serviceType, String appId) { + Properties properties = getConfigurationForAuditService(serviceType); + AuditProviderFactory ret = AuditProviderFactory.getInstance(); + ret.init(properties, appId); + return ret; + } + + private Properties getConfigurationForAuditService(String serviceType) { + Properties ret = getAuditConfig(); + setCloudStorageLocationProperty(ret, serviceType); + setSpoolFolderForDestination(ret, serviceType); + return ret; + } + + private String getServiceType(AuthzAuditEvent auditEvent) { + String ret = null; + String additionalInfo = auditEvent.getAdditionalInfo(); + Map addInfoMap = JsonUtils.jsonToMapStringString(additionalInfo); + if (MapUtils.isNotEmpty(addInfoMap)) { + ret = addInfoMap.get("serviceType"); + } + return ret; + } + + private void setCloudStorageLocationProperty(Properties prop, String serviceType) { + String hdfsDir = prop.getProperty("xasecure.audit.destination.hdfs.dir"); + if (StringUtils.isNotBlank(hdfsDir)) { + StringBuilder sb = new StringBuilder(hdfsDir).append("/").append(serviceType); + setProperty(prop, "xasecure.audit.destination.hdfs.dir", sb.toString()); + } + } + + private void setSpoolFolderForDestination(Properties ret, String serviceType) { + ArrayList enabledDestinations = getAuditDestinationList(ret); + for (String dest : enabledDestinations) { + String spoolDirProp = new StringBuilder("xasecure.audit.destination.").append(dest).append(".batch.filespool.dir").toString(); + String spoolDir = ret.getProperty(spoolDirProp); + if (StringUtils.isNotBlank(spoolDir)) { + spoolDir = replaceToken(spoolDir, serviceType); + setProperty(ret, spoolDirProp, spoolDir); + } + } + } + + private void setProperty(Properties properties, String key, String val) { + properties.setProperty(key, val); + } + + private ArrayList getAuditDestinationList(Properties props) { + ArrayList ret = new ArrayList<>(); + + for (Object propNameObj : props.keySet()) { + String propName = propNameObj.toString(); + if (!propName.startsWith(AuditProviderFactory.AUDIT_DEST_BASE)) { + continue; + } + String destName = propName.substring(AuditProviderFactory.AUDIT_DEST_BASE.length() + 1); + List splits = toArray(destName, "."); + if (splits.size() > 1) { + continue; + } + String value = props.getProperty(propName); + if ("enable".equalsIgnoreCase(value) || "enabled".equalsIgnoreCase(value) || "true".equalsIgnoreCase(value)) { + ret.add(destName); + LOG.info("Audit destination {} is set to {}", propName, value); + } + } + return ret; + } + + private String replaceToken(String str, String appType) { + String ret = str; + for (int startPos = 0; startPos < str.length(); ) { + int tagStartPos = str.indexOf(TOKEN_START, startPos); + + if (tagStartPos == -1) { + break; + } + + int tagEndPos = str.indexOf(TOKEN_END, tagStartPos + TOKEN_START.length()); + + if (tagEndPos == -1) { + break; + } + + String tag = str.substring(tagStartPos, tagEndPos + TOKEN_END.length()); + String token = tag.substring(TOKEN_START.length(), tag.lastIndexOf(TOKEN_END)); + String val = ""; + if (token != null) { + if (token.equals(TOKEN_APP_TYPE)) { + val = appType; + } + } + ret = str.substring(0, tagStartPos) + val + str.substring(tagEndPos + TOKEN_END.length()); + startPos = tagStartPos + val.length(); + } + + return ret; + } + + private void logAuditProviderCreated() { + if (MapUtils.isNotEmpty(auditProviderMap)) { + for (Map.Entry> entry : auditProviderMap.entrySet()) { + String hostname = entry.getKey(); + Map providerFactoryMap = entry.getValue(); + if (MapUtils.isNotEmpty(providerFactoryMap)) { + for (Map.Entry ap : providerFactoryMap.entrySet()) { + String serviceAppId = ap.getKey(); + AuditProviderFactory auditProviderFactoryVal = ap.getValue(); + String apVal = auditProviderFactoryVal.getAuditProvider().getName(); + LOG.debug("AuditProvider created for HostName: {} ServiceAppId: {} AuditProvider: {}", hostname, serviceAppId, apVal); + } + } + } + } + } + + public boolean waitUntilTopicReady(Admin admin, String topic, Duration totalWait) throws Exception { + long endTime = System.nanoTime() + totalWait.toNanos(); + long baseSleepMs = 100L; + long maxSleepMs = 2000L; + + while (System.nanoTime() < endTime) { + try { + DescribeTopicsResult describeTopicsResult = admin.describeTopics(Collections.singleton(topic)); + TopicDescription topicDescription = describeTopicsResult.values().get(topic).get(); + boolean allHaveLeader = topicDescription.partitions().stream().allMatch(partitionInfo -> partitionInfo.leader() != null); + boolean allHaveISR = topicDescription.partitions().stream().allMatch(partitionInfo -> !partitionInfo.isr().isEmpty()); + if (allHaveLeader && allHaveISR) { + return true; + } + } catch (Exception e) { + // If topic hasn't propagated yet, you'll see UnknownTopicOrPartitionException + // continue to wait for topic availability + if (!(rootCause(e) instanceof UnknownTopicOrPartitionException)) { + throw e; + } + } + + // Sleep until the created topic is available for metadata fetch + baseSleepMs = Math.min(maxSleepMs, baseSleepMs * 2); + long sleep = baseSleepMs + ThreadLocalRandom.current().nextLong(0, baseSleepMs / 2 + 1); + Thread.sleep(sleep); + } + return false; + } + + private static Throwable rootCause(Throwable t) { + Throwable throwable = t; + while (throwable.getCause() != null) { + throwable = throwable.getCause(); + } + return throwable; + } +} diff --git a/ranger-audit-server/ranger-audit-consumer-hdfs/pom.xml b/ranger-audit-server/ranger-audit-consumer-hdfs/pom.xml new file mode 100644 index 0000000000..5e80254ccf --- /dev/null +++ b/ranger-audit-server/ranger-audit-consumer-hdfs/pom.xml @@ -0,0 +1,345 @@ + + + + 4.0.0 + + + org.apache.ranger + ranger-audit-server + 3.0.0-SNAPSHOT + .. + + + ranger-audit-consumer-hdfs + war + Ranger Audit Consumer HDFS + Kafka consumer service for writing audits to HDFS/S3/Azure + + + UTF-8 + + + + + + + ch.qos.logback + logback-classic + ${logback.version} + + + com.fasterxml.jackson.core + jackson-databind + ${fasterxml.jackson.version} + + + + com.google.guava + guava + ${google.guava.version} + + + + com.sun.jersey + jersey-bundle + ${jersey-bundle.version} + + + com.sun.jersey.contribs + jersey-spring + ${jersey-spring.version} + + + com.sun.jersey + jersey-server + + + org.springframework + * + + + + + commons-io + commons-io + ${commons.io.version} + + + javax.servlet + javax.servlet-api + ${javax.servlet.version} + + + + org.apache.commons + commons-lang3 + ${commons.lang3.version} + + + + org.apache.hadoop + hadoop-aws + ${hadoop.version} + + + com.google.guava + guava + + + org.apache.hadoop + hadoop-client-api + + + + + org.apache.hadoop + hadoop-azure + ${hadoop.version} + + + com.google.guava + guava + + + org.apache.hadoop + hadoop-client-api + + + + + + org.apache.hadoop + hadoop-common + ${hadoop.version} + + + ch.qos.reload4j + reload4j + + + log4j + * + + + org.apache.hadoop + hadoop-client-api + + + org.slf4j + * + + + + + org.apache.hadoop + hadoop-hdfs-client + ${hadoop.version} + + + com.google.guava + guava + + + org.apache.hadoop + hadoop-client-api + + + + + org.apache.hadoop.thirdparty + hadoop-shaded-guava + ${hadoop-shaded-guava.version} + + + org.apache.hive + hive-exec + ${hive.version} + + + * + * + + + + + org.apache.hive + hive-storage-api + ${hive.version} + + + * + * + + + + + + + org.apache.kafka + kafka-clients + ${kafka.version} + + + log4j + * + + + org.slf4j + * + + + + + + org.apache.orc + orc-core + ${orc.version} + + + * + * + + + + + org.apache.orc + orc-shims + ${orc.version} + + + * + * + + + + + + org.apache.ranger + ranger-audit-common + ${project.version} + + + org.apache.hadoop + hadoop-client-api + + + + + + org.apache.ranger + ranger-audit-core + ${project.version} + + + org.apache.hadoop + hadoop-client-api + + + + + org.apache.ranger + ranger-audit-dest-hdfs + ${project.version} + + + org.apache.hadoop + hadoop-client-api + + + + + org.apache.ranger + ranger-plugins-common + ${project.version} + + + javax.servlet + javax.servlet-api + + + org.apache.hadoop + hadoop-client-api + + + + + org.apache.tomcat + tomcat-annotations-api + ${tomcat.embed.version} + + + + org.apache.tomcat.embed + tomcat-embed-core + ${tomcat.embed.version} + + + org.apache.tomcat.embed + tomcat-embed-el + ${tomcat.embed.version} + + + org.apache.tomcat.embed + tomcat-embed-jasper + ${tomcat.embed.version} + + + + org.slf4j + log4j-over-slf4j + ${slf4j.version} + + + org.slf4j + slf4j-api + ${slf4j.version} + + + org.springframework + spring-beans + ${springframework.version} + + + + org.springframework + spring-context + ${springframework.version} + + + org.springframework + spring-web + ${springframework.version} + + + + + + + true + src/main/resources + + + + + org.apache.maven.plugins + maven-war-plugin + + + + diff --git a/ranger-audit-server/ranger-audit-consumer-hdfs/scripts/start-consumer-hdfs.sh b/ranger-audit-server/ranger-audit-consumer-hdfs/scripts/start-consumer-hdfs.sh new file mode 100755 index 0000000000..a86f7a67dc --- /dev/null +++ b/ranger-audit-server/ranger-audit-consumer-hdfs/scripts/start-consumer-hdfs.sh @@ -0,0 +1,164 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# ======================================== +# Ranger Audit Consumer HDFS Start Script +# ======================================== +# This script starts the HDFS consumer service +# The service consumes audit events from Kafka and writes them to HDFS/S3/Azure + +set -e + +# Script directory +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +SERVICE_DIR="$(dirname "$SCRIPT_DIR")" + +# Default directories - can be overridden by environment variables +AUDIT_CONSUMER_HOME_DIR="${AUDIT_CONSUMER_HOME_DIR:-${SERVICE_DIR}/target}" +AUDIT_CONSUMER_CONF_DIR="${AUDIT_CONSUMER_CONF_DIR:-${SERVICE_DIR}/src/main/resources/conf}" +AUDIT_CONSUMER_LOG_DIR="${AUDIT_CONSUMER_LOG_DIR:-${SERVICE_DIR}/logs}" + +# Create log directory if it doesn't exist +mkdir -p "${AUDIT_CONSUMER_LOG_DIR}" + +echo "==========================================" +echo "Starting Ranger Audit Consumer - HDFS" +echo "==========================================" +echo "Service Directory: ${SERVICE_DIR}" +echo "Home Directory: ${AUDIT_CONSUMER_HOME_DIR}" +echo "Config Directory: ${AUDIT_CONSUMER_CONF_DIR}" +echo "Log Directory: ${AUDIT_CONSUMER_LOG_DIR}" +echo "==========================================" + +# Check if Java is available +if [ -z "$JAVA_HOME" ]; then + JAVA_CMD=$(which java 2>/dev/null || true) + if [ -z "$JAVA_CMD" ]; then + echo "[ERROR] JAVA_HOME is not set and java is not in PATH" + exit 1 + fi + JAVA_HOME=$(dirname $(dirname $(readlink -f "$JAVA_CMD"))) + echo "[INFO] JAVA_HOME not set, detected: ${JAVA_HOME}" +fi + +export JAVA_HOME +export PATH=$JAVA_HOME/bin:$PATH + +echo "[INFO] Java version:" +java -version + +# Set heap size (default: 512MB to 2GB) +AUDIT_CONSUMER_HEAP="${AUDIT_CONSUMER_HEAP:--Xms512m -Xmx2g}" + +# Set JVM options +if [ -z "$AUDIT_CONSUMER_OPTS" ]; then + AUDIT_CONSUMER_OPTS="-Dlogback.configurationFile=${AUDIT_CONSUMER_CONF_DIR}/logback.xml" + AUDIT_CONSUMER_OPTS="${AUDIT_CONSUMER_OPTS} -Daudit.consumer.hdfs.log.dir=${AUDIT_CONSUMER_LOG_DIR}" + AUDIT_CONSUMER_OPTS="${AUDIT_CONSUMER_OPTS} -Daudit.consumer.hdfs.log.file=ranger-audit-consumer-hdfs.log" + AUDIT_CONSUMER_OPTS="${AUDIT_CONSUMER_OPTS} -Djava.net.preferIPv4Stack=true -server" + AUDIT_CONSUMER_OPTS="${AUDIT_CONSUMER_OPTS} -XX:+UseG1GC -XX:MaxGCPauseMillis=200" + AUDIT_CONSUMER_OPTS="${AUDIT_CONSUMER_OPTS} -XX:InitiatingHeapOccupancyPercent=35" + AUDIT_CONSUMER_OPTS="${AUDIT_CONSUMER_OPTS} -XX:ConcGCThreads=4 -XX:ParallelGCThreads=8" +fi + +# Add Kerberos configuration if needed +if [ "${KERBEROS_ENABLED}" == "true" ]; then + AUDIT_CONSUMER_OPTS="${AUDIT_CONSUMER_OPTS} -Djava.security.krb5.conf=/etc/krb5.conf" + echo "[INFO] Kerberos is enabled" +fi + +export AUDIT_CONSUMER_OPTS +export AUDIT_CONSUMER_LOG_DIR + +echo "[INFO] JAVA_HOME: ${JAVA_HOME}" +echo "[INFO] HEAP: ${AUDIT_CONSUMER_HEAP}" +echo "[INFO] JVM_OPTS: ${AUDIT_CONSUMER_OPTS}" + +# Find the WAR file +WAR_FILE=$(find "${AUDIT_CONSUMER_HOME_DIR}" -name "ranger-audit-consumer-hdfs*.war" | head -1) + +if [ -z "$WAR_FILE" ] || [ ! -f "$WAR_FILE" ]; then + echo "[ERROR] WAR file not found in ${AUDIT_CONSUMER_HOME_DIR}" + echo "[ERROR] Please build the project first using: mvn clean package" + exit 1 +fi + +echo "[INFO] Using WAR file: ${WAR_FILE}" + +# Extract WAR if not already extracted +WEBAPP_DIR="${AUDIT_CONSUMER_HOME_DIR}/webapp/ranger-audit-consumer-hdfs" +if [ ! -d "${WEBAPP_DIR}" ]; then + echo "[INFO] Extracting WAR file..." + mkdir -p "${WEBAPP_DIR}" + cd "${WEBAPP_DIR}" + jar xf "${WAR_FILE}" + cd - > /dev/null +fi + +# Build classpath +RANGER_CLASSPATH="${WEBAPP_DIR}/WEB-INF/classes" +for jar in "${WEBAPP_DIR}"/WEB-INF/lib/*.jar; do + RANGER_CLASSPATH="${RANGER_CLASSPATH}:${jar}" +done + +# Add libext directory if it exists +if [ -d "${AUDIT_CONSUMER_HOME_DIR}/libext" ]; then + for jar in "${AUDIT_CONSUMER_HOME_DIR}"/libext/*.jar; do + if [ -f "${jar}" ]; then + RANGER_CLASSPATH="${RANGER_CLASSPATH}:${jar}" + fi + done +fi + +export RANGER_CLASSPATH + +# Check if already running +PID_FILE="${AUDIT_CONSUMER_LOG_DIR}/ranger-audit-consumer-hdfs.pid" +if [ -f "${PID_FILE}" ]; then + OLD_PID=$(cat "${PID_FILE}") + if kill -0 "$OLD_PID" 2>/dev/null; then + echo "[WARNING] Ranger Audit Consumer - HDFS is already running (PID: ${OLD_PID})" + echo "[INFO] Use stop-consumer-hdfs.sh to stop it first" + exit 1 + else + echo "[INFO] Removing stale PID file" + rm -f "${PID_FILE}" + fi +fi + +# Start the service +echo "[INFO] Starting Ranger Audit Consumer - HDFS..." +nohup java ${AUDIT_CONSUMER_HEAP} ${AUDIT_CONSUMER_OPTS} \ + -Daudit.config=${AUDIT_CONSUMER_CONF_DIR}/ranger-audit-consumer-hdfs-site.xml \ + -Dhadoop.config.dir=${AUDIT_CONSUMER_CONF_DIR} \ + -Dranger.audit.consumer.webapp.dir="${WAR_FILE}" \ + -cp "${RANGER_CLASSPATH}" \ + org.apache.ranger.audit.consumer.HdfsConsumerApplication \ + >> "${AUDIT_CONSUMER_LOG_DIR}/catalina.out" 2>&1 & + +PID=$! +echo $PID > "${PID_FILE}" + +echo "[INFO] ✓ Ranger Audit Consumer - HDFS started successfully" +echo "[INFO] PID: ${PID}" +echo "[INFO] Log file: ${AUDIT_CONSUMER_LOG_DIR}/ranger-audit-consumer-hdfs.log" +echo "[INFO] Catalina out: ${AUDIT_CONSUMER_LOG_DIR}/catalina.out" +echo "[INFO] Health check: http://localhost:7092/api/health" +echo "" +echo "To monitor logs: tail -f ${AUDIT_CONSUMER_LOG_DIR}/ranger-audit-consumer-hdfs.log" +echo "To stop service: ${SCRIPT_DIR}/stop-consumer-hdfs.sh" diff --git a/ranger-audit-server/ranger-audit-consumer-hdfs/scripts/stop-consumer-hdfs.sh b/ranger-audit-server/ranger-audit-consumer-hdfs/scripts/stop-consumer-hdfs.sh new file mode 100755 index 0000000000..7e98af9b14 --- /dev/null +++ b/ranger-audit-server/ranger-audit-consumer-hdfs/scripts/stop-consumer-hdfs.sh @@ -0,0 +1,72 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# ======================================== +# Ranger Audit Consumer HDFS Stop Script +# ======================================== + +set -e + +# Script directory +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +SERVICE_DIR="$(dirname "$SCRIPT_DIR")" + +# Default log directory +AUDIT_CONSUMER_LOG_DIR="${AUDIT_CONSUMER_LOG_DIR:-${SERVICE_DIR}/logs}" +PID_FILE="${AUDIT_CONSUMER_LOG_DIR}/ranger-audit-consumer-hdfs.pid" + +echo "==========================================" +echo "Stopping Ranger Audit Consumer - HDFS" +echo "==========================================" + +if [ ! -f "${PID_FILE}" ]; then + echo "[WARNING] PID file not found: ${PID_FILE}" + echo "[INFO] Service may not be running" + exit 0 +fi + +PID=$(cat "${PID_FILE}") + +if ! kill -0 "$PID" 2>/dev/null; then + echo "[WARNING] Process ${PID} is not running" + echo "[INFO] Removing stale PID file" + rm -f "${PID_FILE}" + exit 0 +fi + +echo "[INFO] Stopping process ${PID}..." +kill "$PID" + +# Wait for process to stop (max 30 seconds) +TIMEOUT=30 +COUNT=0 +while kill -0 "$PID" 2>/dev/null && [ $COUNT -lt $TIMEOUT ]; do + sleep 1 + COUNT=$((COUNT + 1)) + echo -n "." +done +echo "" + +if kill -0 "$PID" 2>/dev/null; then + echo "[WARNING] Process did not stop gracefully, forcing shutdown..." + kill -9 "$PID" + sleep 2 +fi + +rm -f "${PID_FILE}" +echo "[INFO] ✓ Ranger Audit Consumer - HDFS stopped successfully" diff --git a/ranger-audit-server/ranger-audit-consumer-hdfs/src/main/java/org/apache/ranger/audit/consumer/HdfsConsumerApplication.java b/ranger-audit-server/ranger-audit-consumer-hdfs/src/main/java/org/apache/ranger/audit/consumer/HdfsConsumerApplication.java new file mode 100644 index 0000000000..51f391ab7e --- /dev/null +++ b/ranger-audit-server/ranger-audit-consumer-hdfs/src/main/java/org/apache/ranger/audit/consumer/HdfsConsumerApplication.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.audit.consumer; + +import org.apache.hadoop.conf.Configuration; +import org.apache.ranger.audit.server.EmbeddedServer; +import org.apache.ranger.audit.server.HdfsConsumerConfig; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Main application class for Audit Consumer HDFS Service. + * This service consumes audit events from Kafka and writes them to HDFS. + * It uses embedded Tomcat server for lifecycle management and health check endpoints. + */ +public class HdfsConsumerApplication { + private static final Logger LOG = LoggerFactory.getLogger(HdfsConsumerApplication.class); + private static final String APP_NAME = "ranger-audit-consumer-hdfs"; + private static final String CONFIG_PREFIX = "ranger.audit.consumer.hdfs.service."; + + private HdfsConsumerApplication() { + } + + public static void main(String[] args) { + LOG.info("=========================================================================="); + LOG.info("==> Starting Ranger Audit Consumer HDFS Service"); + LOG.info("=========================================================================="); + + try { + // Load configuration (includes core-site.xml, hdfs-site.xml, and ranger-audit-consumer-hdfs-site.xml) + Configuration config = HdfsConsumerConfig.getInstance(); + + EmbeddedServer server = new EmbeddedServer(config, APP_NAME, CONFIG_PREFIX); + server.start(); + + LOG.info("<== Ranger Audit Consumer HDFS Service Started Successfully"); + } catch (Exception e) { + LOG.error("<== Failed to start Ranger Audit Consumer HDFS Service", e); + System.exit(1); + } + } +} diff --git a/ranger-audit-server/ranger-audit-consumer-hdfs/src/main/java/org/apache/ranger/audit/consumer/HdfsConsumerManager.java b/ranger-audit-server/ranger-audit-consumer-hdfs/src/main/java/org/apache/ranger/audit/consumer/HdfsConsumerManager.java new file mode 100644 index 0000000000..643de22c7a --- /dev/null +++ b/ranger-audit-server/ranger-audit-consumer-hdfs/src/main/java/org/apache/ranger/audit/consumer/HdfsConsumerManager.java @@ -0,0 +1,205 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.audit.consumer; + +import org.apache.ranger.audit.consumer.kafka.AuditConsumer; +import org.apache.ranger.audit.consumer.kafka.AuditConsumerRegistry; +import org.apache.ranger.audit.provider.MiscUtil; +import org.apache.ranger.audit.server.AuditServerConstants; +import org.apache.ranger.audit.server.HdfsConsumerConfig; +import org.apache.ranger.audit.utils.AuditServerLogFormatter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; + +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; + +import java.util.ArrayList; +import java.util.List; +import java.util.Properties; + +/** + * Spring component that manages the lifecycle of HDFS consumer threads. + * This manager: + * - Initializes the consumer registry + * - Creates HDFS consumer instances + * - Starts consumer threads + * - Handles graceful shutdown + */ +@Component +public class HdfsConsumerManager { + private static final Logger LOG = LoggerFactory.getLogger(HdfsConsumerManager.class); + + private final AuditConsumerRegistry consumerRegistry = AuditConsumerRegistry.getInstance(); + private final List consumers = new ArrayList<>(); + private final List consumerThreads = new ArrayList<>(); + + @PostConstruct + public void init() { + LOG.info("==> HdfsConsumerManager.init()"); + + try { + HdfsConsumerConfig config = HdfsConsumerConfig.getInstance(); + Properties props = config.getProperties(); + + if (props == null) { + LOG.error("Configuration properties are null"); + throw new RuntimeException("Failed to load configuration"); + } + + // Initialize and register HDFS Consumer + initializeConsumerClasses(props, AuditServerConstants.PROP_KAFKA_PROP_PREFIX); + + // Create consumers from registry + List createdConsumers = consumerRegistry.createConsumers(props, AuditServerConstants.PROP_KAFKA_PROP_PREFIX); + consumers.addAll(createdConsumers); + + if (consumers.isEmpty()) { + LOG.warn("No consumers were created! Verify that xasecure.audit.destination.hdfs=true"); + } else { + LOG.info("Created {} HDFS consumer(s)", consumers.size()); + + // Start consumer threads + startConsumers(); + } + } catch (Exception e) { + LOG.error("Failed to initialize HdfsConsumerManager", e); + throw new RuntimeException("Failed to initialize HdfsConsumerManager", e); + } + + LOG.info("<== HdfsConsumerManager.init() - {} consumer thread(s) started", consumerThreads.size()); + } + + private void initializeConsumerClasses(Properties props, String propPrefix) { + LOG.info("==> HdfsConsumerManager.initializeConsumerClasses()"); + + String clsStr = MiscUtil.getStringProperty(props, propPrefix + "." + AuditServerConstants.PROP_CONSUMER_CLASSES, "org.apache.ranger.audit.consumer.kafka.AuditHDFSConsumer"); + + String[] hdfsConsumerClasses = clsStr.split(","); + + LOG.info("Initializing {} consumer class(es)", hdfsConsumerClasses.length); + + for (String hdfsConsumerClassName : hdfsConsumerClasses) { + hdfsConsumerClassName = hdfsConsumerClassName.trim(); + + if (hdfsConsumerClassName.isEmpty()) { + continue; + } + + try { + Class consumerClass = Class.forName(hdfsConsumerClassName); + LOG.info("Successfully initialized consumer class: {}", consumerClass.getName()); + } catch (ClassNotFoundException e) { + LOG.error("Consumer class not found: {}. Ensure the class is on the classpath.", hdfsConsumerClassName, e); + } catch (Exception e) { + LOG.error("Error initializing consumer class: {}", hdfsConsumerClassName, e); + } + } + + LOG.info("Registered consumer factories: {}", consumerRegistry.getRegisteredDestinationTypes()); + LOG.info("<== HdfsConsumerManager.initializeConsumerClasses()"); + } + + /** + * Start all consumer threads + */ + private void startConsumers() { + LOG.info("==> HdfsConsumerManager.startConsumers()"); + + logConsumerStartup(); + + for (AuditConsumer consumer : consumers) { + try { + String consumerName = consumer.getClass().getSimpleName(); + Thread consumerThread = new Thread(consumer, consumerName); + consumerThread.setDaemon(true); + consumerThread.start(); + consumerThreads.add(consumerThread); + + LOG.info("Started {} thread [Thread-ID: {}, Thread-Name: '{}']", + consumerName, consumerThread.getId(), consumerThread.getName()); + } catch (Exception e) { + LOG.error("Error starting consumer: {}", consumer.getClass().getSimpleName(), e); + } + } + + LOG.info("<== HdfsConsumerManager.startConsumers() - {} thread(s) started", consumerThreads.size()); + } + + private void logConsumerStartup() { + LOG.info("################## HDFS CONSUMER SERVICE STARTUP ######################"); + + if (consumers.isEmpty()) { + LOG.warn("WARNING: No HDFS consumers are enabled!"); + LOG.warn("Verify: xasecure.audit.destination.hdfs=true in configuration"); + } else { + AuditServerLogFormatter.LogBuilder builder = AuditServerLogFormatter.builder("HDFS Consumer Status"); + + for (AuditConsumer consumer : consumers) { + String consumerType = consumer.getClass().getSimpleName(); + builder.add(consumerType, "ENABLED"); + builder.add("Topic", consumer.getTopicName()); + } + + builder.logInfo(LOG); + LOG.info("Starting {} HDFS consumer thread(s)...", consumers.size()); + } + LOG.info("########################################################################"); + } + + @PreDestroy + public void shutdown() { + LOG.info("==> HdfsConsumerManager.shutdown()"); + + // Shutdown all consumers + for (AuditConsumer consumer : consumers) { + try { + LOG.info("Shutting down consumer: {}", consumer.getClass().getSimpleName()); + consumer.shutdown(); + LOG.info("Consumer shutdown completed: {}", consumer.getClass().getSimpleName()); + } catch (Exception e) { + LOG.error("Error shutting down consumer: {}", consumer.getClass().getSimpleName(), e); + } + } + + // Wait for threads to terminate + for (Thread thread : consumerThreads) { + if (thread.isAlive()) { + try { + LOG.info("Waiting for thread to terminate: {}", thread.getName()); + thread.join(10000); // Wait up to 10 seconds + if (thread.isAlive()) { + LOG.warn("Thread did not terminate within 10 seconds: {}", thread.getName()); + } + } catch (InterruptedException e) { + LOG.warn("Interrupted while waiting for thread to terminate: {}", thread.getName(), e); + Thread.currentThread().interrupt(); + } + } + } + + consumers.clear(); + consumerThreads.clear(); + consumerRegistry.clearActiveConsumers(); + + LOG.info("<== HdfsConsumerManager.shutdown() - All HDFS consumers stopped"); + } +} diff --git a/ranger-audit-server/ranger-audit-consumer-hdfs/src/main/java/org/apache/ranger/audit/consumer/kafka/AuditHDFSConsumer.java b/ranger-audit-server/ranger-audit-consumer-hdfs/src/main/java/org/apache/ranger/audit/consumer/kafka/AuditHDFSConsumer.java new file mode 100644 index 0000000000..a41d8e87bb --- /dev/null +++ b/ranger-audit-server/ranger-audit-consumer-hdfs/src/main/java/org/apache/ranger/audit/consumer/kafka/AuditHDFSConsumer.java @@ -0,0 +1,497 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ranger.audit.consumer.kafka; + +import org.apache.hadoop.security.SecureClientLogin; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.clients.consumer.ConsumerRecords; +import org.apache.kafka.clients.consumer.KafkaConsumer; +import org.apache.kafka.clients.consumer.OffsetAndMetadata; +import org.apache.kafka.common.TopicPartition; +import org.apache.ranger.audit.provider.AuditProviderFactory; +import org.apache.ranger.audit.provider.MiscUtil; +import org.apache.ranger.audit.server.AuditServerConstants; +import org.apache.ranger.audit.server.HdfsConsumerConfig; +import org.apache.ranger.audit.utils.AuditServerLogFormatter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; + +public class AuditHDFSConsumer extends AuditConsumerBase implements AuditConsumer { + private static final Logger LOG = LoggerFactory.getLogger(AuditHDFSConsumer.class); + private static final String RANGER_AUDIT_HDFS_CONSUMER_GROUP = AuditServerConstants.DEFAULT_RANGER_AUDIT_HDFS_CONSUMER_GROUP; + + // Register AuditHDFSConsumer factory in the audit consumer registry + static { + try { + AuditConsumerRegistry.getInstance().registerFactory(AuditServerConstants.PROP_HDFS_DEST_PREFIX, (props, propPrefix) -> new AuditHDFSConsumer(props, propPrefix)); + LOG.info("Registered HDFS consumer with AuditConsumerRegistry"); + } catch (Exception e) { + LOG.error("Failed to register HDFS consumer factory", e); + } + } + + private final AtomicBoolean running = new AtomicBoolean(false); + private ExecutorService consumerThreadPool; + private final Map consumerWorkers = new ConcurrentHashMap<>(); + private int consumerThreadCount = 1; + // Offset management configuration (batch or manual only supported) + private String offsetCommitStrategy = AuditServerConstants.DEFAULT_OFFSET_COMMIT_STRATEGY; + private long offsetCommitInterval = AuditServerConstants.DEFAULT_OFFSET_COMMIT_INTERVAL_MS; + // Message routing handler for HDFS destination + private AuditRouterHDFS auditRouterHDFS; + private Properties props; + private String propPrefix; + + public AuditHDFSConsumer(Properties props, String propPrefix) throws Exception { + super(props, propPrefix, RANGER_AUDIT_HDFS_CONSUMER_GROUP); + this.props = new Properties(); + this.props.putAll(props); + this.propPrefix = propPrefix; + init(props, propPrefix); + } + + @Override + public void init(Properties props, String propPrefix) throws Exception { + LOG.info("==> AuditHDFSConsumer.init(): AuditHDFSConsumer initializing with appId-based threading and offset management"); + + consumer = getKafkaConsumer(); + + // Initialize Ranger UGI for HDFS operations if Kerberos is enabled + initializeRangerUGI(); + + // Add Hadoop configuration properties from HdfsConsumerConfig to props + addHadoopConfigToProps(props); + + // Initialize consumer configuration + initConsumerConfig(props, propPrefix); + + // Initialize destination handler for message processing + auditRouterHDFS = new AuditRouterHDFS(); + auditRouterHDFS.init(props, AuditProviderFactory.AUDIT_DEST_BASE + "." + AuditServerConstants.PROP_HDFS_DEST_PREFIX); + + LOG.info("<== AuditHDFSConsumer.init(): AuditHDFSConsumer initialized successfully"); + } + + private void initializeRangerUGI() throws Exception { + LOG.info("==> AuditHDFSConsumer.initializeRangerUGI()"); + + try { + HdfsConsumerConfig auditConfig = HdfsConsumerConfig.getInstance(); + String authType = auditConfig.get(AuditServerConstants.PROP_HADOOP_AUTHENTICATION_TYPE, "simple"); + + if (!AuditServerConstants.PROP_HADOOP_AUTH_TYPE_KERBEROS.equalsIgnoreCase(authType)) { + LOG.info("Hadoop authentication is not Kerberos ({}), skipping Ranger UGI initialization", authType); + return; + } + + String principal = auditConfig.get(AuditServerConstants.AUDIT_SERVER_PROP_PREFIX + AuditServerConstants.PROP_AUDIT_SERVICE_PRINCIPAL); + String keytab = auditConfig.get(AuditServerConstants.AUDIT_SERVER_PROP_PREFIX + AuditServerConstants.PROP_AUDIT_SERVICE_KEYTAB); + String hostName = auditConfig.get(AuditServerConstants.AUDIT_SERVER_PROP_PREFIX + "host"); + + if (principal == null || keytab == null) { + LOG.warn("Kerberos is enabled but principal or keytab is null! principal={}, keytab={}", principal, keytab); + String msg = String.format("Kerberos is enabled but principal or keytab is null! principal={}, keytab={}", principal, keytab); + throw new Exception(msg); + } + + if (principal.contains("_HOST")) { + try { + principal = SecureClientLogin.getPrincipal(principal, hostName); + LOG.info("Resolved principal from [{}] using hostname [{}]", principal, hostName); + } catch (IOException e) { + LOG.error("Failed to resolve principal pattern [{}] with hostname [{}]", principal, hostName, e); + throw e; + } + } + + // Set Hadoop security configuration from core-site.xml + org.apache.hadoop.conf.Configuration coreSite = auditConfig.getCoreSiteConfiguration(); + UserGroupInformation.setConfiguration(coreSite); + + LOG.info("Initializing Ranger UGI for HDFS writes: principal={}, keytab={}", principal, keytab); + + UserGroupInformation rangerUGI = UserGroupInformation.loginUserFromKeytabAndReturnUGI(principal, keytab); + + MiscUtil.setUGILoginUser(rangerUGI, null); + + LOG.info("<== AuditHDFSConsumer.initializeRangerUGI(): Ranger UGI initialized successfully: user={}, auth={}, hasKerberos={}", + rangerUGI.getUserName(), rangerUGI.getAuthenticationMethod(), rangerUGI.hasKerberosCredentials()); + } catch (IOException e) { + LOG.error("Failed to initialize Ranger UGI for HDFS writes", e); + throw e; + } + } + + /** + * Add Hadoop configuration properties from core-site.xml and hdfs-site.xml to props. + */ + private void addHadoopConfigToProps(Properties props) { + LOG.info("==> AuditHDFSConsumer.addHadoopConfigToProps()"); + + try { + HdfsConsumerConfig hdfsConfig = HdfsConsumerConfig.getInstance(); + String configPrefix = "xasecure.audit.destination.hdfs.config."; + + Properties hadoopProps = hdfsConfig.getHadoopPropertiesWithPrefix(configPrefix); + props.putAll(hadoopProps); + + LOG.info("<== AuditHDFSConsumer.addHadoopConfigToProps(): Added {} Hadoop configuration properties from HdfsConsumerConfig", hadoopProps.size()); + } catch (Exception e) { + LOG.error("Failed to add Hadoop configuration properties to props", e); + } + } + + private void initConsumerConfig(Properties props, String propPrefix) { + LOG.info("==> AuditHDFSConsumer.initConsumerConfig()"); + + // Get consumer thread count + this.consumerThreadCount = MiscUtil.getIntProperty(props, propPrefix + "." + AuditServerConstants.PROP_CONSUMER_THREAD_COUNT, 1); + LOG.info("HDFS consumer thread count: {}", consumerThreadCount); + + // Initialize offset management configuration + initializeOffsetManagement(props, propPrefix); + + LOG.info("<== AuditHDFSConsumer.initConsumerConfig()"); + } + + private void initializeOffsetManagement(Properties props, String propPrefix) { + LOG.info("==> AuditHDFSConsumer.initializeOffsetManagement()"); + + // Get offset commit strategy + this.offsetCommitStrategy = MiscUtil.getStringProperty(props, + propPrefix + "." + AuditServerConstants.PROP_CONSUMER_OFFSET_COMMIT_STRATEGY, + AuditServerConstants.DEFAULT_OFFSET_COMMIT_STRATEGY); + + // Get offset commit interval (only used for manual strategy) + this.offsetCommitInterval = MiscUtil.getLongProperty(props, + propPrefix + "." + AuditServerConstants.PROP_CONSUMER_OFFSET_COMMIT_INTERVAL, + AuditServerConstants.DEFAULT_OFFSET_COMMIT_INTERVAL_MS); + + AuditServerLogFormatter.builder("HDFS Consumer Offset Management Configuration") + .add("Commit Strategy", offsetCommitStrategy) + .add("Commit Interval (ms)", offsetCommitInterval + " (used in manual mode only)") + .logInfo(LOG); + + LOG.info("<== AuditHDFSConsumer.initializeOffsetManagement()"); + } + + @Override + public void run() { + try { + if (auditRouterHDFS == null) { + init(this.props, this.propPrefix); + } + + LOG.info("Starting AuditHDFSConsumer with appId-based thread"); + startMultithreadedConsumption(); + + // Keep main thread alive while consumer threads are running + while (running.get()) { + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + LOG.info("HDFS consumer main thread interrupted"); + Thread.currentThread().interrupt(); + break; + } + } + } catch (Throwable e) { + LOG.error("Error in AuditHDFSConsumer", e); + } finally { + shutdown(); + } + } + + @Override + public KafkaConsumer getKafkaConsumer() { + return consumer; + } + + @Override + public void processMessage(String audit) throws Exception { + processMessage(audit, null); + } + + public void processMessage(String message, String partitionKey) throws Exception { + auditRouterHDFS.routeAuditMessage(message, partitionKey); + } + + @Override + public String getTopicName() { + return topicName; + } + + public String getConsumerGroupId() { + return consumerGroupId; + } + + private void startMultithreadedConsumption() { + LOG.debug("==> AuditHDFSConsumer.startMultithreadedConsumption()"); + + if (running.compareAndSet(false, true)) { + startConsumerWorkers(); + } + + LOG.debug("<== AuditHDFSConsumer.startMultithreadedConsumption()"); + } + + private void startConsumerWorkers() { + LOG.info("==> AuditHDFSConsumer.startConsumerWorkers(): Creating {} consumer workers for horizontal scaling", consumerThreadCount); + LOG.info("Each worker will subscribe to topic '{}' and process partitions assigned by Kafka", topicName); + + // Create thread pool sized for consumer workers + consumerThreadPool = Executors.newFixedThreadPool(consumerThreadCount); + LOG.info("Created thread pool with {} threads for scalable HDFS consumption", consumerThreadCount); + + // Create HDFS consumer workers + for (int i = 0; i < consumerThreadCount; i++) { + String workerId = "hdfs-worker-" + i; + ConsumerWorker worker = new ConsumerWorker(workerId, new ArrayList<>()); + consumerWorkers.put(workerId, worker); + consumerThreadPool.submit(worker); + + LOG.info("Started HDFS consumer worker '{}' - will process ANY appId assigned by Kafka", workerId); + } + + LOG.info("<== AuditHDFSConsumer.startConsumerWorkers(): All {} workers started in SUBSCRIBE mode", consumerThreadCount); + } + + private class ConsumerWorker implements Runnable { + private final String workerId; + private final List assignedPartitions; + private KafkaConsumer workerConsumer; + + // Offset management + private final Map pendingOffsets = new HashMap<>(); + private final AtomicLong lastCommitTime = new AtomicLong(System.currentTimeMillis()); + private final AtomicInteger messagesProcessedSinceLastCommit = new AtomicInteger(0); + + public ConsumerWorker(String workerId, List assignedPartitions) { + this.workerId = workerId; + this.assignedPartitions = assignedPartitions; + } + + @Override + public void run() { + try { + // Create consumer for this worker with offset management configuration + Properties workerConsumerProps = new Properties(); + workerConsumerProps.putAll(consumerProps); + + // Configure offset management based on strategy + configureOffsetManagement(workerConsumerProps); + + workerConsumer = new KafkaConsumer<>(workerConsumerProps); + + // Create re-balance listener + AuditConsumerRebalanceListener rebalanceListener = new AuditConsumerRebalanceListener( + workerId, + AuditServerConstants.DESTINATION_HDFS, + topicName, + offsetCommitStrategy, + consumerGroupId, + workerConsumer, + pendingOffsets, + messagesProcessedSinceLastCommit, + lastCommitTime, + assignedPartitions); + + // Subscribe to topic with re-balance listener + workerConsumer.subscribe(Collections.singletonList(topicName), rebalanceListener); + + LOG.info("[HDFS-CONSUMER] Worker '{}' subscribed successfully, waiting for partition assignment from Kafka", workerId); + long threadId = Thread.currentThread().getId(); + String threadName = Thread.currentThread().getName(); + LOG.info("[HDFS-CONSUMER-STARTUP] Worker '{}' [Thread-ID: {}, Thread-Name: '{}'] started | Topic: '{}' | Consumer-Group: {} | Mode: SUBSCRIBE", + workerId, threadId, threadName, topicName, consumerGroupId); + + // Consume messages + while (running.get()) { + ConsumerRecords records = workerConsumer.poll(Duration.ofMillis(100)); + + if (!records.isEmpty()) { + processRecordBatch(records); + // Handle offset committing based on strategy + handleOffsetCommitting(); + } + } + } catch (Exception e) { + LOG.error("Error in HDFS consumer worker '{}'", workerId, e); + } finally { + // Final offset commit before shutdown + commitPendingOffsets(true); + + if (workerConsumer != null) { + try { + LOG.info("HDFS Worker '{}': Unsubscribing from topic", workerId); + workerConsumer.unsubscribe(); + } catch (Exception e) { + LOG.warn("HDFS Worker '{}': Error during unsubscribe", workerId, e); + } + + try { + LOG.info("HDFS Worker '{}': Closing consumer", workerId); + workerConsumer.close(); + } catch (Exception e) { + LOG.error("Error closing consumer for HDFS worker '{}'", workerId, e); + } + } + LOG.info("HDFS consumer worker '{}' stopped", workerId); + } + } + + private void configureOffsetManagement(Properties consumerProps) { + // Always disable auto commit - only batch or manual strategies supported + consumerProps.put("enable.auto.commit", "false"); + LOG.debug("HDFS worker '{}' configured for manual offset commit with strategy: {}", workerId, offsetCommitStrategy); + } + + private void processRecordBatch(ConsumerRecords records) { + for (ConsumerRecord record : records) { + try { + LOG.debug("HDFS worker '{}' consumed: partition={}, key={}, offset={}", + workerId, record.partition(), record.key(), record.offset()); + + // Process the message using the destination handler + // The partition key (record.key()) contains the appId for HDFS path routing + processMessage(record.value(), record.key()); + + // Track offset for manual commit strategies + TopicPartition partition = new TopicPartition(record.topic(), record.partition()); + pendingOffsets.put(partition, new OffsetAndMetadata(record.offset() + 1)); + messagesProcessedSinceLastCommit.incrementAndGet(); + } catch (Exception e) { + LOG.error("Error processing message in HDFS worker '{}': partition={}, key={}, offset={}", + workerId, record.partition(), record.key(), record.offset(), e); + + // On error, track offset to prevent reprocessing + TopicPartition partition = new TopicPartition(record.topic(), record.partition()); + pendingOffsets.put(partition, new OffsetAndMetadata(record.offset())); + } + } + } + + private void handleOffsetCommitting() { + boolean shouldCommit = false; + long currentTime = System.currentTimeMillis(); + + if (AuditServerConstants.PROP_OFFSET_COMMIT_STRATEGY_BATCH.equals(offsetCommitStrategy)) { + // Commit after processing each batch + shouldCommit = !pendingOffsets.isEmpty(); + } else if (AuditServerConstants.PROP_OFFSET_COMMIT_STRATEGY_MANUAL.equals(offsetCommitStrategy)) { + // Commit based on time interval + shouldCommit = (currentTime - lastCommitTime.get()) >= offsetCommitInterval && !pendingOffsets.isEmpty(); + } + + if (shouldCommit) { + commitPendingOffsets(false); + } + } + + private void commitPendingOffsets(boolean isShutdown) { + if (pendingOffsets.isEmpty()) { + return; + } + + try { + workerConsumer.commitSync(pendingOffsets); + + LOG.debug("HDFS worker '{}' committed {} offsets, processed {} messages", + workerId, pendingOffsets.size(), messagesProcessedSinceLastCommit.get()); + + // Clear committed offsets + pendingOffsets.clear(); + lastCommitTime.set(System.currentTimeMillis()); + messagesProcessedSinceLastCommit.set(0); + } catch (Exception e) { + LOG.error("Error committing offsets in HDFS worker '{}': {}", workerId, pendingOffsets, e); + + if (isShutdown) { + // During shutdown, retry to avoid loss of any offsets + try { + Thread.sleep(1000); + workerConsumer.commitSync(pendingOffsets); + LOG.info("Successfully committed offsets on retry during shutdown for HDFS worker '{}'", workerId); + } catch (Exception retryException) { + LOG.error("Failed to commit offsets even on retry during shutdown for HDFS worker '{}'", workerId, retryException); + } + } + } + } + } + + @Override + public void shutdown() { + LOG.info("==> AuditHDFSConsumer.shutdown()"); + + // Stop consumer threads + running.set(false); + + // Shutdown consumer workers + if (consumerThreadPool != null) { + consumerThreadPool.shutdownNow(); + try { + if (!consumerThreadPool.awaitTermination(30, java.util.concurrent.TimeUnit.SECONDS)) { + LOG.warn("HDFS consumer thread pool did not terminate within 30 seconds"); + } + } catch (InterruptedException e) { + LOG.warn("Interrupted while waiting for HDFS consumer thread pool to terminate", e); + Thread.currentThread().interrupt(); + } + } + + consumerWorkers.clear(); + + // Shutdown destination handler + if (auditRouterHDFS != null) { + try { + auditRouterHDFS.shutdown(); + } catch (Exception e) { + LOG.error("Error shutting down HDFS destination handler", e); + } + } + + // Close main Kafka consumer + if (consumer != null) { + try { + consumer.close(); + } catch (Exception e) { + LOG.error("Error closing main Kafka consumer", e); + } + } + + LOG.info("<== AuditHDFSConsumer.shutdown() complete"); + } +} diff --git a/ranger-audit-server/ranger-audit-consumer-hdfs/src/main/java/org/apache/ranger/audit/consumer/kafka/AuditRouterHDFS.java b/ranger-audit-server/ranger-audit-consumer-hdfs/src/main/java/org/apache/ranger/audit/consumer/kafka/AuditRouterHDFS.java new file mode 100644 index 0000000000..49080aacb5 --- /dev/null +++ b/ranger-audit-server/ranger-audit-consumer-hdfs/src/main/java/org/apache/ranger/audit/consumer/kafka/AuditRouterHDFS.java @@ -0,0 +1,354 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.audit.consumer.kafka; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.ranger.audit.destination.HDFSAuditDestination; +import org.apache.ranger.audit.provider.MiscUtil; +import org.apache.ranger.audit.utils.AuditServerLogFormatter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Collections; +import java.util.Map; +import java.util.Properties; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +/** + * Router class that routes audit messages to different HDFSAuditDestination threads + * based on the app_id. + * Each app_id gets its own HDFSAuditDestination instance running in a separate thread. + * + * This router writes audits to HDFS as the rangerauditserver user. The audit folder in HDFS + * should be configured with appropriate permissions to allow rangerauditserver to write audits + * either by Ranger policy or by HDFS acl. + */ +public class AuditRouterHDFS { + private static final Logger LOG = LoggerFactory.getLogger(AuditRouterHDFS.class); + + private final Map destinationMap = new ConcurrentHashMap<>(); + private final Map executorMap = new ConcurrentHashMap<>(); + private final ObjectMapper jsonMapper = new ObjectMapper(); + private Properties props; + private String hdfsPropPrefix; + + public AuditRouterHDFS() { + } + + public void init(Properties props, String hdfsPropPrefix) { + LOG.info("==> AuditRouterHDFS.init()"); + + this.props = new Properties(); + this.props.putAll(props); + this.hdfsPropPrefix = hdfsPropPrefix; + + LOG.info("<== AuditRouterHDFS.init()"); + } + + /** + * Routes audit message to appropriate HDFSAuditDestination based on app_id + * @param message JSON audit message + * @param partitionKey The partition key from Kafka (used as app_id) + */ + public void routeAuditMessage(String message, String partitionKey) { + LOG.debug("==> AuditRouterHDFS:routeAuditMessage(): Message => {}, partitionKey => {}", message, partitionKey); + + try { + String appId = extractAppId(message, partitionKey); + String serviceType = extractServiceType(message); + String agentHostname = extractAgentHostname(message); + + LOG.debug("Routing audit message for app_id: {}, serviceType: {}, agentHostname: {}", appId, serviceType, agentHostname); + + // Get or create HDFSAuditDestination for the appId + if (appId != null) { + HDFSAuditDestination hdfsAuditDestination = getHdfsAuditDestination(appId, serviceType, agentHostname); + + // Submit message to destination's thread for processing + final String finalAppId = appId; + ExecutorService executor = executorMap.get(appId); + + if (executor != null && !executor.isShutdown()) { + executor.submit(() -> { + try { + hdfsAuditDestination.logJSON(Collections.singletonList(message)); + LOG.debug("Successfully wrote audit for app_id: {}", finalAppId); + } catch (Exception e) { + LOG.error("Error processing audit message for app_id: {}", finalAppId, e); + } + }); + } else { + LOG.warn("Executor is null or shutdown for app_id: {}", appId); + } + } + } catch (Exception e) { + LOG.error("AuditRouterHDFS:routeAuditMessage(): Error routing audit message: {}", message, e); + } + + LOG.debug("<== AuditRouterHDFS:routeAuditMessage()"); + } + + /** + * Extract app_id from audit message. First tries partition key, then falls back to parsing JSON. + */ + private String extractAppId(String message, String partitionKey) { + // First try to use partition key as app_id if available + if (partitionKey != null && !partitionKey.trim().isEmpty()) { + return partitionKey; + } + // Fall back to extracting from JSON message + try { + JsonNode rootNode = jsonMapper.readTree(message); + JsonNode agentIdNode = rootNode.get("agent"); + if (agentIdNode != null) { + return agentIdNode.asText(); + } + } catch (Exception e) { + LOG.debug("AuditRouterHDFS:extractAppId(): Failed to parse JSON message for app_id extraction: {}", e.getMessage()); + } + return null; + } + + /** + * Extract serviceType from audit message additional_info + */ + private String extractServiceType(String message) { + try { + JsonNode rootNode = jsonMapper.readTree(message); + JsonNode additionalInfoNode = rootNode.get("additional_info"); + JsonNode additionalInfoJson = jsonMapper.readTree(additionalInfoNode.asText()); + JsonNode serviceTypeNode = additionalInfoJson.get("serviceType"); + if (serviceTypeNode != null) { + return serviceTypeNode.asText(); + } + } catch (Exception e) { + LOG.debug("AuditRouterHDFS:extractServiceType(): Failed to parse JSON message for serviceType extraction: {}", e.getMessage()); + } + return null; + } + + /** + * Extract agentHostname from audit message + */ + private String extractAgentHostname(String message) { + try { + JsonNode rootNode = jsonMapper.readTree(message); + JsonNode agentHostNode = rootNode.get("agentHost"); + if (agentHostNode != null) { + return agentHostNode.asText(); + } + } catch (Exception e) { + LOG.debug("AuditRouterHDFS:extractAgentHostname(): Failed to parse JSON message for agentHostname extraction: {}", e.getMessage()); + } + return null; + } + + /** + * Get or create HDFSAuditDestination for the given app_id + */ + private synchronized HDFSAuditDestination getHdfsAuditDestination(String appId, String serviceType, String agentHostname) { + LOG.debug("==> AuditRouterHDFS:getHdfsAuditDestination() for app_id: {}, serviceType: {} and agentHostname: {}", appId, serviceType, agentHostname); + HDFSAuditDestination destination = destinationMap.get(appId); + if (destination == null) { + try { + destination = createHDFSDestination(appId, serviceType, agentHostname); + destinationMap.put(appId, destination); + + // Create dedicated executor for this destination + ExecutorService executor = Executors.newSingleThreadExecutor(r -> { + Thread t = new Thread(r, "HDFSAuditDestination-" + appId); + t.setDaemon(true); + return t; + }); + executorMap.put(appId, executor); + } catch (Exception e) { + LOG.error("Failed to create HDFSAuditDestination for app_id: {}", appId, e); + throw new RuntimeException("AuditRouterHDFS:getOrCreateDestination() : Failed to create destination for app_id: " + appId, e); + } + } + + LOG.debug("<== AuditRouterHDFS:getHdfsAuditDestination()..got HDFSAuditDestination for app_id: {}", appId); + + return destination; + } + + /** + * Create and initialize HDFSAuditDestination with app_id specific configuration + */ + private HDFSAuditDestination createHDFSDestination(String appId, String serviceType, String agentHostname) throws Exception { + LOG.debug("==> AuditRouterHDFS:createHDFSDestination(): Creating new HDFSAuditDestination for app_id: {}, serviceType: {}, agentHostname: {}", appId, serviceType, agentHostname); + + HDFSAuditDestination destination = new HDFSAuditDestination(); + + // Create app_id specific properties + Properties appSpecificProps = new Properties(); + appSpecificProps.putAll(props); + + // Determine file type and writer implementation + String fileType = MiscUtil.getStringProperty(props, hdfsPropPrefix + ".batch.filequeue.filetype", "json"); + String writerImpl = getWriterImplementation(fileType); + + // Configure directory properties + String baseDir = MiscUtil.getStringProperty(props, hdfsPropPrefix + ".dir", "/ranger/audit/" + serviceType); + String subDir = MiscUtil.getStringProperty(props, hdfsPropPrefix + ".subdir", appId + "/%time:yyyyMMdd%/"); + + // Set file extension based on file type + String fileExtension = getFileExtension(fileType); + // Get unique instance identifier for filename uniqueness across scaled audit services + String instanceId = getUniqueInstanceIdentifier(); + + // Use agentHostname from audit message if available, otherwise fall back to local hostname token + String hostnameValue = (agentHostname != null && !agentHostname.isEmpty()) ? agentHostname : "%hostname%"; + + // Build default filename with agent hostname to properly identify the source system + String defaultFileName = appId + "_ranger_audit_" + hostnameValue + "_" + instanceId + fileExtension; + String fileNameFormat = MiscUtil.getStringProperty(props, hdfsPropPrefix + ".filename.format", defaultFileName); + + // If filename format contains %hostname% and we have agentHostname, replace it + if (agentHostname != null && !agentHostname.isEmpty() && fileNameFormat.contains("%hostname%")) { + fileNameFormat = fileNameFormat.replace("%hostname%", agentHostname); + } + + // Set the enhanced properties + appSpecificProps.setProperty(hdfsPropPrefix + ".dir", baseDir); + appSpecificProps.setProperty(hdfsPropPrefix + ".subdir", subDir); + appSpecificProps.setProperty(hdfsPropPrefix + ".filename.format", fileNameFormat); + appSpecificProps.setProperty(hdfsPropPrefix + ".filewriter.impl", writerImpl); + appSpecificProps.setProperty(hdfsPropPrefix + ".batch.filequeue.filetype", fileType); + // Preserve other properties + preserveFileSystemProperties(appSpecificProps, hdfsPropPrefix); + + // Log configuration + AuditServerLogFormatter.builder("Initializing HDFSAuditDestination for app_id: " + appId) + .add("Base directory", baseDir) + .add("Subdirectory pattern", subDir) + .add("Filename format", fileNameFormat) + .add("Agent hostname", agentHostname) + .add("Instance identifier", instanceId) + .add("File type", fileType) + .add("Writer implementation", writerImpl) + .logInfo(LOG); + + destination.init(appSpecificProps, hdfsPropPrefix); + destination.start(); + + LOG.debug("<== AuditRouterHDFS:createHDFSDestination(): Created new HDFSDestination {}", destination.getName()); + + return destination; + } + + private String getWriterImplementation(String fileType) { + switch (fileType.toLowerCase()) { + case "orc": + return "org.apache.ranger.audit.utils.RangerORCAuditWriter"; + case "json": + default: + return "org.apache.ranger.audit.utils.RangerJSONAuditWriter"; + } + } + + private String getFileExtension(String fileType) { + switch (fileType.toLowerCase()) { + case "orc": + return ".orc"; + case "json": + default: + return ".log"; + } + } + + private void preserveFileSystemProperties(Properties appSpecificProps, String baseDestPrefix) { + String fileRolloverSec = MiscUtil.getStringProperty(props, baseDestPrefix + ".file.rollover.sec"); + if (fileRolloverSec != null) { + appSpecificProps.setProperty(baseDestPrefix + ".file.rollover.sec", fileRolloverSec); + } + + String rolloverPeriod = MiscUtil.getStringProperty(props, baseDestPrefix + ".file.rollover.period"); + if (rolloverPeriod != null) { + appSpecificProps.setProperty(baseDestPrefix + ".file.rollover.period", rolloverPeriod); + } + + String appendEnabled = MiscUtil.getStringProperty(props, baseDestPrefix + ".file.append.enabled"); + if (appendEnabled != null) { + appSpecificProps.setProperty(baseDestPrefix + ".file.append.enabled", appendEnabled); + } + + String periodicRolloverEnabled = MiscUtil.getStringProperty(props, baseDestPrefix + ".file.rollover.enable.periodic.rollover"); + if (periodicRolloverEnabled != null) { + appSpecificProps.setProperty(baseDestPrefix + ".file.rollover.enable.periodic.rollover", periodicRolloverEnabled); + } + + String periodicRolloverCheckTime = MiscUtil.getStringProperty(props, baseDestPrefix + ".file.rollover.periodic.rollover.check.sec"); + if (periodicRolloverCheckTime != null) { + appSpecificProps.setProperty(baseDestPrefix + ".file.rollover.periodic.rollover.check.sec", periodicRolloverCheckTime); + } + } + + private String getUniqueInstanceIdentifier() { + String jvmInstanceId = MiscUtil.getJvmInstanceId(); + LOG.info("Using JVM instance ID as unique identifier: {}", jvmInstanceId); + return jvmInstanceId; + } + + public void shutdown() { + LOG.info("==> AuditRouterHDFS.shutdown()"); + + // Shutdown all executors + for (Map.Entry entry : executorMap.entrySet()) { + String appId = entry.getKey(); + ExecutorService executor = entry.getValue(); + + LOG.info("Shutting down executor for app_id: {}", appId); + executor.shutdown(); + + try { + if (!executor.awaitTermination(30, java.util.concurrent.TimeUnit.SECONDS)) { + LOG.warn("Executor for app_id {} did not terminate gracefully, forcing shutdown", appId); + executor.shutdownNow(); + } + } catch (InterruptedException e) { + LOG.warn("Interrupted while waiting for executor shutdown for app_id: {}", appId); + executor.shutdownNow(); + Thread.currentThread().interrupt(); + } + } + + // Stop all destinations + for (Map.Entry entry : destinationMap.entrySet()) { + String appId = entry.getKey(); + HDFSAuditDestination destination = entry.getValue(); + + LOG.info("Stopping HDFSAuditDestination for app_id: {}", appId); + try { + destination.stop(); + } catch (Exception e) { + LOG.error("Error stopping HDFSAuditDestination for app_id: {}", appId, e); + } + } + + destinationMap.clear(); + executorMap.clear(); + + LOG.info("<== AuditRouterHDFS.shutdown()"); + } +} diff --git a/ranger-audit-server/ranger-audit-consumer-hdfs/src/main/java/org/apache/ranger/audit/rest/HealthCheckREST.java b/ranger-audit-server/ranger-audit-consumer-hdfs/src/main/java/org/apache/ranger/audit/rest/HealthCheckREST.java new file mode 100644 index 0000000000..ba1303582c --- /dev/null +++ b/ranger-audit-server/ranger-audit-consumer-hdfs/src/main/java/org/apache/ranger/audit/rest/HealthCheckREST.java @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.audit.rest; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.ranger.audit.consumer.HdfsConsumerManager; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Scope; +import org.springframework.stereotype.Component; + +import javax.ws.rs.GET; +import javax.ws.rs.Path; +import javax.ws.rs.Produces; +import javax.ws.rs.core.Response; + +import java.util.HashMap; +import java.util.Map; + +/** + * Health check REST endpoint for HDFS Consumer Service + */ +@Path("/health") +@Component +@Scope("request") +public class HealthCheckREST { + private static final Logger LOG = LoggerFactory.getLogger(HealthCheckREST.class); + + @Autowired(required = false) + HdfsConsumerManager hdfsConsumerManager; + + /** + * Health check endpoint + */ + @GET + @Produces("application/json") + public Response healthCheck() { + LOG.debug("==> HealthCheckREST.healthCheck()"); + Response ret; + String jsonString; + + try { + // Check if consumer manager is available and healthy + if (hdfsConsumerManager != null) { + Map resp = new HashMap<>(); + resp.put("status", "UP"); + resp.put("service", "audit-consumer-hdfs"); + resp.put("timestamp", System.currentTimeMillis()); + jsonString = buildResponse(resp); + ret = Response.ok() + .entity(jsonString) + .build(); + } else { + Map resp = new HashMap<>(); + resp.put("status", "DOWN"); + resp.put("service", "audit-consumer-hdfs"); + resp.put("reason", "HdfsConsumerManager not available"); + resp.put("timestamp", System.currentTimeMillis()); + jsonString = buildResponse(resp); + ret = Response.status(Response.Status.SERVICE_UNAVAILABLE) + .entity(jsonString) + .build(); + } + } catch (Exception e) { + LOG.error("Health check failed", e); + Map resp = new HashMap<>(); + resp.put("status", "DOWN"); + resp.put("service", "audit-consumer-hdfs"); + resp.put("reason", e.getMessage()); + resp.put("timestamp", System.currentTimeMillis()); + jsonString = buildResponse(resp); + ret = Response.status(Response.Status.SERVICE_UNAVAILABLE) + .entity(jsonString) + .build(); + } + + LOG.debug("<== HealthCheckREST.healthCheck(): {}", ret); + + return ret; + } + + private String buildResponse(Map respMap) { + String ret; + + try { + ObjectMapper objectMapper = new ObjectMapper(); + ret = objectMapper.writeValueAsString(respMap); + } catch (Exception e) { + ret = "Error: " + e.getMessage(); + } + + return ret; + } +} diff --git a/ranger-audit-server/ranger-audit-consumer-hdfs/src/main/java/org/apache/ranger/audit/server/HdfsConsumerConfig.java b/ranger-audit-server/ranger-audit-consumer-hdfs/src/main/java/org/apache/ranger/audit/server/HdfsConsumerConfig.java new file mode 100644 index 0000000000..6c38a3ac95 --- /dev/null +++ b/ranger-audit-server/ranger-audit-consumer-hdfs/src/main/java/org/apache/ranger/audit/server/HdfsConsumerConfig.java @@ -0,0 +1,146 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.audit.server; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Loads HDFS consumer-specific configuration files including Hadoop configs. + */ +public class HdfsConsumerConfig extends AuditConfig { + private static final Logger LOG = LoggerFactory.getLogger(HdfsConsumerConfig.class); + private static final String CONFIG_FILE_PATH = "conf/ranger-audit-consumer-hdfs-site.xml"; + private static final String CORE_SITE_FILE_PATH = "conf/core-site.xml"; + private static final String HDFS_SITE_FILE_PATH = "conf/hdfs-site.xml"; + private static volatile HdfsConsumerConfig sInstance; + + private HdfsConsumerConfig() { + super(); + addHdfsConsumerResources(); + } + + public static HdfsConsumerConfig getInstance() { + HdfsConsumerConfig ret = HdfsConsumerConfig.sInstance; + + if (ret == null) { + synchronized (HdfsConsumerConfig.class) { + ret = HdfsConsumerConfig.sInstance; + + if (ret == null) { + ret = new HdfsConsumerConfig(); + HdfsConsumerConfig.sInstance = ret; + } + } + } + + return ret; + } + + private boolean addHdfsConsumerResources() { + LOG.debug("==> HdfsConsumerConfig.addHdfsConsumerResources()"); + + boolean ret = true; + + if (!addAuditResource(CORE_SITE_FILE_PATH, false)) { + LOG.warn("Could not load required configuration: {}", CORE_SITE_FILE_PATH); + ret = false; + } + + if (!addAuditResource(HDFS_SITE_FILE_PATH, true)) { + LOG.error("Could not load required configuration: {}", HDFS_SITE_FILE_PATH); + ret = false; + } + + if (!addAuditResource(CONFIG_FILE_PATH, true)) { + LOG.error("Could not load required configuration: {}", CONFIG_FILE_PATH); + ret = false; + } + + LOG.debug("<== HdfsConsumerConfig.addHdfsConsumerResources(), result={}", ret); + + return ret; + } + + /** + * Get Hadoop configuration properties (from core-site.xml and hdfs-site.xml) with a specific prefix. + * @param prefix The prefix to add to each property name ("xasecure.audit.destination.hdfs.config.") + * @return Properties from core-site.xml and hdfs-site.xml with the specified prefix + */ + public java.util.Properties getHadoopPropertiesWithPrefix(String prefix) { + LOG.debug("==> HdfsConsumerConfig.getHadoopPropertiesWithPrefix(prefix={})", prefix); + + java.util.Properties prefixedProps = new java.util.Properties(); + int propsAdded = 0; + + try { + // Load core-site.xml separately to get pure Hadoop security properties + org.apache.hadoop.conf.Configuration coreSite = new org.apache.hadoop.conf.Configuration(false); + coreSite.addResource(CORE_SITE_FILE_PATH); + + for (java.util.Map.Entry entry : coreSite) { + String propName = entry.getKey(); + String propValue = entry.getValue(); + + if (propValue != null && !propValue.trim().isEmpty()) { + prefixedProps.setProperty(prefix + propName, propValue); + LOG.trace("Added from core-site.xml: {} = {}", propName, propValue); + propsAdded++; + } + } + + // Load hdfs-site.xml separately to get pure HDFS client properties + org.apache.hadoop.conf.Configuration hdfsSite = new org.apache.hadoop.conf.Configuration(false); + hdfsSite.addResource(HDFS_SITE_FILE_PATH); + + for (java.util.Map.Entry entry : hdfsSite) { + String propName = entry.getKey(); + String propValue = entry.getValue(); + + if (propValue != null && !propValue.trim().isEmpty()) { + prefixedProps.setProperty(prefix + propName, propValue); + LOG.trace("Added from hdfs-site.xml: {} = {}", propName, propValue); + propsAdded++; + } + } + + LOG.debug("<== HdfsConsumerConfig.getHadoopPropertiesWithPrefix(): Added {} Hadoop properties with prefix '{}'", propsAdded, prefix); + } catch (Exception e) { + LOG.error("Failed to load Hadoop properties from {} and {}", CORE_SITE_FILE_PATH, HDFS_SITE_FILE_PATH, e); + } + + return prefixedProps; + } + + /** + * Get core-site.xml Configuration for UGI initialization. + * @return Configuration loaded from core-site.xml + */ + public org.apache.hadoop.conf.Configuration getCoreSiteConfiguration() { + LOG.debug("==> HdfsConsumerConfig.getCoreSiteConfiguration()"); + + org.apache.hadoop.conf.Configuration coreSite = new org.apache.hadoop.conf.Configuration(false); + coreSite.addResource(CORE_SITE_FILE_PATH); + + LOG.debug("<== HdfsConsumerConfig.getCoreSiteConfiguration(): authentication={}", coreSite.get("hadoop.security.authentication")); + + return coreSite; + } +} diff --git a/ranger-audit-server/ranger-audit-consumer-hdfs/src/main/resources/conf/core-site.xml b/ranger-audit-server/ranger-audit-consumer-hdfs/src/main/resources/conf/core-site.xml new file mode 100644 index 0000000000..3e8bc00a16 --- /dev/null +++ b/ranger-audit-server/ranger-audit-consumer-hdfs/src/main/resources/conf/core-site.xml @@ -0,0 +1,39 @@ + + + + + + hadoop.security.authentication + kerberos + Authentication method for Hadoop services: simple or kerberos + + + + hadoop.security.auth_to_local + + RULE:[1:$1@$0](rangerauditserver@EXAMPLE\.COM)s/.*/rangerauditserver/ + RULE:[1:$1/$2@$0](rangerauditserver/ranger-audit-consumer-hdfs.rangernw@EXAMPLE\.COM)s/.*/rangerauditserver/ + DEFAULT + + + Kerberos principal to local user name mapping rules for Ranger Audit HDFS Consumer. + Note: rangerauditserver/hostname@REALM and rangerauditserver@REALM both map to 'ranger' OS user + + + diff --git a/ranger-audit-server/ranger-audit-consumer-hdfs/src/main/resources/conf/hdfs-site.xml b/ranger-audit-server/ranger-audit-consumer-hdfs/src/main/resources/conf/hdfs-site.xml new file mode 100644 index 0000000000..26e604dcfe --- /dev/null +++ b/ranger-audit-server/ranger-audit-consumer-hdfs/src/main/resources/conf/hdfs-site.xml @@ -0,0 +1,52 @@ + + + + + + + + dfs.namenode.kerberos.principal + nn/_HOST@EXAMPLE.COM + + HDFS NameNode Kerberos principal. Required for Kerberos RPC authentication to NameNode. + Common formats: + - hdfs/_HOST@REALM + - nn/_HOST@REALM + - hdfs/namenode.fqdn@REALM (fully qualified, for specific NameNode) + + + + + + dfs.client.use.datanode.hostname + false + + Whether clients should use the datanode hostname when connecting. + Set to true if using hostname-based Kerberos principals. + + + + + dfs.client.failover.proxy.provider + org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider + + Failover proxy provider for HA NameNode clusters. + Adjust if your cluster uses a different HA configuration. + + + diff --git a/ranger-audit-server/ranger-audit-consumer-hdfs/src/main/resources/conf/logback.xml b/ranger-audit-server/ranger-audit-consumer-hdfs/src/main/resources/conf/logback.xml new file mode 100644 index 0000000000..e8a08e8ea8 --- /dev/null +++ b/ranger-audit-server/ranger-audit-consumer-hdfs/src/main/resources/conf/logback.xml @@ -0,0 +1,71 @@ + + + + + + + + + + + %d{yyyy-MM-dd HH:mm:ss.SSS} %-5p - %m%n + + + + + + ${LOG_DIR}/${LOG_FILE} + + %d %p %c{1} [%t] %m%n + + + ${LOG_DIR}/${LOG_FILE}-%d{MM-dd-yyyy}-%i.log.gz + + 200MB + + 10 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/ranger-audit-server/ranger-audit-consumer-hdfs/src/main/resources/conf/ranger-audit-consumer-hdfs-site.xml b/ranger-audit-server/ranger-audit-consumer-hdfs/src/main/resources/conf/ranger-audit-consumer-hdfs-site.xml new file mode 100644 index 0000000000..0cfb6688bc --- /dev/null +++ b/ranger-audit-server/ranger-audit-consumer-hdfs/src/main/resources/conf/ranger-audit-consumer-hdfs-site.xml @@ -0,0 +1,156 @@ + + + + + + log.dir + ${audit.consumer.hdfs.log.dir} + Log directory for HDFS consumer service + + + + ranger.audit.consumer.hdfs.service.webapp.dir + webapp/ranger-audit-consumer-hdfs + Path to the extracted webapp directory + + + + ranger.audit.consumer.hdfs.service.contextName + / + + + + ranger.audit.consumer.hdfs.service.host + ranger-audit-consumer-hdfs.rangernw + Service hostname + + + + ranger.audit.consumer.hdfs.service.http.port + 7092 + Health check endpoint port (different from Solr: 7091) + + + + + ranger.audit.service.authentication.method + KERBEROS + + + + ranger.audit.service.host + ranger-audit-consumer-hdfs.rangernw + Hostname for resolving _HOST in Kerberos principal + + + + ranger.audit.service.kerberos.principal + rangerauditserver/_HOST@EXAMPLE.COM + Principal for Kafka consumer and HDFS writes + + + + ranger.audit.service.kerberos.keytab + /etc/keytabs/rangerauditserver.keytab + + + + + xasecure.audit.destination.kafka.bootstrap.servers + ranger-kafka:9092 + + + + xasecure.audit.destination.kafka.topic.name + ranger_audits + + + + xasecure.audit.destination.kafka.security.protocol + SASL_PLAINTEXT + + + + xasecure.audit.destination.kafka.sasl.mechanism + GSSAPI + + + + + xasecure.audit.destination.kafka.consumer.thread.count + 3 + Number of HDFS consumer worker threads + + + + + xasecure.audit.destination.kafka.consumer.offset.commit.strategy + batch + batch or manual + + + + xasecure.audit.destination.kafka.consumer.offset.commit.interval.ms + 30000 + Used only if strategy is 'manual' + + + + xasecure.audit.destination.kafka.consumer.max.poll.records + 500 + Maximum records per poll + + + + + xasecure.audit.destination.kafka.consumer.classes + org.apache.ranger.audit.consumer.kafka.AuditHDFSConsumer + + + + + xasecure.audit.destination.hdfs + true + MUST be true for HDFS consumer to work + + + + xasecure.audit.destination.hdfs.dir + hdfs://ranger-hadoop:9000/ranger/audit + + Namenode host for HDFS audit destination. + example: hdfs://hdfs-namenode:8020/ranger/audit + In Docker, use the service name of the namenode: hdfs://ranger-hadoop:9000/ranger/audit + + + + + xasecure.audit.destination.hdfs.batch.filespool.dir + /var/log/audit-consumer-hdfs/spool + + + + + xasecure.audit.destination.hdfs.batch.filequeue.filetype + json + File format: json or orc + + + + xasecure.audit.destination.hdfs.file.rollover.sec + 86400 + File rollover time in seconds (default: 24 hours) + + diff --git a/ranger-audit-server/ranger-audit-consumer-hdfs/src/main/webapp/WEB-INF/applicationContext.xml b/ranger-audit-server/ranger-audit-consumer-hdfs/src/main/webapp/WEB-INF/applicationContext.xml new file mode 100644 index 0000000000..5a7696647f --- /dev/null +++ b/ranger-audit-server/ranger-audit-consumer-hdfs/src/main/webapp/WEB-INF/applicationContext.xml @@ -0,0 +1,39 @@ + + + + + + + + + + + + + + + + + + diff --git a/ranger-audit-server/ranger-audit-consumer-hdfs/src/main/webapp/WEB-INF/web.xml b/ranger-audit-server/ranger-audit-consumer-hdfs/src/main/webapp/WEB-INF/web.xml new file mode 100644 index 0000000000..995d91281a --- /dev/null +++ b/ranger-audit-server/ranger-audit-consumer-hdfs/src/main/webapp/WEB-INF/web.xml @@ -0,0 +1,60 @@ + + + + + Apache Ranger - Audit Consumer HDFS Service + Apache Ranger - Audit Consumer HDFS Service + + + contextConfigLocation + + WEB-INF/applicationContext.xml + + + + + org.springframework.web.context.ContextLoaderListener + + + + org.springframework.web.context.request.RequestContextListener + + + + Health Check Service + com.sun.jersey.spi.spring.container.servlet.SpringServlet + + + com.sun.jersey.config.property.packages + org.apache.ranger.audit.rest + + + + com.sun.jersey.api.json.POJOMappingFeature + true + + + 1 + + + + Health Check Service + /api/* + + + diff --git a/ranger-audit-server/ranger-audit-consumer-solr/pom.xml b/ranger-audit-server/ranger-audit-consumer-solr/pom.xml new file mode 100644 index 0000000000..1e5db3f1ab --- /dev/null +++ b/ranger-audit-server/ranger-audit-consumer-solr/pom.xml @@ -0,0 +1,336 @@ + + + + 4.0.0 + + + org.apache.ranger + ranger-audit-server + 3.0.0-SNAPSHOT + .. + + + ranger-audit-consumer-solr + war + Ranger Audit Consumer Solr + Kafka consumer service for indexing audits into Solr/OpenSearch + + + UTF-8 + + + + + + ch.qos.logback + logback-classic + ${logback.version} + + + + + com.fasterxml.jackson.core + jackson-databind + ${fasterxml.jackson.version} + + + com.google.guava + guava + ${google.guava.version} + + + + + com.sun.jersey + jersey-bundle + ${jersey-bundle.version} + + + com.sun.jersey.contribs + jersey-spring + ${jersey-spring.version} + + + com.sun.jersey + jersey-server + + + org.springframework + * + + + + + commons-io + commons-io + ${commons.io.version} + + + javax.servlet + javax.servlet-api + ${javax.servlet.version} + + + + + org.apache.commons + commons-lang3 + ${commons.lang3.version} + + + + + org.apache.httpcomponents + httpasyncclient + ${httpcomponents.httpasyncclient.version} + + + commons-logging + * + + + + + org.apache.httpcomponents + httpclient + ${httpcomponents.httpclient.version} + + + commons-logging + * + + + + + org.apache.httpcomponents + httpcore + ${httpcomponents.httpcore.version} + + + org.apache.httpcomponents + httpcore-nio + ${httpcomponents.httpcore.version} + + + + + org.apache.kafka + kafka-clients + ${kafka.version} + + + log4j + * + + + org.slf4j + * + + + + + org.apache.lucene + lucene-analyzers-common + ${lucene.version} + + + org.apache.lucene + lucene-backward-codecs + ${lucene.version} + + + + + org.apache.lucene + lucene-core + ${lucene.version} + + + org.apache.lucene + lucene-grouping + ${lucene.version} + + + org.apache.lucene + lucene-highlighter + ${lucene.version} + + + org.apache.lucene + lucene-join + ${lucene.version} + + + org.apache.lucene + lucene-memory + ${lucene.version} + + + org.apache.lucene + lucene-misc + ${lucene.version} + + + org.apache.lucene + lucene-queries + ${lucene.version} + + + org.apache.lucene + lucene-queryparser + ${lucene.version} + + + org.apache.lucene + lucene-sandbox + ${lucene.version} + + + org.apache.lucene + lucene-spatial-extras + ${lucene.version} + + + org.apache.lucene + lucene-spatial3d + ${lucene.version} + + + org.apache.lucene + lucene-suggest + ${lucene.version} + + + + org.apache.ranger + ranger-audit-common + ${project.version} + + + org.apache.hadoop + hadoop-client-api + + + + + + + org.apache.ranger + ranger-audit-core + ${project.version} + + + org.apache.hadoop + hadoop-client-api + + + + + org.apache.ranger + ranger-audit-dest-solr + ${project.version} + + + org.apache.hadoop + hadoop-client-api + + + + + org.apache.ranger + ranger-plugins-common + ${project.version} + + + javax.servlet + javax.servlet-api + + + org.apache.hadoop + hadoop-client-api + + + + + org.apache.tomcat + tomcat-annotations-api + ${tomcat.embed.version} + + + + + org.apache.tomcat.embed + tomcat-embed-core + ${tomcat.embed.version} + + + org.apache.tomcat.embed + tomcat-embed-el + ${tomcat.embed.version} + + + org.apache.tomcat.embed + tomcat-embed-jasper + ${tomcat.embed.version} + + + + org.slf4j + log4j-over-slf4j + ${slf4j.version} + + + org.slf4j + slf4j-api + ${slf4j.version} + + + org.springframework + spring-beans + ${springframework.version} + + + + + org.springframework + spring-context + ${springframework.version} + + + org.springframework + spring-web + ${springframework.version} + + + + + + + true + src/main/resources + + + + + org.apache.maven.plugins + maven-war-plugin + + + + diff --git a/ranger-audit-server/ranger-audit-consumer-solr/scripts/start-consumer-solr.sh b/ranger-audit-server/ranger-audit-consumer-solr/scripts/start-consumer-solr.sh new file mode 100755 index 0000000000..88f64e4231 --- /dev/null +++ b/ranger-audit-server/ranger-audit-consumer-solr/scripts/start-consumer-solr.sh @@ -0,0 +1,163 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# ======================================== +# Ranger Audit Consumer Solr Start Script +# ======================================== +# This script starts the Solr consumer service +# The service consumes audit events from Kafka and indexes them to Solr + +set -e + +# Script directory +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +SERVICE_DIR="$(dirname "$SCRIPT_DIR")" + +# Default directories - can be overridden by environment variables +AUDIT_CONSUMER_HOME_DIR="${AUDIT_CONSUMER_HOME_DIR:-${SERVICE_DIR}/target}" +AUDIT_CONSUMER_CONF_DIR="${AUDIT_CONSUMER_CONF_DIR:-${SERVICE_DIR}/src/main/resources/conf}" +AUDIT_CONSUMER_LOG_DIR="${AUDIT_CONSUMER_LOG_DIR:-${SERVICE_DIR}/logs}" + +# Create log directory if it doesn't exist +mkdir -p "${AUDIT_CONSUMER_LOG_DIR}" + +echo "==========================================" +echo "Starting Ranger Audit Consumer - Solr" +echo "==========================================" +echo "Service Directory: ${SERVICE_DIR}" +echo "Home Directory: ${AUDIT_CONSUMER_HOME_DIR}" +echo "Config Directory: ${AUDIT_CONSUMER_CONF_DIR}" +echo "Log Directory: ${AUDIT_CONSUMER_LOG_DIR}" +echo "==========================================" + +# Check if Java is available +if [ -z "$JAVA_HOME" ]; then + JAVA_CMD=$(which java 2>/dev/null || true) + if [ -z "$JAVA_CMD" ]; then + echo "[ERROR] JAVA_HOME is not set and java is not in PATH" + exit 1 + fi + JAVA_HOME=$(dirname $(dirname $(readlink -f "$JAVA_CMD"))) + echo "[INFO] JAVA_HOME not set, detected: ${JAVA_HOME}" +fi + +export JAVA_HOME +export PATH=$JAVA_HOME/bin:$PATH + +echo "[INFO] Java version:" +java -version + +# Set heap size (default: 512MB to 2GB) +AUDIT_CONSUMER_HEAP="${AUDIT_CONSUMER_HEAP:--Xms512m -Xmx2g}" + +# Set JVM options +if [ -z "$AUDIT_CONSUMER_OPTS" ]; then + AUDIT_CONSUMER_OPTS="-Dlogback.configurationFile=${AUDIT_CONSUMER_CONF_DIR}/logback.xml" + AUDIT_CONSUMER_OPTS="${AUDIT_CONSUMER_OPTS} -Daudit.consumer.solr.log.dir=${AUDIT_CONSUMER_LOG_DIR}" + AUDIT_CONSUMER_OPTS="${AUDIT_CONSUMER_OPTS} -Daudit.consumer.solr.log.file=ranger-audit-consumer-solr.log" + AUDIT_CONSUMER_OPTS="${AUDIT_CONSUMER_OPTS} -Djava.net.preferIPv4Stack=true -server" + AUDIT_CONSUMER_OPTS="${AUDIT_CONSUMER_OPTS} -XX:+UseG1GC -XX:MaxGCPauseMillis=200" + AUDIT_CONSUMER_OPTS="${AUDIT_CONSUMER_OPTS} -XX:InitiatingHeapOccupancyPercent=35" + AUDIT_CONSUMER_OPTS="${AUDIT_CONSUMER_OPTS} -XX:ConcGCThreads=4 -XX:ParallelGCThreads=8" +fi + +# Add Kerberos configuration if needed +if [ "${KERBEROS_ENABLED}" == "true" ]; then + AUDIT_CONSUMER_OPTS="${AUDIT_CONSUMER_OPTS} -Djava.security.krb5.conf=/etc/krb5.conf" + echo "[INFO] Kerberos is enabled" +fi + +export AUDIT_CONSUMER_OPTS +export AUDIT_CONSUMER_LOG_DIR + +echo "[INFO] JAVA_HOME: ${JAVA_HOME}" +echo "[INFO] HEAP: ${AUDIT_CONSUMER_HEAP}" +echo "[INFO] JVM_OPTS: ${AUDIT_CONSUMER_OPTS}" + +# Find the WAR file +WAR_FILE=$(find "${AUDIT_CONSUMER_HOME_DIR}" -name "ranger-audit-consumer-solr*.war" | head -1) + +if [ -z "$WAR_FILE" ] || [ ! -f "$WAR_FILE" ]; then + echo "[ERROR] WAR file not found in ${AUDIT_CONSUMER_HOME_DIR}" + echo "[ERROR] Please build the project first using: mvn clean package" + exit 1 +fi + +echo "[INFO] Using WAR file: ${WAR_FILE}" + +# Extract WAR if not already extracted +WEBAPP_DIR="${AUDIT_CONSUMER_HOME_DIR}/webapp/ranger-audit-consumer-solr" +if [ ! -d "${WEBAPP_DIR}" ]; then + echo "[INFO] Extracting WAR file..." + mkdir -p "${WEBAPP_DIR}" + cd "${WEBAPP_DIR}" + jar xf "${WAR_FILE}" + cd - > /dev/null +fi + +# Build classpath +RANGER_CLASSPATH="${WEBAPP_DIR}/WEB-INF/classes" +for jar in "${WEBAPP_DIR}"/WEB-INF/lib/*.jar; do + RANGER_CLASSPATH="${RANGER_CLASSPATH}:${jar}" +done + +# Add libext directory if it exists +if [ -d "${AUDIT_CONSUMER_HOME_DIR}/libext" ]; then + for jar in "${AUDIT_CONSUMER_HOME_DIR}"/libext/*.jar; do + if [ -f "${jar}" ]; then + RANGER_CLASSPATH="${RANGER_CLASSPATH}:${jar}" + fi + done +fi + +export RANGER_CLASSPATH + +# Check if already running +PID_FILE="${AUDIT_CONSUMER_LOG_DIR}/ranger-audit-consumer-solr.pid" +if [ -f "${PID_FILE}" ]; then + OLD_PID=$(cat "${PID_FILE}") + if kill -0 "$OLD_PID" 2>/dev/null; then + echo "[WARNING] Ranger Audit Consumer - Solr is already running (PID: ${OLD_PID})" + echo "[INFO] Use stop-consumer-solr.sh to stop it first" + exit 1 + else + echo "[INFO] Removing stale PID file" + rm -f "${PID_FILE}" + fi +fi + +# Start the service +echo "[INFO] Starting Ranger Audit Consumer - Solr..." +nohup java ${AUDIT_CONSUMER_HEAP} ${AUDIT_CONSUMER_OPTS} \ + -Daudit.config=${AUDIT_CONSUMER_CONF_DIR}/ranger-audit-consumer-solr-site.xml \ + -Dranger.audit.consumer.webapp.dir="${WAR_FILE}" \ + -cp "${RANGER_CLASSPATH}" \ + org.apache.ranger.audit.consumer.SolrConsumerApplication \ + >> "${AUDIT_CONSUMER_LOG_DIR}/catalina.out" 2>&1 & + +PID=$! +echo $PID > "${PID_FILE}" + +echo "[INFO] ✓ Ranger Audit Consumer - Solr started successfully" +echo "[INFO] PID: ${PID}" +echo "[INFO] Log file: ${AUDIT_CONSUMER_LOG_DIR}/ranger-audit-consumer-solr.log" +echo "[INFO] Catalina out: ${AUDIT_CONSUMER_LOG_DIR}/catalina.out" +echo "[INFO] Health check: http://localhost:7091/api/health" +echo "" +echo "To monitor logs: tail -f ${AUDIT_CONSUMER_LOG_DIR}/ranger-audit-consumer-solr.log" +echo "To stop service: ${SCRIPT_DIR}/stop-consumer-solr.sh" diff --git a/ranger-audit-server/ranger-audit-consumer-solr/scripts/stop-consumer-solr.sh b/ranger-audit-server/ranger-audit-consumer-solr/scripts/stop-consumer-solr.sh new file mode 100755 index 0000000000..8e2049ac4b --- /dev/null +++ b/ranger-audit-server/ranger-audit-consumer-solr/scripts/stop-consumer-solr.sh @@ -0,0 +1,72 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# ======================================== +# Ranger Audit Consumer Solr Stop Script +# ======================================== + +set -e + +# Script directory +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +SERVICE_DIR="$(dirname "$SCRIPT_DIR")" + +# Default log directory +AUDIT_CONSUMER_LOG_DIR="${AUDIT_CONSUMER_LOG_DIR:-${SERVICE_DIR}/logs}" +PID_FILE="${AUDIT_CONSUMER_LOG_DIR}/ranger-audit-consumer-solr.pid" + +echo "==========================================" +echo "Stopping Ranger Audit Consumer - Solr" +echo "==========================================" + +if [ ! -f "${PID_FILE}" ]; then + echo "[WARNING] PID file not found: ${PID_FILE}" + echo "[INFO] Service may not be running" + exit 0 +fi + +PID=$(cat "${PID_FILE}") + +if ! kill -0 "$PID" 2>/dev/null; then + echo "[WARNING] Process ${PID} is not running" + echo "[INFO] Removing stale PID file" + rm -f "${PID_FILE}" + exit 0 +fi + +echo "[INFO] Stopping process ${PID}..." +kill "$PID" + +# Wait for process to stop (max 30 seconds) +TIMEOUT=30 +COUNT=0 +while kill -0 "$PID" 2>/dev/null && [ $COUNT -lt $TIMEOUT ]; do + sleep 1 + COUNT=$((COUNT + 1)) + echo -n "." +done +echo "" + +if kill -0 "$PID" 2>/dev/null; then + echo "[WARNING] Process did not stop gracefully, forcing shutdown..." + kill -9 "$PID" + sleep 2 +fi + +rm -f "${PID_FILE}" +echo "[INFO] ✓ Ranger Audit Consumer - Solr stopped successfully" diff --git a/ranger-audit-server/ranger-audit-consumer-solr/src/main/java/org/apache/ranger/audit/consumer/SolrConsumerApplication.java b/ranger-audit-server/ranger-audit-consumer-solr/src/main/java/org/apache/ranger/audit/consumer/SolrConsumerApplication.java new file mode 100644 index 0000000000..601f37b475 --- /dev/null +++ b/ranger-audit-server/ranger-audit-consumer-solr/src/main/java/org/apache/ranger/audit/consumer/SolrConsumerApplication.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.audit.consumer; + +import org.apache.hadoop.conf.Configuration; +import org.apache.ranger.audit.server.EmbeddedServer; +import org.apache.ranger.audit.server.SolrConsumerConfig; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Main application class for Audit Consumer Solr Service. + * This service consumes audit events from Kafka and indexes them into Solr + * It uses embedded Tomcat server for lifecycle management and health check endpoints. + */ +public class SolrConsumerApplication { + private static final Logger LOG = LoggerFactory.getLogger(SolrConsumerApplication.class); + + private static final String APP_NAME = "ranger-audit-consumer-solr"; + private static final String CONFIG_PREFIX = "ranger.audit.consumer.solr.service."; + + private SolrConsumerApplication() { + } + + public static void main(String[] args) { + LOG.info("=========================================================================="); + LOG.info("==> Starting Ranger Audit Consumer Solr Service"); + LOG.info("=========================================================================="); + + try { + // Load configuration + Configuration config = SolrConsumerConfig.getInstance(); + + LOG.info("Configuration loaded successfully"); + + // Create and start embedded server + // The server will load Spring context which initializes SolrConsumerManager + // which then creates and starts the Solr consumer threads + EmbeddedServer server = new EmbeddedServer(config, APP_NAME, CONFIG_PREFIX); + server.start(); + + LOG.info("==> Ranger Audit Consumer Solr Service Started Successfully"); + } catch (Exception e) { + LOG.error("Failed to start Ranger Audit Consumer Solr Service", e); + System.exit(1); + } + } +} diff --git a/ranger-audit-server/ranger-audit-consumer-solr/src/main/java/org/apache/ranger/audit/consumer/SolrConsumerManager.java b/ranger-audit-server/ranger-audit-consumer-solr/src/main/java/org/apache/ranger/audit/consumer/SolrConsumerManager.java new file mode 100644 index 0000000000..2024f2ddcf --- /dev/null +++ b/ranger-audit-server/ranger-audit-consumer-solr/src/main/java/org/apache/ranger/audit/consumer/SolrConsumerManager.java @@ -0,0 +1,210 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.audit.consumer; + +import org.apache.ranger.audit.consumer.kafka.AuditConsumer; +import org.apache.ranger.audit.consumer.kafka.AuditConsumerRegistry; +import org.apache.ranger.audit.provider.MiscUtil; +import org.apache.ranger.audit.server.AuditServerConstants; +import org.apache.ranger.audit.server.SolrConsumerConfig; +import org.apache.ranger.audit.utils.AuditServerLogFormatter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; + +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; + +import java.util.ArrayList; +import java.util.List; +import java.util.Properties; + +/** + * Spring component that manages the lifecycle of Solr consumer threads. + * This manager: + * - Initializes the consumer registry + * - Creates Solr consumer instances + * - Starts consumer threads + * - Handles graceful shutdown + */ +@Component +public class SolrConsumerManager { + private static final Logger LOG = LoggerFactory.getLogger(SolrConsumerManager.class); + + private final AuditConsumerRegistry consumerRegistry = AuditConsumerRegistry.getInstance(); + private final List consumers = new ArrayList<>(); + private final List consumerThreads = new ArrayList<>(); + + @PostConstruct + public void init() { + LOG.info("==> SolrConsumerManager.init()"); + + try { + SolrConsumerConfig config = SolrConsumerConfig.getInstance(); + Properties props = config.getProperties(); + + if (props == null) { + LOG.error("Configuration properties are null"); + throw new RuntimeException("Failed to load configuration"); + } + + // Initialize and register Solr Consumer + initializeConsumerClasses(props, AuditServerConstants.PROP_KAFKA_PROP_PREFIX); + + // Create consumers from registry + List createdConsumers = consumerRegistry.createConsumers(props, AuditServerConstants.PROP_KAFKA_PROP_PREFIX); + consumers.addAll(createdConsumers); + + if (consumers.isEmpty()) { + LOG.warn("No consumers were created! Verify that xasecure.audit.destination.solr=true"); + } else { + LOG.info("Created {} Solr consumer(s)", consumers.size()); + + // Start consumer threads + startConsumers(); + } + } catch (Exception e) { + LOG.error("Failed to initialize SolrConsumerManager", e); + throw new RuntimeException("Failed to initialize SolrConsumerManager", e); + } + + LOG.info("<== SolrConsumerManager.init() - {} consumer thread(s) started", consumerThreads.size()); + } + + private void initializeConsumerClasses(Properties props, String propPrefix) { + LOG.info("==> SolrConsumerManager.initializeConsumerClasses()"); + + // Get consumer classes from configuration + String clsStr = MiscUtil.getStringProperty(props, propPrefix + "." + AuditServerConstants.PROP_CONSUMER_CLASSES, + "org.apache.ranger.audit.consumer.kafka.AuditSolrConsumer"); + + String[] solrConsumerClasses = clsStr.split(","); + + LOG.info("Initializing {} consumer class(es)", solrConsumerClasses.length); + + for (String solrConsumerClassName : solrConsumerClasses) { + solrConsumerClassName = solrConsumerClassName.trim(); + + if (solrConsumerClassName.isEmpty()) { + continue; + } + + try { + Class consumerClass = Class.forName(solrConsumerClassName); + LOG.info("Successfully initialized consumer class: {}", consumerClass.getName()); + } catch (ClassNotFoundException e) { + LOG.error("Consumer class not found: {}. Ensure the class is on the classpath.", solrConsumerClassName, e); + } catch (Exception e) { + LOG.error("Error initializing consumer class: {}", solrConsumerClassName, e); + } + } + + LOG.info("Registered consumer factories: {}", consumerRegistry.getRegisteredDestinationTypes()); + LOG.info("<== SolrConsumerManager.initializeConsumerClasses()"); + } + + /** + * Start all consumer threads + */ + private void startConsumers() { + LOG.info("==> SolrConsumerManager.startConsumers()"); + + logSolrConsumerStartup(); + + for (AuditConsumer consumer : consumers) { + try { + String consumerName = consumer.getClass().getSimpleName(); + Thread consumerThread = new Thread(consumer, consumerName); + consumerThread.setDaemon(true); + consumerThread.start(); + consumerThreads.add(consumerThread); + + LOG.info("Started {} thread [Thread-ID: {}, Thread-Name: '{}']", + consumerName, consumerThread.getId(), consumerThread.getName()); + } catch (Exception e) { + LOG.error("Error starting consumer: {}", consumer.getClass().getSimpleName(), e); + } + } + + LOG.info("<== SolrConsumerManager.startConsumers() - {} thread(s) started", consumerThreads.size()); + } + + /** + * Log startup banner with consumer information + */ + private void logSolrConsumerStartup() { + LOG.info("################## SOLR CONSUMER SERVICE STARTUP ######################"); + + if (consumers.isEmpty()) { + LOG.warn("WARNING: No Solr consumers are enabled!"); + LOG.warn("Verify: xasecure.audit.destination.solr=true in configuration"); + } else { + AuditServerLogFormatter.LogBuilder builder = AuditServerLogFormatter.builder("Solr Consumer Status"); + + for (AuditConsumer consumer : consumers) { + String consumerType = consumer.getClass().getSimpleName(); + builder.add(consumerType, "ENABLED"); + builder.add("Topic", consumer.getTopicName()); + } + + builder.logInfo(LOG); + LOG.info("Starting {} Solr consumer thread(s)...", consumers.size()); + } + LOG.info("########################################################################"); + } + + @PreDestroy + public void shutdown() { + LOG.info("==> SolrConsumerManager.shutdown()"); + + // Shutdown all consumers + for (AuditConsumer consumer : consumers) { + try { + LOG.info("Shutting down consumer: {}", consumer.getClass().getSimpleName()); + consumer.shutdown(); + LOG.info("Consumer shutdown completed: {}", consumer.getClass().getSimpleName()); + } catch (Exception e) { + LOG.error("Error shutting down consumer: {}", consumer.getClass().getSimpleName(), e); + } + } + + // Wait for threads to terminate + for (Thread thread : consumerThreads) { + if (thread.isAlive()) { + try { + LOG.info("Waiting for thread to terminate: {}", thread.getName()); + thread.join(10000); // Wait up to 10 seconds + if (thread.isAlive()) { + LOG.warn("Thread did not terminate within 10 seconds: {}", thread.getName()); + } + } catch (InterruptedException e) { + LOG.warn("Interrupted while waiting for thread to terminate: {}", thread.getName(), e); + Thread.currentThread().interrupt(); + } + } + } + + consumers.clear(); + consumerThreads.clear(); + consumerRegistry.clearActiveConsumers(); + + LOG.info("<== SolrConsumerManager.shutdown() - All Solr consumers stopped"); + } +} diff --git a/ranger-audit-server/ranger-audit-consumer-solr/src/main/java/org/apache/ranger/audit/consumer/kafka/AuditSolrConsumer.java b/ranger-audit-server/ranger-audit-consumer-solr/src/main/java/org/apache/ranger/audit/consumer/kafka/AuditSolrConsumer.java new file mode 100644 index 0000000000..043231f572 --- /dev/null +++ b/ranger-audit-server/ranger-audit-consumer-solr/src/main/java/org/apache/ranger/audit/consumer/kafka/AuditSolrConsumer.java @@ -0,0 +1,457 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.audit.consumer.kafka; + +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.clients.consumer.ConsumerRecords; +import org.apache.kafka.clients.consumer.KafkaConsumer; +import org.apache.kafka.clients.consumer.OffsetAndMetadata; +import org.apache.kafka.common.TopicPartition; +import org.apache.ranger.audit.destination.SolrAuditDestination; +import org.apache.ranger.audit.provider.AuditProviderFactory; +import org.apache.ranger.audit.provider.MiscUtil; +import org.apache.ranger.audit.server.AuditServerConstants; +import org.apache.ranger.audit.utils.AuditServerLogFormatter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; + +/** + * Solr consumer that writes audits into Solr index using rangerauditserver user + */ +public class AuditSolrConsumer extends AuditConsumerBase implements AuditConsumer { + private static final Logger LOG = LoggerFactory.getLogger(AuditSolrConsumer.class); + private static final String RANGER_AUDIT_SOLR_CONSUMER_GROUP = AuditServerConstants.DEFAULT_RANGER_AUDIT_SOLR_CONSUMER_GROUP; + + // Register AuditSolrConsumer factory in the audit consumer registry + static { + try { + AuditConsumerRegistry.getInstance().registerFactory(AuditServerConstants.PROP_SOLR_DEST_PREFIX, (props, propPrefix) -> new AuditSolrConsumer(props, propPrefix)); + LOG.info("Registered Solr consumer factory with AuditConsumerRegistry"); + } catch (Exception e) { + LOG.error("Failed to register Solr consumer factory", e); + } + } + + private final AtomicBoolean running = new AtomicBoolean(false); + private final Map consumerWorkers = new ConcurrentHashMap<>(); + private ExecutorService consumerThreadPool; + private int consumerThreadCount = 1; + // Offset management configuration (batch or manual only supported) + private String offsetCommitStrategy = AuditServerConstants.DEFAULT_OFFSET_COMMIT_STRATEGY; + private long offsetCommitInterval = AuditServerConstants.DEFAULT_OFFSET_COMMIT_INTERVAL_MS; + + public SolrAuditDestination solrAuditDestination; + public Properties props; + public String propPrefix; + + public AuditSolrConsumer(Properties props, String propPrefix) throws Exception { + super(props, propPrefix, RANGER_AUDIT_SOLR_CONSUMER_GROUP); + this.props = props; + this.propPrefix = propPrefix; + init(props, propPrefix); + } + + @Override + public void init(Properties props, String propPrefix) throws Exception { + LOG.info("==> AuditSolrConsumer.init()"); + + consumer = getConsumer(); + solrAuditDestination = new SolrAuditDestination(); + solrAuditDestination.init(props, AuditProviderFactory.AUDIT_DEST_BASE + "." + AuditServerConstants.PROP_SOLR_DEST_PREFIX); + + // Initialize consumer configuration + initConsumerConfig(props, propPrefix); + + LOG.info("<== AuditSolrConsumer.init()"); + } + + private void initConsumerConfig(Properties props, String propPrefix) { + LOG.info("==> AuditSolrConsumer.initConsumerConfig()"); + + // Get consumer thread count + this.consumerThreadCount = MiscUtil.getIntProperty(props, propPrefix + "." + AuditServerConstants.PROP_CONSUMER_THREAD_COUNT, 1); + LOG.info("Consumer thread count: {}", consumerThreadCount); + + // Initialize offset management configuration + initializeOffsetManagement(props, propPrefix); + + LOG.info("<== AuditSolrConsumer.initConsumerConfig()"); + } + + private void initializeOffsetManagement(Properties props, String propPrefix) { + LOG.info("==> AuditSolrConsumer.initializeOffsetManagement()"); + + this.offsetCommitStrategy = MiscUtil.getStringProperty(props, + propPrefix + "." + AuditServerConstants.PROP_CONSUMER_OFFSET_COMMIT_STRATEGY, + AuditServerConstants.DEFAULT_OFFSET_COMMIT_STRATEGY); + + // Get offset commit interval (only used for manual strategy) + this.offsetCommitInterval = MiscUtil.getLongProperty(props, + propPrefix + "." + AuditServerConstants.PROP_CONSUMER_OFFSET_COMMIT_INTERVAL, + AuditServerConstants.DEFAULT_OFFSET_COMMIT_INTERVAL_MS); + + AuditServerLogFormatter.builder("AuditSolrConsumer Offset Management Configuration") + .add("Commit Strategy", offsetCommitStrategy) + .add("Commit Interval (ms)", offsetCommitInterval + " (used in manual mode only)") + .logInfo(LOG); + + LOG.info("<== AuditSolrConsumer.initializeOffsetManagement()"); + } + + @Override + public void run() { + try { + if (solrAuditDestination == null) { + init(this.props, this.propPrefix); + } + + startMultithreadedConsumption(); + + // Keep main thread alive while consumer threads are running + while (running.get()) { + Thread.sleep(1000); + } + } catch (Throwable e) { + LOG.error("Error in AuditSolrConsumer", e); + } finally { + shutdown(); + } + } + + /** + * Start multithreaded consumption with generic workers for horizontal scaling. + * Creates configured number of generic workers that can process ANY partition assigned by Kafka. + * This enables true horizontal scaling across multiple audit-server instances. + */ + private void startMultithreadedConsumption() { + LOG.info("==> AuditSolrConsumer.startMultithreadedConsumption()"); + + if (running.compareAndSet(false, true)) { + startConsumerWorkers(); + } + + LOG.info("<== AuditSolrConsumer.startMultithreadedConsumption()"); + } + + /** + * Start consumer workers for horizontal scaling. + * Each worker subscribes to the topic and Kafka automatically assigns partitions. + * Workers can process messages from ANY appId. + */ + private void startConsumerWorkers() { + int workerCount = consumerThreadCount; + + LOG.info("==> AuditSolrConsumer.startConsumerWorkers(): Creating {} generic workers for horizontal scaling", workerCount); + LOG.info("Each worker will subscribe to topic '{}' and process partitions assigned by Kafka", topicName); + + // Create thread pool sized for generic workers + consumerThreadPool = Executors.newFixedThreadPool(workerCount); + LOG.info("Created thread pool with {} threads for scalable SOLR consumption", workerCount); + + // Create generic workers (no appId pre-assignment) + for (int i = 0; i < workerCount; i++) { + String workerId = "solr-worker-" + i; + // Pass empty list for partitions (Kafka will assign dynamically) + ConsumerWorker worker = new ConsumerWorker(workerId, new ArrayList<>()); + consumerWorkers.put(workerId, worker); + consumerThreadPool.submit(worker); + + LOG.info("Started SOLR consumer worker '{}' - will process ANY appId assigned by Kafka", workerId); + } + + LOG.info("<== AuditSolrConsumer.startConsumerWorkers(): All {} workers started in SUBSCRIBE mode", workerCount); + } + + private class ConsumerWorker implements Runnable { + private final String workerId; + private final List assignedPartitions; + private KafkaConsumer workerConsumer; + + // Offset management + private final Map pendingOffsets = new HashMap<>(); + private final AtomicLong lastCommitTime = new AtomicLong(System.currentTimeMillis()); + private final AtomicInteger messagesProcessedSinceLastCommit = new AtomicInteger(0); + + public ConsumerWorker(String workerId, List assignedPartitions) { + this.workerId = workerId; + this.assignedPartitions = assignedPartitions; + } + + @Override + public void run() { + try { + // Create consumer for this worker with offset management configuration + Properties workerConsumerProps = new Properties(); + workerConsumerProps.putAll(consumerProps); + + // Configure offset management based on strategy + configureOffsetManagement(workerConsumerProps); + + workerConsumer = new KafkaConsumer<>(workerConsumerProps); + + // Create re-balance listener + AuditConsumerRebalanceListener reBalanceListener = new AuditConsumerRebalanceListener( + workerId, + AuditServerConstants.DESTINATION_SOLR, + topicName, + offsetCommitStrategy, + consumerGroupId, + workerConsumer, + pendingOffsets, + messagesProcessedSinceLastCommit, + lastCommitTime, + assignedPartitions); + + // Subscribe to topic with re-balance listener and let kafka automatically assign partitions + workerConsumer.subscribe(Collections.singletonList(topicName), reBalanceListener); + + LOG.info("[SOLR-CONSUMER] Worker '{}' subscribed successfully, waiting for partition assignment from Kafka", workerId); + long threadId = Thread.currentThread().getId(); + String threadName = Thread.currentThread().getName(); + LOG.info("[SOLR-CONSUMER-STARTUP] Worker '{}' [Thread-ID: {}, Thread-Name: '{}'] started | Topic: '{}' | Consumer-Group: {} | Mode: SUBSCRIBE", + workerId, threadId, threadName, topicName, consumerGroupId); + + // Consume messages + while (running.get()) { + ConsumerRecords records = workerConsumer.poll(Duration.ofMillis(100)); + + if (!records.isEmpty()) { + processRecordBatch(records); + + // Handle offset committing based on strategy + handleOffsetCommitting(); + } + } + } catch (Exception e) { + LOG.error("Error in consumer worker '{}'", workerId, e); + } finally { + // Final offset commit before shutdown + commitPendingOffsets(true); + + if (workerConsumer != null) { + try { + // Unsubscribe before closing + LOG.info("Worker '{}': Unsubscribing from topic", workerId); + workerConsumer.unsubscribe(); + } catch (Exception e) { + LOG.warn("Worker '{}': Error during unsubscribe", workerId, e); + } + + try { + LOG.info("Worker '{}': Closing consumer", workerId); + workerConsumer.close(); + } catch (Exception e) { + LOG.error("Error closing consumer for worker '{}'", workerId, e); + } + } + LOG.info("Consumer worker '{}' stopped", workerId); + } + } + + /** + * Configure offset management for this worker's consumer (batch or manual) + */ + private void configureOffsetManagement(Properties consumerProps) { + // Always disable auto commit - only batch or manual strategies supported + consumerProps.put("enable.auto.commit", "false"); + LOG.debug("Worker '{}' configured for manual offset commit with strategy: {}", workerId, offsetCommitStrategy); + } + + /** + * Process a batch of records. + * In generic worker mode, processes messages from ANY appId. + * Uses batch processing to send all records to Solr in one request for efficiency. + */ + private void processRecordBatch(ConsumerRecords records) { + // Collect all audit messages for batch processing + List auditBatch = new ArrayList<>(); + List> recordList = new ArrayList<>(); + + for (ConsumerRecord record : records) { + LOG.debug("Worker '{}' consumed: partition={}, key={}, offset={}", + workerId, record.partition(), record.key(), record.offset()); + + auditBatch.add(record.value()); + recordList.add(record); + } + + // Process entire batch at once + try { + if (!auditBatch.isEmpty()) { + processMessageBatch(auditBatch); + + // Track offsets for all successfully processed messages + for (ConsumerRecord record : recordList) { + TopicPartition partition = new TopicPartition(record.topic(), record.partition()); + pendingOffsets.put(partition, new OffsetAndMetadata(record.offset() + 1)); + messagesProcessedSinceLastCommit.incrementAndGet(); + } + } + } catch (Exception e) { + LOG.error("Error processing batch in worker '{}', batch size: {}", + workerId, auditBatch.size(), e); + + // On batch error, track offsets up to the first message to avoid reprocessing + // (Note: This is a simplistic approach - could be enhanced to track last successful offset) + if (!recordList.isEmpty()) { + ConsumerRecord firstRecord = recordList.get(0); + TopicPartition partition = new TopicPartition(firstRecord.topic(), firstRecord.partition()); + pendingOffsets.put(partition, new OffsetAndMetadata(firstRecord.offset())); + } + } + } + + /** + * Handle offset committing based on the configured strategy (batch or manual only) + */ + private void handleOffsetCommitting() { + boolean shouldCommit = false; + long currentTime = System.currentTimeMillis(); + + if (AuditServerConstants.PROP_OFFSET_COMMIT_STRATEGY_BATCH.equals(offsetCommitStrategy)) { + // Commit after processing each batch + shouldCommit = !pendingOffsets.isEmpty(); + } else if (AuditServerConstants.PROP_OFFSET_COMMIT_STRATEGY_MANUAL.equals(offsetCommitStrategy)) { + // Commit based on time interval + shouldCommit = (currentTime - lastCommitTime.get()) >= offsetCommitInterval && !pendingOffsets.isEmpty(); + } + + if (shouldCommit) { + commitPendingOffsets(false); + } + } + + /** + * Commit pending offsets + */ + private void commitPendingOffsets(boolean isShutdown) { + if (pendingOffsets.isEmpty()) { + return; + } + + try { + workerConsumer.commitSync(pendingOffsets); + + LOG.debug("Worker '{}' committed {} offsets, processed {} messages", + workerId, pendingOffsets.size(), messagesProcessedSinceLastCommit.get()); + + // Clear committed offsets + pendingOffsets.clear(); + lastCommitTime.set(System.currentTimeMillis()); + messagesProcessedSinceLastCommit.set(0); + } catch (Exception e) { + LOG.error("Error committing offsets in worker '{}': {}", workerId, pendingOffsets, e); + + if (isShutdown) { + // During shutdown, retry to avoid loss of any offsets + try { + Thread.sleep(1000); + workerConsumer.commitSync(pendingOffsets); + LOG.info("Successfully committed offsets on retry during shutdown for worker '{}'", workerId); + } catch (Exception retryException) { + LOG.error("Failed to commit offsets even on retry during shutdown for worker '{}'", workerId, retryException); + } + } + } + } + } + + @Override + public KafkaConsumer getKafkaConsumer() { + return consumer; + } + + @Override + public void processMessage(String audit) throws Exception { + if (solrAuditDestination != null) { + solrAuditDestination.logJSON(audit); + } + } + + /** + * Process a batch of audit messages. + * This method leverages SolrAuditDestination's batch processing capability + * to send multiple audits to Solr in a single request, improving performance. + * + * @param audits Collection of audit messages in JSON format + * @throws Exception if batch processing fails + */ + public void processMessageBatch(Collection audits) throws Exception { + if (solrAuditDestination != null && audits != null && !audits.isEmpty()) { + solrAuditDestination.logJSON(audits); + } + } + + @Override + public String getTopicName() { + return topicName; + } + + @Override + public void shutdown() { + LOG.info("Shutting down AuditSolrConsumer..."); + + running.set(false); + + // Shutdown consumer workers + if (consumerThreadPool != null) { + consumerThreadPool.shutdownNow(); + try { + if (!consumerThreadPool.awaitTermination(30, java.util.concurrent.TimeUnit.SECONDS)) { + LOG.warn("Consumer thread pool did not terminate within 30 seconds"); + } + } catch (InterruptedException e) { + LOG.warn("Interrupted while waiting for consumer thread pool to terminate", e); + Thread.currentThread().interrupt(); + } + } + + // Close main consumer + if (consumer != null) { + try { + consumer.close(); + } catch (Exception e) { + LOG.error("Error closing main consumer", e); + } + } + + consumerWorkers.clear(); + LOG.info("AuditSolrConsumer shutdown complete"); + } + + public String getConsumerGroupId() { + return consumerGroupId; + } +} diff --git a/ranger-audit-server/ranger-audit-consumer-solr/src/main/java/org/apache/ranger/audit/rest/HealthCheckREST.java b/ranger-audit-server/ranger-audit-consumer-solr/src/main/java/org/apache/ranger/audit/rest/HealthCheckREST.java new file mode 100644 index 0000000000..3245c14471 --- /dev/null +++ b/ranger-audit-server/ranger-audit-consumer-solr/src/main/java/org/apache/ranger/audit/rest/HealthCheckREST.java @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.audit.rest; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.ranger.audit.consumer.SolrConsumerManager; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Scope; +import org.springframework.stereotype.Component; + +import javax.ws.rs.GET; +import javax.ws.rs.Path; +import javax.ws.rs.Produces; +import javax.ws.rs.core.Response; + +import java.util.HashMap; +import java.util.Map; + +/** + * Health check REST endpoint for Solr Consumer Service + */ +@Path("/health") +@Component +@Scope("request") +public class HealthCheckREST { + private static final Logger LOG = LoggerFactory.getLogger(HealthCheckREST.class); + + @Autowired(required = false) + SolrConsumerManager solrConsumerManager; + + /** + * Health check endpoint + */ + @GET + @Produces("application/json") + public Response healthCheck() { + LOG.debug("==> HealthCheckREST.healthCheck()"); + Response ret; + String jsonString; + + try { + // Check if consumer manager is available and healthy + if (solrConsumerManager != null) { + Map resp = new HashMap<>(); + resp.put("status", "UP"); + resp.put("service", "audit-consumer-solr"); + resp.put("timestamp", System.currentTimeMillis()); + jsonString = buildResponse(resp); + ret = Response.ok() + .entity(jsonString) + .build(); + } else { + Map resp = new HashMap<>(); + resp.put("status", "DOWN"); + resp.put("service", "audit-consumer-solr"); + resp.put("reason", "SolrConsumerManager not available"); + resp.put("timestamp", System.currentTimeMillis()); + jsonString = buildResponse(resp); + ret = Response.status(Response.Status.SERVICE_UNAVAILABLE) + .entity(jsonString) + .build(); + } + } catch (Exception e) { + LOG.error("Health check failed", e); + Map resp = new HashMap<>(); + resp.put("status", "DOWN"); + resp.put("service", "audit-consumer-solr"); + resp.put("reason", e.getMessage()); + resp.put("timestamp", System.currentTimeMillis()); + jsonString = buildResponse(resp); + ret = Response.status(Response.Status.SERVICE_UNAVAILABLE) + .entity(jsonString) + .build(); + } + + LOG.debug("<== HealthCheckREST.healthCheck(): {}", ret); + + return ret; + } + + private String buildResponse(Map respMap) { + String ret; + + try { + ObjectMapper objectMapper = new ObjectMapper(); + ret = objectMapper.writeValueAsString(respMap); + } catch (Exception e) { + ret = "Error: " + e.getMessage(); + } + + return ret; + } +} diff --git a/ranger-audit-server/ranger-audit-consumer-solr/src/main/java/org/apache/ranger/audit/server/SolrConsumerConfig.java b/ranger-audit-server/ranger-audit-consumer-solr/src/main/java/org/apache/ranger/audit/server/SolrConsumerConfig.java new file mode 100644 index 0000000000..837a6fc734 --- /dev/null +++ b/ranger-audit-server/ranger-audit-consumer-solr/src/main/java/org/apache/ranger/audit/server/SolrConsumerConfig.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.audit.server; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Configuration class for Solr Consumer Service. + * Loads Solr consumer-specific configuration files. + */ +public class SolrConsumerConfig extends AuditConfig { + private static final Logger LOG = LoggerFactory.getLogger(SolrConsumerConfig.class); + private static final String CONFIG_FILE_PATH = "conf/ranger-audit-consumer-solr-site.xml"; + private static volatile SolrConsumerConfig sInstance; + + private SolrConsumerConfig() { + super(); + addSolrConsumerResources(); + } + + public static SolrConsumerConfig getInstance() { + SolrConsumerConfig ret = SolrConsumerConfig.sInstance; + + if (ret == null) { + synchronized (SolrConsumerConfig.class) { + ret = SolrConsumerConfig.sInstance; + + if (ret == null) { + ret = new SolrConsumerConfig(); + SolrConsumerConfig.sInstance = ret; + } + } + } + + return ret; + } + + private boolean addSolrConsumerResources() { + LOG.debug("==> SolrConsumerConfig.addSolrConsumerResources()"); + + boolean ret = true; + + // Load ranger-audit-consumer-solr-site.xml + if (!addAuditResource(CONFIG_FILE_PATH, true)) { + LOG.error("Could not load required configuration: {}", CONFIG_FILE_PATH); + ret = false; + } + + LOG.debug("<== SolrConsumerConfig.addSolrConsumerResources(), result={}", ret); + + return ret; + } +} diff --git a/ranger-audit-server/ranger-audit-consumer-solr/src/main/resources/conf/logback.xml b/ranger-audit-server/ranger-audit-consumer-solr/src/main/resources/conf/logback.xml new file mode 100644 index 0000000000..fb7eced612 --- /dev/null +++ b/ranger-audit-server/ranger-audit-consumer-solr/src/main/resources/conf/logback.xml @@ -0,0 +1,76 @@ + + + + + + + + + + + %d{yyyy-MM-dd HH:mm:ss.SSS} %-5p - %m%n + + + + + + ${LOG_DIR}/${LOG_FILE} + + %d %p %c{1} [%t] %m%n + + + ${LOG_DIR}/${LOG_FILE}-%d{MM-dd-yyyy}-%i.log.gz + + 200MB + + 10 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/ranger-audit-server/ranger-audit-consumer-solr/src/main/resources/conf/ranger-audit-consumer-solr-site.xml b/ranger-audit-server/ranger-audit-consumer-solr/src/main/resources/conf/ranger-audit-consumer-solr-site.xml new file mode 100644 index 0000000000..8b07c153f7 --- /dev/null +++ b/ranger-audit-server/ranger-audit-consumer-solr/src/main/resources/conf/ranger-audit-consumer-solr-site.xml @@ -0,0 +1,204 @@ + + + + + + log.dir + ${audit.consumer.solr.log.dir} + Log directory for Solr consumer service + + + + ranger.audit.consumer.solr.service.webapp.dir + webapp/ranger-audit-consumer-solr + Path to the extracted webapp directory + + + + ranger.audit.consumer.solr.service.contextName + / + + + + ranger.audit.consumer.solr.service.host + ranger-audit-consumer-solr.rangernw + Service hostname + + + + ranger.audit.consumer.solr.service.http.port + 7091 + Health check endpoint port + + + + + ranger.audit.service.authentication.method + KERBEROS + + + + ranger.audit.service.host + ranger-audit-consumer-solr.rangernw + Hostname for resolving _HOST in Kerberos principal + + + + ranger.audit.service.kerberos.principal + rangerauditserver/_HOST@EXAMPLE.COM + Principal for Kafka consumer and Solr authentication + + + + ranger.audit.service.kerberos.keytab + /etc/keytabs/rangerauditserver.keytab + + + + + xasecure.audit.destination.kafka.bootstrap.servers + ranger-kafka:9092 + + + + xasecure.audit.destination.kafka.topic.name + ranger_audits + + + + xasecure.audit.destination.kafka.security.protocol + SASL_PLAINTEXT + + + + xasecure.audit.destination.kafka.sasl.mechanism + GSSAPI + + + + + xasecure.audit.destination.kafka.consumer.thread.count + 5 + Number of Solr consumer worker threads (higher for indexing throughput) + + + + + xasecure.audit.destination.kafka.consumer.offset.commit.strategy + batch + batch or manual + + + + xasecure.audit.destination.kafka.consumer.offset.commit.interval.ms + 30000 + Used only if strategy is 'manual' + + + + xasecure.audit.destination.kafka.consumer.max.poll.records + 500 + Maximum records per poll for batch processing + + + + + xasecure.audit.destination.kafka.consumer.classes + org.apache.ranger.audit.consumer.kafka.AuditSolrConsumer + + + + + + xasecure.audit.destination.solr + true + MUST be true for Solr consumer to work + + + + xasecure.audit.destination.solr.urls + http://ranger-solr:8983/solr/ranger_audits + + Solr URLs for audits when SolrCloud is not enabled. + Docker supports only standalone mode for now, + configure http://ranger-solr:8983/solr/ranger_audits + + + + + xasecure.audit.destination.solr.zookeepers + + + Zookeepers for Solr audit destination (SolrCloud mode). + example: zookeeper-host1:2181,zookeeper-host2:2181,zookeeper-host3:2181 + In Docker, use: ranger-zk:2181/ranger_audits + + + + + xasecure.audit.destination.solr.batch.filespool.dir + /var/log/audit-consumer-solr/spool + + + + + xasecure.audit.destination.solr.force.use.inmemory.jaas.config + true + + + + xasecure.audit.jaas.Client.loginModuleName + com.sun.security.auth.module.Krb5LoginModule + + + + xasecure.audit.jaas.Client.loginModuleControlFlag + required + + + + xasecure.audit.jaas.Client.option.useKeyTab + true + + + + xasecure.audit.jaas.Client.option.storeKey + true + + + + xasecure.audit.jaas.Client.option.useTicketCache + true + Allow use of cached Kerberos tickets for Solr authentication + + + + xasecure.audit.jaas.Client.option.serviceName + solr + Kerberos service name for Solr + + + + xasecure.audit.jaas.Client.option.principal + rangerauditserver/_HOST@EXAMPLE.COM + Principal for Solr authentication + + + + xasecure.audit.jaas.Client.option.keyTab + /etc/keytabs/rangerauditserver.keytab + Keytab for Solr authentication + + diff --git a/ranger-audit-server/ranger-audit-consumer-solr/src/main/webapp/WEB-INF/applicationContext.xml b/ranger-audit-server/ranger-audit-consumer-solr/src/main/webapp/WEB-INF/applicationContext.xml new file mode 100644 index 0000000000..b6913c970e --- /dev/null +++ b/ranger-audit-server/ranger-audit-consumer-solr/src/main/webapp/WEB-INF/applicationContext.xml @@ -0,0 +1,39 @@ + + + + + + + + + + + + + + + + + + diff --git a/ranger-audit-server/ranger-audit-consumer-solr/src/main/webapp/WEB-INF/web.xml b/ranger-audit-server/ranger-audit-consumer-solr/src/main/webapp/WEB-INF/web.xml new file mode 100644 index 0000000000..db19d1326b --- /dev/null +++ b/ranger-audit-server/ranger-audit-consumer-solr/src/main/webapp/WEB-INF/web.xml @@ -0,0 +1,60 @@ + + + + + Apache Ranger - Audit Consumer Solr Service + Apache Ranger - Audit Consumer Solr Service (Kafka to Solr) + + + contextConfigLocation + + WEB-INF/applicationContext.xml + + + + + org.springframework.web.context.ContextLoaderListener + + + + org.springframework.web.context.request.RequestContextListener + + + + Health Check Service + com.sun.jersey.spi.spring.container.servlet.SpringServlet + + + com.sun.jersey.config.property.packages + org.apache.ranger.audit.rest + + + + com.sun.jersey.api.json.POJOMappingFeature + true + + + 1 + + + + Health Check Service + /api/* + + + diff --git a/ranger-audit-server/ranger-audit-server-service/pom.xml b/ranger-audit-server/ranger-audit-server-service/pom.xml new file mode 100644 index 0000000000..3cae25fc8a --- /dev/null +++ b/ranger-audit-server/ranger-audit-server-service/pom.xml @@ -0,0 +1,383 @@ + + + + 4.0.0 + + + org.apache.ranger + ranger-audit-server + 3.0.0-SNAPSHOT + .. + + + ranger-audit-server-service + war + Ranger Audit Server Service + Core audit service: REST API for receiving audit events and producing to Kafka + + + UTF-8 + false + + + + + + + ch.qos.logback + logback-classic + ${logback.version} + + + + com.fasterxml.jackson.core + jackson-annotations + ${fasterxml.jackson.version} + + + com.fasterxml.jackson.core + jackson-core + ${fasterxml.jackson.version} + + + com.fasterxml.jackson.core + jackson-databind + ${fasterxml.jackson.version} + + + com.google.guava + guava + ${google.guava.version} + + + com.google.inject + guice + ${guice.version} + + + com.google.guava + guava + + + + + + + com.sun.jersey + jersey-bundle + ${jersey-bundle.version} + + + + com.sun.jersey + jersey-core + ${jersey-bundle.version} + + + + com.sun.jersey.contribs + jersey-spring + ${jersey-spring.version} + + + com.sun.jersey + jersey-server + + + org.springframework + * + + + + + + com.sun.xml.bind + jaxb-impl + ${jaxb-impl.version} + + + + commons-codec + commons-codec + ${commons.codec.version} + + + + commons-io + commons-io + ${commons.io.version} + + + + javax.servlet + javax.servlet-api + ${javax.servlet.version} + + + + javax.xml.bind + jaxb-api + ${jaxb.api.version} + + + + org.apache.commons + commons-lang3 + ${commons.lang3.version} + + + + org.apache.hadoop + hadoop-common + ${hadoop.version} + + + ch.qos.reload4j + reload4j + + + log4j + * + + + org.apache.hadoop + hadoop-client-api + + + org.apache.hadoop + hadoop-hdfs + + + org.slf4j + * + + + + + + + org.apache.httpcomponents + httpclient + ${httpcomponents.httpclient.version} + + + commons-logging + * + + + + + org.apache.httpcomponents + httpcore + ${httpcomponents.httpcore.version} + + + + + org.apache.kafka + kafka-clients + ${kafka.version} + + + log4j + * + + + org.slf4j + * + + + + + + + org.apache.ranger + ranger-audit-common + ${project.version} + + + org.apache.hadoop + hadoop-client-api + + + + + org.apache.ranger + ranger-audit-core + ${project.version} + + + org.apache.hadoop + hadoop-client-api + + + + + org.apache.ranger + ranger-authn + ${project.version} + + + org.apache.hadoop + hadoop-client-api + + + + + org.apache.ranger + ranger-plugins-common + ${project.version} + + + javax.servlet + javax.servlet-api + + + org.apache.hadoop + hadoop-client-api + + + org.apache.shiro + shiro-core + + + + + org.apache.ranger + ranger-plugins-cred + ${project.version} + + + org.apache.hadoop + hadoop-client-api + + + + + + + org.apache.tomcat + tomcat-annotations-api + ${tomcat.embed.version} + + + org.apache.tomcat.embed + tomcat-embed-core + ${tomcat.embed.version} + + + org.apache.tomcat.embed + tomcat-embed-el + ${tomcat.embed.version} + + + org.apache.tomcat.embed + tomcat-embed-jasper + ${tomcat.embed.version} + + + + org.slf4j + log4j-over-slf4j + ${slf4j.version} + + + + + org.slf4j + log4j-over-slf4j + ${slf4j.version} + + + org.slf4j + slf4j-api + ${slf4j.version} + + + + + org.springframework + spring-beans + ${springframework.version} + + + org.springframework + spring-jcl + + + + + org.springframework + spring-context + ${springframework.version} + + + org.springframework + spring-context-support + ${springframework.version} + + + org.springframework + spring-tx + ${springframework.version} + + + org.springframework + spring-web + ${springframework.version} + + + org.springframework.security + spring-security-config + ${springframework.security.version} + + + org.springframework + * + + + + + org.springframework.security + spring-security-web + ${springframework.security.version} + + + org.springframework + * + + + + + + + + + true + src/main/resources + + + + + org.apache.maven.plugins + maven-war-plugin + + + + diff --git a/ranger-audit-server/ranger-audit-server-service/scripts/start-audit-server.sh b/ranger-audit-server/ranger-audit-server-service/scripts/start-audit-server.sh new file mode 100755 index 0000000000..52b9a00f26 --- /dev/null +++ b/ranger-audit-server/ranger-audit-server-service/scripts/start-audit-server.sh @@ -0,0 +1,162 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# ======================================== +# Ranger Audit Server Service Start Script +# ======================================== +# This script starts the Ranger Audit Server service +# The service receives audit events from Ranger plugins and produces them to Kafka + +set -e + +# Script directory +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +SERVICE_DIR="$(dirname "$SCRIPT_DIR")" + +# Default directories - can be overridden by environment variables +AUDIT_SERVER_HOME_DIR="${AUDIT_SERVER_HOME_DIR:-${SERVICE_DIR}/target}" +AUDIT_SERVER_CONF_DIR="${AUDIT_SERVER_CONF_DIR:-${SERVICE_DIR}/src/main/resources/conf}" +AUDIT_SERVER_LOG_DIR="${AUDIT_SERVER_LOG_DIR:-${SERVICE_DIR}/logs}" + +# Create log directory if it doesn't exist +mkdir -p "${AUDIT_SERVER_LOG_DIR}" + +echo "==========================================" +echo "Starting Ranger Audit Server Service" +echo "==========================================" +echo "Service Directory: ${SERVICE_DIR}" +echo "Home Directory: ${AUDIT_SERVER_HOME_DIR}" +echo "Config Directory: ${AUDIT_SERVER_CONF_DIR}" +echo "Log Directory: ${AUDIT_SERVER_LOG_DIR}" +echo "==========================================" + +# Check if Java is available +if [ -z "$JAVA_HOME" ]; then + JAVA_CMD=$(which java 2>/dev/null || true) + if [ -z "$JAVA_CMD" ]; then + echo "[ERROR] JAVA_HOME is not set and java is not in PATH" + exit 1 + fi + JAVA_HOME=$(dirname $(dirname $(readlink -f "$JAVA_CMD"))) + echo "[INFO] JAVA_HOME not set, detected: ${JAVA_HOME}" +fi + +export JAVA_HOME +export PATH=$JAVA_HOME/bin:$PATH + +echo "[INFO] Java version:" +java -version + +# Set heap size (default: 512MB to 2GB) +AUDIT_SERVER_HEAP="${AUDIT_SERVER_HEAP:--Xms512m -Xmx2g}" + +# Set JVM options +if [ -z "$AUDIT_SERVER_OPTS" ]; then + AUDIT_SERVER_OPTS="-Dlogback.configurationFile=${AUDIT_SERVER_CONF_DIR}/logback.xml" + AUDIT_SERVER_OPTS="${AUDIT_SERVER_OPTS} -Daudit.server.log.dir=${AUDIT_SERVER_LOG_DIR}" + AUDIT_SERVER_OPTS="${AUDIT_SERVER_OPTS} -Daudit.server.log.file=ranger-audit-server.log" + AUDIT_SERVER_OPTS="${AUDIT_SERVER_OPTS} -Djava.net.preferIPv4Stack=true -server" + AUDIT_SERVER_OPTS="${AUDIT_SERVER_OPTS} -XX:+UseG1GC -XX:MaxGCPauseMillis=200" + AUDIT_SERVER_OPTS="${AUDIT_SERVER_OPTS} -XX:InitiatingHeapOccupancyPercent=35" + AUDIT_SERVER_OPTS="${AUDIT_SERVER_OPTS} -XX:ConcGCThreads=4 -XX:ParallelGCThreads=8" +fi + +# Add Kerberos configuration if needed +if [ "${KERBEROS_ENABLED}" == "true" ]; then + AUDIT_SERVER_OPTS="${AUDIT_SERVER_OPTS} -Djava.security.krb5.conf=/etc/krb5.conf" + echo "[INFO] Kerberos is enabled" +fi + +export AUDIT_SERVER_OPTS +export AUDIT_SERVER_LOG_DIR + +echo "[INFO] JAVA_HOME: ${JAVA_HOME}" +echo "[INFO] HEAP: ${AUDIT_SERVER_HEAP}" +echo "[INFO] JVM_OPTS: ${AUDIT_SERVER_OPTS}" + +# Find the WAR file +WAR_FILE=$(find "${AUDIT_SERVER_HOME_DIR}" -name "ranger-audit-server-service*.war" | head -1) + +if [ -z "$WAR_FILE" ] || [ ! -f "$WAR_FILE" ]; then + echo "[ERROR] WAR file not found in ${AUDIT_SERVER_HOME_DIR}" + echo "[ERROR] Please build the project first using: mvn clean package" + exit 1 +fi + +echo "[INFO] Using WAR file: ${WAR_FILE}" + +# Extract WAR if not already extracted +WEBAPP_DIR="${AUDIT_SERVER_HOME_DIR}/webapp/ranger-audit-server-service" +if [ ! -d "${WEBAPP_DIR}" ]; then + echo "[INFO] Extracting WAR file..." + mkdir -p "${WEBAPP_DIR}" + cd "${WEBAPP_DIR}" + jar xf "${WAR_FILE}" + cd - > /dev/null +fi + +# Build classpath +RANGER_CLASSPATH="${WEBAPP_DIR}/WEB-INF/classes" +for jar in "${WEBAPP_DIR}"/WEB-INF/lib/*.jar; do + RANGER_CLASSPATH="${RANGER_CLASSPATH}:${jar}" +done + +# Add libext directory if it exists +if [ -d "${AUDIT_SERVER_HOME_DIR}/libext" ]; then + for jar in "${AUDIT_SERVER_HOME_DIR}"/libext/*.jar; do + if [ -f "${jar}" ]; then + RANGER_CLASSPATH="${RANGER_CLASSPATH}:${jar}" + fi + done +fi + +export RANGER_CLASSPATH + +# Check if already running +PID_FILE="${AUDIT_SERVER_LOG_DIR}/ranger-audit-server.pid" +if [ -f "${PID_FILE}" ]; then + OLD_PID=$(cat "${PID_FILE}") + if kill -0 "$OLD_PID" 2>/dev/null; then + echo "[WARNING] Ranger Audit Server is already running (PID: ${OLD_PID})" + echo "[INFO] Use stop-audit-server.sh to stop it first" + exit 1 + else + echo "[INFO] Removing stale PID file" + rm -f "${PID_FILE}" + fi +fi + +# Start the service +echo "[INFO] Starting Ranger Audit Server Service..." +nohup java ${AUDIT_SERVER_HEAP} ${AUDIT_SERVER_OPTS} \ + -Daudit.config=${AUDIT_SERVER_CONF_DIR}/ranger-audit-server-site.xml \ + -cp "${RANGER_CLASSPATH}" \ + org.apache.ranger.audit.server.AuditServerApplication \ + >> "${AUDIT_SERVER_LOG_DIR}/catalina.out" 2>&1 & + +PID=$! +echo $PID > "${PID_FILE}" + +echo "[INFO] ✓ Ranger Audit Server started successfully" +echo "[INFO] PID: ${PID}" +echo "[INFO] Log file: ${AUDIT_SERVER_LOG_DIR}/ranger-audit-server.log" +echo "[INFO] Catalina out: ${AUDIT_SERVER_LOG_DIR}/catalina.out" +echo "[INFO] Health check: http://localhost:7081/api/audit/health" +echo "" +echo "To monitor logs: tail -f ${AUDIT_SERVER_LOG_DIR}/ranger-audit-server.log" +echo "To stop service: ${SCRIPT_DIR}/stop-audit-server.sh" diff --git a/ranger-audit-server/ranger-audit-server-service/scripts/stop-audit-server.sh b/ranger-audit-server/ranger-audit-server-service/scripts/stop-audit-server.sh new file mode 100755 index 0000000000..87ff4af9ad --- /dev/null +++ b/ranger-audit-server/ranger-audit-server-service/scripts/stop-audit-server.sh @@ -0,0 +1,72 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# ======================================== +# Ranger Audit Server Service Stop Script +# ======================================== + +set -e + +# Script directory +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +SERVICE_DIR="$(dirname "$SCRIPT_DIR")" + +# Default log directory +AUDIT_SERVER_LOG_DIR="${AUDIT_SERVER_LOG_DIR:-${SERVICE_DIR}/logs}" +PID_FILE="${AUDIT_SERVER_LOG_DIR}/ranger-audit-server.pid" + +echo "==========================================" +echo "Stopping Ranger Audit Server Service" +echo "==========================================" + +if [ ! -f "${PID_FILE}" ]; then + echo "[WARNING] PID file not found: ${PID_FILE}" + echo "[INFO] Service may not be running" + exit 0 +fi + +PID=$(cat "${PID_FILE}") + +if ! kill -0 "$PID" 2>/dev/null; then + echo "[WARNING] Process ${PID} is not running" + echo "[INFO] Removing stale PID file" + rm -f "${PID_FILE}" + exit 0 +fi + +echo "[INFO] Stopping process ${PID}..." +kill "$PID" + +# Wait for process to stop (max 30 seconds) +TIMEOUT=30 +COUNT=0 +while kill -0 "$PID" 2>/dev/null && [ $COUNT -lt $TIMEOUT ]; do + sleep 1 + COUNT=$((COUNT + 1)) + echo -n "." +done +echo "" + +if kill -0 "$PID" 2>/dev/null; then + echo "[WARNING] Process did not stop gracefully, forcing shutdown..." + kill -9 "$PID" + sleep 2 +fi + +rm -f "${PID_FILE}" +echo "[INFO] ✓ Ranger Audit Server stopped successfully" diff --git a/ranger-audit-server/ranger-audit-server-service/src/main/java/org/apache/ranger/audit/producer/AuditDestinationMgr.java b/ranger-audit-server/ranger-audit-server-service/src/main/java/org/apache/ranger/audit/producer/AuditDestinationMgr.java new file mode 100644 index 0000000000..73cdc0428c --- /dev/null +++ b/ranger-audit-server/ranger-audit-server-service/src/main/java/org/apache/ranger/audit/producer/AuditDestinationMgr.java @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.audit.producer; + +import org.apache.ranger.audit.model.AuthzAuditEvent; +import org.apache.ranger.audit.producer.kafka.AuditMessageQueue; +import org.apache.ranger.audit.provider.AuditHandler; +import org.apache.ranger.audit.provider.AuditProviderFactory; +import org.apache.ranger.audit.provider.MiscUtil; +import org.apache.ranger.audit.server.AuditServerConfig; +import org.apache.ranger.audit.server.AuditServerConstants; +import org.apache.ranger.audit.utils.AuditServerUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; + +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; + +import java.util.Properties; + +@Component +public class AuditDestinationMgr { + private static final Logger LOG = LoggerFactory.getLogger(AuditDestinationMgr.class); + + public AuditHandler auditHandler; + public AuditServerUtils auditServerUtils; + + @PostConstruct + public void configure() { + init(); + } + + public void init() { + LOG.info("==> AuditDestinationMgr.init()"); + + auditServerUtils = new AuditServerUtils(); + AuditServerConfig auditConfig = AuditServerConfig.getInstance(); + Properties properties = auditConfig.getProperties(); + if (properties != null) { + auditServerUtils.setAuditConfig(properties); + } + + String kafkaDestPrefix = AuditProviderFactory.AUDIT_DEST_BASE + "." + AuditServerConstants.DEFAULT_SERVICE_NAME; + boolean isAuditToKafkaDestinationEnabled = MiscUtil.getBooleanProperty(properties, kafkaDestPrefix, false); + if (isAuditToKafkaDestinationEnabled) { + auditHandler = new AuditMessageQueue(); + auditHandler.init(properties, kafkaDestPrefix); + auditHandler.start(); + LOG.info("Kafka producer initialized and started"); + } else { + LOG.warn("Kafka audit destination is not enabled. Producer service will not function."); + } + + LOG.info("<== AuditDestinationMgr.init() AuditDestination: {} ", kafkaDestPrefix); + } + + public boolean log(AuthzAuditEvent authzAuditEvent) throws Exception { + boolean ret = false; + + if (auditHandler == null) { + init(); + } + ret = auditHandler.log(authzAuditEvent); + + return ret; + } + + @PreDestroy + public void shutdown() { + LOG.info("==> AuditDestinationMgr.shutdown()"); + + if (auditHandler != null) { + try { + LOG.info("Shutting down audit handler: {}", auditHandler.getClass().getSimpleName()); + auditHandler.stop(); + LOG.info("Audit handler shutdown completed successfully"); + } catch (Exception e) { + LOG.error("Error shutting down audit handler", e); + } + } + + LOG.info("<== AuditDestinationMgr.shutdown()"); + } +} diff --git a/ranger-audit-server/ranger-audit-server-service/src/main/java/org/apache/ranger/audit/producer/kafka/AuditMessageQueue.java b/ranger-audit-server/ranger-audit-server-service/src/main/java/org/apache/ranger/audit/producer/kafka/AuditMessageQueue.java new file mode 100644 index 0000000000..a4b387ff9e --- /dev/null +++ b/ranger-audit-server/ranger-audit-server-service/src/main/java/org/apache/ranger/audit/producer/kafka/AuditMessageQueue.java @@ -0,0 +1,277 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.audit.producer.kafka; + +import org.apache.kafka.clients.producer.KafkaProducer; +import org.apache.ranger.audit.destination.AuditDestination; +import org.apache.ranger.audit.model.AuditEventBase; +import org.apache.ranger.audit.model.AuthzAuditEvent; +import org.apache.ranger.audit.provider.MiscUtil; +import org.apache.ranger.audit.utils.AuditMessageQueueUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.security.PrivilegedExceptionAction; +import java.util.Collection; +import java.util.Properties; + +/** + * AuditMessageQueue creates the necessary kafka queue for http post to relay the audit events into kafka. + * It creates the necessary audit topics, producer threads and recovery threads. + */ +public class AuditMessageQueue extends AuditDestination { + public KafkaProducer kafkaProducer; + public AuditProducer auditProducerRunnable; + public AuditMessageQueueUtils auditMessageQueueUtils; + public String topicName; + + private Thread producerThread; + private AuditRecoveryManager recoveryManager; + + private static final Logger LOG = LoggerFactory.getLogger(AuditMessageQueue.class); + + @Override + public void init(Properties props, String propPrefix) { + LOG.info("==> AuditMessageQueue.init() [CORE AUDIT SERVER]"); + + super.init(props, propPrefix); + + auditMessageQueueUtils = new AuditMessageQueueUtils(props); + + createAuditsTopic(props, propPrefix); + createKafkaProducer(props, propPrefix); + createRecoveryManager(props, propPrefix); + + LOG.info("<== AuditMessageQueue.init() [CORE AUDIT SERVER]: created topic: {}, producer: {}", + topicName, (kafkaProducer != null) ? kafkaProducer.getClass() : ""); + } + + @Override + public void start() { + LOG.debug("==> AuditMessageQueue.start() - Starting Audit Producer and Recovery threads"); + startRangerAuditRecoveryThread(); + startRangerAuditProducer(); + LOG.debug("<== AuditMessageQueue.start()"); + } + + @Override + public void stop() { + LOG.info("==> AuditMessageQueue.stop() [CORE AUDIT SERVER]"); + + // Shutdown recovery manager first to process any remaining messages + if (recoveryManager != null) { + try { + LOG.info("Shutting down Audit recovery manager..."); + recoveryManager.stop(); + LOG.info("Audit recovery manager shutdown completed"); + } catch (Exception e) { + LOG.error("Error shutting down Audit recovery manager", e); + } + } + + // Shutdown producer thread + if (auditProducerRunnable != null) { + try { + LOG.info("Shutting down Audit producer..."); + auditProducerRunnable.shutdown(); + + // Interrupt and wait for producer thread to finish + if (producerThread != null && producerThread.isAlive()) { + producerThread.interrupt(); + try { + producerThread.join(5000); // Wait up to 5 seconds + if (producerThread.isAlive()) { + LOG.warn("Audit Producer thread did not terminate within 5 seconds"); + } else { + LOG.info("Audit Producer thread terminated successfully"); + } + } catch (InterruptedException e) { + LOG.warn("Interrupted while waiting for producer thread to terminate", e); + Thread.currentThread().interrupt(); + } + } + LOG.info("Audit producer shutdown completed"); + } catch (Exception e) { + LOG.error("Error shutting down Kafka audit producer", e); + } + } + + // Close producer + if (kafkaProducer != null) { + try { + LOG.info("Closing Kafka producer..."); + kafkaProducer.close(); + LOG.info("Kafka producer shutdown completed"); + } catch (Exception e) { + LOG.error("Error shutting down Kafka producer", e); + } + } + + LOG.info("<== AuditMessageQueue.stop() [CORE AUDIT SERVER]"); + } + + @Override + public synchronized boolean log(final AuditEventBase event) { + boolean ret = false; + if (event instanceof AuthzAuditEvent) { + AuthzAuditEvent authzEvent = (AuthzAuditEvent) event; + + if (authzEvent.getAgentHostname() == null) { + authzEvent.setAgentHostname(MiscUtil.getHostname()); + } + + if (authzEvent.getLogType() == null) { + authzEvent.setLogType("RangerAudit"); + } + + if (authzEvent.getEventId() == null) { + authzEvent.setEventId(MiscUtil.generateUniqueId()); + } + + // Partition key is agentId (aka plugin ID) used by Kafka's default partitioner + // for load balancing across partitions. Messages with the same key go to the same partition, + // providing ordering guarantees per appId while enabling full horizontal scaling + + final String key = authzEvent.getAgentId(); + final String message = MiscUtil.stringify(event); + + try { + if (topicName == null || kafkaProducer == null) { + init(props, propPrefix); + } + if (kafkaProducer != null) { + MiscUtil.executePrivilegedAction((PrivilegedExceptionAction) () -> { + AuditProducer.send(kafkaProducer, topicName, key, message); + return null; + }); + ret = true; + } else { + // Kafka producer not available - spool to file for recovery + LOG.warn("Kafka producer not available, spooling message to recovery"); + spoolToRecovery(key, message); + } + } catch (Throwable t) { + LOG.error("Error sending message to Kafka topic. topic={}, key={}, message={}", topicName, key, t.getMessage()); + // Spool to file for recovery + spoolToRecovery(key, message); + } + } + return ret; + } + + @Override + public boolean log(final Collection events) { + return false; + } + + private void startRangerAuditRecoveryThread() { + LOG.info("==> AuditMessageQueue.startRangerAuditRecoveryThread()"); + + try { + if (recoveryManager != null) { + recoveryManager.start(); + LOG.info("Audit Recovery Manager started with writer and retry threads"); + } else { + LOG.warn("==== Recovery manager is null; recovery threads not started"); + } + } catch (Exception e) { + LOG.error("Error starting Audit Recovery Manager", e); + } + + LOG.info("<== AuditMessageQueue.startRangerAuditRecoveryThread()"); + } + + private void startRangerAuditProducer() { + LOG.info("==> AuditMessageQueue.startRangerAuditProducer()"); + + try { + if (auditProducerRunnable != null) { + producerThread = new Thread(auditProducerRunnable, "AuditProducer"); + producerThread.setDaemon(true); + producerThread.start(); + LOG.info("==== AuditProducer Thread started: {}", producerThread.getName()); + } else { + LOG.warn("AuditProducer runnable is null; producer thread not started"); + } + } catch (Exception e) { + LOG.error("Error Starting Ranger Audit Producer", e); + } + + LOG.info("<== AuditMessageQueue.startRangerAuditProducer()"); + } + + private void createKafkaProducer(final Properties props, final String propPrefix) { + if (auditProducerRunnable == null) { + try { + auditProducerRunnable = new AuditProducer(props, propPrefix); + if (auditProducerRunnable != null) { + kafkaProducer = auditProducerRunnable.getKafkaProducer(); + } + } catch (Exception e) { + LOG.error("Error creating Kafka producer", e); + } + } + } + + private void createAuditsTopic(final Properties props, final String propPrefix) { + if (topicName == null) { + topicName = auditMessageQueueUtils.createAuditsTopicIfNotExists(props, propPrefix); + } + } + + private void createRecoveryManager(final Properties props, final String propPrefix) { + // Create recovery manager even if kafkaProducer is null - it will handle null producer gracefully + // This ensures audits are spooled when Kafka is unavailable during startup + if (recoveryManager == null && topicName != null) { + try { + recoveryManager = new AuditRecoveryManager(props, propPrefix, this, topicName); + LOG.info("Created Audit Recovery Manager (Kafka producer available: {})", (kafkaProducer != null)); + } catch (Exception e) { + LOG.error("Error creating Audit Recovery Manager", e); + } + } + } + + /** + * Spool failed audit message to recovery system + */ + private void spoolToRecovery(String key, String message) { + if (recoveryManager != null) { + boolean queued = recoveryManager.addFailedMessage(key, message); + if (queued) { + LOG.debug("Spooled failed message to recovery system"); + } else { + LOG.warn("Failed to spool message to recovery system - queue may be full or recovery disabled"); + } + } else { + LOG.warn("Recovery manager not initialized - cannot spool failed message"); + } + } + + /** + * Get recovery statistics for monitoring + */ + public AuditRecoveryManager.RecoveryStats getRecoveryStats() { + if (recoveryManager != null) { + return recoveryManager.getStats(); + } + return null; + } +} diff --git a/ranger-audit-server/ranger-audit-server-service/src/main/java/org/apache/ranger/audit/producer/kafka/AuditProducer.java b/ranger-audit-server/ranger-audit-server-service/src/main/java/org/apache/ranger/audit/producer/kafka/AuditProducer.java new file mode 100644 index 0000000000..e8c13d76fb --- /dev/null +++ b/ranger-audit-server/ranger-audit-server-service/src/main/java/org/apache/ranger/audit/producer/kafka/AuditProducer.java @@ -0,0 +1,132 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.audit.producer.kafka; + +import org.apache.kafka.clients.admin.AdminClientConfig; +import org.apache.kafka.clients.producer.Callback; +import org.apache.kafka.clients.producer.KafkaProducer; +import org.apache.kafka.clients.producer.ProducerConfig; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.clients.producer.RecordMetadata; +import org.apache.ranger.audit.provider.MiscUtil; +import org.apache.ranger.audit.server.AuditServerConstants; +import org.apache.ranger.audit.utils.AuditMessageQueueUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.security.PrivilegedExceptionAction; +import java.util.Properties; + +public class AuditProducer implements Runnable { + private static final Logger LOG = LoggerFactory.getLogger(AuditProducer.class); + + public Properties producerProps = new Properties(); + public KafkaProducer kafkaProducer; + private volatile boolean running = true; + + public AuditProducer(Properties props, String propPrefix) throws Exception { + LOG.debug("==> AuditProducer()"); + + AuditMessageQueueUtils auditMessageQueueUtils = new AuditMessageQueueUtils(props); + producerProps.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, MiscUtil.getStringProperty(props, propPrefix + "." + AuditServerConstants.PROP_BOOTSTRAP_SERVERS)); + producerProps.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, Class.forName("org.apache.kafka.common.serialization.StringSerializer")); + producerProps.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, Class.forName("org.apache.kafka.common.serialization.StringSerializer")); + producerProps.put(ProducerConfig.ENABLE_IDEMPOTENCE_CONFIG, "true"); + producerProps.put(ProducerConfig.ACKS_CONFIG, "all"); + + String securityProtocol = MiscUtil.getStringProperty(props, propPrefix + "." + AuditServerConstants.PROP_SECURITY_PROTOCOL, AuditServerConstants.DEFAULT_SECURITY_PROTOCOL); + producerProps.put(AdminClientConfig.SECURITY_PROTOCOL_CONFIG, securityProtocol); + + producerProps.put(AuditServerConstants.PROP_SASL_MECHANISM, MiscUtil.getStringProperty(props, propPrefix + "." + AuditServerConstants.PROP_SASL_MECHANISM, AuditServerConstants.DEFAULT_SASL_MECHANISM)); + producerProps.put(AuditServerConstants.PROP_SASL_KERBEROS_SERVICE_NAME, AuditServerConstants.DEFAULT_SERVICE_NAME); + + if (securityProtocol.toUpperCase().contains(AuditServerConstants.PROP_SECURITY_PROTOCOL_VALUE)) { + producerProps.put(AuditServerConstants.PROP_SASL_JAAS_CONFIG, auditMessageQueueUtils.getJAASConfig(props, propPrefix)); + } + + producerProps.put(ProducerConfig.LINGER_MS_CONFIG, 5); + producerProps.put(ProducerConfig.BATCH_SIZE_CONFIG, 32 * 1024); + + try { + kafkaProducer = MiscUtil.executePrivilegedAction((PrivilegedExceptionAction>) () -> new KafkaProducer<>(producerProps)); + LOG.info("AuditProducer(): KafkaProducer created successfully!"); + } catch (Exception ex) { + LOG.warn("AuditProducer(): Unable to create KafkaProducer - Kafka may not be available. " + + "Audit messages will be spooled to recovery system for retry. Error: {}", ex.getMessage()); + LOG.debug("Full exception details:", ex); + } + + LOG.debug("<== AuditProducer()"); + } + + @Override + public void run() { + LOG.info("AuditProducer thread started"); + while (running) { + try { + Thread.sleep(100); // keep thread alive + } catch (InterruptedException e) { + LOG.info("AuditProducer: Thread interrupted. Exiting..."); + Thread.currentThread().interrupt(); + break; + } + } + LOG.info("AuditProducer thread stopped"); + } + + public void shutdown() { + LOG.info("==> AuditProducer.shutdown()"); + running = false; + + if (kafkaProducer != null) { + try { + LOG.info("Closing Kafka producer..."); + kafkaProducer.close(); + LOG.info("Kafka producer closed successfully"); + } catch (Exception e) { + LOG.error("Error closing Kafka producer", e); + } + } + + LOG.info("<== AuditProducer.shutdown()"); + } + + public KafkaProducer getKafkaProducer() { + return kafkaProducer; + } + + public static void send(KafkaProducer producer, String topic, String key, String value) throws Exception { + ProducerRecord auditEvent = new ProducerRecord<>(topic, key, value); + try { + producer.send(auditEvent, new Callback() { + @Override + public void onCompletion(RecordMetadata metadata, Exception e) { + if (e != null) { + LOG.error("Error sending Ranger Audit logs to Kafka....", e); + } else { + LOG.debug("Ranger Audit sent to Topic: {} Partition: {} Offset: {}", metadata.topic(), metadata.partition(), metadata.offset()); + } + } + }); + } catch (Exception e) { + throw new Exception(e); + } + } +} diff --git a/ranger-audit-server/ranger-audit-server-service/src/main/java/org/apache/ranger/audit/producer/kafka/AuditRecoveryManager.java b/ranger-audit-server/ranger-audit-server-service/src/main/java/org/apache/ranger/audit/producer/kafka/AuditRecoveryManager.java new file mode 100644 index 0000000000..28210d2e6e --- /dev/null +++ b/ranger-audit-server/ranger-audit-server-service/src/main/java/org/apache/ranger/audit/producer/kafka/AuditRecoveryManager.java @@ -0,0 +1,222 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.audit.producer.kafka; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Properties; + +/** + * AuditRecoveryManager - Manages the audit recovery system with two threads: + * 1. AuditRecoveryWriter - Writes failed messages to spool files with .failed file extension + * 2. AuditRecoveryRetry - Reads .failed files and retries to Kafka and if successful move to archive folder with .processed file extension + */ +public class AuditRecoveryManager { + private static final Logger LOG = LoggerFactory.getLogger(AuditRecoveryManager.class); + + private final AuditRecoveryWriter recoveryWriter; + private final AuditRecoveryRetry recoveryRetry; + private Thread writerThread; + private Thread retryThread; + private volatile boolean started; + + public AuditRecoveryManager(Properties props, String propPrefix, AuditMessageQueue messageQueue, String topicName) { + LOG.info("==> AuditRecoveryManager()"); + + // Initialize writer and retry components + // Pass messageQueue reference so retry can get current producer (may be recreated when Kafka comes online) + this.recoveryWriter = new AuditRecoveryWriter(props, propPrefix); + this.recoveryRetry = new AuditRecoveryRetry(props, propPrefix, messageQueue, topicName); + + LOG.info("<== AuditRecoveryManager() initialized"); + } + + /** + * Start the recovery threads + */ + public void start() { + if (started) { + LOG.info("AuditRecoveryManager already started"); + return; + } + + LOG.info("==> AuditRecoveryManager.start()"); + + try { + // Start recovery writer thread + if (recoveryWriter.isEnabled()) { + writerThread = new Thread(recoveryWriter, "AuditRecoveryWriter"); + writerThread.setDaemon(true); + writerThread.start(); + LOG.info("Started AuditRecoveryWriter thread"); + } else { + LOG.info("AuditRecoveryWriter is disabled"); + } + + // Start recovery retry thread + if (recoveryRetry.isEnabled()) { + retryThread = new Thread(recoveryRetry, "AuditRecoveryRetry"); + retryThread.setDaemon(true); + retryThread.start(); + LOG.info("Started AuditRecoveryRetry thread"); + } else { + LOG.info("AuditRecoveryRetry is disabled"); + } + + started = true; + LOG.info("<== AuditRecoveryManager.start() completed successfully"); + } catch (Exception e) { + LOG.error("Error starting AuditRecoveryManager", e); + throw new RuntimeException("Failed to start AuditRecoveryManager", e); + } + } + + /** + * Stop the recovery threads gracefully + */ + public void stop() { + if (!started) { + LOG.warn("AuditRecoveryManager not started"); + return; + } + + LOG.info("==> AuditRecoveryManager.stop()"); + + try { + // Shutdown recovery writer + if (recoveryWriter != null) { + LOG.info("Shutting down AuditRecoveryWriter..."); + recoveryWriter.shutdown(); + + if (writerThread != null && writerThread.isAlive()) { + writerThread.interrupt(); + try { + writerThread.join(10000); // Wait up to 10 seconds + if (writerThread.isAlive()) { + LOG.warn("AuditRecoveryWriter thread did not terminate within 10 seconds"); + } else { + LOG.info("AuditRecoveryWriter thread terminated successfully"); + } + } catch (InterruptedException e) { + LOG.warn("Interrupted while waiting for writer thread to terminate", e); + Thread.currentThread().interrupt(); + } + } + } + + // Shutdown recovery retry + if (recoveryRetry != null) { + LOG.info("Shutting down AuditRecoveryRetry..."); + recoveryRetry.shutdown(); + + if (retryThread != null && retryThread.isAlive()) { + retryThread.interrupt(); + try { + //Wait up to 10 seconds + retryThread.join(10000); + if (retryThread.isAlive()) { + LOG.warn("AuditRecoveryRetry thread did not terminate within 10 seconds"); + } else { + LOG.info("AuditRecoveryRetry thread terminated successfully"); + } + } catch (InterruptedException e) { + LOG.warn("Interrupted while waiting for retry thread to terminate", e); + Thread.currentThread().interrupt(); + } + } + } + + started = false; + + LOG.info("<== AuditRecoveryManager.stop() completed"); + } catch (Exception e) { + LOG.error("Error stopping AuditRecoveryManager", e); + } + } + + /** + * Add a failed audit message to the recovery queue + * @param key The partition key (agentId) + * @param message The serialized audit message + * @return true if message was queued successfully + */ + public boolean addFailedMessage(String key, String message) { + boolean ret; + if (recoveryWriter == null || !recoveryWriter.isEnabled()) { + ret = false; + } else { + ret = recoveryWriter.addFailedMessage(key, message); + } + return ret; + } + + /** + * Get recovery statistics for monitoring + */ + public RecoveryStats getStats() { + RecoveryStats ret = new RecoveryStats(); + + if (recoveryWriter != null) { + ret.writerQueueSize = recoveryWriter.getQueueSize(); + ret.writerCurrentFile = recoveryWriter.getCurrentFileName(); + ret.writerMessageCount = recoveryWriter.getCurrentFileMessageCount(); + } + + if (recoveryRetry != null) { + ret.retryTotalRetried = recoveryRetry.getTotalMessagesRetried(); + ret.retryTotalSucceeded = recoveryRetry.getTotalMessagesSucceeded(); + ret.retryTotalFailed = recoveryRetry.getTotalMessagesFailed(); + } + + return ret; + } + + /** + * Check if recovery system is enabled and running + */ + public boolean isRunning() { + return started; + } + + /** + * Statistics class for monitoring + */ + public static class RecoveryStats { + public int writerQueueSize; + public int writerMessageCount; + public String writerCurrentFile; + public long retryTotalRetried; + public long retryTotalSucceeded; + public long retryTotalFailed; + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("RecoveryStats{") + .append(" writerQueue= ").append(writerQueueSize) + .append(" writerFile= ").append(writerCurrentFile) + .append(" writerMsgCount= ").append(writerMessageCount) + .append(" retryRetried= ").append(retryTotalRetried) + .append(" retrySucceeded= ").append(retryTotalSucceeded) + .append(" retryFailed= ").append(retryTotalFailed); + return sb.toString(); + } + } +} diff --git a/ranger-audit-server/ranger-audit-server-service/src/main/java/org/apache/ranger/audit/producer/kafka/AuditRecoveryRetry.java b/ranger-audit-server/ranger-audit-server-service/src/main/java/org/apache/ranger/audit/producer/kafka/AuditRecoveryRetry.java new file mode 100644 index 0000000000..35ee4bcbc9 --- /dev/null +++ b/ranger-audit-server/ranger-audit-server-service/src/main/java/org/apache/ranger/audit/producer/kafka/AuditRecoveryRetry.java @@ -0,0 +1,362 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.audit.producer.kafka; + +import org.apache.kafka.clients.producer.KafkaProducer; +import org.apache.ranger.audit.provider.MiscUtil; +import org.apache.ranger.audit.utils.AuditServerLogFormatter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStreamReader; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.security.PrivilegedExceptionAction; +import java.util.Arrays; +import java.util.Comparator; +import java.util.Properties; + +/** + * AuditRecoveryRetry - Thread that reads .failed files from spool directory and retries sending to Kafka + * Successfully processed files are moved to archive directory with .processed extension + * Manages archive retention by deleting old .processed files + */ +public class AuditRecoveryRetry implements Runnable { + private static final Logger LOG = LoggerFactory.getLogger(AuditRecoveryRetry.class); + + // Configuration keys + private static final String PROP_RECOVERY_ENABLED = "recovery.enabled"; + private static final String PROP_SPOOL_DIR = "recovery.spool.dir"; + private static final String PROP_ARCHIVE_DIR = "recovery.archive.dir"; + private static final String PROP_RETRY_INTERVAL_SEC = "recovery.retry.interval.sec"; + private static final String PROP_MAX_ARCHIVE_FILES = "recovery.archive.max.processed.files"; + private static final String PROP_RETRY_MAX_ATTEMPTS = "recovery.retry.max.attempts"; + private static final boolean DEFAULT_RECOVERY_ENABLED = true; + private static final String DEFAULT_SPOOL_DIR = "/var/log/ranger/ranger-audit-server/audit/spool"; + private static final String DEFAULT_ARCHIVE_DIR = "/var/log/ranger/ranger-audit-server/audit/archive"; + private static final long DEFAULT_RETRY_INTERVAL_SEC = 60; // 1 minute + private static final int DEFAULT_MAX_ARCHIVE_FILES = 100; + private static final int DEFAULT_RETRY_MAX_ATTEMPTS = 3; + + private final Properties props; + private final String propPrefix; + private final String topicName; + private final AuditMessageQueue messageQueue; // Hold reference to parent to get current producer + + private volatile boolean running = true; + private boolean enabled; + private String spoolDir; + private String archiveDir; + private long retryIntervalSec; + private long totalMessagesRetried; + private long totalMessagesSucceeded; + private long totalMessagesFailed; + private int maxArchiveFiles; + private int retryMaxAttempts; + + public AuditRecoveryRetry(Properties props, String propPrefix, AuditMessageQueue messageQueue, String topicName) { + this.props = new Properties(); + this.props.putAll(props); + this.propPrefix = propPrefix; + this.messageQueue = messageQueue; // Store parent reference to get current producer + this.topicName = topicName; + + init(); + } + + private void init() { + enabled = MiscUtil.getBooleanProperty(props, propPrefix + "." + PROP_RECOVERY_ENABLED, DEFAULT_RECOVERY_ENABLED); + spoolDir = MiscUtil.getStringProperty(props, propPrefix + "." + PROP_SPOOL_DIR, DEFAULT_SPOOL_DIR); + archiveDir = MiscUtil.getStringProperty(props, propPrefix + "." + PROP_ARCHIVE_DIR, DEFAULT_ARCHIVE_DIR); + retryIntervalSec = MiscUtil.getLongProperty(props, propPrefix + "." + PROP_RETRY_INTERVAL_SEC, DEFAULT_RETRY_INTERVAL_SEC); + maxArchiveFiles = MiscUtil.getIntProperty(props, propPrefix + "." + PROP_MAX_ARCHIVE_FILES, DEFAULT_MAX_ARCHIVE_FILES); + retryMaxAttempts = MiscUtil.getIntProperty(props, propPrefix + "." + PROP_RETRY_MAX_ATTEMPTS, DEFAULT_RETRY_MAX_ATTEMPTS); + + AuditServerLogFormatter.builder("AuditRecoveryRetry Configuration:") + .add("enabled: ", enabled) + .add("spoolDir: ", spoolDir) + .add("archiveDir: ", archiveDir) + .add("retryIntervalSec: ", retryIntervalSec) + .add("maxArchiveFiles: ", maxArchiveFiles) + .add("retryMaxAttempts: ", retryMaxAttempts) + .logInfo(LOG); + } + + public boolean isEnabled() { + return enabled; + } + + @Override + public void run() { + if (!enabled) { + LOG.info("AuditRecoveryRetry is disabled. Thread will not start."); + return; + } + + LOG.info("==> AuditRecoveryRetry thread started"); + + try { + // Create archive directory if needed + try { + createDirectories(); + } catch (IOException e) { + LOG.error("Failed to create archive directory", e); + return; + } + + while (running) { + try { + // Check and clean up archive + cleanupArchive(); + + // Process failed files from spool directory + processFailedFiles(); + + // Sleep before next retry cycle + Thread.sleep(retryIntervalSec * 1000); + } catch (InterruptedException e) { + if (running) { + LOG.warn("AuditRecoveryRetry interrupted", e); + } + Thread.currentThread().interrupt(); + break; + } catch (Exception e) { + LOG.error("Error in AuditRecoveryRetry loop", e); + } + } + } finally { + LOG.info("<== AuditRecoveryRetry thread stopped"); + LOG.info("Recovery Statistics - Retried: {}, Succeeded: {}, Failed: {}", totalMessagesRetried, totalMessagesSucceeded, totalMessagesFailed); + } + } + + private void createDirectories() throws IOException { + Path archivePath = Paths.get(archiveDir); + if (!Files.exists(archivePath)) { + Files.createDirectories(archivePath); + LOG.info("Created archive directory: {}", archiveDir); + } + } + + private void processFailedFiles() { + File spoolDirectory = new File(spoolDir); + File[] files = spoolDirectory.listFiles((dir, name) -> name.startsWith("spool_audit_") && name.endsWith(".failed")); + + if (files == null || files.length == 0) { + LOG.debug("No failed files to process in spool directory"); + return; + } + + // Sort files by modification time (oldest first) + Arrays.sort(files, Comparator.comparingLong(File::lastModified)); + + LOG.info("Processing {} failed files from spool directory", files.length); + + for (File file : files) { + if (!running) { + break; + } + + try { + processFailedFile(file); + } catch (Exception e) { + LOG.error("Error processing failed file: {}", file.getName(), e); + } + } + } + + private void processFailedFile(File file) { + LOG.info("Processing failed file: {}", file.getName()); + + int messagesProcessed = 0; + int messagesSucceeded = 0; + int messagesFailed = 0; + + try (BufferedReader reader = new BufferedReader(new InputStreamReader(new FileInputStream(file), "UTF-8"))) { + String line; + + while ((line = reader.readLine()) != null && running) { + if (line.trim().isEmpty()) { + continue; + } + + try { + // Parse line: timestamp|key|message + String[] parts = line.split("\\|", 3); + if (parts.length < 3) { + LOG.warn("Invalid line format in file {}: {}", file.getName(), line); + continue; + } + + // parts[0] is timestamp - not used but kept for future metrics + String key = parts[1].isEmpty() ? null : parts[1]; + String message = parts[2]; + + messagesProcessed++; + totalMessagesRetried++; + + // Retry sending to Kafka + boolean success = retrySendToKafka(key, message); + + if (success) { + messagesSucceeded++; + totalMessagesSucceeded++; + } else { + messagesFailed++; + totalMessagesFailed++; + } + } catch (Exception e) { + LOG.error("Error processing line from file {}: {}", file.getName(), line, e); + messagesFailed++; + totalMessagesFailed++; + } + } + + LOG.info("Completed processing file: {} - Processed: {}, Succeeded: {}, Failed: {}", + file.getName(), messagesProcessed, messagesSucceeded, messagesFailed); + + // Move file to archive if all messages succeeded + if (messagesFailed == 0 && messagesSucceeded > 0) { + moveToArchive(file); + } else if (messagesFailed > 0) { + // Keep file in spool directory as .failed for retry in next cycle + LOG.warn("File {} has {} failed messages - keeping in spool for retry", + file.getName(), messagesFailed); + } + } catch (IOException e) { + LOG.error("Error reading failed file: {}", file.getName(), e); + } + } + + private boolean retrySendToKafka(String key, String message) { + // Get current producer from parent - may have been recreated after Kafka came back online + KafkaProducer currentProducer = (messageQueue != null) ? messageQueue.kafkaProducer : null; + + if (currentProducer == null) { + LOG.warn("Kafka producer is null, cannot retry message - will retry on next cycle"); + return false; + } + + for (int attempt = 1; attempt <= retryMaxAttempts; attempt++) { + try { + MiscUtil.executePrivilegedAction((PrivilegedExceptionAction) () -> { + AuditProducer.send(currentProducer, topicName, key, message); + return null; + }); + + LOG.debug("Successfully retried message to Kafka (attempt {})", attempt); + return true; + } catch (Exception e) { + LOG.warn("Failed to retry message to Kafka (attempt {}): {}", attempt, e.getMessage()); + if (attempt < retryMaxAttempts) { + try { + // Wait before retry with exponential backoff + long waitTime = (long) Math.min(1000 * Math.pow(2, attempt - 1), 10000); + Thread.sleep(waitTime); + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + return false; + } + } + } + } + + LOG.error("Failed to retry message after {} attempts", retryMaxAttempts); + return false; + } + + private void moveToArchive(File file) { + try { + // Change extension from .failed to .processed + String processedFileName = file.getName().replace(".failed", ".processed"); + Path targetPath = Paths.get(archiveDir, processedFileName); + + if (Files.move(file.toPath(), targetPath) != null) { + LOG.info("Moved successfully processed file to archive: {} -> {}", + file.getName(), processedFileName); + } else { + LOG.warn("Failed to move file to archive: {}", file.getName()); + } + } catch (Exception e) { + LOG.error("Error moving file to archive: {}", file.getName(), e); + } + } + + private void cleanupArchive() { + try { + File archiveDirectory = new File(archiveDir); + File[] files = archiveDirectory.listFiles((dir, name) -> + (name.startsWith("spool_audit_") && name.endsWith(".processed"))); + + if (files == null || files.length <= maxArchiveFiles) { + LOG.debug("Archive cleanup: {} .processed files, max: {} - no cleanup needed", + files != null ? files.length : 0, maxArchiveFiles); + return; + } + + // Sort by modification time (oldest first) + Arrays.sort(files, Comparator.comparingLong(File::lastModified)); + + // Delete oldest .processed files beyond retention limit + int filesToDelete = files.length - maxArchiveFiles; + LOG.info("Archive cleanup: Deleting {} oldest .processed files (current: {}, max: {})", + filesToDelete, files.length, maxArchiveFiles); + + for (int i = 0; i < filesToDelete; i++) { + File file = files[i]; + try { + if (file.delete()) { + LOG.info("Deleted old processed archive file: {}", file.getName()); + } else { + LOG.warn("Failed to delete old processed archive file: {}", file.getName()); + } + } catch (Exception e) { + LOG.error("Error deleting old processed archive file: {}", file.getName(), e); + } + } + } catch (Exception e) { + LOG.error("Error during archive cleanup", e); + } + } + + public void shutdown() { + LOG.info("==> AuditRecoveryRetry.shutdown()"); + running = false; + LOG.info("<== AuditRecoveryRetry.shutdown() completed"); + } + + // Getters for metrics/monitoring + public long getTotalMessagesRetried() { + return totalMessagesRetried; + } + + public long getTotalMessagesSucceeded() { + return totalMessagesSucceeded; + } + + public long getTotalMessagesFailed() { + return totalMessagesFailed; + } +} diff --git a/ranger-audit-server/ranger-audit-server-service/src/main/java/org/apache/ranger/audit/producer/kafka/AuditRecoveryWriter.java b/ranger-audit-server/ranger-audit-server-service/src/main/java/org/apache/ranger/audit/producer/kafka/AuditRecoveryWriter.java new file mode 100644 index 0000000000..43a5c00d34 --- /dev/null +++ b/ranger-audit-server/ranger-audit-server-service/src/main/java/org/apache/ranger/audit/producer/kafka/AuditRecoveryWriter.java @@ -0,0 +1,319 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.audit.producer.kafka; + +import org.apache.ranger.audit.provider.MiscUtil; +import org.apache.ranger.audit.utils.AuditServerLogFormatter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.BufferedWriter; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStreamWriter; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.Properties; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; + +/** + * AuditRecoveryWriter - Thread that receives failed audit messages and writes them to local spool files with .failed file extension + * Processed files are rotated based on time configuration and moved to archive with .processed file extension after writing + */ +public class AuditRecoveryWriter implements Runnable { + private static final Logger LOG = LoggerFactory.getLogger(AuditRecoveryWriter.class); + + private static final String PROP_RECOVERY_ENABLED = "recovery.enabled"; + private static final String PROP_SPOOL_DIR = "recovery.spool.dir"; + private static final String PROP_FILE_ROTATION_INTERVAL_SEC = "recovery.file.rotation.interval.sec"; + private static final String PROP_MAX_MESSAGES_PER_FILE = "recovery.max.messages.per.file"; + private static final boolean DEFAULT_RECOVERY_ENABLED = true; + private static final String DEFAULT_SPOOL_DIR = "/var/log/ranger/ranger-audit-server/audit/spool"; + private static final long DEFAULT_FILE_ROTATION_INTERVAL_SEC = 300; + private static final int DEFAULT_MAX_MESSAGES_PER_FILE = 10000; + + private final BlockingQueue messageQueue = new LinkedBlockingQueue<>(100000); + private final Properties props; + private final String propPrefix; + + private volatile boolean running = true; + private boolean enabled; + private String spoolDir; + private long fileRotationIntervalSec; + private int maxMessagesPerFile; + + private BufferedWriter currentWriter; + private String currentFileName; + private long currentFileStartTime; + private int currentFileMessageCount; + + public AuditRecoveryWriter(Properties props, String propPrefix) { + this.props = new Properties(); + this.props.putAll(props); + this.propPrefix = propPrefix; + init(); + } + + private void init() { + enabled = MiscUtil.getBooleanProperty(props, propPrefix + "." + PROP_RECOVERY_ENABLED, DEFAULT_RECOVERY_ENABLED); + spoolDir = MiscUtil.getStringProperty(props, propPrefix + "." + PROP_SPOOL_DIR, DEFAULT_SPOOL_DIR); + fileRotationIntervalSec = MiscUtil.getLongProperty(props, propPrefix + "." + PROP_FILE_ROTATION_INTERVAL_SEC, DEFAULT_FILE_ROTATION_INTERVAL_SEC); + maxMessagesPerFile = MiscUtil.getIntProperty(props, propPrefix + "." + PROP_MAX_MESSAGES_PER_FILE, DEFAULT_MAX_MESSAGES_PER_FILE); + + AuditServerLogFormatter.builder("AuditRecoveryWriter Configuration:") + .add("enabled: ", enabled) + .add("spoolDir: ", spoolDir) + .add("fileRotationIntervalSec: ", fileRotationIntervalSec) + .add("enabled: ", enabled) + .add("maxMessagesPerFile: ", maxMessagesPerFile); + } + + public boolean isEnabled() { + return enabled; + } + + /** + * Add a failed audit message to the queue for writing to spool file + */ + public boolean addFailedMessage(String key, String message) { + boolean ret; + + if (!enabled || !running) { + ret = false; + } else { + try { + FailedAuditMessage failedMsg = new FailedAuditMessage(key, message, System.currentTimeMillis()); + boolean added = messageQueue.offer(failedMsg, 1, TimeUnit.SECONDS); + if (!added) { + LOG.warn("Failed to add message to recovery queue - queue is full"); + } + ret = added; + } catch (InterruptedException e) { + LOG.warn("Interrupted while adding message to recovery queue", e); + Thread.currentThread().interrupt(); + ret = false; + } + } + + return ret; + } + + @Override + public void run() { + if (!enabled) { + LOG.info("AuditRecoveryWriter is disabled. Thread will not start."); + return; + } + + LOG.info("==> AuditRecoveryWriter thread started"); + + try { + // Create spool directory + try { + createDirectories(); + } catch (IOException e) { + LOG.error("Failed to create spool directory", e); + return; + } + + while (running) { + try { + // Poll for messages with timeout + FailedAuditMessage failedMsg = messageQueue.poll(1, TimeUnit.SECONDS); + + if (failedMsg != null) { + writeMessageToFile(failedMsg); + } + + // Check if file needs rotation + checkAndRotateFile(); + } catch (InterruptedException e) { + if (running) { + LOG.warn("AuditRecoveryWriter interrupted", e); + } + Thread.currentThread().interrupt(); + break; + } catch (Exception e) { + LOG.error("Error in AuditRecoveryWriter loop", e); + } + } + } finally { + // Cleanup + closeCurrentFile(); + LOG.info("<== AuditRecoveryWriter thread stopped"); + } + } + + public void shutdown() { + LOG.info("==> AuditRecoveryWriter.shutdown()"); + running = false; + + // Process remaining messages in queue + int remainingMessages = messageQueue.size(); + if (remainingMessages > 0) { + LOG.info("Processing {} remaining messages before shutdown", remainingMessages); + + long shutdownStartTime = System.currentTimeMillis(); + long shutdownTimeout = 30000; // 30 seconds + + while (!messageQueue.isEmpty() && (System.currentTimeMillis() - shutdownStartTime < shutdownTimeout)) { + try { + FailedAuditMessage failedMsg = messageQueue.poll(100, TimeUnit.MILLISECONDS); + if (failedMsg != null) { + writeMessageToFile(failedMsg); + } + } catch (Exception e) { + LOG.error("Error processing remaining messages during shutdown", e); + break; + } + } + } + + // Close current file + closeCurrentFile(); + + LOG.info("<== AuditRecoveryWriter.shutdown() completed"); + } + + private void createDirectories() throws IOException { + Path spoolPath = Paths.get(spoolDir); + + if (!Files.exists(spoolPath)) { + Files.createDirectories(spoolPath); + LOG.info("Created spool directory: {}", spoolDir); + } + } + + private void writeMessageToFile(FailedAuditMessage failedMsg) { + try { + // Open new file if needed + if (currentWriter == null) { + openNewFile(); + } + + // Write message: timestamp|key|message + String line = String.format("%d|%s|%s%n", failedMsg.timestamp, failedMsg.key != null ? failedMsg.key : "", failedMsg.message); + + currentWriter.write(line); + currentFileMessageCount++; + + // Flush periodically for safety + if (currentFileMessageCount % 100 == 0) { + currentWriter.flush(); + } + } catch (IOException e) { + LOG.error("Error writing message to spool file: {}", currentFileName, e); + // Try to close and reopen file + closeCurrentFile(); + } + } + + private void openNewFile() throws IOException { + SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMdd_HHmmss_SSS"); + String timestamp = sdf.format(new Date()); + // Use .tmp extension while writing to prevent retry thread from picking it up + currentFileName = String.format("spool_audit_%s.tmp", timestamp); + String filePath = Paths.get(spoolDir, currentFileName).toString(); + currentWriter = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(filePath, true), "UTF-8")); + currentFileStartTime = System.currentTimeMillis(); + currentFileMessageCount = 0; + + LOG.info("Opened new spool file (temp): {}", currentFileName); + } + + private void checkAndRotateFile() { + if (currentWriter == null) { + return; + } + + long currentTime = System.currentTimeMillis(); + long elapsedSec = (currentTime - currentFileStartTime) / 1000; + + // Rotate if time elapsed or message count exceeded + boolean rotateByTime = elapsedSec >= fileRotationIntervalSec; + boolean rotateByCount = currentFileMessageCount >= maxMessagesPerFile; + + if (rotateByTime || rotateByCount) { + if (rotateByTime) { + LOG.info("Rotating spool file due to time: {} seconds elapsed", elapsedSec); + } + if (rotateByCount) { + LOG.info("Rotating spool file due to message count: {} messages written", currentFileMessageCount); + } + + closeCurrentFile(); + } + } + + private void closeCurrentFile() { + if (currentWriter != null) { + String tempFileName = currentFileName; + try { + currentWriter.flush(); + currentWriter.close(); + + // Rename from .tmp to .failed to signal it's ready for retry + String failedFileName = tempFileName.replace(".tmp", ".failed"); + Path tempPath = Paths.get(spoolDir, tempFileName); + Path failedPath = Paths.get(spoolDir, failedFileName); + + Files.move(tempPath, failedPath); + + LOG.info("Closed and finalized spool file: {} ({} messages) - Ready for retry processing", failedFileName, currentFileMessageCount); + } catch (IOException e) { + LOG.error("Error closing/renaming spool file: {}", tempFileName, e); + } finally { + currentWriter = null; + currentFileName = null; + currentFileStartTime = 0; + currentFileMessageCount = 0; + } + } + } + + private static class FailedAuditMessage { + final String key; + final String message; + final long timestamp; + + FailedAuditMessage(String key, String message, long timestamp) { + this.key = key; + this.message = message; + this.timestamp = timestamp; + } + } + + public int getQueueSize() { + return messageQueue.size(); + } + + public String getCurrentFileName() { + return currentFileName; + } + + public int getCurrentFileMessageCount() { + return currentFileMessageCount; + } +} diff --git a/ranger-audit-server/ranger-audit-server-service/src/main/java/org/apache/ranger/audit/rest/AuditREST.java b/ranger-audit-server/ranger-audit-server-service/src/main/java/org/apache/ranger/audit/rest/AuditREST.java new file mode 100644 index 0000000000..f8751c1cbc --- /dev/null +++ b/ranger-audit-server/ranger-audit-server-service/src/main/java/org/apache/ranger/audit/rest/AuditREST.java @@ -0,0 +1,219 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.audit.rest; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonSyntaxException; +import com.google.gson.reflect.TypeToken; +import org.apache.ranger.audit.model.AuthzAuditEvent; +import org.apache.ranger.audit.producer.AuditDestinationMgr; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Scope; +import org.springframework.stereotype.Component; + +import javax.ws.rs.Consumes; +import javax.ws.rs.GET; +import javax.ws.rs.POST; +import javax.ws.rs.Path; +import javax.ws.rs.Produces; +import javax.ws.rs.core.Response; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +@Path("/audit") +@Component +@Scope("request") +public class AuditREST { + private static final Logger LOG = LoggerFactory.getLogger(AuditREST.class); + private Gson gson = new GsonBuilder().setDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSZ").create(); + @Autowired + AuditDestinationMgr auditDestinationMgr; + + /** + * Health check endpoint + */ + @GET + @Path("/health") + @Produces("application/json") + public Response healthCheck() { + LOG.debug("==> AuditREST.healthCheck()"); + Response ret; + String jsonString; + + try { + // Check if audit destination manager is available and healthy + if (auditDestinationMgr != null) { + Map resp = new HashMap<>(); + resp.put("status", "UP"); + resp.put("service", "ranger-audit-server"); + jsonString = buildResponse(resp); + ret = Response.ok() + .entity(jsonString) + .build(); + } else { + Map resp = new HashMap<>(); + resp.put("status", "DOWN"); + resp.put("service", "ranger-audit-server"); + resp.put("reason", "AuditDestinationMgr not available"); + jsonString = buildResponse(resp); + ret = Response.status(Response.Status.SERVICE_UNAVAILABLE) + .entity(jsonString) + .build(); + } + } catch (Exception e) { + LOG.error("Health check failed", e); + Map resp = new HashMap<>(); + resp.put("status", "DOWN"); + resp.put("service", "ranger-audit-server"); + resp.put("reason", e.getMessage()); + jsonString = buildResponse(resp); + ret = Response.status(Response.Status.SERVICE_UNAVAILABLE) + .entity(jsonString) + .build(); + } + + LOG.debug("<== AuditREST.healthCheck(): {}", ret); + + return ret; + } + + /** + * Status endpoint for monitoring + */ + @GET + @Path("/status") + @Produces("application/json") + public Response getStatus() { + LOG.debug("==> AuditREST.getStatus()"); + + Response ret; + String jsonString; + + try { + String status = (auditDestinationMgr != null) ? "READY" : "NOT_READY"; + + Map resp = new HashMap<>(); + resp.put("status", status); + resp.put("timestamp", System.currentTimeMillis()); + resp.put("service", "ranger-audit-server"); + jsonString = buildResponse(resp); + ret = Response.status(Response.Status.OK) + .entity(jsonString) + .build(); + } catch (Exception e) { + LOG.error("Status check failed", e); + ret = Response.status(Response.Status.INTERNAL_SERVER_ERROR) + .entity("{\"status\":\"ERROR\",\"error\":\"" + e.getMessage() + "\"}") + .build(); + } + + LOG.debug("<== AuditREST.getStatus(): {}", ret); + + return ret; + } + + /** + * Access Audits producer endpoint. + */ + @POST + @Path("/post") + @Consumes("application/json") + @Produces("application/json") + public Response postAudit(String accessAudits) { + LOG.debug("==> AuditREST.postAudit(): {}", accessAudits); + + Response ret; + if (accessAudits == null || accessAudits.trim().isEmpty()) { + LOG.warn("Empty or null audit events batch received"); + ret = Response.status(Response.Status.BAD_REQUEST) + .entity(buildErrorResponse("Audit events cannot be empty")) + .build(); + } else if (auditDestinationMgr == null) { + LOG.error("AuditDestinationMgr not initialized"); + ret = Response.status(Response.Status.SERVICE_UNAVAILABLE) + .entity(buildErrorResponse("Audit service not available")) + .build(); + } else { + try { + String jsonString; + List events = gson.fromJson(accessAudits, new TypeToken>() {}.getType()); + + for (AuthzAuditEvent event : events) { + auditDestinationMgr.log(event); + } + + Map response = new HashMap<>(); + response.put("total", events.size()); + response.put("timestamp", System.currentTimeMillis()); + jsonString = buildResponse(response); + ret = Response.status(Response.Status.OK) + .entity(jsonString) + .build(); + } catch (JsonSyntaxException e) { + LOG.error("Invalid Access audit JSON string...", e); + ret = Response.status(Response.Status.BAD_REQUEST) + .entity(buildErrorResponse("Invalid Access audit JSON string..." + e.getMessage())) + .build(); + } catch (Exception e) { + LOG.error("Error processing access audits batch...", e); + ret = Response.status(Response.Status.INTERNAL_SERVER_ERROR) + .entity(buildErrorResponse("Invalid Access audit JSON string..." + e.getMessage())) + .build(); + } + } + + LOG.debug("<== AuditREST.postAudit(): HttpStatus {}", ret.getStatus()); + + return ret; + } + + private String buildResponse(Map respMap) { + String ret; + + try { + ObjectMapper objectMapper = new ObjectMapper(); + ret = objectMapper.writeValueAsString(respMap); + } catch (Exception e) { + ret = "Error: " + e.getMessage(); + } + + return ret; + } + + private String buildErrorResponse(String message) { + try { + Map error = new HashMap<>(); + error.put("status", "error"); + error.put("message", message); + error.put("timestamp", System.currentTimeMillis()); + ObjectMapper mapper = new ObjectMapper(); + return mapper.writeValueAsString(error); + } catch (Exception e) { + LOG.error("Error building error response", e); + return "{\"status\":\"error\",\"message\":\"" + message.replace("\"", "'") + "\"}"; + } + } +} diff --git a/ranger-audit-server/ranger-audit-server-service/src/main/java/org/apache/ranger/audit/security/AuditAuthEntryPoint.java b/ranger-audit-server/ranger-audit-server-service/src/main/java/org/apache/ranger/audit/security/AuditAuthEntryPoint.java new file mode 100644 index 0000000000..0df4824bc7 --- /dev/null +++ b/ranger-audit-server/ranger-audit-server-service/src/main/java/org/apache/ranger/audit/security/AuditAuthEntryPoint.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ranger.audit.security; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.security.core.AuthenticationException; +import org.springframework.security.web.authentication.LoginUrlAuthenticationEntryPoint; + +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import java.io.IOException; + +public class AuditAuthEntryPoint extends LoginUrlAuthenticationEntryPoint { + private static final Logger LOG = LoggerFactory.getLogger(AuditAuthEntryPoint.class); + + public AuditAuthEntryPoint(String loginFormUrl) { + super(loginFormUrl); + } + + @Override + public void commence(HttpServletRequest request, HttpServletResponse response, AuthenticationException authException) throws IOException, ServletException { + //any task for entry filter + response.setHeader("X-Frame-Options", "DENY"); + response.setHeader("Strict-Transport-Security", "max-age=31536000; includeSubDomains; preload"); + + LOG.debug("commence() X-Requested-With=[{}]", request.getHeader("X-Requested-With")); + LOG.debug("requestURL : [{}]", (request.getRequestURL() != null) ? request.getRequestURL().toString() : ""); + + /* This is invoked when user tries to access a secured REST resource without supplying any credentials + We should just send a 401 Unauthorized response because there is no 'login page' to redirect to */ + response.sendError(HttpServletResponse.SC_UNAUTHORIZED, "Authorization Error"); + } +} diff --git a/ranger-audit-server/ranger-audit-server-service/src/main/java/org/apache/ranger/audit/security/AuditDelegationTokenFilter.java b/ranger-audit-server/ranger-audit-server-service/src/main/java/org/apache/ranger/audit/security/AuditDelegationTokenFilter.java new file mode 100644 index 0000000000..db8b8c28d9 --- /dev/null +++ b/ranger-audit-server/ranger-audit-server-service/src/main/java/org/apache/ranger/audit/security/AuditDelegationTokenFilter.java @@ -0,0 +1,316 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.audit.security; + +import org.apache.commons.collections.iterators.IteratorEnumeration; +import org.apache.hadoop.security.SecurityUtil; +import org.apache.hadoop.security.authentication.server.KerberosAuthenticationHandler; +import org.apache.hadoop.security.authentication.server.PseudoAuthenticationHandler; +import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticationFilter; +import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticationHandler; +import org.apache.hadoop.security.token.delegation.web.KerberosDelegationTokenAuthenticationHandler; +import org.apache.hadoop.security.token.delegation.web.PseudoDelegationTokenAuthenticationHandler; +import org.apache.ranger.audit.server.AuditServerConfig; +import org.apache.ranger.audit.utils.AuditServerLogFormatter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.security.core.Authentication; +import org.springframework.security.core.context.SecurityContextHolder; + +import javax.annotation.PostConstruct; +import javax.servlet.FilterChain; +import javax.servlet.FilterConfig; +import javax.servlet.ServletContext; +import javax.servlet.ServletException; +import javax.servlet.ServletRequest; +import javax.servlet.ServletResponse; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import java.io.IOException; +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.Enumeration; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; +import java.util.Properties; + +public class AuditDelegationTokenFilter extends DelegationTokenAuthenticationFilter { + private static final Logger LOG = LoggerFactory.getLogger(AuditDelegationTokenFilter.class); + + private final ServletContext nullContext = new NullServletContext(); + + protected AuditServerConfig auditConfig; + + private String tokenKindStr; + + private static final String AUDIT_DELEGATION_TOKEN_KIND_DEFAULT = "AUDIT_DELEGATION_TOKEN"; + + // Uses "ranger.audit" prefix for KerberosAuthenticationHandler + private static final String DEFAULT_CONFIG_PREFIX = "ranger.audit"; + private static final String PROP_CONFIG_PREFIX = "config.prefix"; + + /** + * Initialized this filter. Only PROP_CONFIG_PREFIX needs to be set. + * Rest other configurations will be fetched from {@link java.util.Properties config} + * return by {@link #getConfiguration(String, FilterConfig)}. + */ + @PostConstruct + public void initialize() { + LOG.debug("==> AuditDelegationTokenFilter.initialize()"); + + this.auditConfig = AuditServerConfig.getInstance(); + LOG.debug("AuditServerConfig instance obtained: [{}]", auditConfig); + + try { + // Currently setting tokenKindStr to default value. Later we can configure as per requirement. + tokenKindStr = AUDIT_DELEGATION_TOKEN_KIND_DEFAULT; + LOG.debug("Token kind set to: [{}]", tokenKindStr); + + final Map params = new HashMap<>(); + params.put(PROP_CONFIG_PREFIX, DEFAULT_CONFIG_PREFIX); + LOG.debug("Config Prefix used for AuditDelegationTokenFilter is [{}].", params.get(PROP_CONFIG_PREFIX)); + + // Log all audit config properties for debugging + AuditServerLogFormatter.LogBuilder logBuilder = AuditServerLogFormatter.builder("===> All audit config properties:"); + Properties allProps = auditConfig.getProperties(); + for (String key : allProps.stringPropertyNames()) { + logBuilder.add(key, allProps.getProperty(key)); + } + logBuilder.logDebug(LOG); + + FilterConfig filterConfigNew = new FilterConfig() { + @Override + public ServletContext getServletContext() { + return nullContext; + } + + @SuppressWarnings("unchecked") + @Override + public Enumeration getInitParameterNames() { + return new IteratorEnumeration(params.keySet().iterator()); + } + + @Override + public String getInitParameter(String param) { + String value = params.get(param); + LOG.debug("FilterConfig.getInitParameter({}) = [{}]", param, value); + return value; + } + + @Override + public String getFilterName() { + return "AuditDelegationTokenFilter"; + } + }; + + LOG.debug("Initializing parent DelegationTokenAuthenticationFilter"); + super.init(filterConfigNew); + LOG.debug("Parent DelegationTokenAuthenticationFilter initialized successfully"); + } catch (ServletException e) { + LOG.error("AuditDelegationTokenFilter(): initialization failure", e); + throw new RuntimeException("Failed to initialize AuditDelegationTokenFilter", e); + } + + LOG.debug("<== AuditDelegationTokenFilter.initialize()"); + } + + /** + * Return {@link java.util.Properties configuration} required for this filter to work properly. + */ + @Override + protected Properties getConfiguration(String configPrefix, FilterConfig filterConfig) throws ServletException { + LOG.debug("==> getConfiguration() - configPrefix: [{}]", configPrefix); + + Properties props = new Properties(); + Properties allProps = auditConfig.getProperties(); + + // Log all available properties for debugging + AuditServerLogFormatter.LogBuilder confLogBuilder = AuditServerLogFormatter.builder("====> All available audit config properties:"); + for (String key : allProps.stringPropertyNames()) { + if (key.startsWith(configPrefix)) { + confLogBuilder.add(key, auditConfig.get(key)); + } + } + confLogBuilder.logDebug(LOG); + + // Get all properties with the given prefix + // configPrefix comes with trailing dot (e.g., "ranger.audit.") + for (String key : allProps.stringPropertyNames()) { + if (key.startsWith(configPrefix)) { + // Remove the prefix - configPrefix already includes the trailing dot + String shortKey = key.substring(configPrefix.length()); + String value = auditConfig.get(key); + props.put(shortKey, value); + LOG.debug("Added property: {} = {}", shortKey, value); + } + } + + // Map kerberos.type to type (Hadoop expects "type" property) + String kerberosType = props.getProperty("kerberos.type"); + if (kerberosType != null) { + props.setProperty(AUTH_TYPE, kerberosType); + LOG.debug("Mapped kerberos.type [{}] to AUTH_TYPE", kerberosType); + } + + String authType = props.getProperty(AUTH_TYPE, "simple"); + LOG.debug("Authentication type: [{}]", authType); + + if (authType.equals(PseudoAuthenticationHandler.TYPE)) { + props.setProperty(AUTH_TYPE, PseudoDelegationTokenAuthenticationHandler.class.getName()); + LOG.debug("Using PseudoDelegationTokenAuthenticationHandler"); + } else if (authType.equals(KerberosAuthenticationHandler.TYPE)) { + props.setProperty(AUTH_TYPE, KerberosDelegationTokenAuthenticationHandler.class.getName()); + LOG.debug("Using KerberosDelegationTokenAuthenticationHandler"); + } + + props.setProperty(DelegationTokenAuthenticationHandler.TOKEN_KIND, tokenKindStr); + + // Resolve _HOST into bind address + String bindAddress = auditConfig.get("ranger.audit.server.bind.address"); + LOG.debug("Bind address from config: [{}]", bindAddress); + + if (Objects.isNull(bindAddress)) { + LOG.warn("No host name configured. Defaulting to local host name."); + + try { + bindAddress = InetAddress.getLocalHost().getHostName(); + LOG.debug("Resolved bind address to: [{}]", bindAddress); + } catch (UnknownHostException e) { + LOG.error("Unable to obtain host name", e); + throw new ServletException("Unable to obtain host name", e); + } + } + + // Resolve principle using latest bindAddress + String principal = props.getProperty(KerberosAuthenticationHandler.PRINCIPAL); + LOG.debug("Original Kerberos principal: [{}]", principal); + + if (Objects.nonNull(principal)) { + try { + String resolvedPrincipal = SecurityUtil.getServerPrincipal(principal, bindAddress); + LOG.debug("Resolved Kerberos principal: [{}] -> [{}]", principal, resolvedPrincipal); + props.put(KerberosAuthenticationHandler.PRINCIPAL, resolvedPrincipal); + } catch (IOException ex) { + LOG.error("Could not resolve Kerberos principal name: [{}] with bind address: [{}]", principal, bindAddress, ex); + throw new RuntimeException("Could not resolve Kerberos principal name: " + ex.toString(), ex); + } + } + + // Log final authentication configuration + AuditServerLogFormatter.LogBuilder finalAuthnConfLogBuilder = AuditServerLogFormatter.builder("====> Final authentication configuration:"); + for (String key : props.stringPropertyNames()) { + finalAuthnConfLogBuilder.add(key, props.getProperty(key)); + } + finalAuthnConfLogBuilder.logDebug(LOG); + + LOG.debug("<== getConfiguration()"); + + return props; + } + + @Override + public void doFilter(ServletRequest request, ServletResponse response, FilterChain filterChain) throws IOException, ServletException { + HttpServletRequest httpRequest = (HttpServletRequest) request; + HttpServletResponse httpResponse = (HttpServletResponse) response; + String requestURI = httpRequest.getRequestURI(); + + // Log request details and authentication headers + AuditServerLogFormatter.builder("===>>> AuditDelegationTokenFilter.doFilter()") + .add("Request URI", requestURI) + .add("Request method", httpRequest.getMethod()) + .add("Remote address", httpRequest.getRemoteAddr()) + .add("Remote host", httpRequest.getRemoteHost()) + .add("User-Agent", httpRequest.getHeader("User-Agent")) + .add("Authorization header", httpRequest.getHeader("Authorization")) + .add("WWW-Authenticate header", httpRequest.getHeader("WWW-Authenticate")) + .logDebug(LOG); + + // Skip authentication for anonymous/public endpoints (health check, status) + // These are marked as permitAll() in Spring Security config + if (requestURI != null && (requestURI.endsWith("/api/audit/health") || requestURI.endsWith("/api/audit/status"))) { + LOG.debug("Skipping authentication for public endpoint: {}", requestURI); + filterChain.doFilter(request, response); + return; + } + + // Log all headers for debugging + AuditServerLogFormatter.LogBuilder logBuilder = AuditServerLogFormatter.builder("===>>> All request headers:"); + Enumeration headerNames = httpRequest.getHeaderNames(); + while (headerNames.hasMoreElements()) { + String headerName = headerNames.nextElement(); + String headerValue = httpRequest.getHeader(headerName); + logBuilder.add(headerName, headerValue); + } + logBuilder.logDebug(LOG); + + Authentication previousAuth = SecurityContextHolder.getContext().getAuthentication(); + LOG.debug("Previous authentication: [{}]", previousAuth); + + if (previousAuth == null || !previousAuth.isAuthenticated()) { + FilterChainWrapper filterChainWrapper = new FilterChainWrapper(filterChain); + + LOG.debug("No previous authentication found, attempting authentication via AuditDelegationTokenFilter"); + + try { + super.doFilter(request, response, filterChainWrapper); + LOG.debug("Authentication attempt completed"); + } catch (Exception e) { + LOG.error("Authentication failed with exception", e); + throw e; + } + } else { + // Already authenticated + LOG.debug("User [{}] is already authenticated, proceeding with filter chain.", previousAuth.getPrincipal()); + LOG.debug("Authentication details: name=[{}], authorities=[{}], authenticated=[{}]", previousAuth.getName(), previousAuth.getAuthorities(), previousAuth.isAuthenticated()); + filterChain.doFilter(request, response); + } + + Authentication authentication = SecurityContextHolder.getContext().getAuthentication(); + boolean isAuthenticated = (authentication != null && authentication.isAuthenticated()); + + //LOG Final Authentication status + AuditServerLogFormatter.LogBuilder authnStatusLogBuilder = AuditServerLogFormatter.builder("===>>> Final Authentication "); + authnStatusLogBuilder.add("status", isAuthenticated); + if (authentication != null) { + authnStatusLogBuilder.add("Authenticated user", authentication.getName()); + authnStatusLogBuilder.add("Authentication class", authentication.getClass().getSimpleName()); + authnStatusLogBuilder.add("Authentication authorities", authentication.getAuthorities()); + authnStatusLogBuilder.logDebug(LOG); + } else { + LOG.debug("No authentication found in SecurityContext"); + } + + // Log response status and headers + LOG.debug("Response status: [{}]", httpResponse.getStatus()); + LOG.debug("Response WWW-Authenticate header: [{}]", httpResponse.getHeader("WWW-Authenticate")); + LOG.debug("Response Set-Cookie header: [{}]", httpResponse.getHeader("Set-Cookie")); + + // Log all response headers for debugging + AuditServerLogFormatter.LogBuilder responseHeadersLog = AuditServerLogFormatter.builder("===>>> Response headers:"); + for (String headerName : httpResponse.getHeaderNames()) { + responseHeadersLog.add(headerName, httpResponse.getHeader(headerName)); + } + responseHeadersLog.logDebug(LOG); + + LOG.debug("<== AuditDelegationTokenFilter.doFilter()"); + } +} diff --git a/ranger-audit-server/ranger-audit-server-service/src/main/java/org/apache/ranger/audit/security/AuditJwtAuthFilter.java b/ranger-audit-server/ranger-audit-server-service/src/main/java/org/apache/ranger/audit/security/AuditJwtAuthFilter.java new file mode 100644 index 0000000000..8a245bf11f --- /dev/null +++ b/ranger-audit-server/ranger-audit-server-service/src/main/java/org/apache/ranger/audit/security/AuditJwtAuthFilter.java @@ -0,0 +1,196 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.audit.security; + +import org.apache.ranger.audit.server.AuditServerConfig; +import org.apache.ranger.authz.handler.RangerAuth; +import org.apache.ranger.authz.handler.jwt.RangerDefaultJwtAuthHandler; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.security.authentication.AbstractAuthenticationToken; +import org.springframework.security.authentication.UsernamePasswordAuthenticationToken; +import org.springframework.security.core.Authentication; +import org.springframework.security.core.GrantedAuthority; +import org.springframework.security.core.authority.SimpleGrantedAuthority; +import org.springframework.security.core.context.SecurityContextHolder; +import org.springframework.security.core.userdetails.User; +import org.springframework.security.core.userdetails.UserDetails; +import org.springframework.security.web.authentication.WebAuthenticationDetails; + +import javax.annotation.PostConstruct; +import javax.servlet.Filter; +import javax.servlet.FilterChain; +import javax.servlet.FilterConfig; +import javax.servlet.ServletException; +import javax.servlet.ServletRequest; +import javax.servlet.ServletResponse; +import javax.servlet.http.HttpServletRequest; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.Properties; + +public class AuditJwtAuthFilter extends RangerDefaultJwtAuthHandler implements Filter { + private static final Logger LOG = LoggerFactory.getLogger(AuditJwtAuthFilter.class); + private static final String DEFAULT_AUDIT_ROLE = "ROLE_USER"; + private static final String CONFIG_PREFIX = "config.prefix"; + private static final String DEFAULT_JWT_PREFIX = "ranger.audit.jwt.auth"; + + @PostConstruct + public void initialize() { + if (LOG.isDebugEnabled()) { + LOG.debug("==> AuditJwtAuthFilter.initialize()"); + } + + AuditServerConfig auditConfig = AuditServerConfig.getInstance(); + + // Check if JWT authentication is enabled + String configPrefix = auditConfig.get(CONFIG_PREFIX, DEFAULT_JWT_PREFIX).concat("."); + boolean jwtEnabled = auditConfig.getBoolean(configPrefix + "enabled", false); + + if (!jwtEnabled) { + if (LOG.isDebugEnabled()) { + LOG.debug("JWT authentication is disabled, skipping initialization"); + } + if (LOG.isDebugEnabled()) { + LOG.debug("<<<=== AuditJwtAuthFilter.initialize()"); + } + return; + } + + /** + * If this filter is configured in spring security. The + * {@link org.springframework.web.filter.DelegatingFilterProxy + * DelegatingFilterProxy} does not invoke init method (like Servlet container). + */ + try { + Properties config = new Properties(); + Properties allProps = auditConfig.getProperties(); + + for (String key : allProps.stringPropertyNames()) { + if (key.startsWith(configPrefix)) { + String mappedKey = key.substring(configPrefix.length()); + // Map audit-specific property names to RangerJwtAuthHandler expected names + if ("provider-url".equals(mappedKey)) { + mappedKey = "jwks.provider-url"; + } else if ("public-key".equals(mappedKey)) { + mappedKey = "jwt.public-key"; + } else if ("cookie.name".equals(mappedKey)) { + mappedKey = "jwt.cookie-name"; + } else if ("audiences".equals(mappedKey)) { + mappedKey = "jwt.audiences"; + } + config.put(mappedKey, auditConfig.get(key)); + } + } + + if (LOG.isDebugEnabled()) { + LOG.debug("JWT Auth configs : {}", config.toString()); + } + + super.initialize(config); + } catch (Exception e) { + LOG.error("Failed to initialize Audit JWT Auth Filter.", e); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== AuditJwtAuthFilter.initialize()"); + } + } + + @Override + public void init(FilterConfig filterConfig) throws ServletException { + } + + @Override + public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain) + throws IOException, ServletException { + if (LOG.isDebugEnabled()) { + LOG.debug("==> AuditJwtAuthFilter.doFilter({}, {}, {})", request, response, chain); + } + + // Check if JWT authentication is enabled + AuditServerConfig auditConfig = AuditServerConfig.getInstance(); + String configPrefix = auditConfig.get(CONFIG_PREFIX, DEFAULT_JWT_PREFIX).concat("."); + boolean jwtEnabled = auditConfig.getBoolean(configPrefix + "enabled", false); + + if (!jwtEnabled) { + if (LOG.isDebugEnabled()) { + LOG.debug("JWT authentication is disabled, passing request to next filter"); + } + chain.doFilter(request, response); + return; + } + + if (request != null) { + if (canAuthenticateRequest(request)) { + Authentication previousAuth = SecurityContextHolder.getContext().getAuthentication(); + + // check if not already authenticated + if (previousAuth == null || !previousAuth.isAuthenticated()) { + HttpServletRequest httpServletRequest = (HttpServletRequest) request; + + RangerAuth rangerAuth = authenticate(httpServletRequest); + + if (rangerAuth != null) { + final List grantedAuths = Arrays.asList(new SimpleGrantedAuthority(DEFAULT_AUDIT_ROLE)); + final UserDetails principal = new User(rangerAuth.getUserName(), "", grantedAuths); + final Authentication finalAuthentication = new UsernamePasswordAuthenticationToken(principal, "", grantedAuths); + final WebAuthenticationDetails webDetails = new WebAuthenticationDetails(httpServletRequest); + + ((AbstractAuthenticationToken) finalAuthentication).setDetails(webDetails); + SecurityContextHolder.getContext().setAuthentication(finalAuthentication); + } + } else { + // Already authenticated + if (LOG.isDebugEnabled()) { + LOG.debug("User [{}] is already authenticated, proceeding with filter chain.", previousAuth.getPrincipal()); + } + } + + // Log final status of request. + Authentication finalAuth = SecurityContextHolder.getContext().getAuthentication(); + if (finalAuth != null) { + if (LOG.isDebugEnabled()) { + LOG.debug("AuditJwtAuthFilter.doFilter() - user=[{}], isUserAuthenticated? [{}]", finalAuth.getPrincipal(), finalAuth.isAuthenticated()); + } + } else { + LOG.warn("AuditJwtAuthFilter.doFilter() - Failed to authenticate request using Audit JWT authentication framework."); + } + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("Skipping JWT Audit auth for request."); + } + } + } + + chain.doFilter(request, response); // proceed with filter chain + + if (LOG.isDebugEnabled()) { + LOG.debug("<== AuditJwtAuthFilter.doFilter()"); + } + } + + @Override + public void destroy() { + // Auto-generated method stub + } +} diff --git a/ranger-audit-server/ranger-audit-server-service/src/main/java/org/apache/ranger/audit/security/FilterChainWrapper.java b/ranger-audit-server/ranger-audit-server-service/src/main/java/org/apache/ranger/audit/security/FilterChainWrapper.java new file mode 100644 index 0000000000..3346ec29e9 --- /dev/null +++ b/ranger-audit-server/ranger-audit-server-service/src/main/java/org/apache/ranger/audit/security/FilterChainWrapper.java @@ -0,0 +1,159 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.audit.security; + +import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.security.authentication.client.AuthenticatedURL; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.security.authentication.AbstractAuthenticationToken; +import org.springframework.security.authentication.UsernamePasswordAuthenticationToken; +import org.springframework.security.core.Authentication; +import org.springframework.security.core.GrantedAuthority; +import org.springframework.security.core.authority.SimpleGrantedAuthority; +import org.springframework.security.core.context.SecurityContextHolder; +import org.springframework.security.core.userdetails.User; +import org.springframework.security.core.userdetails.UserDetails; +import org.springframework.security.web.authentication.WebAuthenticationDetails; + +import javax.servlet.FilterChain; +import javax.servlet.ServletException; +import javax.servlet.ServletRequest; +import javax.servlet.ServletResponse; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; + +/** + * Ranger Audit specific {@link javax.servlet.FilterChain filter-chain} wrapper. + * This wrapper class will be used to perform additional activities after + * {@link javax.servlet.Filter filter} based authentication. + */ +public class FilterChainWrapper implements FilterChain { + private static final Logger LOG = LoggerFactory.getLogger(FilterChainWrapper.class); + + private static final String DEFAULT_AUDIT_ROLE = "ROLE_USER"; + private static final String COOKIE_PARAM = "Set-Cookie"; + private FilterChain filterChain; + + // Constructor + public FilterChainWrapper(final FilterChain filterChain) { + this.filterChain = filterChain; + } + + /** + * Read user from cookie. + * + * @param httpResponse + * @return + */ + private static String readUserFromCookie(HttpServletResponse httpResponse) { + String userName = null; + boolean isCookieSet = httpResponse.containsHeader(COOKIE_PARAM); + + if (isCookieSet) { + Collection cookies = httpResponse.getHeaders(COOKIE_PARAM); + + if (cookies != null) { + for (String cookie : cookies) { + if (!StringUtils.startsWithIgnoreCase(cookie, AuthenticatedURL.AUTH_COOKIE) + || !cookie.contains("u=")) { + continue; + } + + String[] split = cookie.split(";"); + + for (String s : split) { + if (StringUtils.startsWithIgnoreCase(s, AuthenticatedURL.AUTH_COOKIE)) { + int idxUEq = s.indexOf("u="); + + if (idxUEq == -1) { + continue; + } + + int idxAmp = s.indexOf("&", idxUEq); + + if (idxAmp == -1) { + continue; + } + + try { + userName = s.substring(idxUEq + 2, idxAmp); + break; + } catch (Exception e) { + userName = null; + } + } + } + } + } + } + + return userName; + } + + /** + * This method sets spring security {@link org.springframework.security.core.context.SecurityContextHolder context} + * with {@link org.springframework.security.core.Authentication authentication}, + * if not already authenticated by other filter. And carried on with {@link javax.servlet.FilterChain filter-chain}. + * This method will be invoked by {@link org.apache.hadoop.security.authentication.server.AuthenticationFilter AuthenticationFilter}, + * only when user is authenticated. + */ + @Override + public void doFilter(ServletRequest servletRequest, ServletResponse servletResponse) throws IOException, ServletException { + final HttpServletRequest httpRequest = (HttpServletRequest) servletRequest; + final HttpServletResponse httpResponse = (HttpServletResponse) servletResponse; + final Authentication existingAuth = SecurityContextHolder.getContext().getAuthentication(); + String userName = readUserFromCookie(httpResponse); + + if (StringUtils.isEmpty(userName) && StringUtils.isNotEmpty(httpRequest.getRemoteUser())) { + userName = httpRequest.getRemoteUser(); + } + + if ((existingAuth == null || !existingAuth.isAuthenticated()) && StringUtils.isNotEmpty(userName)) { + String doAsUser = httpRequest.getParameter("doAs"); + if (StringUtils.isNotEmpty(doAsUser)) { + userName = doAsUser; + } + + final List grantedAuths = Arrays.asList(new SimpleGrantedAuthority(DEFAULT_AUDIT_ROLE)); + final UserDetails principal = new User(userName, "", grantedAuths); + final Authentication finalAuthentication = new UsernamePasswordAuthenticationToken(principal, "", grantedAuths); + final WebAuthenticationDetails webDetails = new WebAuthenticationDetails(httpRequest); + + ((AbstractAuthenticationToken) finalAuthentication).setDetails(webDetails); + + SecurityContextHolder.getContext().setAuthentication(finalAuthentication); + + if (LOG.isDebugEnabled()) { + LOG.debug("User [{}] is authenticated via AuditDelegationTokenFilter.", finalAuthentication.getPrincipal()); + } + } + + if (LOG.isDebugEnabled()) { + LOG.debug("As user [{}] is authenticated, proceeding with filter chain.", userName); + } + filterChain.doFilter(httpRequest, httpResponse); + } +} diff --git a/ranger-audit-server/ranger-audit-server-service/src/main/java/org/apache/ranger/audit/security/NullServletContext.java b/ranger-audit-server/ranger-audit-server-service/src/main/java/org/apache/ranger/audit/security/NullServletContext.java new file mode 100644 index 0000000000..e5df8e03a2 --- /dev/null +++ b/ranger-audit-server/ranger-audit-server-service/src/main/java/org/apache/ranger/audit/security/NullServletContext.java @@ -0,0 +1,240 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.audit.security; + +import javax.servlet.Filter; +import javax.servlet.FilterRegistration; +import javax.servlet.FilterRegistration.Dynamic; +import javax.servlet.RequestDispatcher; +import javax.servlet.Servlet; +import javax.servlet.ServletContext; +import javax.servlet.ServletException; +import javax.servlet.SessionCookieConfig; +import javax.servlet.SessionTrackingMode; +import javax.servlet.descriptor.JspConfigDescriptor; + +import java.io.InputStream; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.Enumeration; +import java.util.EventListener; +import java.util.Map; +import java.util.Set; + +public class NullServletContext implements ServletContext { + public void setSessionTrackingModes(Set sessionTrackingModes) { + } + + public boolean setInitParameter(String name, String value) { + return false; + } + + public void setAttribute(String name, Object object) { + } + + public void removeAttribute(String name) { + } + + public void log(String message, Throwable throwable) { + } + + public void log(Exception exception, String msg) { + } + + public void log(String msg) { + } + + public String getVirtualServerName() { + return null; + } + + public SessionCookieConfig getSessionCookieConfig() { + return null; + } + + public Enumeration getServlets() { + return null; + } + + public Map getServletRegistrations() { + return null; + } + + public javax.servlet.ServletRegistration getServletRegistration(String servletName) { + return null; + } + + public Enumeration getServletNames() { + return null; + } + + public String getServletContextName() { + return null; + } + + public Servlet getServlet(String name) throws ServletException { + return null; + } + + public String getServerInfo() { + return null; + } + + public Set getResourcePaths(String path) { + return null; + } + + public InputStream getResourceAsStream(String path) { + return null; + } + + public URL getResource(String path) throws MalformedURLException { + return null; + } + + public RequestDispatcher getRequestDispatcher(String path) { + return null; + } + + public String getRealPath(String path) { + return null; + } + + public RequestDispatcher getNamedDispatcher(String name) { + return null; + } + + public int getMinorVersion() { + return 0; + } + + public String getMimeType(String file) { + return null; + } + + public int getMajorVersion() { + return 0; + } + + public JspConfigDescriptor getJspConfigDescriptor() { + return null; + } + + public Enumeration getInitParameterNames() { + return null; + } + + public String getInitParameter(String name) { + return null; + } + + public Map getFilterRegistrations() { + return null; + } + + public FilterRegistration getFilterRegistration(String filterName) { + return null; + } + + public Set getEffectiveSessionTrackingModes() { + return null; + } + + public int getEffectiveMinorVersion() { + return 0; + } + + public int getEffectiveMajorVersion() { + return 0; + } + + public Set getDefaultSessionTrackingModes() { + return null; + } + + public String getContextPath() { + return null; + } + + public ServletContext getContext(String uripath) { + return null; + } + + public ClassLoader getClassLoader() { + return null; + } + + public Enumeration getAttributeNames() { + return null; + } + + public Object getAttribute(String name) { + return null; + } + + public void declareRoles(String... roleNames) { + } + + public T createServlet(Class clazz) throws ServletException { + return null; + } + + public T createListener(Class clazz) throws ServletException { + return null; + } + + public T createFilter(Class clazz) throws ServletException { + return null; + } + + public javax.servlet.ServletRegistration.Dynamic addServlet(String servletName, + Class servletClass) { + return null; + } + + public javax.servlet.ServletRegistration.Dynamic addServlet(String servletName, Servlet servlet) { + return null; + } + + public javax.servlet.ServletRegistration.Dynamic addServlet(String servletName, String className) { + return null; + } + + public void addListener(Class listenerClass) { + } + + public void addListener(T t) { + } + + public void addListener(String className) { + } + + public Dynamic addFilter(String filterName, Class filterClass) { + return null; + } + + public Dynamic addFilter(String filterName, Filter filter) { + return null; + } + + public Dynamic addFilter(String filterName, String className) { + return null; + } +} diff --git a/ranger-audit-server/ranger-audit-server-service/src/main/java/org/apache/ranger/audit/server/AuditServerApplication.java b/ranger-audit-server/ranger-audit-server-service/src/main/java/org/apache/ranger/audit/server/AuditServerApplication.java new file mode 100644 index 0000000000..a9d6664c26 --- /dev/null +++ b/ranger-audit-server/ranger-audit-server-service/src/main/java/org/apache/ranger/audit/server/AuditServerApplication.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.audit.server; + +import org.apache.hadoop.conf.Configuration; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Main application class for Ranger Audit Server Service. + * + * This service receives audit events from Ranger plugins via REST API + * and produces them to Kafka for consumption by downstream services. + * It uses embedded Tomcat server for the REST API and health check endpoints. + */ +public class AuditServerApplication { + private static final Logger LOG = LoggerFactory.getLogger(AuditServerApplication.class); + + private static final String APP_NAME = "ranger-audit-server"; + private static final String CONFIG_PREFIX = "ranger.audit.service."; + + private AuditServerApplication() { + } + + public static void main(String[] args) { + LOG.info("=========================================================================="); + LOG.info("==> Starting Ranger Audit Server Service"); + LOG.info("=========================================================================="); + + try { + Configuration config = AuditServerConfig.getInstance(); + + LOG.info("Configuration loaded successfully"); + + EmbeddedServer server = new EmbeddedServer(config, APP_NAME, CONFIG_PREFIX); + server.start(); + + LOG.info("==> Ranger Audit Server Service Started Successfully"); + } catch (Exception e) { + LOG.error("Failed to start Ranger Audit Server Service", e); + System.exit(1); + } + } +} diff --git a/ranger-audit-server/ranger-audit-server-service/src/main/java/org/apache/ranger/audit/server/AuditServerConfig.java b/ranger-audit-server/ranger-audit-server-service/src/main/java/org/apache/ranger/audit/server/AuditServerConfig.java new file mode 100644 index 0000000000..afc7bbb844 --- /dev/null +++ b/ranger-audit-server/ranger-audit-server-service/src/main/java/org/apache/ranger/audit/server/AuditServerConfig.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.audit.server; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Configuration class for Ranger Audit Server Service. + * Loads audit server configuration files. + */ +public class AuditServerConfig extends AuditConfig { + private static final Logger LOG = LoggerFactory.getLogger(AuditServerConfig.class); + private static final String CONFIG_FILE_PATH = "conf/ranger-audit-server-site.xml"; + private static volatile AuditServerConfig sInstance; + + private AuditServerConfig() { + super(); + addAuditServerResources(); + } + + public static AuditServerConfig getInstance() { + AuditServerConfig ret = AuditServerConfig.sInstance; + + if (ret == null) { + synchronized (AuditServerConfig.class) { + ret = AuditServerConfig.sInstance; + + if (ret == null) { + ret = new AuditServerConfig(); + AuditServerConfig.sInstance = ret; + } + } + } + + return ret; + } + + private boolean addAuditServerResources() { + LOG.debug("==> AuditServerConfig.addAuditServerResources()"); + + boolean ret = true; + + // Load ranger-audit-server-site.xml + if (!addAuditResource(CONFIG_FILE_PATH, true)) { + LOG.error("Could not load required configuration: {}", CONFIG_FILE_PATH); + ret = false; + } + + LOG.debug("<== AuditServerConfig.addAuditServerResources(), result={}", ret); + + return ret; + } +} diff --git a/ranger-audit-server/ranger-audit-server-service/src/main/resources/conf/logback.xml b/ranger-audit-server/ranger-audit-server-service/src/main/resources/conf/logback.xml new file mode 100644 index 0000000000..4f1aa0500a --- /dev/null +++ b/ranger-audit-server/ranger-audit-server-service/src/main/resources/conf/logback.xml @@ -0,0 +1,71 @@ + + + + + + + + + + + %d{yyyy-MM-dd HH:mm:ss.SSS} %-5p - %m%n + + + + + + ${LOG_DIR}/${LOG_FILE} + + %d %p %c{1} [%t] %m%n + + + ${LOG_DIR}/${LOG_FILE}-%d{MM-dd-yyyy}-%i.log.gz + + 200MB + + 10 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/ranger-audit-server/ranger-audit-server-service/src/main/resources/conf/ranger-audit-server-site.xml b/ranger-audit-server/ranger-audit-server-service/src/main/resources/conf/ranger-audit-server-site.xml new file mode 100644 index 0000000000..fe019818c0 --- /dev/null +++ b/ranger-audit-server/ranger-audit-server-service/src/main/resources/conf/ranger-audit-server-site.xml @@ -0,0 +1,359 @@ + + + + + + log.dir + ${audit.server.log.dir} + Log directory for Ranger Audit Server service + + + + ranger.audit.service.webapp.dir + webapp/ranger-audit-server-service + Path to the extracted webapp directory (relative to AUDIT_SERVER_HOME_DIR). Must be extracted directory, not WAR file, because Jersey cannot scan packages in unexpanded WAR files. + + + + ranger.audit.service.contextName + / + + + + ranger.audit.service.host + ranger-audit-server.rangernw + + - In Docker: Use full service name with domain (e.g., ranger-audit-server.rangernw) + - In VM: Use the actual FQDN (e.g., ranger.example.com) + + + + + ranger.audit.service.http.port + 7081 + + + + ranger.audit.service.https.port + 7182 + + + + ranger.audit.service.http.enabled + true + + + + ranger.audit.service.enableTLS + false + + + + ranger.audit.service.https.attrib.ssl.enabled + false + + + + + ranger.audit.service.https.attrib.keystore.keyalias + myKey + + + + ranger.audit.service.https.attrib.keystore.pass + _ + + + + ranger.audit.service.https.attrib.keystore.file + /etc/ranger/ranger-audit-server/keys/server.jks + + + + ranger.audit.service.https.attrib.keystore.credential.alias + keyStoreCredentialAlias + + + + ranger.audit.service.tomcat.ciphers + + + + + + hadoop.security.authentication + kerberos + + Hadoop authentication type. Must be set to "kerberos" for EmbeddedServer to call + UGI.loginUserFromKeytab() and load principal credentials for SPNEGO authentication. + + + + + ranger.audit.kerberos.enabled + true + Enable Kerberos authentication for incoming REST API requests + + + + ranger.audit.kerberos.type + kerberos + Authentication type: kerberos or simple + + + + ranger.audit.kerberos.principal + HTTP/_HOST@EXAMPLE.COM + + INCOMING: SPNEGO HTTP principal for authenticating REST API requests FROM plugins. + This principal must be present in the keytab file. + + + + + ranger.audit.kerberos.keytab + /etc/keytabs/HTTP.keytab + + Keytab file containing HTTP principal for SPNEGO authentication + + + + + ranger.audit.kerberos.name.rules + DEFAULT + Kerberos name rules for principal mapping + + + + ranger.audit.server.bind.address + ranger-audit-server.rangernw + + Hostname for Kerberos principal _HOST resolution in HTTP principal + - Example: If set to "myhost.example.com" and principal is "HTTP/_HOST@REALM", + it becomes "HTTP/myhost.example.com@REALM" + - In Docker: Use full service name with domain (e.g., ranger-audit-server.rangernw) + - In VM: Use the actual FQDN (e.g., ranger.example.com) + - MUST match the hostname in the keytab principal + + + + + ranger.audit.kerberos.cookie.path + / + Cookie path for delegation token + + + + + config.prefix + ranger.audit.jwt.auth + Configuration prefix for JWT authentication + + + + ranger.audit.jwt.auth.enabled + false + Enable JWT authentication for audit REST API + + + + ranger.audit.jwt.auth.provider-url + http://localhost:9180/rest/jwks + JWKS provider URL for JWT token validation + + + + ranger.audit.jwt.auth.public-key + + JWT public key in PEM format (alternative to provider-url) + + + + ranger.audit.jwt.auth.cookie.name + hadoop-jwt + Name of the cookie containing JWT token + + + + ranger.audit.jwt.auth.audiences + + Expected audiences for JWT validation (comma-separated) + + + + + ranger.audit.service.authentication.method + KERBEROS + + + + ranger.audit.service.kerberos.principal + rangerauditserver/_HOST@EXAMPLE.COM + + rangerauditserver user kerberos principal for authentication into kafka + + + + + ranger.audit.service.kerberos.keytab + /etc/keytabs/rangerauditserver.keytab + + keytab of the rangerauditserver principal + + + + + + xasecure.audit.destination.kafka + true + + + + xasecure.audit.destination.kafka.bootstrap.servers + ranger-kafka:9092 + + Kafka broker hosts for Kafka audit destination. + example: kafka-broker1:9092,kafka-broker2:9092,kafka-broker3:9092 + In Docker, use the service name of the kafka brokers: ranger-kafka:9092 + + + + + xasecure.audit.destination.kafka.topic.name + ranger_audits + + + + xasecure.audit.destination.kafka.topic.partitions + 30 + + Number of partitions for the Kafka audit topic. + Recommendations: + - Small deployments: 30 partitions (default) + - Medium deployments: 50 partitions + - Large deployments: 100+ partitions + + + + + xasecure.audit.destination.kafka.replication.factor + 1 + + Replication factor for the Kafka audit topic. + Default: 3 (production with multiple brokers) + Development/Single broker: 1 (current setting for Docker environment) + Must not exceed the number of available Kafka brokers. + + + + + xasecure.audit.destination.kafka.security.protocol + SASL_PLAINTEXT + + Security protocol for Kafka audit destination. + example: PLAINTEXT for non-Kerberos mode, SASL_PLAINTEXT for Kerberos mode, SASL_SSL for SSL mode + + + + + xasecure.audit.destination.kafka.sasl.mechanism + GSSAPI + + + + xasecure.audit.destination.kafka.request.timeout.ms + 60000 + + + + xasecure.audit.destination.kafka.connections.max.idle.ms + 90000 + + + + + ranger.audit.kafka.recovery.enabled + true + + Enable audit message recovery system. When enabled, failed audit messages are written to local spool files + and retried to Kafka automatically. + + + + + ranger.audit.kafka.recovery.spool.dir + /var/log/ranger/ranger-audit-server/audit/spool + + Directory for storing .failed files with audit messages that failed to send to Kafka. + Files remain here until successfully processed, then moved to archive as .processed files. + + + + + ranger.audit.kafka.recovery.archive.dir + /var/log/ranger/ranger-audit-server/audit/archive + + Directory for successfully processed files. Files are moved here with .processed extension + after all messages are successfully retried to Kafka. Old .processed files are automatically + deleted based on retention policy (max.processed.files). + + + + + ranger.audit.kafka.recovery.file.rotation.interval.sec + 300 + + Time interval in seconds for rotating spool files. Default is 300 seconds (5 minutes). + A new spool file is created after this interval. + + + + + ranger.audit.kafka.recovery.max.messages.per.file + 10000 + + Maximum number of messages to write per spool file before rotation. + File is closed and moved to archive after reaching this limit. + + + + + ranger.audit.kafka.recovery.retry.interval.sec + 60 + + Time interval in seconds between retry attempts for archived files. + Default is 60 seconds (1 minute). + + + + + ranger.audit.kafka.recovery.archive.max.processed.files + 100 + + Maximum number of .processed files to retain in archive directory. + Oldest .processed files are deleted when this limit is exceeded. + Failed messages remain in spool directory as .failed files for retry. + + + + + ranger.audit.kafka.recovery.retry.max.attempts + 3 + + Maximum number of retry attempts per message before keeping file as .failed in spool. + Default is 3 attempts with exponential backoff. + + + diff --git a/ranger-audit-server/ranger-audit-server-service/src/main/webapp/WEB-INF/applicationContext.xml b/ranger-audit-server/ranger-audit-server-service/src/main/webapp/WEB-INF/applicationContext.xml new file mode 100644 index 0000000000..93de022cd8 --- /dev/null +++ b/ranger-audit-server/ranger-audit-server-service/src/main/webapp/WEB-INF/applicationContext.xml @@ -0,0 +1,54 @@ + + + + + + + + + + + + + + + + + + + + + diff --git a/ranger-audit-server/ranger-audit-server-service/src/main/webapp/WEB-INF/security-applicationContext.xml b/ranger-audit-server/ranger-audit-server-service/src/main/webapp/WEB-INF/security-applicationContext.xml new file mode 100644 index 0000000000..30a9fa5e2e --- /dev/null +++ b/ranger-audit-server/ranger-audit-server-service/src/main/webapp/WEB-INF/security-applicationContext.xml @@ -0,0 +1,63 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/ranger-audit-server/ranger-audit-server-service/src/main/webapp/WEB-INF/web.xml b/ranger-audit-server/ranger-audit-server-service/src/main/webapp/WEB-INF/web.xml new file mode 100644 index 0000000000..612d595471 --- /dev/null +++ b/ranger-audit-server/ranger-audit-server-service/src/main/webapp/WEB-INF/web.xml @@ -0,0 +1,73 @@ + + + + + Apache Ranger - Audit Server Service + Apache Ranger - Audit Server Service (Core: REST API + Kafka Producer) + + + contextConfigLocation + + WEB-INF/applicationContext.xml + WEB-INF/security-applicationContext.xml + + + + + + springSecurityFilterChain + org.springframework.web.filter.DelegatingFilterProxy + + + + springSecurityFilterChain + /api/audit/* + + + + + org.springframework.web.context.ContextLoaderListener + + + + org.springframework.web.context.request.RequestContextListener + + + + REST Service + com.sun.jersey.spi.spring.container.servlet.SpringServlet + + + com.sun.jersey.config.property.packages + org.apache.ranger.audit.rest + + + + com.sun.jersey.api.json.POJOMappingFeature + true + + + 1 + + + + REST Service + /api/* + + + diff --git a/ranger-audit-server/scripts/README.md b/ranger-audit-server/scripts/README.md new file mode 100644 index 0000000000..509b778048 --- /dev/null +++ b/ranger-audit-server/scripts/README.md @@ -0,0 +1,549 @@ + +# Ranger Audit Server + +This directory contains shell scripts to start and stop the Ranger Audit Server services locally (outside of Docker). + +## Overview + +The Ranger Audit Server consists of three microservices: + +1. **ranger-audit-server-service** - Core audit server that receives audit events via REST API and produces them to Kafka +2. **ranger-audit-consumer-solr** - Consumer service that reads from Kafka and indexes audits to Solr +3. **ranger-audit-consumer-hdfs** - Consumer service that reads from Kafka and writes audits to HDFS/S3/Azure + +Each service has its own `scripts` folder with start/stop scripts in its main directory. + +## Prerequisites + +Before running these scripts, ensure you have: + +1. **Java 8 or higher** installed and `JAVA_HOME` set +2. **Built the project** using Maven: + ```bash + cd /path/to/ranger-audit-server + mvn clean package -DskipTests + ``` +3. **Kafka running** (required for all services) +4. **Solr running** (required for Solr consumer) +5. **HDFS/Hadoop running** (required for HDFS consumer) + +## Quick Start - All Services + +### Start All Services + +```bash +./scripts/start-all-services.sh +``` + +This script will start all three services in the correct order: +1. Audit Server (waits 10 seconds) +2. Solr Consumer (waits 5 seconds) +3. HDFS Consumer + +### Stop All Services + +```bash +./scripts/stop-all-services.sh +``` + +This script will stop all three services in reverse order. + +## Individual Service Scripts + +Each service can also be started/stopped individually: + +### Audit Server Service + +```bash +# Start +./ranger-audit-server-service/scripts/start-audit-server.sh + +# Stop +./ranger-audit-server-service/scripts/stop-audit-server.sh +``` + +**Default Ports:** 7081 (HTTP), 7182 (HTTPS) +**Health Check:** http://localhost:7081/api/audit/health + +### Solr Consumer + +```bash +# Start +./ranger-audit-consumer-solr/scripts/start-consumer-solr.sh + +# Stop +./ranger-audit-consumer-solr/scripts/stop-consumer-solr.sh +``` + +**Default Port:** 7091 +**Health Check:** http://localhost:7091/api/health + +### HDFS Consumer + +```bash +# Start +./ranger-audit-consumer-hdfs/scripts/start-consumer-hdfs.sh + +# Stop +./ranger-audit-consumer-hdfs/scripts/stop-consumer-hdfs.sh +``` + +**Default Port:** 7092 +**Health Check:** http://localhost:7092/api/health + +## Configuration + +### Environment Variables + +Each script supports the following environment variables: + +#### Audit Server +- `AUDIT_SERVER_HOME_DIR` - Home directory (default: `target/`) +- `AUDIT_SERVER_CONF_DIR` - Configuration directory (default: `src/main/resources/conf/`) +- `AUDIT_SERVER_LOG_DIR` - Log directory (default: `logs/`) +- `AUDIT_SERVER_HEAP` - JVM heap settings (default: `-Xms512m -Xmx2g`) +- `AUDIT_SERVER_OPTS` - Additional JVM options +- `KERBEROS_ENABLED` - Enable Kerberos authentication (default: `false`) + +#### Consumers (HDFS and Solr) +- `AUDIT_CONSUMER_HOME_DIR` - Home directory (default: `target/`) +- `AUDIT_CONSUMER_CONF_DIR` - Configuration directory (default: `src/main/resources/conf/`) +- `AUDIT_CONSUMER_LOG_DIR` - Log directory (default: `logs/`) +- `AUDIT_CONSUMER_HEAP` - JVM heap settings (default: `-Xms512m -Xmx2g`) +- `AUDIT_CONSUMER_OPTS` - Additional JVM options +- `KERBEROS_ENABLED` - Enable Kerberos authentication (default: `false`) + +### Example with Custom Settings + +```bash +# Set custom heap size and log directory +export AUDIT_SERVER_HEAP="-Xms1g -Xmx4g" +export AUDIT_SERVER_LOG_DIR="/var/log/ranger/range-audit-server" + +./ranger-audit-server-service/scripts/start-audit-server.sh +``` + +## Log Files + +Each service creates logs in its respective `logs/` directory (or custom location if set): + +- **Audit Server:** + - Application logs: `logs/ranger-audit-server.log` + - Catalina output: `logs/catalina.out` + - PID file: `logs/ranger-audit-server.pid` + +- **Solr Consumer:** + - Application logs: `logs/ranger-audit-consumer-solr.log` + - Catalina output: `logs/catalina.out` + - PID file: `logs/ranger-audit-consumer-solr.pid` + +- **HDFS Consumer:** + - Application logs: `logs/ranger-audit-consumer-hdfs.log` + - Catalina output: `logs/catalina.out` + - PID file: `logs/ranger-audit-consumer-hdfs.pid` + +### Monitoring Logs + +```bash +# Tail audit server logs +tail -f ranger-audit-server-service/logs/ranger-audit-server.log + +# Tail Solr consumer logs +tail -f ranger-audit-consumer-solr/logs/ranger-audit-consumer-solr.log + +# Tail HDFS consumer logs +tail -f ranger-audit-consumer-hdfs/logs/ranger-audit-consumer-hdfs.log +``` + +### Enabling Debug Logging + +To enable debug logging for troubleshooting, modify the `logback.xml` configuration file in the service's `conf/` directory: + +**For Audit Server:** +Edit `ranger-audit-server-service/src/main/resources/conf/logback.xml` (or `/opt/ranger-audit-server/conf/logback.xml` in Docker): + +```xml + + + + + + + + + + + +``` + +**For Consumers (HDFS/Solr):** +Similarly, edit the `logback.xml` in their respective `conf/` directories. + +**Available log levels:** `TRACE`, `DEBUG`, `INFO`, `WARN`, `ERROR` + +After modifying the logback configuration, restart the service for changes to take effect. + +## Troubleshooting + +### Service Won't Start + +1. **Check if already running:** + ```bash + ps aux | grep ranger-audit + ``` + +2. **Check for port conflicts:** + ```bash + lsof -i :7081 # Audit Server + lsof -i :7091 # Solr Consumer + lsof -i :7092 # HDFS Consumer + ``` + +3. **Verify WAR file exists:** + ```bash + find ./target -name "*.war" + ``` + +4. **Check logs for errors:** + ```bash + tail -100 logs/catalina.out + ``` + +### Service Won't Stop + +If a service doesn't stop gracefully, the script will force kill after 30 seconds. You can also manually kill: + +```bash +# Find and kill the process +ps aux | grep "AuditServerApplication" +kill + +# Or force kill +kill -9 + +# Remove stale PID file +rm -f logs/ranger-audit-server.pid +``` + +### Java Not Found + +If Java is not detected: + +```bash +# Set JAVA_HOME +export JAVA_HOME=/path/to/java +export PATH=$JAVA_HOME/bin:$PATH + +# Verify +java -version +``` + +### Kafka Connection Issues + +Check Kafka bootstrap servers configuration in: +- `ranger-audit-server-service/src/main/resources/conf/ranger-audit-server-site.xml` +- `ranger-audit-consumer-solr/src/main/resources/conf/ranger-audit-consumer-solr-site.xml` +- `ranger-audit-consumer-hdfs/src/main/resources/conf/ranger-audit-consumer-hdfs-site.xml` + +## Architecture + +``` +┌─────────────────────┐ +│ Ranger Plugins │ +│ (HDFS, Hive, etc.) │ +└──────────┬──────────┘ + │ REST API + ▼ +┌─────────────────────┐ +│ Audit Server │ Port 7081 +│ (Producer) │ +└──────────┬──────────┘ + │ Kafka + ▼ + ┌──────────────┐ + │ Kafka │ + │ (Topic) │ + └──────┬───────┘ + │ + ┌────┴────┬──────┬─────────┐ + │ │ │ │ + ▼ ▼ ▼ ▼ +┌──────────┐ ┌──────────┐ ┌──────────┐ ┌──────────┐ +│ Solr │ │ HDFS │ │ New │ ... │ Nth │ +│ Consumer │ │ Consumer │ │ Consumer │ │ Consumer │ +│ (7091) │ │ (7092) │ │ (709N) │ │ (709N+1) │ +└────┬─────┘ └────┬─────┘ └────┬─────┘ └────┬─────┘ + │ │ │ │ + ▼ ▼ ▼ ▼ +┌─────────┐ ┌──────────┐ ┌──────────┐ ┌──────────┐ +│ Solr │ │ HDFS │ │ New │ │ Nth │ +│ (Index) │ │ (Storage)│ │(Dest) │ │ (Dest) │ +└─────────┘ └──────────┘ └──────────┘ └──────────┘ +``` + +## Adding a New Destination + +To add a new audit destination (e.g., Elasticsearch, MongoDB, Cloud Storage, etc.), follow these steps: + +### 1. Create a New Consumer Module + +Create a new Maven module in the `ranger-audit-server` directory: + +```bash +cd ranger-audit-server +mkdir ranger-audit-consumer- +cd ranger-audit-consumer- +``` + +Create a `pom.xml` based on the existing consumers (Solr or HDFS). Key dependencies: +- Spring Boot Starter +- Spring Kafka +- Your destination-specific client library (e.g., Elasticsearch client, MongoDB driver) + +### 2. Implement the Consumer Application + +Create the main Spring Boot application class: + +```java +package org.apache.ranger.audit.consumer; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; + +@SpringBootApplication +public class YourDestinationConsumerApplication { + public static void main(String[] args) { + SpringApplication.run(YourDestinationConsumerApplication.class, args); + } +} +``` + +### 3. Create the Kafka Consumer + +Implement a Kafka consumer to read audit events: + +```java +package org.apache.ranger.audit.consumer; + +import org.springframework.kafka.annotation.KafkaListener; +import org.springframework.stereotype.Service; + +@Service +public class YourDestinationConsumer { + @KafkaListener(topics = "${ranger.audit.kafka.topic:ranger_audits}", groupId = "${ranger.audit.kafka.consumer.group:audit-consumer-your-destination}") + public void consumeAudit(String auditEvent) { + // Parse audit event + // Transform if needed + // Write to your destination + } +} +``` + +### 4. Add Configuration Files + +Create configuration files in `src/main/resources/conf/`: + +**ranger-audit-consumer--site.xml:** +```xml + + + + ranger.audit.kafka.bootstrap.servers + localhost:9092 + + + ranger.audit.kafka.topic + ranger_audits + + + ranger.audit.your-destination.url + http://localhost:PORT + + + +``` + +**application.yml:** +```yaml +server: + port: 709X # Choose next available port (e.g., 7093, 7094...) + +spring: + kafka: + bootstrap-servers: ${ranger.audit.kafka.bootstrap.servers:localhost:9092} + consumer: + group-id: audit-consumer-your-destination + auto-offset-reset: earliest + key-deserializer: org.apache.kafka.common.serialization.StringDeserializer + value-deserializer: org.apache.kafka.common.serialization.StringDeserializer + +# Add destination-specific Spring configurations +``` + +### 5. Create Start/Stop Scripts + +Create a `scripts` directory with start/stop scripts: + +**scripts/start-consumer-.sh:** +```bash +#!/bin/bash + +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +SERVICE_DIR="$(dirname "$SCRIPT_DIR")" + +# Environment variables +AUDIT_CONSUMER_HOME_DIR="${AUDIT_CONSUMER_HOME_DIR:-$SERVICE_DIR/target}" +AUDIT_CONSUMER_CONF_DIR="${AUDIT_CONSUMER_CONF_DIR:-$SERVICE_DIR/src/main/resources/conf}" +AUDIT_CONSUMER_LOG_DIR="${AUDIT_CONSUMER_LOG_DIR:-$SERVICE_DIR/logs}" +AUDIT_CONSUMER_HEAP="${AUDIT_CONSUMER_HEAP:--Xms512m -Xmx2g}" +AUDIT_CONSUMER_OPTS="${AUDIT_CONSUMER_OPTS:-}" +KERBEROS_ENABLED="${KERBEROS_ENABLED:-false}" + +# Find WAR file +WAR_FILE=$(find "$AUDIT_CONSUMER_HOME_DIR" -name "ranger-audit-consumer-*.war" | head -1) + +if [ -z "$WAR_FILE" ]; then + echo "Error: WAR file not found in $AUDIT_CONSUMER_HOME_DIR" + exit 1 +fi + +# Start service +java $AUDIT_CONSUMER_HEAP $AUDIT_CONSUMER_OPTS \ + -Dlog.dir="$AUDIT_CONSUMER_LOG_DIR" \ + -Dconf.dir="$AUDIT_CONSUMER_CONF_DIR" \ + -jar "$WAR_FILE" > "$AUDIT_CONSUMER_LOG_DIR/catalina.out" 2>&1 & + +echo $! > "$AUDIT_CONSUMER_LOG_DIR/ranger-audit-consumer-.pid" +echo "Started Ranger Audit Consumer () with PID: $(cat $AUDIT_CONSUMER_LOG_DIR/ranger-audit-consumer-.pid)" +``` + +**scripts/stop-consumer-.sh:** +```bash +#!/bin/bash + +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +SERVICE_DIR="$(dirname "$SCRIPT_DIR")" +AUDIT_CONSUMER_LOG_DIR="${AUDIT_CONSUMER_LOG_DIR:-$SERVICE_DIR/logs}" +PID_FILE="$AUDIT_CONSUMER_LOG_DIR/ranger-audit-consumer-.pid" + +if [ -f "$PID_FILE" ]; then + PID=$(cat "$PID_FILE") + kill "$PID" + echo "Stopped Ranger Audit Consumer () with PID: $PID" + rm -f "$PID_FILE" +else + echo "PID file not found. Service may not be running." +fi +``` + +Make scripts executable: +```bash +chmod +x scripts/*.sh +``` + +### 6. Update Parent POM + +Add the new module to the parent `ranger-audit-server/pom.xml`: + +```xml + + ranger-audit-server-service + ranger-audit-consumer-solr + ranger-audit-consumer-hdfs + ranger-audit-consumer- + +``` + +### 7. Update Start/Stop All Scripts + +Add your consumer to `scripts/start-all-services.sh`: + +```bash +# Start Your Destination Consumer +echo "Starting Ranger Audit Consumer ()..." +cd "$BASE_DIR/ranger-audit-consumer-" +./scripts/start-consumer-.sh +echo "Waiting 5 seconds for consumer to initialize..." +sleep 5 +``` + +Add to `scripts/stop-all-services.sh`: + +```bash +# Stop Your Destination Consumer +echo "Stopping Ranger Audit Consumer ()..." +cd "$BASE_DIR/ranger-audit-consumer-" +./scripts/stop-consumer-.sh +``` + +### 8. Build and Test + +```bash +# Build the new consumer +cd ranger-audit-consumer- +mvn clean package -DskipTests + +# Test individually +./scripts/start-consumer-.sh + +# Check health (implement a health endpoint) +curl http://localhost:709X/api/health + +# View logs +tail -f logs/ranger-audit-consumer-.log + +# Stop when done +./scripts/stop-consumer-.sh +``` + +### 9. Add Documentation + +Update this README to include: +- The new consumer in the "Overview" section +- Individual start/stop commands +- Default port and health check endpoint +- Configuration details specific to the destination +- Any prerequisite services required + + +## Development + +### Building Individual Services + +```bash +# Build specific service +cd ranger-audit-server-service +mvn clean package + +cd ../ranger-audit-consumer-solr +mvn clean package + +cd ../ranger-audit-consumer-hdfs +mvn clean package +``` + +### Running in Debug Mode + +Add debug options to the `OPTS` environment variable: + +```bash +export AUDIT_SERVER_OPTS="-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5005" +./ranger-audit-server-service/scripts/start-audit-server.sh +``` +Then attach your IDE debugger to port 5005. + diff --git a/ranger-audit-server/scripts/start-all-services.sh b/ranger-audit-server/scripts/start-all-services.sh new file mode 100755 index 0000000000..cd172f8f18 --- /dev/null +++ b/ranger-audit-server/scripts/start-all-services.sh @@ -0,0 +1,78 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# ======================================== +# Start All Ranger Audit Services +# ======================================== +# This script starts all three audit services in the correct order: +# 1. Audit Server (receives audits and produces to Kafka) +# 2. Solr Consumer (consumes from Kafka and indexes to Solr) +# 3. HDFS Consumer (consumes from Kafka and writes to HDFS) + +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PARENT_DIR="$(dirname "$SCRIPT_DIR")" + +echo "==========================================" +echo "Starting All Ranger Audit Services" +echo "==========================================" +echo "" + +# Start Audit Server +echo "[1/3] Starting Ranger Audit Server Service..." +if [ -f "${PARENT_DIR}/ranger-audit-server-service/scripts/start-audit-server.sh" ]; then + bash "${PARENT_DIR}/ranger-audit-server-service/scripts/start-audit-server.sh" + echo "" + echo "Waiting 10 seconds for Audit Server to initialize..." + sleep 10 +else + echo "[ERROR] Audit Server start script not found" + exit 1 +fi + +# Start Solr Consumer +echo "[2/3] Starting Ranger Audit Consumer - Solr..." +if [ -f "${PARENT_DIR}/ranger-audit-consumer-solr/scripts/start-consumer-solr.sh" ]; then + bash "${PARENT_DIR}/ranger-audit-consumer-solr/scripts/start-consumer-solr.sh" + echo "" + echo "Waiting 5 seconds for Solr Consumer to initialize..." + sleep 5 +else + echo "[WARNING] Solr Consumer start script not found, skipping..." +fi + +# Start HDFS Consumer +echo "[3/3] Starting Ranger Audit Consumer - HDFS..." +if [ -f "${PARENT_DIR}/ranger-audit-consumer-hdfs/scripts/start-consumer-hdfs.sh" ]; then + bash "${PARENT_DIR}/ranger-audit-consumer-hdfs/scripts/start-consumer-hdfs.sh" + echo "" +else + echo "[WARNING] HDFS Consumer start script not found, skipping..." +fi + +echo "==========================================" +echo "✓ All Ranger Audit Services Started" +echo "==========================================" +echo "" +echo "Service Endpoints:" +echo " - Audit Server: http://localhost:7081/api/audit/health" +echo " - Solr Consumer: http://localhost:7091/api/health" +echo " - HDFS Consumer: http://localhost:7092/api/health" +echo "" +echo "To stop all services: ${SCRIPT_DIR}/stop-all-services.sh" diff --git a/ranger-audit-server/scripts/stop-all-services.sh b/ranger-audit-server/scripts/stop-all-services.sh new file mode 100755 index 0000000000..6bd0124c19 --- /dev/null +++ b/ranger-audit-server/scripts/stop-all-services.sh @@ -0,0 +1,66 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# ======================================== +# Stop All Ranger Audit Services +# ======================================== +# This script stops all three audit services in the reverse order: +# 1. HDFS Consumer +# 2. Solr Consumer +# 3. Audit Server + +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PARENT_DIR="$(dirname "$SCRIPT_DIR")" + +echo "==========================================" +echo "Stopping All Ranger Audit Services" +echo "==========================================" +echo "" + +# Stop HDFS Consumer +echo "[1/3] Stopping Ranger Audit Consumer - HDFS..." +if [ -f "${PARENT_DIR}/ranger-audit-consumer-hdfs/scripts/stop-consumer-hdfs.sh" ]; then + bash "${PARENT_DIR}/ranger-audit-consumer-hdfs/scripts/stop-consumer-hdfs.sh" || true + echo "" +else + echo "[WARNING] HDFS Consumer stop script not found, skipping..." +fi + +# Stop Solr Consumer +echo "[2/3] Stopping Ranger Audit Consumer - Solr..." +if [ -f "${PARENT_DIR}/ranger-audit-consumer-solr/scripts/stop-consumer-solr.sh" ]; then + bash "${PARENT_DIR}/ranger-audit-consumer-solr/scripts/stop-consumer-solr.sh" || true + echo "" +else + echo "[WARNING] Solr Consumer stop script not found, skipping..." +fi + +# Stop Audit Server +echo "[3/3] Stopping Ranger Audit Server Service..." +if [ -f "${PARENT_DIR}/ranger-audit-server-service/scripts/stop-audit-server.sh" ]; then + bash "${PARENT_DIR}/ranger-audit-server-service/scripts/stop-audit-server.sh" || true + echo "" +else + echo "[WARNING] Audit Server stop script not found, skipping..." +fi + +echo "==========================================" +echo "✓ All Ranger Audit Services Stopped" +echo "==========================================" From ce3542b4585d8abc9525c21aa3765358a2a2a2b0 Mon Sep 17 00:00:00 2001 From: Ramesh Mani Date: Wed, 11 Feb 2026 13:42:47 -0800 Subject: [PATCH 02/12] RANGER-5482:Create Ranger Audit Server with SOLR and HDFS as audit consumer - fix failing testing --- .../ranger/plugin/audit/TestRangerDefaultAuditHandler.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/agents-common/src/test/java/org/apache/ranger/plugin/audit/TestRangerDefaultAuditHandler.java b/agents-common/src/test/java/org/apache/ranger/plugin/audit/TestRangerDefaultAuditHandler.java index 54a63c42d7..249f7fdf6f 100644 --- a/agents-common/src/test/java/org/apache/ranger/plugin/audit/TestRangerDefaultAuditHandler.java +++ b/agents-common/src/test/java/org/apache/ranger/plugin/audit/TestRangerDefaultAuditHandler.java @@ -22,6 +22,7 @@ import org.apache.ranger.audit.model.AuthzAuditEvent; import org.apache.ranger.audit.provider.AuditHandler; import org.apache.ranger.authorization.hadoop.constants.RangerHadoopConstants; +import org.apache.ranger.authorization.utils.*; import org.apache.ranger.plugin.contextenricher.RangerTagForEval; import org.apache.ranger.plugin.model.RangerServiceDef; import org.apache.ranger.plugin.model.RangerServiceDef.RangerResourceDef; @@ -289,7 +290,7 @@ public void test08_getAdditionalInfo_nullAndNonNull() { RangerAccessResult res = new RangerAccessResult(0, "svc", null, req); req.setRemoteIPAddress(null); req.setForwardedAddresses(new ArrayList<>()); - Assertions.assertNull(handler.getAdditionalInfo(req, res)); + Assertions.assertTrue(handler.getAdditionalInfo(req, res).isEmpty()); RangerAccessRequestImpl req2 = new RangerAccessRequestImpl(); RangerAccessResult res2 = new RangerAccessResult(0, "svc", null, req); From 0004b637e858b86b1e4077d6a961488cb7566eeb Mon Sep 17 00:00:00 2001 From: Ramesh Mani Date: Wed, 11 Feb 2026 13:45:12 -0800 Subject: [PATCH 03/12] RANGER-5482:Create Ranger Audit Server with SOLR and HDFS as audit consumer - fix pmd issue --- .../ranger/plugin/audit/TestRangerDefaultAuditHandler.java | 1 - 1 file changed, 1 deletion(-) diff --git a/agents-common/src/test/java/org/apache/ranger/plugin/audit/TestRangerDefaultAuditHandler.java b/agents-common/src/test/java/org/apache/ranger/plugin/audit/TestRangerDefaultAuditHandler.java index 249f7fdf6f..7fbe23909d 100644 --- a/agents-common/src/test/java/org/apache/ranger/plugin/audit/TestRangerDefaultAuditHandler.java +++ b/agents-common/src/test/java/org/apache/ranger/plugin/audit/TestRangerDefaultAuditHandler.java @@ -22,7 +22,6 @@ import org.apache.ranger.audit.model.AuthzAuditEvent; import org.apache.ranger.audit.provider.AuditHandler; import org.apache.ranger.authorization.hadoop.constants.RangerHadoopConstants; -import org.apache.ranger.authorization.utils.*; import org.apache.ranger.plugin.contextenricher.RangerTagForEval; import org.apache.ranger.plugin.model.RangerServiceDef; import org.apache.ranger.plugin.model.RangerServiceDef.RangerResourceDef; From e6bdea259635bbd3aa5d10a6900b3839ecfcdbe1 Mon Sep 17 00:00:00 2001 From: Ramesh Mani Date: Tue, 17 Feb 2026 11:33:34 -0800 Subject: [PATCH 04/12] RANGER-5482:Create Ranger Audit Server with SOLR and HDFS as audit consumer - Fix audit commit failure propagation and recovery in the consumers --- .../audit/consumer/kafka/AuditRouterHDFS.java | 65 +++++-------------- .../consumer/kafka/AuditSolrConsumer.java | 18 ++++- 2 files changed, 31 insertions(+), 52 deletions(-) diff --git a/ranger-audit-server/ranger-audit-consumer-hdfs/src/main/java/org/apache/ranger/audit/consumer/kafka/AuditRouterHDFS.java b/ranger-audit-server/ranger-audit-consumer-hdfs/src/main/java/org/apache/ranger/audit/consumer/kafka/AuditRouterHDFS.java index 49080aacb5..061894bdd5 100644 --- a/ranger-audit-server/ranger-audit-consumer-hdfs/src/main/java/org/apache/ranger/audit/consumer/kafka/AuditRouterHDFS.java +++ b/ranger-audit-server/ranger-audit-consumer-hdfs/src/main/java/org/apache/ranger/audit/consumer/kafka/AuditRouterHDFS.java @@ -31,13 +31,13 @@ import java.util.Map; import java.util.Properties; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; /** - * Router class that routes audit messages to different HDFSAuditDestination threads + * Router class that routes audit messages to different HDFSAuditDestination instances * based on the app_id. - * Each app_id gets its own HDFSAuditDestination instance running in a separate thread. + * Each app_id gets its own HDFSAuditDestination instance for separate path configuration. + * Writes are synchronous to ensure Kafka offset commits only happen after successful HDFS writes, + * enabling automatic recovery via Kafka redelivery on failures. * * This router writes audits to HDFS as the rangerauditserver user. The audit folder in HDFS * should be configured with appropriate permissions to allow rangerauditserver to write audits @@ -47,7 +47,6 @@ public class AuditRouterHDFS { private static final Logger LOG = LoggerFactory.getLogger(AuditRouterHDFS.class); private final Map destinationMap = new ConcurrentHashMap<>(); - private final Map executorMap = new ConcurrentHashMap<>(); private final ObjectMapper jsonMapper = new ObjectMapper(); private Properties props; private String hdfsPropPrefix; @@ -70,7 +69,7 @@ public void init(Properties props, String hdfsPropPrefix) { * @param message JSON audit message * @param partitionKey The partition key from Kafka (used as app_id) */ - public void routeAuditMessage(String message, String partitionKey) { + public void routeAuditMessage(String message, String partitionKey) throws Exception { LOG.debug("==> AuditRouterHDFS:routeAuditMessage(): Message => {}, partitionKey => {}", message, partitionKey); try { @@ -84,25 +83,20 @@ public void routeAuditMessage(String message, String partitionKey) { if (appId != null) { HDFSAuditDestination hdfsAuditDestination = getHdfsAuditDestination(appId, serviceType, agentHostname); - // Submit message to destination's thread for processing - final String finalAppId = appId; - ExecutorService executor = executorMap.get(appId); - - if (executor != null && !executor.isShutdown()) { - executor.submit(() -> { - try { - hdfsAuditDestination.logJSON(Collections.singletonList(message)); - LOG.debug("Successfully wrote audit for app_id: {}", finalAppId); - } catch (Exception e) { - LOG.error("Error processing audit message for app_id: {}", finalAppId, e); - } - }); - } else { - LOG.warn("Executor is null or shutdown for app_id: {}", appId); + boolean success = hdfsAuditDestination.logJSON(Collections.singletonList(message)); + + if (!success) { + throw new Exception("Failed to write audit to HDFS for app_id: " + appId); } + + LOG.debug("Successfully wrote audit for app_id: {}", appId); + } else { + LOG.warn("Unable to extract app_id from message, skipping audit write"); } } catch (Exception e) { + String errorMessage = "Error routing audit message to HDFS"; LOG.error("AuditRouterHDFS:routeAuditMessage(): Error routing audit message: {}", message, e); + throw new Exception(errorMessage, e); } LOG.debug("<== AuditRouterHDFS:routeAuditMessage()"); @@ -173,14 +167,6 @@ private synchronized HDFSAuditDestination getHdfsAuditDestination(String appId, try { destination = createHDFSDestination(appId, serviceType, agentHostname); destinationMap.put(appId, destination); - - // Create dedicated executor for this destination - ExecutorService executor = Executors.newSingleThreadExecutor(r -> { - Thread t = new Thread(r, "HDFSAuditDestination-" + appId); - t.setDaemon(true); - return t; - }); - executorMap.put(appId, executor); } catch (Exception e) { LOG.error("Failed to create HDFSAuditDestination for app_id: {}", appId, e); throw new RuntimeException("AuditRouterHDFS:getOrCreateDestination() : Failed to create destination for app_id: " + appId, e); @@ -313,26 +299,6 @@ private String getUniqueInstanceIdentifier() { public void shutdown() { LOG.info("==> AuditRouterHDFS.shutdown()"); - // Shutdown all executors - for (Map.Entry entry : executorMap.entrySet()) { - String appId = entry.getKey(); - ExecutorService executor = entry.getValue(); - - LOG.info("Shutting down executor for app_id: {}", appId); - executor.shutdown(); - - try { - if (!executor.awaitTermination(30, java.util.concurrent.TimeUnit.SECONDS)) { - LOG.warn("Executor for app_id {} did not terminate gracefully, forcing shutdown", appId); - executor.shutdownNow(); - } - } catch (InterruptedException e) { - LOG.warn("Interrupted while waiting for executor shutdown for app_id: {}", appId); - executor.shutdownNow(); - Thread.currentThread().interrupt(); - } - } - // Stop all destinations for (Map.Entry entry : destinationMap.entrySet()) { String appId = entry.getKey(); @@ -347,7 +313,6 @@ public void shutdown() { } destinationMap.clear(); - executorMap.clear(); LOG.info("<== AuditRouterHDFS.shutdown()"); } diff --git a/ranger-audit-server/ranger-audit-consumer-solr/src/main/java/org/apache/ranger/audit/consumer/kafka/AuditSolrConsumer.java b/ranger-audit-server/ranger-audit-consumer-solr/src/main/java/org/apache/ranger/audit/consumer/kafka/AuditSolrConsumer.java index 043231f572..12fae9e542 100644 --- a/ranger-audit-server/ranger-audit-consumer-solr/src/main/java/org/apache/ranger/audit/consumer/kafka/AuditSolrConsumer.java +++ b/ranger-audit-server/ranger-audit-consumer-solr/src/main/java/org/apache/ranger/audit/consumer/kafka/AuditSolrConsumer.java @@ -395,8 +395,15 @@ public KafkaConsumer getKafkaConsumer() { @Override public void processMessage(String audit) throws Exception { + boolean processed = false; + if (solrAuditDestination != null) { - solrAuditDestination.logJSON(audit); + processed = solrAuditDestination.logJSON(audit); + } + + if (!processed) { + String errorMessage = "Failure in committing audits into Solr"; + throw new Exception(errorMessage); } } @@ -409,8 +416,15 @@ public void processMessage(String audit) throws Exception { * @throws Exception if batch processing fails */ public void processMessageBatch(Collection audits) throws Exception { + boolean processed = false; + if (solrAuditDestination != null && audits != null && !audits.isEmpty()) { - solrAuditDestination.logJSON(audits); + processed = solrAuditDestination.logJSON(audits); + } + + if (!processed) { + String errorMessage = "Failure in sending audits into Solr"; + throw new Exception(errorMessage); } } From bbbb191d0863cef44e71a276b71edb7ab1bafa1f Mon Sep 17 00:00:00 2001 From: Ramesh Mani Date: Fri, 20 Feb 2026 00:02:23 -0800 Subject: [PATCH 05/12] RANGER-5482:Create Ranger Audit Server with SOLR and HDFS as audit consumer - audit server partition management enhancement --- .../audit/server/AuditServerConstants.java | 19 +- .../audit/utils/AuditMessageQueueUtils.java | 49 +++-- .../producer/kafka/AuditMessageQueue.java | 9 +- .../producer/kafka/AuditPartitioner.java | 187 ++++++++++++++++++ .../audit/producer/kafka/AuditProducer.java | 21 ++ .../conf/ranger-audit-server-site.xml | 79 +++++++- 6 files changed, 337 insertions(+), 27 deletions(-) create mode 100644 ranger-audit-server/ranger-audit-server-service/src/main/java/org/apache/ranger/audit/producer/kafka/AuditPartitioner.java diff --git a/ranger-audit-server/ranger-audit-common/src/main/java/org/apache/ranger/audit/server/AuditServerConstants.java b/ranger-audit-server/ranger-audit-common/src/main/java/org/apache/ranger/audit/server/AuditServerConstants.java index 9b3141bc71..24674b1e7f 100644 --- a/ranger-audit-server/ranger-audit-common/src/main/java/org/apache/ranger/audit/server/AuditServerConstants.java +++ b/ranger-audit-server/ranger-audit-common/src/main/java/org/apache/ranger/audit/server/AuditServerConstants.java @@ -56,9 +56,21 @@ private AuditServerConstants() {} public static final String PROP_KAFKA_STARTUP_RETRY_DELAY_MS = "kafka.startup.retry.delay.ms"; // ranger_audits topic configuration - public static final String PROP_TOPIC_PARTITIONS = "topic.partitions"; - public static final String PROP_REPLICATION_FACTOR = "replication.factor"; - public static final short DEFAULT_REPLICATION_FACTOR = 3; + public static final String PROP_TOPIC_PARTITIONS = "topic.partitions"; + public static final String PROP_PARTITIONER_CLASS = "partitioner.class"; + public static final String PROP_CONFIGURED_PLUGINS = "configured.plugins"; + public static final String PROP_TOPIC_PARTITIONS_PER_CONFIGURED_PLUGIN = "topic.partitions.per.configured.plugin"; + public static final String PROP_PLUGIN_PARTITION_OVERRIDE_PREFIX = "plugin.partition.overrides."; + public static final String PROP_BUFFER_PARTITIONS = "topic.partitions.buffer"; + public static final String PROP_REPLICATION_FACTOR = "replication.factor"; + + // Default configured plugins: each gets allocated partitions from the topic + public static final String DEFAULT_PARTITIONER_CLASS = "org.apache.ranger.audit.producer.kafka.AuditPartitioner"; + public static final String DEFAULT_CONFIGURED_PLUGINS = "hdfs,yarn,knox,hiveServer2,hiveMetastore,kafka,hbaseRegional,hbaseMaster,solr,trino,ozone,kudu,nifi"; + public static final short DEFAULT_REPLICATION_FACTOR = 2; + public static final int DEFAULT_TOPIC_PARTITIONS = 10; // For hash-based partitioning + public static final int DEFAULT_PARTITIONS_PER_CONFIGURED_PLUGIN = 3; // For plugin-based partitioning + public static final int DEFAULT_BUFFER_PARTITIONS = 9; // For plugin-based partitioning // Hadoop security configuration for UGI public static final String PROP_HADOOP_AUTHENTICATION_TYPE = "hadoop.security.authentication"; @@ -95,5 +107,4 @@ private AuditServerConstants() {} // Consumer Registry Configuration public static final String PROP_CONSUMER_CLASSES = "consumer.classes"; - public static final String DEFAULT_CONSUMER_CLASSES = "org.apache.ranger.audit.consumer.kafka.AuditSolrConsumer, org.apache.ranger.audit.consumer.kafka.AuditHDFSConsumer"; } diff --git a/ranger-audit-server/ranger-audit-common/src/main/java/org/apache/ranger/audit/utils/AuditMessageQueueUtils.java b/ranger-audit-server/ranger-audit-common/src/main/java/org/apache/ranger/audit/utils/AuditMessageQueueUtils.java index de8a5a7244..281ac8e417 100644 --- a/ranger-audit-server/ranger-audit-common/src/main/java/org/apache/ranger/audit/utils/AuditMessageQueueUtils.java +++ b/ranger-audit-server/ranger-audit-common/src/main/java/org/apache/ranger/audit/utils/AuditMessageQueueUtils.java @@ -323,26 +323,47 @@ public String updateExistingTopicPartitions(AdminClient admin, String topicName, } /** - * Get the number of partitions for the Kafka topic + * Get the number of partitions for the Kafka topic. * - * Configurable via xasecure.audit.destination.kafka.topic.partitions property. - * Default: 30 partitions for balanced distribution with Kafka's default partitioner. - * - * With default partitioner (murmur2 hash), messages with same key (appId) always - * go to the same partition, providing ordering per appId while distributing load - * evenly across all partitions. + * If configured.plugins is NOT set, uses topic.partitions property (default: 10). + * If configured.plugins is set, auto-calculates: (plugin partitions + overrides) + buffer partitions. * * @return Number of partitions for the topic */ private int getPartitions(Properties prop, String propPrefix) { - int defaultPartitions = 30; - int partitions = MiscUtil.getIntProperty(prop, - propPrefix + "." + AuditServerConstants.PROP_TOPIC_PARTITIONS, - defaultPartitions); + // Check if configured.plugins is set (use empty string as default to detect when not configured) + String configuredPlugins = MiscUtil.getStringProperty(prop, propPrefix + "." + AuditServerConstants.PROP_CONFIGURED_PLUGINS, AuditServerConstants.DEFAULT_CONFIGURED_PLUGINS); + + // If no configured plugins, use simple hash-based partitioning with topic.partitions + if (configuredPlugins == null || configuredPlugins.trim().isEmpty()) { + int partitions = MiscUtil.getIntProperty(prop, propPrefix + "." + AuditServerConstants.PROP_TOPIC_PARTITIONS, AuditServerConstants.DEFAULT_TOPIC_PARTITIONS); + LOG.info("No configured plugins - using hash-based partitioning with {} partitions", partitions); + return partitions; + } + + // Auto-calculate based on plugin configuration + String[] plugins = configuredPlugins.split(","); + int defaultPartitionsPerPlugin = MiscUtil.getIntProperty(prop, propPrefix + "." + AuditServerConstants.PROP_TOPIC_PARTITIONS_PER_CONFIGURED_PLUGIN, AuditServerConstants.DEFAULT_PARTITIONS_PER_CONFIGURED_PLUGIN); + + // Calculate total partitions needed + AuditServerLogFormatter.LogBuilder logBuilder = AuditServerLogFormatter.builder("Kafka Topic Partition Allocation (Plugin-based)"); + int totalPartitions = 0; + for (String plugin : plugins) { + String pluginTrimmed = plugin.trim(); + String overrideKey = propPrefix + "." + AuditServerConstants.PROP_PLUGIN_PARTITION_OVERRIDE_PREFIX + pluginTrimmed; + int partitionCount = MiscUtil.getIntProperty(prop, overrideKey, defaultPartitionsPerPlugin); + totalPartitions += partitionCount; + logBuilder.add("Plugin '" + pluginTrimmed + "'", partitionCount + " partitions"); + } + + // Add buffer partitions for unconfigured plugins + int bufferPartitions = MiscUtil.getIntProperty(prop, propPrefix + "." + AuditServerConstants.PROP_BUFFER_PARTITIONS, AuditServerConstants.DEFAULT_BUFFER_PARTITIONS); + totalPartitions += bufferPartitions; - LOG.info("Kafka topic partition count: {} (configured: {})", - partitions, prop.getProperty(propPrefix + "." + AuditServerConstants.PROP_TOPIC_PARTITIONS, "default")); + logBuilder.add("Buffer partitions", bufferPartitions + " partitions"); + logBuilder.add("Total topic partitions (calculated)", totalPartitions); + logBuilder.logInfo(LOG); - return partitions; + return totalPartitions; } } diff --git a/ranger-audit-server/ranger-audit-server-service/src/main/java/org/apache/ranger/audit/producer/kafka/AuditMessageQueue.java b/ranger-audit-server/ranger-audit-server-service/src/main/java/org/apache/ranger/audit/producer/kafka/AuditMessageQueue.java index a4b387ff9e..a80ac77978 100644 --- a/ranger-audit-server/ranger-audit-server-service/src/main/java/org/apache/ranger/audit/producer/kafka/AuditMessageQueue.java +++ b/ranger-audit-server/ranger-audit-server-service/src/main/java/org/apache/ranger/audit/producer/kafka/AuditMessageQueue.java @@ -145,10 +145,11 @@ public synchronized boolean log(final AuditEventBase event) { authzEvent.setEventId(MiscUtil.generateUniqueId()); } - // Partition key is agentId (aka plugin ID) used by Kafka's default partitioner - // for load balancing across partitions. Messages with the same key go to the same partition, - // providing ordering guarantees per appId while enabling full horizontal scaling - + /** + Partition key is agentId (aka plugin ID). AuditPartitioner allocates configured plugins + (hdfs, hiveServer2, etc.) to fixed partition sets of partitions. + If unconfigured plugins are there in the audit message it uses buffer partitions. + **/ final String key = authzEvent.getAgentId(); final String message = MiscUtil.stringify(event); diff --git a/ranger-audit-server/ranger-audit-server-service/src/main/java/org/apache/ranger/audit/producer/kafka/AuditPartitioner.java b/ranger-audit-server/ranger-audit-server-service/src/main/java/org/apache/ranger/audit/producer/kafka/AuditPartitioner.java new file mode 100644 index 0000000000..c2438b3e02 --- /dev/null +++ b/ranger-audit-server/ranger-audit-server-service/src/main/java/org/apache/ranger/audit/producer/kafka/AuditPartitioner.java @@ -0,0 +1,187 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.audit.producer.kafka; + +import org.apache.kafka.clients.producer.Partitioner; +import org.apache.kafka.common.Cluster; +import org.apache.kafka.common.PartitionInfo; +import org.apache.ranger.audit.server.AuditServerConstants; +import org.apache.ranger.audit.utils.AuditServerLogFormatter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * Plugin-based Kafka partitioner for Ranger audit events. + *

+ * Configured plugins (hdfs, hiveServer, hiveMetastore, kafka, hbaseRegion, hbaseMaster, solr) + * each get a fixed number of partitions. Unconfigured plugins use buffer partitions. + * Within each plugin's partition set, messages are round-robin distributed for load balancing. + */ +public class AuditPartitioner implements Partitioner { + private static final Logger LOG = LoggerFactory.getLogger(AuditPartitioner.class); + + private String[] configuredPlugins; + private int defaultPartitionsPerPlugin; + private int totalPartitions; + private int[] pluginPartitionCounts; + private int[] configuredPluginPartitionStart; + private int[] configuredPluginPartitionEnd; + private int bufferPartitionStart; + private int bufferPartitionCount; + private final ConcurrentHashMap appIdCounters = new ConcurrentHashMap<>(); + + @Override + public void configure(Map configs) { + String propPrefix = AuditServerConstants.PROP_KAFKA_PROP_PREFIX; + + String pluginsStr = getConfig(configs, propPrefix + "." + AuditServerConstants.PROP_CONFIGURED_PLUGINS, AuditServerConstants.DEFAULT_CONFIGURED_PLUGINS); + configuredPlugins = pluginsStr.split(","); + for (int i = 0; i < configuredPlugins.length; i++) { + configuredPlugins[i] = configuredPlugins[i].trim(); + } + + defaultPartitionsPerPlugin = getIntConfig(configs, propPrefix + "." + AuditServerConstants.PROP_TOPIC_PARTITIONS_PER_CONFIGURED_PLUGIN, AuditServerConstants.DEFAULT_PARTITIONS_PER_CONFIGURED_PLUGIN); + if (defaultPartitionsPerPlugin < 1) { + defaultPartitionsPerPlugin = 1; + } + + totalPartitions = getIntConfig(configs, propPrefix + "." + AuditServerConstants.PROP_TOPIC_PARTITIONS, AuditServerConstants.DEFAULT_TOPIC_PARTITIONS); + if (totalPartitions < 1) { + totalPartitions = AuditServerConstants.DEFAULT_TOPIC_PARTITIONS; + } + + // Build per-plugin partition counts - check for individual overrides per plugin + pluginPartitionCounts = new int[configuredPlugins.length]; + for (int i = 0; i < configuredPlugins.length; i++) { + String plugin = configuredPlugins[i]; + String overrideKey = propPrefix + "." + AuditServerConstants.PROP_PLUGIN_PARTITION_OVERRIDE_PREFIX + plugin; + int partitionCount = getIntConfig(configs, overrideKey, defaultPartitionsPerPlugin); + pluginPartitionCounts[i] = partitionCount; + if (partitionCount != defaultPartitionsPerPlugin) { + LOG.info("Plugin '{}' partition override: {} partitions (default: {})", plugin, partitionCount, defaultPartitionsPerPlugin); + } + } + + // Calculate partition ranges for each plugin + configuredPluginPartitionStart = new int[configuredPlugins.length]; + configuredPluginPartitionEnd = new int[configuredPlugins.length]; + int currentPartition = 0; + int configuredPartitionCount = 0; + + for (int i = 0; i < configuredPlugins.length; i++) { + configuredPluginPartitionStart[i] = currentPartition; + configuredPluginPartitionEnd[i] = currentPartition + pluginPartitionCounts[i] - 1; + currentPartition += pluginPartitionCounts[i]; + configuredPartitionCount += pluginPartitionCounts[i]; + } + + // Buffer partitions start after configured plugins + bufferPartitionStart = configuredPartitionCount; + bufferPartitionCount = Math.max(1, totalPartitions - configuredPartitionCount); + + // Log allocated partition + AuditServerLogFormatter.LogBuilder logBuilder = AuditServerLogFormatter.builder("****** AuditPartitioner Configuration ******"); + logBuilder.add("Total partitions: ", totalPartitions); + logBuilder.add("Configured plugins: ", configuredPlugins.length); + logBuilder.add("Default partitions per plugin: ", defaultPartitionsPerPlugin); + for (int i = 0; i < configuredPlugins.length; i++) { + String rangeInfo = String.format("%d partitions (range: %d-%d): ", pluginPartitionCounts[i], configuredPluginPartitionStart[i], configuredPluginPartitionEnd[i]); + logBuilder.add("Plugin '" + configuredPlugins[i] + "'", rangeInfo); + } + logBuilder.add("Buffer partitions (unconfigured): ", String.format("%d partitions (range: %d-%d)", bufferPartitionCount, bufferPartitionStart, totalPartitions - 1)); + logBuilder.logInfo(LOG); + } + + @Override + public int partition(String topic, Object key, byte[] keyBytes, Object value, byte[] valueBytes, Cluster cluster) { + int numPartitions = totalPartitions; + List partitions = cluster.partitionsForTopic(topic); + if (partitions != null && !partitions.isEmpty()) { + numPartitions = partitions.size(); + } + + String appId = key != null ? key.toString() : null; + if (appId == null || appId.isEmpty()) { + return Math.abs(System.identityHashCode(key) % numPartitions); + } + + int pluginIndex = indexOfConfiguredPlugin(appId); + if (pluginIndex >= 0) { + int start = configuredPluginPartitionStart[pluginIndex]; + int end = Math.min(configuredPluginPartitionEnd[pluginIndex], numPartitions - 1); + if (end < start) { + end = start; + } + int rangeSize = end - start + 1; + int subPartition = appIdCounters + .computeIfAbsent(appId, k -> new AtomicInteger(0)) + .getAndIncrement() % rangeSize; + return start + subPartition; + } else { + // Unconfigured plugin - use buffer partitions + int start = Math.min(bufferPartitionStart, numPartitions - 1); + int count = Math.min(bufferPartitionCount, numPartitions - start); + if (count <= 0) { + return start; + } + int p = Math.abs(appId.hashCode() % count) + start; + return Math.min(p, numPartitions - 1); + } + } + + @Override + public void close() { + appIdCounters.clear(); + } + + private int indexOfConfiguredPlugin(String appId) { + for (int i = 0; i < configuredPlugins.length; i++) { + if (configuredPlugins[i].equalsIgnoreCase(appId)) { + return i; + } + } + return -1; + } + + private static String getConfig(Map configs, String key, String defaultValue) { + Object val = configs.get(key); + return val != null ? val.toString() : defaultValue; + } + + private static int getIntConfig(Map configs, String key, int defaultValue) { + Object val = configs.get(key); + if (val == null) { + return defaultValue; + } + if (val instanceof Number) { + return ((Number) val).intValue(); + } + try { + return Integer.parseInt(val.toString()); + } catch (NumberFormatException e) { + return defaultValue; + } + } +} diff --git a/ranger-audit-server/ranger-audit-server-service/src/main/java/org/apache/ranger/audit/producer/kafka/AuditProducer.java b/ranger-audit-server/ranger-audit-server-service/src/main/java/org/apache/ranger/audit/producer/kafka/AuditProducer.java index e8c13d76fb..879424a6e1 100644 --- a/ranger-audit-server/ranger-audit-server-service/src/main/java/org/apache/ranger/audit/producer/kafka/AuditProducer.java +++ b/ranger-audit-server/ranger-audit-server-service/src/main/java/org/apache/ranger/audit/producer/kafka/AuditProducer.java @@ -64,6 +64,27 @@ public AuditProducer(Properties props, String propPrefix) throws Exception { producerProps.put(ProducerConfig.LINGER_MS_CONFIG, 5); producerProps.put(ProducerConfig.BATCH_SIZE_CONFIG, 32 * 1024); + // Check if configured.plugins is set to determine partitioning strategy + // 1) configured.plugins is set then Custom AuditPartitioner is used, it allocates predefined set of partitions to each appId. + // 2) if configured.plugins is not set then default kafka hash based partitioner is used with initial quota of 10 partition. + String configuredPlugins = MiscUtil.getStringProperty(props, propPrefix + "." + AuditServerConstants.PROP_CONFIGURED_PLUGINS, ""); + if (configuredPlugins != null && !configuredPlugins.trim().isEmpty()) { + // Plugin-based partitioning: use AuditPartitioner + String partitionerClass = MiscUtil.getStringProperty(props, propPrefix + "." + AuditServerConstants.PROP_PARTITIONER_CLASS, AuditServerConstants.DEFAULT_PARTITIONER_CLASS); + producerProps.put(ProducerConfig.PARTITIONER_CLASS_CONFIG, partitionerClass); + LOG.info("Configured plugins detected - using plugin-based partitioner: {}", partitionerClass); + + // Pass all xasecure.audit.destination.kafka.* properties to partitioner (no namespace translation) + for (String propName : props.stringPropertyNames()) { + if (propName.startsWith(propPrefix + ".")) { + producerProps.put(propName, props.getProperty(propName)); + } + } + } else { + // No configured plugins: use Kafka default hash-based partitioner + LOG.info("No configured plugins - using Kafka default hash-based partitioner"); + } + try { kafkaProducer = MiscUtil.executePrivilegedAction((PrivilegedExceptionAction>) () -> new KafkaProducer<>(producerProps)); LOG.info("AuditProducer(): KafkaProducer created successfully!"); diff --git a/ranger-audit-server/ranger-audit-server-service/src/main/resources/conf/ranger-audit-server-site.xml b/ranger-audit-server/ranger-audit-server-service/src/main/resources/conf/ranger-audit-server-site.xml index fe019818c0..49f134a077 100644 --- a/ranger-audit-server/ranger-audit-server-service/src/main/resources/conf/ranger-audit-server-site.xml +++ b/ranger-audit-server/ranger-audit-server-service/src/main/resources/conf/ranger-audit-server-site.xml @@ -235,15 +235,84 @@ ranger_audits + + xasecure.audit.destination.kafka.partitioner.class + org.apache.ranger.audit.producer.kafka.AuditPartitioner + + Kafka partitioner class for distributing audit messages across partitions. + Default: org.apache.ranger.audit.producer.kafka.AuditPartitioner (plugin-based partitioning) + Note: If configured.plugins is not set, uses Kafka's default hash-based partitioner instead. + Can be overridden with custom implementation of org.apache.kafka.clients.producer.Partitioner + + + + + xasecure.audit.destination.kafka.configured.plugins + hdfs,yarn,knox,hiveServer2,hiveMetastore,kafka,hbaseRegional,hbaseMaster,solr,trino,ozone,kudu,nifi + + Comma-separated list of configured plugin IDs. + If set: Uses AuditPartitioner with auto-calculated partitions (sum of plugin allocations + buffer). + If empty/not set: Uses Kafka default hash-based partitioner with topic.partitions value. + Each plugin receives dedicated partitions based on topic.partitions.per.configured.plugin (default: 3). + + + + + xasecure.audit.destination.kafka.topic.partitions.per.configured.plugin + 3 + + Default number of partitions per configured plugin. + Can be overridden per plugin using plugin.partition.overrides.{pluginName} properties. + Used ONLY when configured.plugins is set (for plugin-based partitioning). + Ignored when configured.plugins is empty. + + + + + xasecure.audit.destination.kafka.topic.partitions.buffer + 9 + + Number of buffer partitions reserved for unconfigured plugins. + Used ONLY when configured.plugins is set (for plugin-based partitioning). + Total = (sum of plugin partitions) + (buffer partitions). + Example: 7 plugins × 3 + 9 buffer = 30 total. + Ignored when configured.plugins is empty. + + + + + + + + xasecure.audit.destination.kafka.topic.partitions - 30 + 10 Number of partitions for the Kafka audit topic. - Recommendations: - - Small deployments: 30 partitions (default) - - Medium deployments: 50 partitions - - Large deployments: 100+ partitions + Used ONLY when configured.plugins is NOT set (empty) for hash-based partitioning. + When configured.plugins is set, partitions are auto-calculated from plugin configuration. + Default: 10 partitions (sufficient for small-medium deployments) + Recommendations for hash-based mode: + - Small deployments (1-3 consumers): 10 partitions + - Medium deployments (4-10 consumers): 20-30 partitions + - Large deployments (10+ consumers): 50+ partitions From cb086a089e1693ef5da2c9c79c5eaa5a5df4f2f6 Mon Sep 17 00:00:00 2001 From: Ramesh Mani Date: Mon, 23 Feb 2026 00:20:34 -0800 Subject: [PATCH 06/12] RANGER-5482:Create Ranger Audit Server with SOLR and HDFS as audit consumer - Fix review comments --- .../audit/provider/BaseAuditHandler.java | 39 +- agents-audit/dest-auditserver/pom.xml | 7 +- .../RangerAuditServerDestination.java | 668 +++++------------- .../plugin/service/RangerBasePlugin.java | 9 +- .../assembly/ranger-audit-consumer-hdfs.xml | 10 - .../assembly/ranger-audit-consumer-solr.xml | 10 - .../assembly/ranger-audit-server-service.xml | 10 - .../audit/server/AuditServerConstants.java | 5 + .../ranger-audit-server-service/pom.xml | 1 + .../apache/ranger/audit/rest/AuditREST.java | 213 +++++- .../conf/ranger-audit-server-site.xml | 34 + .../src/main/webapp/WEB-INF/web.xml | 5 - 12 files changed, 425 insertions(+), 586 deletions(-) diff --git a/agents-audit/core/src/main/java/org/apache/ranger/audit/provider/BaseAuditHandler.java b/agents-audit/core/src/main/java/org/apache/ranger/audit/provider/BaseAuditHandler.java index 5a23b37e5c..4dc35cb458 100644 --- a/agents-audit/core/src/main/java/org/apache/ranger/audit/provider/BaseAuditHandler.java +++ b/agents-audit/core/src/main/java/org/apache/ranger/audit/provider/BaseAuditHandler.java @@ -72,18 +72,19 @@ public abstract class BaseAuditHandler implements AuditHandler { protected Map configProps = new HashMap<>(); protected Properties props; - int errorLogIntervalMS = 30 * 1000; // Every 30 seconds - long lastErrorLogMS; + int errorLogIntervalMS = 30 * 1000; // Every 30 seconds + long lastErrorLogMS; + long lastIntervalCount; + long lastIntervalSuccessCount; + long lastIntervalFailedCount; + long lastStashedCount; + long lastDeferredCount; AtomicLong totalCount = new AtomicLong(0); AtomicLong totalSuccessCount = new AtomicLong(0); AtomicLong totalFailedCount = new AtomicLong(0); AtomicLong totalStashedCount = new AtomicLong(0); AtomicLong totalDeferredCount = new AtomicLong(0); - AtomicLong lastIntervalCount = new AtomicLong(0); - AtomicLong lastIntervalSuccessCount = new AtomicLong(0); - AtomicLong lastIntervalFailedCount = new AtomicLong(0); - AtomicLong lastStashedCount = new AtomicLong(0); - AtomicLong lastDeferredCount = new AtomicLong(0); + boolean statusLogEnabled = DEFAULT_AUDIT_LOG_STATUS_LOG_ENABLED; long statusLogIntervalMS = DEFAULT_AUDIT_LOG_STATUS_LOG_INTERVAL_SEC * 1000; long lastStatusLogTime = System.currentTimeMillis(); @@ -273,7 +274,7 @@ public long getTotalStashedCount() { } public long getLastStashedCount() { - return lastStashedCount.get(); + return lastStashedCount; } public long getTotalDeferredCount() { @@ -281,7 +282,7 @@ public long getTotalDeferredCount() { } public long getLastDeferredCount() { - return lastDeferredCount.get(); + return lastDeferredCount; } public boolean isStatusLogEnabled() { @@ -308,21 +309,21 @@ public void logStatus() { long currentStashedCount = totalStashedCount.get(); long currentDeferredCount = totalDeferredCount.get(); - long diffCount = currentTotalCount - lastIntervalCount.get(); - long diffSuccess = currentSuccessCount - lastIntervalSuccessCount.get(); - long diffFailed = currentFailedCount - lastIntervalFailedCount.get(); - long diffStashed = currentStashedCount - lastStashedCount.get(); - long diffDeferred = currentDeferredCount - lastDeferredCount.get(); + long diffCount = currentTotalCount - lastIntervalCount; + long diffSuccess = currentSuccessCount - lastIntervalSuccessCount; + long diffFailed = currentFailedCount - lastIntervalFailedCount; + long diffStashed = currentStashedCount - lastStashedCount; + long diffDeferred = currentDeferredCount - lastDeferredCount; if (diffCount == 0 && diffSuccess == 0 && diffFailed == 0 && diffStashed == 0 && diffDeferred == 0) { return; } - lastIntervalCount.set(currentTotalCount); - lastIntervalSuccessCount.set(currentSuccessCount); - lastIntervalFailedCount.set(currentFailedCount); - lastStashedCount.set(currentStashedCount); - lastDeferredCount.set(currentDeferredCount); + lastIntervalCount = currentTotalCount; + lastIntervalSuccessCount = currentSuccessCount; + lastIntervalFailedCount = currentFailedCount; + lastStashedCount = currentStashedCount; + lastDeferredCount = currentDeferredCount; if (statusLogEnabled) { String finalPath = ""; diff --git a/agents-audit/dest-auditserver/pom.xml b/agents-audit/dest-auditserver/pom.xml index c489e1da9c..80ebe09d1b 100644 --- a/agents-audit/dest-auditserver/pom.xml +++ b/agents-audit/dest-auditserver/pom.xml @@ -41,6 +41,11 @@ ranger-audit-core ${project.version} + + org.apache.ranger + ranger-plugins-common + ${project.version} + org.slf4j slf4j-api @@ -60,4 +65,4 @@ test - \ No newline at end of file + diff --git a/agents-audit/dest-auditserver/src/main/java/org/apache/ranger/audit/destination/RangerAuditServerDestination.java b/agents-audit/dest-auditserver/src/main/java/org/apache/ranger/audit/destination/RangerAuditServerDestination.java index 572b3c430d..b81aec02af 100644 --- a/agents-audit/dest-auditserver/src/main/java/org/apache/ranger/audit/destination/RangerAuditServerDestination.java +++ b/agents-audit/dest-auditserver/src/main/java/org/apache/ranger/audit/destination/RangerAuditServerDestination.java @@ -19,69 +19,24 @@ package org.apache.ranger.audit.destination; -import com.google.gson.Gson; -import com.google.gson.GsonBuilder; +import com.sun.jersey.api.client.ClientResponse; +import com.sun.jersey.api.client.WebResource; import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; -import org.apache.http.HttpResponse; -import org.apache.http.HttpStatus; -import org.apache.http.auth.AuthSchemeProvider; -import org.apache.http.auth.AuthScope; -import org.apache.http.auth.KerberosCredentials; -import org.apache.http.auth.UsernamePasswordCredentials; -import org.apache.http.client.config.AuthSchemes; -import org.apache.http.client.config.CookieSpecs; -import org.apache.http.client.config.RequestConfig; -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.client.methods.HttpRequestBase; -import org.apache.http.client.protocol.HttpClientContext; -import org.apache.http.client.utils.URIBuilder; -import org.apache.http.config.Lookup; -import org.apache.http.config.Registry; -import org.apache.http.config.RegistryBuilder; -import org.apache.http.conn.socket.ConnectionSocketFactory; -import org.apache.http.conn.ssl.NoopHostnameVerifier; -import org.apache.http.conn.ssl.SSLConnectionSocketFactory; -import org.apache.http.entity.StringEntity; -import org.apache.http.impl.auth.SPNegoSchemeFactory; -import org.apache.http.impl.client.BasicCredentialsProvider; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClientBuilder; -import org.apache.http.impl.client.HttpClients; -import org.apache.http.impl.client.StandardHttpRequestRetryHandler; -import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; -import org.apache.http.protocol.HttpContext; import org.apache.ranger.audit.model.AuditEventBase; import org.apache.ranger.audit.provider.MiscUtil; +import org.apache.ranger.plugin.util.RangerRESTClient; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.net.ssl.KeyManager; -import javax.net.ssl.KeyManagerFactory; -import javax.net.ssl.SSLContext; -import javax.net.ssl.TrustManager; -import javax.net.ssl.TrustManagerFactory; +import javax.servlet.http.HttpServletResponse; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; -import java.io.InputStreamReader; -import java.net.URI; -import java.net.URISyntaxException; import java.nio.charset.Charset; -import java.security.KeyManagementException; -import java.security.KeyStore; -import java.security.KeyStoreException; -import java.security.NoSuchAlgorithmException; import java.security.PrivilegedExceptionAction; -import java.security.SecureRandom; -import java.security.UnrecoverableKeyException; -import java.security.cert.CertificateException; -import java.util.Arrays; import java.util.Collection; import java.util.HashMap; import java.util.Map; @@ -96,50 +51,49 @@ public class RangerAuditServerDestination extends AuditDestination { public static final String PROP_AUDITSERVER_JWT_TOKEN_FILE = "xasecure.audit.destination.auditserver.jwt.token.file"; public static final String PROP_AUDITSERVER_CLIENT_CONN_TIMEOUT_MS = "xasecure.audit.destination.auditserver.connection.timeout.ms"; public static final String PROP_AUDITSERVER_CLIENT_READ_TIMEOUT_MS = "xasecure.audit.destination.auditserver.read.timeout.ms"; - public static final String PROP_AUDITSERVER_MAX_CONNECTION = "xasecure.audit.destination.auditserver.max.connections"; - public static final String PROP_AUDITSERVER_MAX_CONNECTION_PER_HOST = "xasecure.audit.destination.auditserver.max.connections.per.host"; - public static final String PROP_AUDITSERVER_VALIDATE_INACTIVE_MS = "xasecure.audit.destination.auditserver.validate.inactivity.ms"; - public static final String PROP_AUDITSERVER_POOL_RETRY_COUNT = "xasecure.audit.destination.auditserver.pool.retry.count"; - public static final String GSON_DATE_FORMAT = "yyyy-MM-dd'T'HH:mm:ss.SSSZ"; - public static final String REST_ACCEPTED_MIME_TYPE_JSON = "application/json"; - public static final String REST_CONTENT_TYPE_MIME_TYPE_JSON = "application/json"; - public static final String REST_HEADER_ACCEPT = "Accept"; - public static final String REST_HEADER_CONTENT_TYPE = "Content-type"; - public static final String REST_HEADER_AUTHORIZATION = "Authorization"; - public static final String REST_RELATIVE_PATH_POST = "/api/audit/post"; + public static final String PROP_AUDITSERVER_SSL_CONFIG_FILE = "xasecure.audit.destination.auditserver.ssl.config.file"; + public static final String PROP_AUDITSERVER_MAX_RETRY_ATTEMPTS = "xasecure.audit.destination.auditserver.max.retry.attempts"; + public static final String PROP_AUDITSERVER_RETRY_INTERVAL_MS = "xasecure.audit.destination.auditserver.retry.interval.ms"; + public static final String PROP_SERVICE_TYPE = "ranger.plugin.audit.service.type"; + public static final String REST_RELATIVE_PATH_POST = "/api/audit/access"; + public static final String QUERY_PARAM_SERVICE_NAME = "serviceName"; // Authentication types public static final String AUTH_TYPE_KERBEROS = "kerberos"; public static final String AUTH_TYPE_BASIC = "basic"; public static final String AUTH_TYPE_JWT = "jwt"; - private static final Logger LOG = LoggerFactory.getLogger(RangerAuditServerDestination.class); - private static final String PROTOCOL_HTTPS = "https"; - private volatile CloseableHttpClient httpClient; - private volatile Gson gsonBuilder; - private String httpURL; - private String authType; - private String jwtToken; + private static final Logger LOG = LoggerFactory.getLogger(RangerAuditServerDestination.class); + private RangerRESTClient restClient; + private String authType; + private String serviceType; @Override public void init(Properties props, String propPrefix) { LOG.info("==> RangerAuditServerDestination:init()"); super.init(props, propPrefix); - this.authType = MiscUtil.getStringProperty(props, PROP_AUDITSERVER_AUTH_TYPE); - if (AUTH_TYPE_JWT.equalsIgnoreCase(authType)) { - LOG.info("JWT authentication configured...."); - initJwtToken(); - } else if (StringUtils.isEmpty(authType)) { + String url = MiscUtil.getStringProperty(props, PROP_AUDITSERVER_URL); + String sslConfigFileName = MiscUtil.getStringProperty(props, PROP_AUDITSERVER_SSL_CONFIG_FILE); + String userName = MiscUtil.getStringProperty(props, PROP_AUDITSERVER_USER_NAME); + String password = MiscUtil.getStringProperty(props, PROP_AUDITSERVER_USER_PASSWORD); + int connTimeoutMs = MiscUtil.getIntProperty(props, PROP_AUDITSERVER_CLIENT_CONN_TIMEOUT_MS, 120000); + int readTimeoutMs = MiscUtil.getIntProperty(props, PROP_AUDITSERVER_CLIENT_READ_TIMEOUT_MS, 30000); + int maxRetryAttempts = MiscUtil.getIntProperty(props, PROP_AUDITSERVER_MAX_RETRY_ATTEMPTS, 3); + int retryIntervalMs = MiscUtil.getIntProperty(props, PROP_AUDITSERVER_RETRY_INTERVAL_MS, 1000); + + this.authType = MiscUtil.getStringProperty(props, PROP_AUDITSERVER_AUTH_TYPE); + this.serviceType = MiscUtil.getStringProperty(props, PROP_SERVICE_TYPE); + + if (StringUtils.isEmpty(authType)) { // Authentication priority: JWT → Kerberos → Basic try { if (StringUtils.isNotEmpty(MiscUtil.getStringProperty(props, PROP_AUDITSERVER_JWT_TOKEN)) || StringUtils.isNotEmpty(MiscUtil.getStringProperty(props, PROP_AUDITSERVER_JWT_TOKEN_FILE))) { this.authType = AUTH_TYPE_JWT; - initJwtToken(); } else if (isKerberosAuthenticated()) { this.authType = AUTH_TYPE_KERBEROS; - } else if (StringUtils.isNotEmpty(MiscUtil.getStringProperty(props, PROP_AUDITSERVER_USER_NAME))) { + } else if (StringUtils.isNotEmpty(userName)) { this.authType = AUTH_TYPE_BASIC; } } catch (Exception e) { @@ -149,24 +103,36 @@ public void init(Properties props, String propPrefix) { LOG.info("Audit destination authentication type: {}", authType); + if (StringUtils.isEmpty(this.serviceType)) { + LOG.error("Service type not available in audit properties. This is a configuration error. Audit destination will not function correctly."); + LOG.error("Ensure that RangerBasePlugin is properly initialized with a valid serviceType (hdfs, hive, kafka, etc.)"); + } else { + LOG.info("Audit destination configured for service type: {}", this.serviceType); + } + if (AUTH_TYPE_KERBEROS.equalsIgnoreCase(authType)) { preAuthenticateKerberos(); } - this.httpClient = buildHTTPClient(); - this.gsonBuilder = new GsonBuilder().setDateFormat(GSON_DATE_FORMAT).create(); + Configuration config = createConfigurationFromProperties(props, authType, userName, password); + this.restClient = new RangerRESTClient(url, sslConfigFileName, config); + this.restClient.setRestClientConnTimeOutMs(connTimeoutMs); + this.restClient.setRestClientReadTimeOutMs(readTimeoutMs); + this.restClient.setMaxRetryAttempts(maxRetryAttempts); + this.restClient.setRetryIntervalMs(retryIntervalMs); + LOG.info("<== RangerAuditServerDestination:init()"); } - private void initJwtToken() { + private String initJwtToken() { LOG.info("==> RangerAuditServerDestination:initJwtToken()"); - this.jwtToken = MiscUtil.getStringProperty(props, PROP_AUDITSERVER_JWT_TOKEN); + String jwtToken = MiscUtil.getStringProperty(props, PROP_AUDITSERVER_JWT_TOKEN); if (StringUtils.isEmpty(jwtToken)) { String jwtTokenFile = MiscUtil.getStringProperty(props, PROP_AUDITSERVER_JWT_TOKEN_FILE); if (StringUtils.isNotEmpty(jwtTokenFile)) { try { - this.jwtToken = readJwtTokenFromFile(jwtTokenFile); + jwtToken = readJwtTokenFromFile(jwtTokenFile); LOG.info("JWT token loaded from file: {}", jwtTokenFile); } catch (Exception e) { LOG.error("Failed to read JWT token from file: {}", jwtTokenFile, e); @@ -181,27 +147,20 @@ private void initJwtToken() { } LOG.info("<== RangerAuditServerDestination:initJwtToken()"); + + return jwtToken; } private String readJwtTokenFromFile(String tokenFile) throws IOException { - InputStream in = null; - try { - in = getFileInputStream(tokenFile); + try (InputStream in = getFileInputStream(tokenFile)) { if (in != null) { - String token = IOUtils.toString(in, Charset.defaultCharset()).trim(); - return token; + return IOUtils.toString(in, Charset.defaultCharset()).trim(); } else { throw new IOException("Unable to read JWT token file: " + tokenFile); } - } finally { - close(in, tokenFile); } } - /** - * This method proactively obtains the TGT and service ticket for the audit server - * during initialization, so they are cached and ready when the first audit event arrives. - */ private void preAuthenticateKerberos() { LOG.info("==> RangerAuditServerDestination:preAuthenticateKerberos()"); @@ -223,30 +182,7 @@ private void preAuthenticateKerberos() { ugi.checkTGTAndReloginFromKeytab(); LOG.debug("TGT verified and refreshed if needed for user: {}", ugi.getUserName()); - // Get the audit server URL to determine the target hostname for service ticket - String auditServerUrl = MiscUtil.getStringProperty(props, PROP_AUDITSERVER_URL); - - if (StringUtils.isNotEmpty(auditServerUrl)) { - try { - URI uri = new URI(auditServerUrl); - String hostname = uri.getHost(); - - LOG.info("Pre-fetching Kerberos service ticket for HTTP/{}", hostname); - - ugi.doAs((PrivilegedExceptionAction) () -> { - LOG.debug("Kerberos security context initialized for audit server: {}", hostname); - return null; - }); - - LOG.info("Kerberos pre-authentication completed successfully. Service ticket cached for HTTP/{}", hostname); - } catch (URISyntaxException e) { - LOG.warn("Invalid audit server URL format: {}. Skipping service ticket pre-fetch", auditServerUrl, e); - } catch (Exception e) { - LOG.warn("Failed to pre-fetch service ticket for audit server: {}. First request may need to obtain ticket", auditServerUrl, e); - } - } else { - LOG.warn("Audit server URL not configured. Cannot pre-fetch service ticket"); - } + LOG.info("Kerberos pre-authentication completed successfully"); } catch (Exception e) { LOG.warn("Kerberos pre-authentication failed. First request will retry authentication", e); } @@ -258,14 +194,9 @@ private void preAuthenticateKerberos() { public void stop() { LOG.info("==> RangerAuditServerDestination.stop() called.."); logStatus(); - if (httpClient != null) { - try { - httpClient.close(); - } catch (IOException ioe) { - LOG.error("Error while closing httpclient in RangerAuditServerDestination!", ioe); - } finally { - httpClient = null; - } + if (restClient != null) { + restClient.resetClient(); + restClient = null; } } @@ -285,18 +216,16 @@ public boolean log(Collection events) { logStatusIfRequired(); addTotalCount(events.size()); - if (httpClient == null) { - httpClient = buildHTTPClient(); - if (httpClient == null) { - // HTTP Server is still not initialized. So need return error - addDeferredCount(events.size()); - return ret; - } + if (restClient == null) { + LOG.error("REST client is not initialized. Cannot send audit events"); + addDeferredCount(events.size()); + return ret; } + ret = logAsBatch(events); } catch (Throwable t) { addDeferredCount(events.size()); - logError("Error sending audit to HTTP Server", t); + logError("Error sending audit to Audit Server", t); } return ret; } @@ -327,423 +256,164 @@ private boolean logAsBatch(Collection events) { private boolean sendBatch(Collection events) { boolean ret = false; Map queryParams = new HashMap<>(); - try { - UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); - LOG.debug("Sending audit batch of {} events as user: {}", events.size(), ugi.getUserName()); - - PrivilegedExceptionAction> action = - () -> executeHttpBatchRequest(REST_RELATIVE_PATH_POST, queryParams, events, Map.class); - Map response = executeAction(action, ugi); - if (response != null) { - LOG.info("Audit batch sent successfully. {} events delivered. Response: {}", events.size(), response); - ret = true; - } else { - LOG.error("Received null response from audit server for batch of {} events", events.size()); - ret = false; - } - } catch (Exception e) { - LOG.error("Failed to send audit batch of {} events to {}. Error: {}", events.size(), httpURL, e.getMessage(), e); - - // Log additional context for authentication errors - if (e.getMessage() != null && e.getMessage().contains("401")) { - LOG.error("Authentication failure detected. Verify Kerberos credentials are valid and audit server is reachable."); - } - ret = false; + // Add serviceName to query parameters + if (StringUtils.isNotEmpty(serviceType)) { + queryParams.put(QUERY_PARAM_SERVICE_NAME, serviceType); + LOG.debug("Adding serviceName={} to audit request", serviceType); + } else { + LOG.error("Cannot send audit batch: serviceType is not set. This indicates a configuration error."); + LOG.error("Audit server requires serviceName parameter. Please ensure RangerBasePlugin is properly initialized."); + return false; } - return ret; - } - - public T executeHttpBatchRequest(String relativeUrl, Map params, - Collection events, Class clazz) throws Exception { - T finalResponse = postAndParse(httpURL + relativeUrl, params, events, clazz); - return finalResponse; - } - - public T executeHttpRequestPOST(String relativeUrl, Map params, Object obj, Class clazz) throws Exception { - LOG.debug("==> RangerAuditServerDestination().executeHttpRequestPOST()"); - - T finalResponse = postAndParse(httpURL + relativeUrl, params, obj, clazz); - - LOG.debug("<== RangerAuditServerDestination().executeHttpRequestPOST()"); - return finalResponse; - } - - public T postAndParse(String url, Map queryParams, Object obj, Class clazz) throws Exception { - HttpPost httpPost = new HttpPost(buildURI(url, queryParams)); - StringEntity entity = new StringEntity(gsonBuilder.toJson(obj)); - httpPost.setEntity(entity); - return executeAndParseResponse(httpPost, clazz, queryParams); - } - - synchronized CloseableHttpClient buildHTTPClient() { - Lookup authRegistry = RegistryBuilder.create().register(AuthSchemes.SPNEGO, new SPNegoSchemeFactory(true, true)).build(); - HttpClientBuilder clientBuilder = HttpClients.custom().setDefaultAuthSchemeRegistry(authRegistry); - String userName = MiscUtil.getStringProperty(props, PROP_AUDITSERVER_USER_NAME); - String passWord = MiscUtil.getStringProperty(props, PROP_AUDITSERVER_USER_PASSWORD); - int clientConnTimeOutMs = MiscUtil.getIntProperty(props, PROP_AUDITSERVER_CLIENT_CONN_TIMEOUT_MS, 1000); - int clientReadTimeOutMs = MiscUtil.getIntProperty(props, PROP_AUDITSERVER_CLIENT_READ_TIMEOUT_MS, 1000); - int maxConnections = MiscUtil.getIntProperty(props, PROP_AUDITSERVER_MAX_CONNECTION, 10); - int maxConnectionsPerHost = MiscUtil.getIntProperty(props, PROP_AUDITSERVER_MAX_CONNECTION_PER_HOST, 10); - int validateAfterInactivityMs = MiscUtil.getIntProperty(props, PROP_AUDITSERVER_VALIDATE_INACTIVE_MS, 1000); - int poolRetryCount = MiscUtil.getIntProperty(props, PROP_AUDITSERVER_POOL_RETRY_COUNT, 5); - httpURL = MiscUtil.getStringProperty(props, PROP_AUDITSERVER_URL); - - LOG.info("Building HTTP client for audit destination: url={}, connTimeout={}ms, readTimeout={}ms, maxConn={}", httpURL, clientConnTimeOutMs, clientReadTimeOutMs, maxConnections); - try { - if (AUTH_TYPE_JWT.equalsIgnoreCase(authType)) { - // JWT authentication - token will be added to request headers directly - LOG.info("HTTP client configured for JWT Bearer token authentication"); + final UserGroupInformation user = MiscUtil.getUGILoginUser(); + final boolean isSecureMode = isKerberosAuthenticated(); + + if (isSecureMode && user != null) { + LOG.debug("Sending audit batch of {} events using Kerberos. Principal: {}, AuthMethod: {}", events.size(), user.getUserName(), user.getAuthenticationMethod()); } else { - // Kerberos or Basic authentication - use credentials provider - BasicCredentialsProvider credentialsProvider = new BasicCredentialsProvider(); + LOG.debug("Sending audit batch of {} events. SecureMode: {}, User: {}", events.size(), isSecureMode, user != null ? user.getUserName() : "null"); + } - if (AUTH_TYPE_KERBEROS.equalsIgnoreCase(authType) || isKerberosAuthenticated()) { - credentialsProvider.setCredentials(AuthScope.ANY, new KerberosCredentials(null)); - LOG.info("HTTP client configured for Kerberos authentication (SPNEGO)...."); + final ClientResponse response; + + if (isSecureMode) { + response = MiscUtil.executePrivilegedAction((PrivilegedExceptionAction) () -> { try { - UserGroupInformation ugi = UserGroupInformation.getLoginUser(); - LOG.info("Current Kerberos principal: {}, authMethod: {}, hasKerberosCredentials: {}", ugi.getUserName(), ugi.getAuthenticationMethod(), ugi.hasKerberosCredentials()); + return postAuditEvents(REST_RELATIVE_PATH_POST, queryParams, events); } catch (Exception e) { - LOG.warn("Failed to get current UGI details", e); + LOG.error("Failed to post audit events in privileged action: {}", e.getMessage()); + throw e; } - } else if (AUTH_TYPE_BASIC.equalsIgnoreCase(authType) || (StringUtils.isNotEmpty(userName) && StringUtils.isNotEmpty(passWord))) { - credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(userName, passWord)); - LOG.info("HTTP client configured for basic authentication with username: {}", userName); - } else { - LOG.warn("No authentication credentials configured for HTTP audit destination"); - } - - clientBuilder.setDefaultCredentialsProvider(credentialsProvider); + }); + } else { + response = postAuditEvents(REST_RELATIVE_PATH_POST, queryParams, events); } - } catch (Exception excp) { - LOG.error("Exception while configuring authentication credentials. Audits may fail to send!", excp); - } - - PoolingHttpClientConnectionManager connectionManager; - - KeyManager[] kmList = getKeyManagers(); - TrustManager[] tmList = getTrustManagers(); - SSLContext sslContext = getSSLContext(kmList, tmList); - - if (sslContext != null) { - SSLContext.setDefault(sslContext); - } - - boolean isSSL = (httpURL != null && httpURL.contains("https://")); - if (isSSL) { - SSLConnectionSocketFactory sslsf = new SSLConnectionSocketFactory(sslContext, NoopHostnameVerifier.INSTANCE); - Registry socketFactoryRegistry = RegistryBuilder.create().register(PROTOCOL_HTTPS, sslsf).build(); - - connectionManager = new PoolingHttpClientConnectionManager(socketFactoryRegistry); - clientBuilder.setSSLSocketFactory(sslsf); - } else { - connectionManager = new PoolingHttpClientConnectionManager(); - } + if (response != null) { + int status = response.getStatus(); - connectionManager.setMaxTotal(maxConnections); - connectionManager.setDefaultMaxPerRoute(maxConnectionsPerHost); - connectionManager.setValidateAfterInactivity(validateAfterInactivityMs); + if (status == HttpServletResponse.SC_OK) { + String responseStr = response.getEntity(String.class); + LOG.debug("Audit batch sent successfully. {} events delivered. Response: {}", events.size(), responseStr); + ret = true; + } else { + String errorBody = ""; + try { + if (response.hasEntity()) { + errorBody = response.getEntity(String.class); + } + } catch (Exception e) { + LOG.debug("Failed to read error response body", e); + } - RequestConfig.Builder requestConfigBuilder = RequestConfig.custom() - .setCookieSpec(CookieSpecs.DEFAULT) - .setConnectTimeout(clientConnTimeOutMs) - .setSocketTimeout(clientReadTimeOutMs); + LOG.error("Failed to send audit batch. HTTP status: {}, Response: {}", status, errorBody); - // Configure authentication based on auth type - if (AUTH_TYPE_JWT.equalsIgnoreCase(authType)) { - // JWT doesn't use HttpClient's authentication mechanism - token is added to headers - LOG.info("RequestConfig configured for JWT authentication...."); - } else { - // For Kerberos and Basic auth, enable HttpClient's authentication mechanism - requestConfigBuilder.setAuthenticationEnabled(true); + if (status == HttpServletResponse.SC_UNAUTHORIZED) { + LOG.error("Authentication failure (401). Verify credentials are valid and audit server is properly configured."); + } - try { - // For Kerberos authentication, specify SPNEGO as target auth scheme - if (isKerberosAuthenticated() || (AUTH_TYPE_KERBEROS.equalsIgnoreCase(authType))) { - requestConfigBuilder.setTargetPreferredAuthSchemes(Arrays.asList(AuthSchemes.SPNEGO)); - LOG.info("Configured SPNEGO as target authentication scheme for audit HTTP client"); + ret = false; } - } catch (Exception e) { - LOG.warn("Failed to check Kerberos authentication status for request config", e); - } - } - - RequestConfig customizedRequestConfig = requestConfigBuilder.build(); - - CloseableHttpClient httpClient = clientBuilder.setConnectionManager(connectionManager).setRetryHandler(new AuditHTTPRetryHandler(poolRetryCount, true)).setDefaultRequestConfig(customizedRequestConfig).build(); - - return httpClient; - } - - boolean isKerberosAuthenticated() throws Exception { - boolean status = false; - try { - UserGroupInformation loggedInUser = UserGroupInformation.getLoginUser(); - boolean isSecurityEnabled = UserGroupInformation.isSecurityEnabled(); - boolean hasKerberosCredentials = loggedInUser.hasKerberosCredentials(); - UserGroupInformation.AuthenticationMethod loggedInUserAuthMethod = loggedInUser.getAuthenticationMethod(); - - status = isSecurityEnabled && hasKerberosCredentials && loggedInUserAuthMethod.equals(UserGroupInformation.AuthenticationMethod.KERBEROS); - } catch (IOException e) { - throw new Exception("Failed to get authentication details.", e); - } - return status; - } - - private void close(InputStream str, String filename) { - if (str != null) { - try { - str.close(); - } catch (IOException excp) { - LOG.error("Error while closing file: [{}]", filename, excp); - } - } - } - - private PrivilegedExceptionAction getPrivilegedAction(String relativeUrl, Map queryParams, Object postParam, Class clazz) { - return () -> executeHttpRequestPOST(relativeUrl, queryParams, postParam, clazz); - } - - private R executeAndParseResponse(T request, Class responseClazz, Map queryParams) throws Exception { - R ret = null; - CloseableHttpClient client = getCloseableHttpClient(); - - if (client != null) { - addCommonHeaders(request, queryParams); - // Create an HttpClientContext to maintain authentication state across potential retries - HttpClientContext context = HttpClientContext.create(); - try (CloseableHttpResponse response = client.execute(request, context)) { - ret = parseResponse(response, responseClazz); + } else { + LOG.error("Received null response from audit server for batch of {} events", events.size()); + ret = false; } - } else { - LOG.error("Cannot process request as Audit HTTPClient is null..."); + } catch (Exception e) { + LOG.error("Failed to send audit batch of {} events. Error: {}", events.size(), e.getMessage(), e); + ret = false; } return ret; } - private T addCommonHeaders(T t, Map queryParams) { - t.addHeader(REST_HEADER_ACCEPT, REST_ACCEPTED_MIME_TYPE_JSON); - t.setHeader(REST_HEADER_CONTENT_TYPE, REST_CONTENT_TYPE_MIME_TYPE_JSON); - - // Add JWT Bearer token if JWT authentication is configured - if (AUTH_TYPE_JWT.equalsIgnoreCase(authType) && StringUtils.isNotEmpty(jwtToken)) { - t.setHeader(REST_HEADER_AUTHORIZATION, "Bearer " + jwtToken); - LOG.debug("Added JWT Bearer token to request Authorization header"); - } - - return t; - } + private ClientResponse postAuditEvents(String relativeUrl, Map params, Collection events) throws Exception { + LOG.debug("Posting {} audit events to {}", events.size(), relativeUrl); - @SuppressWarnings("unchecked") - private R parseResponse(HttpResponse response, Class clazz) throws Exception { - R type = null; + String jsonPayload = MiscUtil.stringify(events); - if (response == null) { - String responseError = "Received NULL response from server"; - LOG.error(responseError); - throw new Exception(responseError); + if (LOG.isDebugEnabled()) { + LOG.debug("Serialized {} events to JSON payload (length: {} bytes)", events.size(), jsonPayload.length()); } - int httpStatus = response.getStatusLine().getStatusCode(); - - if (httpStatus == HttpStatus.SC_OK) { - InputStream responseInputStream = response.getEntity().getContent(); - if (clazz.equals(String.class)) { - type = (R) IOUtils.toString(responseInputStream, Charset.defaultCharset()); - } else { - type = gsonBuilder.fromJson(new InputStreamReader(responseInputStream), clazz); + WebResource webResource = restClient.getResource(relativeUrl); + if (params != null && !params.isEmpty()) { + for (Map.Entry entry : params.entrySet()) { + webResource = webResource.queryParam(entry.getKey(), entry.getValue()); } - responseInputStream.close(); - } else { - String responseBody = ""; - try { - if (response.getEntity() != null && response.getEntity().getContent() != null) { - responseBody = IOUtils.toString(response.getEntity().getContent(), Charset.defaultCharset()); - } - } catch (Exception e) { - LOG.debug("Failed to read error response body", e); - } - - String error = String.format("Request failed with HTTP status %d. Response body: %s", httpStatus, responseBody); - if (httpStatus == HttpStatus.SC_UNAUTHORIZED) { - LOG.error("{} - Authentication failed. Verify Kerberos credentials are valid and properly configured.", error); - } else { - LOG.error(error); - } - throw new Exception(error); - } - return type; - } - - private T executeAction(PrivilegedExceptionAction action, UserGroupInformation owner) throws Exception { - T ret = null; - if (owner != null) { - ret = owner.doAs(action); - } else { - ret = action.run(); } - return ret; - } - private URI buildURI(String url, Map queryParams) throws URISyntaxException { - URIBuilder builder = new URIBuilder(url); - if (queryParams != null) { - for (Map.Entry param : queryParams.entrySet()) { - builder.addParameter(param.getKey(), param.getValue()); - } - } - return builder.build(); + return webResource + .accept("application/json") + .type("application/json") + .entity(jsonPayload) + .post(ClientResponse.class); } - private CloseableHttpClient getCloseableHttpClient() { - CloseableHttpClient client = httpClient; - if (client == null) { - synchronized (this) { - client = httpClient; + private Configuration createConfigurationFromProperties(Properties props, String authType, String userName, String password) { + Configuration config = new Configuration(); - if (client == null) { - client = buildHTTPClient(); - httpClient = client; - } - } + for (String key : props.stringPropertyNames()) { + config.set(key, props.getProperty(key)); } - return client; - } - - private KeyManager[] getKeyManagers() { - KeyManager[] kmList = null; - String credentialProviderPath = MiscUtil.getStringProperty(props, RANGER_POLICYMGR_CLIENT_KEY_FILE_CREDENTIAL); - String keyStoreAlias = RANGER_POLICYMGR_CLIENT_KEY_FILE_CREDENTIAL_ALIAS; - String keyStoreFile = MiscUtil.getStringProperty(props, RANGER_POLICYMGR_CLIENT_KEY_FILE); - String keyStoreFilepwd = MiscUtil.getCredentialString(credentialProviderPath, keyStoreAlias); - if (StringUtils.isNotEmpty(keyStoreFile) && StringUtils.isNotEmpty(keyStoreFilepwd)) { - InputStream in = null; - try { - in = getFileInputStream(keyStoreFile); - if (in != null) { - String keyStoreType = MiscUtil.getStringProperty(props, RANGER_POLICYMGR_CLIENT_KEY_FILE_TYPE); - keyStoreType = StringUtils.isNotEmpty(keyStoreType) ? keyStoreType : RANGER_POLICYMGR_CLIENT_KEY_FILE_TYPE_DEFAULT; - KeyStore keyStore = KeyStore.getInstance(keyStoreType); - - keyStore.load(in, keyStoreFilepwd.toCharArray()); - - KeyManagerFactory keyManagerFactory = KeyManagerFactory.getInstance(RANGER_SSL_KEYMANAGER_ALGO_TYPE); - - keyManagerFactory.init(keyStore, keyStoreFilepwd.toCharArray()); - - kmList = keyManagerFactory.getKeyManagers(); + final String restClientPrefix = "ranger.plugin"; + if (AUTH_TYPE_JWT.equalsIgnoreCase(authType)) { + String jwtToken = initJwtToken(); + if (StringUtils.isNotEmpty(jwtToken)) { + String jwtTokenFile = MiscUtil.getStringProperty(props, PROP_AUDITSERVER_JWT_TOKEN_FILE); + if (StringUtils.isNotEmpty(jwtTokenFile)) { + config.set(restClientPrefix + ".policy.rest.client.jwt.source", "file"); + config.set(restClientPrefix + ".policy.rest.client.jwt.file", jwtTokenFile); + LOG.info("JWT authentication configured via file: {}", jwtTokenFile); } else { - LOG.error("Unable to obtain keystore from file [{}]", keyStoreFile); + LOG.warn("JWT token is set but file path is not available. JWT may not work as expected"); } - } catch (KeyStoreException e) { - LOG.error("Unable to obtain from KeyStore :{}", e.getMessage(), e); - } catch (NoSuchAlgorithmException e) { - LOG.error("SSL algorithm is NOT available in the environment", e); - } catch (CertificateException e) { - LOG.error("Unable to obtain the requested certification ", e); - } catch (FileNotFoundException e) { - LOG.error("Unable to find the necessary SSL Keystore Files", e); - } catch (IOException e) { - LOG.error("Unable to read the necessary SSL Keystore Files", e); - } catch (UnrecoverableKeyException e) { - LOG.error("Unable to recover the key from keystore", e); - } finally { - close(in, keyStoreFile); } + } else if (AUTH_TYPE_KERBEROS.equalsIgnoreCase(authType)) { + // For Kerberos, no additional configuration is needed in RangerRESTClient + // Authentication happens via Subject.doAs() security context + LOG.info("Kerberos authentication will be used via privileged action (Subject.doAs)"); + } else if (AUTH_TYPE_BASIC.equalsIgnoreCase(authType) && StringUtils.isNotEmpty(userName) && StringUtils.isNotEmpty(password)) { + config.set(restClientPrefix + ".policy.rest.client.username", userName); + config.set(restClientPrefix + ".policy.rest.client.password", password); + LOG.info("Basic authentication configured for user: {}", userName); } - return kmList; - } - - private TrustManager[] getTrustManagers() { - TrustManager[] tmList = null; - String credentialProviderPath = MiscUtil.getStringProperty(props, RANGER_POLICYMGR_TRUSTSTORE_FILE_CREDENTIAL); - String trustStoreAlias = RANGER_POLICYMGR_TRUSTSTORE_FILE_CREDENTIAL_ALIAS; - String trustStoreFile = MiscUtil.getStringProperty(props, RANGER_POLICYMGR_TRUSTSTORE_FILE); - String trustStoreFilepwd = MiscUtil.getCredentialString(credentialProviderPath, trustStoreAlias); - if (StringUtils.isNotEmpty(trustStoreFile) && StringUtils.isNotEmpty(trustStoreFilepwd)) { - InputStream in = null; - try { - in = getFileInputStream(trustStoreFile); - - if (in != null) { - String trustStoreType = MiscUtil.getStringProperty(props, RANGER_POLICYMGR_TRUSTSTORE_FILE_TYPE); - trustStoreType = StringUtils.isNotEmpty(trustStoreType) ? trustStoreType : RANGER_POLICYMGR_TRUSTSTORE_FILE_TYPE_DEFAULT; - KeyStore trustStore = KeyStore.getInstance(trustStoreType); - trustStore.load(in, trustStoreFilepwd.toCharArray()); - - TrustManagerFactory trustManagerFactory = TrustManagerFactory.getInstance(RANGER_SSL_TRUSTMANAGER_ALGO_TYPE); - - trustManagerFactory.init(trustStore); - - tmList = trustManagerFactory.getTrustManagers(); - } else { - LOG.error("Unable to obtain truststore from file [{}]", trustStoreFile); - } - } catch (KeyStoreException e) { - LOG.error("Unable to obtain from KeyStore", e); - } catch (NoSuchAlgorithmException e) { - LOG.error("SSL algorithm is NOT available in the environment :{}", e.getMessage(), e); - } catch (CertificateException e) { - LOG.error("Unable to obtain the requested certification :{}", e.getMessage(), e); - } catch (FileNotFoundException e) { - LOG.error("Unable to find the necessary SSL TrustStore File:{}", trustStoreFile, e); - } catch (IOException e) { - LOG.error("Unable to read the necessary SSL TrustStore Files :{}", trustStoreFile, e); - } finally { - close(in, trustStoreFile); - } - } - return tmList; + return config; } - private SSLContext getSSLContext(KeyManager[] kmList, TrustManager[] tmList) { - SSLContext sslContext = null; + private boolean isKerberosAuthenticated() { try { - sslContext = SSLContext.getInstance(RANGER_SSL_CONTEXT_ALGO_TYPE); - if (sslContext != null) { - sslContext.init(kmList, tmList, new SecureRandom()); - } - } catch (NoSuchAlgorithmException e) { - LOG.error("SSL algorithm is not available in the environment", e); - } catch (KeyManagementException e) { - LOG.error("Unable to initialise the SSLContext", e); - } - return sslContext; - } + UserGroupInformation loggedInUser = UserGroupInformation.getLoginUser(); - private InputStream getFileInputStream(String fileName) throws IOException { - InputStream in = null; - if (StringUtils.isNotEmpty(fileName)) { - File file = new File(fileName); - if (file != null && file.exists()) { - in = new FileInputStream(file); - } else { - in = ClassLoader.getSystemResourceAsStream(fileName); + if (loggedInUser == null) { + return false; } + + boolean isSecurityEnabled = UserGroupInformation.isSecurityEnabled(); + boolean hasKerberosCredentials = loggedInUser.hasKerberosCredentials(); + UserGroupInformation.AuthenticationMethod authMethod = loggedInUser.getAuthenticationMethod(); + + return isSecurityEnabled && hasKerberosCredentials && authMethod.equals(UserGroupInformation.AuthenticationMethod.KERBEROS); + } catch (Exception e) { + LOG.warn("Failed to check Kerberos authentication status", e); + return false; } - return in; } - private class AuditHTTPRetryHandler extends StandardHttpRequestRetryHandler { - public AuditHTTPRetryHandler(Integer poolRetryCount, boolean isIdempotentRequest) { - super(poolRetryCount, isIdempotentRequest); + private InputStream getFileInputStream(String fileName) throws IOException { + if (StringUtils.isEmpty(fileName)) { + return null; } - @Override - public boolean retryRequest(IOException exception, int executionCount, HttpContext context) { - LOG.debug("==> AuditHTTPRetryHandler.retryRequest {} Execution Count = {}", exception.getMessage(), executionCount); - - boolean ret = super.retryRequest(exception, executionCount, context); + java.io.File file = new java.io.File(fileName); - LOG.debug("<== AuditHTTPRetryHandler.retryRequest(): ret= {}", ret); - - return ret; + if (file.exists()) { + return new java.io.FileInputStream(file); + } else { + return ClassLoader.getSystemResourceAsStream(fileName); } } } diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java b/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java index fac244d449..a1b49db0ea 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java @@ -85,6 +85,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Properties; import java.util.Set; public class RangerBasePlugin { @@ -400,7 +401,13 @@ public void init() { if (!providerFactory.isInitDone()) { if (pluginConfig.getProperties() != null) { - providerFactory.init(pluginConfig.getProperties(), getAppId()); + Properties auditProps = pluginConfig.getProperties(); + String serviceType = getServiceType(); + if (StringUtils.isNotEmpty(serviceType)) { + auditProps.setProperty("ranger.plugin.audit.service.type", serviceType); + LOG.info("Added serviceType={} to audit properties for audit destination", serviceType); + } + providerFactory.init(auditProps, getAppId()); } else { LOG.error("Audit subsystem is not initialized correctly. Please check audit configuration. "); LOG.error("No authorization audits will be generated. "); diff --git a/distro/src/main/assembly/ranger-audit-consumer-hdfs.xml b/distro/src/main/assembly/ranger-audit-consumer-hdfs.xml index 0130321a67..29641141b8 100644 --- a/distro/src/main/assembly/ranger-audit-consumer-hdfs.xml +++ b/distro/src/main/assembly/ranger-audit-consumer-hdfs.xml @@ -72,16 +72,6 @@ **/* - - - - - ${project.parent.basedir}/ranger-audit-server/ranger-audit-consumer-hdfs/ - - NOTICE.txt - - 644 - diff --git a/distro/src/main/assembly/ranger-audit-consumer-solr.xml b/distro/src/main/assembly/ranger-audit-consumer-solr.xml index edd06938a5..51679d2f21 100644 --- a/distro/src/main/assembly/ranger-audit-consumer-solr.xml +++ b/distro/src/main/assembly/ranger-audit-consumer-solr.xml @@ -70,16 +70,6 @@ **/* - - - - - ${project.parent.basedir}/ranger-audit-server/ranger-audit-consumer-solr/ - - NOTICE.txt - - 644 - diff --git a/distro/src/main/assembly/ranger-audit-server-service.xml b/distro/src/main/assembly/ranger-audit-server-service.xml index 53540f40f5..f91063dc42 100644 --- a/distro/src/main/assembly/ranger-audit-server-service.xml +++ b/distro/src/main/assembly/ranger-audit-server-service.xml @@ -70,16 +70,6 @@ **/* - - - - - ${project.parent.basedir}/ranger-audit-server/ranger-audit-server-service/ - - NOTICE.txt - - 644 - diff --git a/ranger-audit-server/ranger-audit-common/src/main/java/org/apache/ranger/audit/server/AuditServerConstants.java b/ranger-audit-server/ranger-audit-common/src/main/java/org/apache/ranger/audit/server/AuditServerConstants.java index 24674b1e7f..b930a246ed 100644 --- a/ranger-audit-server/ranger-audit-common/src/main/java/org/apache/ranger/audit/server/AuditServerConstants.java +++ b/ranger-audit-server/ranger-audit-common/src/main/java/org/apache/ranger/audit/server/AuditServerConstants.java @@ -24,6 +24,11 @@ private AuditServerConstants() {} public static final String AUDIT_SERVER_APPNAME = "ranger-audit"; public static final String AUDIT_SERVER_PROP_PREFIX = "ranger.audit.service."; + + public static final String PROP_ALLOWED_USERS = "ranger.audit.service.allowed.users"; + public static final String PROP_AUTH_TO_LOCAL = "ranger.audit.service.auth.to.local"; + public static final String DEFAULT_ALLOWED_USERS = "hdfs,hive,hbase,kafka,yarn,solr,knox,storm,atlas,nifi,ozone,kudu,presto,trino"; + public static final String JAAS_KRB5_MODULE = "com.sun.security.auth.module.Krb5LoginModule required"; public static final String JAAS_USE_KEYTAB = "useKeyTab=true"; public static final String JAAS_KEYTAB = "keyTab=\""; diff --git a/ranger-audit-server/ranger-audit-server-service/pom.xml b/ranger-audit-server/ranger-audit-server-service/pom.xml index 3cae25fc8a..4950522cec 100644 --- a/ranger-audit-server/ranger-audit-server-service/pom.xml +++ b/ranger-audit-server/ranger-audit-server-service/pom.xml @@ -60,6 +60,7 @@ jackson-databind ${fasterxml.jackson.version} + com.google.guava guava diff --git a/ranger-audit-server/ranger-audit-server-service/src/main/java/org/apache/ranger/audit/rest/AuditREST.java b/ranger-audit-server/ranger-audit-server-service/src/main/java/org/apache/ranger/audit/rest/AuditREST.java index f8751c1cbc..0ae167e116 100644 --- a/ranger-audit-server/ranger-audit-server-service/src/main/java/org/apache/ranger/audit/rest/AuditREST.java +++ b/ranger-audit-server/ranger-audit-server-service/src/main/java/org/apache/ranger/audit/rest/AuditREST.java @@ -19,36 +19,50 @@ package org.apache.ranger.audit.rest; +import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.gson.Gson; -import com.google.gson.GsonBuilder; -import com.google.gson.JsonSyntaxException; -import com.google.gson.reflect.TypeToken; +import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.security.authentication.util.KerberosName; import org.apache.ranger.audit.model.AuthzAuditEvent; import org.apache.ranger.audit.producer.AuditDestinationMgr; +import org.apache.ranger.audit.provider.MiscUtil; +import org.apache.ranger.audit.server.AuditServerConfig; +import org.apache.ranger.audit.server.AuditServerConstants; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Scope; import org.springframework.stereotype.Component; +import javax.servlet.http.HttpServletRequest; import javax.ws.rs.Consumes; import javax.ws.rs.GET; import javax.ws.rs.POST; import javax.ws.rs.Path; import javax.ws.rs.Produces; +import javax.ws.rs.QueryParam; +import javax.ws.rs.core.Context; import javax.ws.rs.core.Response; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; @Path("/audit") @Component @Scope("request") public class AuditREST { - private static final Logger LOG = LoggerFactory.getLogger(AuditREST.class); - private Gson gson = new GsonBuilder().setDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSZ").create(); + private static final Logger LOG = LoggerFactory.getLogger(AuditREST.class); + private static final TypeReference> AUDIT_EVENT_LIST_TYPE = new TypeReference>() {}; + private static final Set allowedServiceUsers; + + static { + allowedServiceUsers = initializeAllowedUsers(); + initializeAuthToLocal(); + } + @Autowired AuditDestinationMgr auditDestinationMgr; @@ -137,17 +151,55 @@ public Response getStatus() { /** * Access Audits producer endpoint. + * @param serviceName Required query parameter to identify the source service (hdfs, hive, kafka, solr, etc.) + * @param accessAudits JSON payload containing audit events + * @param request HTTP request to extract authenticated user */ @POST - @Path("/post") + @Path("/access") @Consumes("application/json") @Produces("application/json") - public Response postAudit(String accessAudits) { - LOG.debug("==> AuditREST.postAudit(): {}", accessAudits); + public Response accessAudit(@QueryParam("serviceName") String serviceName, String accessAudits, @Context HttpServletRequest request) { + String authenticatedUser = getAuthenticatedUser(request); + + LOG.debug("==> AuditREST.accessAudit(): received JSON payload from service: {}, authenticatedUser: {}", StringUtils.isNotEmpty(serviceName) ? serviceName : "unknown", authenticatedUser); Response ret; + + if (StringUtils.isEmpty(serviceName)) { + LOG.error("serviceName query parameter is required. Rejecting audit request."); + ret = Response.status(Response.Status.BAD_REQUEST) + .entity(buildErrorResponse("serviceName query parameter is required")) + .build(); + return ret; + } + + if (StringUtils.isEmpty(authenticatedUser)) { + LOG.error("No authenticated user found in request for service: {}. Rejecting audit request.", serviceName); + ret = Response.status(Response.Status.UNAUTHORIZED) + .entity(buildErrorResponse("Authentication required to send audit events")) + .build(); + return ret; + } + + if (!serviceName.equals(authenticatedUser)) { + LOG.error("Authentication mismatch: serviceName={} but authenticatedUser={}. Rejecting audit request.", serviceName, authenticatedUser); + ret = Response.status(Response.Status.FORBIDDEN) + .entity(buildErrorResponse("Service name does not match authenticated user")) + .build(); + return ret; + } + + if (!isAllowedServiceUser(authenticatedUser)) { + LOG.error("Unauthorized user: authenticatedUser={} is not in the allowed service users list. Rejecting audit request.", authenticatedUser); + ret = Response.status(Response.Status.FORBIDDEN) + .entity(buildErrorResponse("User is not authorized to send audit events")) + .build(); + return ret; + } + if (accessAudits == null || accessAudits.trim().isEmpty()) { - LOG.warn("Empty or null audit events batch received"); + LOG.warn("Empty or null audit events batch received from service: {}, user: {}", serviceName, authenticatedUser); ret = Response.status(Response.Status.BAD_REQUEST) .entity(buildErrorResponse("Audit events cannot be empty")) .build(); @@ -158,8 +210,10 @@ public Response postAudit(String accessAudits) { .build(); } else { try { - String jsonString; - List events = gson.fromJson(accessAudits, new TypeToken>() {}.getType()); + ObjectMapper mapper = MiscUtil.getMapper(); + List events = mapper.readValue(accessAudits, AUDIT_EVENT_LIST_TYPE); + + LOG.debug("Successfully deserialized {} audit events from service: {}", events.size(), serviceName); for (AuthzAuditEvent event : events) { auditDestinationMgr.log(event); @@ -168,39 +222,35 @@ public Response postAudit(String accessAudits) { Map response = new HashMap<>(); response.put("total", events.size()); response.put("timestamp", System.currentTimeMillis()); - jsonString = buildResponse(response); + if (StringUtils.isNotEmpty(serviceName)) { + response.put("serviceName", serviceName); + } + if (StringUtils.isNotEmpty(authenticatedUser)) { + response.put("authenticatedUser", authenticatedUser); + } + String jsonString = buildResponse(response); ret = Response.status(Response.Status.OK) .entity(jsonString) .build(); - } catch (JsonSyntaxException e) { - LOG.error("Invalid Access audit JSON string...", e); - ret = Response.status(Response.Status.BAD_REQUEST) - .entity(buildErrorResponse("Invalid Access audit JSON string..." + e.getMessage())) - .build(); } catch (Exception e) { - LOG.error("Error processing access audits batch...", e); - ret = Response.status(Response.Status.INTERNAL_SERVER_ERROR) - .entity(buildErrorResponse("Invalid Access audit JSON string..." + e.getMessage())) + LOG.error("Error processing access audits batch from service: {}", serviceName, e); + ret = Response.status(Response.Status.BAD_REQUEST) + .entity(buildErrorResponse("Failed to process audit events: " + e.getMessage())) .build(); } } - LOG.debug("<== AuditREST.postAudit(): HttpStatus {}", ret.getStatus()); + LOG.debug("<== AuditREST.accessAudit(): HttpStatus {} for service: {}, user: {}", ret.getStatus(), serviceName, authenticatedUser); return ret; } private String buildResponse(Map respMap) { - String ret; - try { - ObjectMapper objectMapper = new ObjectMapper(); - ret = objectMapper.writeValueAsString(respMap); + return MiscUtil.getMapper().writeValueAsString(respMap); } catch (Exception e) { - ret = "Error: " + e.getMessage(); + return "Error: " + e.getMessage(); } - - return ret; } private String buildErrorResponse(String message) { @@ -209,11 +259,112 @@ private String buildErrorResponse(String message) { error.put("status", "error"); error.put("message", message); error.put("timestamp", System.currentTimeMillis()); - ObjectMapper mapper = new ObjectMapper(); - return mapper.writeValueAsString(error); + return MiscUtil.getMapper().writeValueAsString(error); } catch (Exception e) { LOG.error("Error building error response", e); return "{\"status\":\"error\",\"message\":\"" + message.replace("\"", "'") + "\"}"; } } + + private String getAuthenticatedUser(HttpServletRequest request) { + if (request != null && request.getUserPrincipal() != null) { + String principalName = request.getUserPrincipal().getName(); + LOG.debug("Authenticated user from Principal: {}", principalName); + String shortName = applyAuthToLocal(principalName); + if (!shortName.equals(principalName)) { + LOG.debug("Applied auth_to_local: '{}' -> '{}'", principalName, shortName); + } + + return shortName; + } + + LOG.debug("No authenticated user found in request"); + return null; + } + + /** + * Apply Hadoop's auth_to_local rules to convert Kerberos principal to short username. + * For JWT or basic auth, the username is already in short form and returned as-is. + */ + private String applyAuthToLocal(String principal) { + if (StringUtils.isEmpty(principal)) { + return principal; + } + + // Check if this looks like a Kerberos principal (has @ or /) + if (!principal.contains("@") && !principal.contains("/")) { + LOG.debug("Username '{}' is already a short name (JWT/basic auth), no auth_to_local mapping needed", principal); + return principal; + } + + try { + KerberosName kerberosName = new KerberosName(principal); + String shortName = kerberosName.getShortName(); + return shortName; + } catch (Exception e) { + LOG.warn("Failed to apply auth_to_local rules to principal '{}': {}. Using original principal.", principal, e.getMessage()); + return principal; + } + } + + /** + * Rules are loaded from ranger.audit.service.auth.to.local property in ranger-audit-server-site.xml. + */ + private static void initializeAuthToLocal() { + AuditServerConfig config = AuditServerConfig.getInstance(); + String authToLocalRules = config.get(AuditServerConstants.PROP_AUTH_TO_LOCAL); + if (StringUtils.isNotEmpty(authToLocalRules)) { + try { + KerberosName.setRules(authToLocalRules); + LOG.debug("Auth_to_local rules: {}", authToLocalRules); + } catch (Exception e) { + LOG.error("Failed to set auth_to_local rules from configuration: {}", e.getMessage(), e); + } + } else { + LOG.warn("No auth_to_local rules configured. Kerberos principal mapping may not work correctly."); + LOG.warn("Set property '{}' in ranger-audit-server-site.xml", AuditServerConstants.PROP_AUTH_TO_LOCAL); + } + } + + /** + * Check if the user login into audit server for posting audits is in the allowed service users list + * @param userName The username to check + * @return true if user is allowed, false otherwise + */ + private boolean isAllowedServiceUser(String userName) { + boolean ret; + + if (StringUtils.isEmpty(userName)) { + ret = false; + } else { + ret = allowedServiceUsers.contains(userName); + LOG.debug("User '{}' allowed: {}", userName, ret); + } + + return ret; + } + + /** + * Initialize the set of allowed service users from configuration + * @return Set of allowed usernames + */ + private static Set initializeAllowedUsers() { + Set ret = new HashSet<>(); + AuditServerConfig config = AuditServerConfig.getInstance(); + + String allowedUsersStr = config.get(AuditServerConstants.PROP_ALLOWED_USERS, AuditServerConstants.DEFAULT_ALLOWED_USERS); + if (StringUtils.isNotEmpty(allowedUsersStr)) { + String[] users = allowedUsersStr.split(","); + for (String user : users) { + String trimmedUser = user.trim(); + if (StringUtils.isNotEmpty(trimmedUser)) { + ret.add(trimmedUser); + } + } + + LOG.debug("Allowed service users: {}", ret); + } + + return ret; + } } diff --git a/ranger-audit-server/ranger-audit-server-service/src/main/resources/conf/ranger-audit-server-site.xml b/ranger-audit-server/ranger-audit-server-service/src/main/resources/conf/ranger-audit-server-site.xml index 49f134a077..9bad9acae3 100644 --- a/ranger-audit-server/ranger-audit-server-service/src/main/resources/conf/ranger-audit-server-site.xml +++ b/ranger-audit-server/ranger-audit-server-service/src/main/resources/conf/ranger-audit-server-site.xml @@ -192,6 +192,40 @@ Expected audiences for JWT validation (comma-separated) + + ranger.audit.service.allowed.users + hdfs,hive,hbase,kafka,yarn,solr,knox,storm,atlas,nifi,ozone,kudu,presto,trino + + Comma-separated list of allowed service users that can send audits to Ranger Audit Server. + + + + + + ranger.audit.service.auth.to.local + + RULE:[2:$1/$2@$0]([ndj]n/.*@.*|hdfs/.*@.*)s/.*/hdfs/ + RULE:[2:$1/$2@$0]([rn]m/.*@.*|yarn/.*@.*)s/.*/yarn/ + RULE:[2:$1/$2@$0](jhs/.*@.*)s/.*/mapred/ + RULE:[1:$1@$0](.*@.*)s/@.*// + DEFAULT + + + Kerberos auth_to_local rules for mapping authenticated principals to service names. + Uses Hadoop KerberosName syntax to convert full Kerberos principals to short usernames. + + IMPORTANT: These rules only apply to Kerberos/SPNEGO authentication. JWT and basic auth + usernames are already in short form and pass through unchanged. + + Default rules provided: + - RULE:[2:$1/$2@$0]([ndj]n/.*@.*|hdfs/.*@.*)s/.*/hdfs/ # nn,dn,jn,hdfs/* -> hdfs + - RULE:[2:$1/$2@$0]([rn]m/.*@.*|yarn/.*@.*)s/.*/yarn/ # rm,nm,yarn/* -> yarn + - RULE:[2:$1/$2@$0](jhs/.*@.*)s/.*/mapred/ # jhs/* -> mapred + - RULE:[1:$1@$0](.*@.*)s/@.*// # user@REALM -> user + - DEFAULT + + + ranger.audit.service.authentication.method diff --git a/ranger-audit-server/ranger-audit-server-service/src/main/webapp/WEB-INF/web.xml b/ranger-audit-server/ranger-audit-server-service/src/main/webapp/WEB-INF/web.xml index 612d595471..06ff7bb7ec 100644 --- a/ranger-audit-server/ranger-audit-server-service/src/main/webapp/WEB-INF/web.xml +++ b/ranger-audit-server/ranger-audit-server-service/src/main/webapp/WEB-INF/web.xml @@ -57,11 +57,6 @@ org.apache.ranger.audit.rest - - com.sun.jersey.api.json.POJOMappingFeature - true - - 1 From 878b4ab3a8fe8234964ef8169ac2d2d86d58ad43 Mon Sep 17 00:00:00 2001 From: Ramesh Mani Date: Tue, 24 Feb 2026 11:41:26 -0800 Subject: [PATCH 07/12] RANGER-5482:Create Ranger Audit Server with SOLR and HDFS as audit consumer - PojoMappingFeature for AuditEvent Object for serialization --- agents-audit/core/pom.xml | 5 +++ .../ranger/audit/model/AuthzAuditEvent.java | 12 ++++++ agents-audit/dest-auditserver/pom.xml | 5 +++ .../RangerAuditServerDestination.java | 8 +--- distro/src/main/assembly/hdfs-agent.xml | 3 ++ distro/src/main/assembly/hive-agent.xml | 3 ++ .../ranger-audit-server-service/pom.xml | 43 +++++++++++++++++++ .../javax/ws/rs/core/NoContentException.java | 43 +++++++++++++++++++ .../apache/ranger/audit/rest/AuditREST.java | 19 +++----- .../src/main/webapp/WEB-INF/web.xml | 5 +++ 10 files changed, 126 insertions(+), 20 deletions(-) create mode 100644 ranger-audit-server/ranger-audit-server-service/src/main/java/javax/ws/rs/core/NoContentException.java diff --git a/agents-audit/core/pom.xml b/agents-audit/core/pom.xml index d11b3bf7c8..e6c8c37926 100644 --- a/agents-audit/core/pom.xml +++ b/agents-audit/core/pom.xml @@ -63,6 +63,11 @@ + + javax.xml.bind + jaxb-api + ${jaxb.api.version} + joda-time joda-time diff --git a/agents-audit/core/src/main/java/org/apache/ranger/audit/model/AuthzAuditEvent.java b/agents-audit/core/src/main/java/org/apache/ranger/audit/model/AuthzAuditEvent.java index 1b26d64d6a..e27a45de0b 100644 --- a/agents-audit/core/src/main/java/org/apache/ranger/audit/model/AuthzAuditEvent.java +++ b/agents-audit/core/src/main/java/org/apache/ranger/audit/model/AuthzAuditEvent.java @@ -19,16 +19,28 @@ package org.apache.ranger.audit.model; +import com.fasterxml.jackson.annotation.JsonAutoDetect; import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.annotation.JsonSerialize; import org.apache.commons.lang3.StringUtils; +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; + import java.util.Date; import java.util.HashSet; import java.util.Set; +@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.ANY) +@JsonInclude(JsonInclude.Include.NON_EMPTY) +@JsonIgnoreProperties(ignoreUnknown = true) @JsonSerialize +@XmlRootElement +@XmlAccessorType(XmlAccessType.FIELD) public class AuthzAuditEvent extends AuditEventBase { protected static final int MAX_ACTION_FIELD_SIZE = 1800; protected static final int MAX_REQUEST_DATA_FIELD_SIZE = 1800; diff --git a/agents-audit/dest-auditserver/pom.xml b/agents-audit/dest-auditserver/pom.xml index 80ebe09d1b..5fa95e1845 100644 --- a/agents-audit/dest-auditserver/pom.xml +++ b/agents-audit/dest-auditserver/pom.xml @@ -32,6 +32,11 @@ 1.2 + + com.fasterxml.jackson.jaxrs + jackson-jaxrs-json-provider + ${fasterxml.jackson.version} + com.google.code.gson gson diff --git a/agents-audit/dest-auditserver/src/main/java/org/apache/ranger/audit/destination/RangerAuditServerDestination.java b/agents-audit/dest-auditserver/src/main/java/org/apache/ranger/audit/destination/RangerAuditServerDestination.java index b81aec02af..625651cfcf 100644 --- a/agents-audit/dest-auditserver/src/main/java/org/apache/ranger/audit/destination/RangerAuditServerDestination.java +++ b/agents-audit/dest-auditserver/src/main/java/org/apache/ranger/audit/destination/RangerAuditServerDestination.java @@ -331,12 +331,6 @@ private boolean sendBatch(Collection events) { private ClientResponse postAuditEvents(String relativeUrl, Map params, Collection events) throws Exception { LOG.debug("Posting {} audit events to {}", events.size(), relativeUrl); - String jsonPayload = MiscUtil.stringify(events); - - if (LOG.isDebugEnabled()) { - LOG.debug("Serialized {} events to JSON payload (length: {} bytes)", events.size(), jsonPayload.length()); - } - WebResource webResource = restClient.getResource(relativeUrl); if (params != null && !params.isEmpty()) { for (Map.Entry entry : params.entrySet()) { @@ -347,7 +341,7 @@ private ClientResponse postAuditEvents(String relativeUrl, Map p return webResource .accept("application/json") .type("application/json") - .entity(jsonPayload) + .entity(events) .post(ClientResponse.class); } diff --git a/distro/src/main/assembly/hdfs-agent.xml b/distro/src/main/assembly/hdfs-agent.xml index 56b3b14e95..1bfc77ec92 100644 --- a/distro/src/main/assembly/hdfs-agent.xml +++ b/distro/src/main/assembly/hdfs-agent.xml @@ -116,6 +116,9 @@ org.apache.orc:orc-shims:jar:${orc.version} io.airlift:aircompressor:jar:${aircompressor.version} org.apache.hadoop.thirdparty:hadoop-shaded-guava:jar:${hadoop-shaded-guava.version} + com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider:jar:${fasterxml.jackson.version} + com.fasterxml.jackson.jaxrs:jackson-jaxrs-base:jar:${fasterxml.jackson.version} + com.fasterxml.jackson.module:jackson-module-jaxb-annotations:jar:${fasterxml.jackson.version} diff --git a/distro/src/main/assembly/hive-agent.xml b/distro/src/main/assembly/hive-agent.xml index d20d3f7cfd..c29d6b26a8 100644 --- a/distro/src/main/assembly/hive-agent.xml +++ b/distro/src/main/assembly/hive-agent.xml @@ -81,6 +81,9 @@ joda-time:joda-time com.carrotsearch:hppc org.apache.hadoop.thirdparty:hadoop-shaded-guava:jar:${hadoop-shaded-guava.version} + com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider:jar:${fasterxml.jackson.version} + com.fasterxml.jackson.jaxrs:jackson-jaxrs-base:jar:${fasterxml.jackson.version} + com.fasterxml.jackson.module:jackson-module-jaxb-annotations:jar:${fasterxml.jackson.version} diff --git a/ranger-audit-server/ranger-audit-server-service/pom.xml b/ranger-audit-server/ranger-audit-server-service/pom.xml index 4950522cec..dc1d77c438 100644 --- a/ranger-audit-server/ranger-audit-server-service/pom.xml +++ b/ranger-audit-server/ranger-audit-server-service/pom.xml @@ -60,6 +60,11 @@ jackson-databind ${fasterxml.jackson.version} + + com.fasterxml.jackson.jaxrs + jackson-jaxrs-json-provider + ${fasterxml.jackson.version} + com.google.guava @@ -83,6 +88,16 @@ com.sun.jersey jersey-bundle ${jersey-bundle.version} + + + javax.ws.rs + jsr311-api + + + org.codehaus.jackson + * + + @@ -91,6 +106,18 @@ ${jersey-bundle.version} + + com.sun.jersey + jersey-server + ${jersey-bundle.version} + + + + com.sun.jersey + jersey-servlet + ${jersey-bundle.version} + + com.sun.jersey.contribs jersey-spring @@ -100,6 +127,10 @@ com.sun.jersey jersey-server + + org.codehaus.jackson + * + org.springframework * @@ -152,6 +183,10 @@ ch.qos.reload4j reload4j + + com.github.pjfanning + jersey-json + log4j * @@ -164,6 +199,10 @@ org.apache.hadoop hadoop-hdfs + + org.codehaus.jackson + * + org.slf4j * @@ -245,6 +284,10 @@ ranger-plugins-common ${project.version} + + com.sun.jersey + jersey-bundle + javax.servlet javax.servlet-api diff --git a/ranger-audit-server/ranger-audit-server-service/src/main/java/javax/ws/rs/core/NoContentException.java b/ranger-audit-server/ranger-audit-server-service/src/main/java/javax/ws/rs/core/NoContentException.java new file mode 100644 index 0000000000..2a85d5877e --- /dev/null +++ b/ranger-audit-server/ranger-audit-server-service/src/main/java/javax/ws/rs/core/NoContentException.java @@ -0,0 +1,43 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package javax.ws.rs.core; + +import java.io.IOException; + +/* Workaround: use of JacksonJaxbJsonProvider in RangerJsonProvider along with jersey-bundle results in following initialization failure: + * SEVERE: Allocate exception for servlet [REST Service] + * java.lang.ClassNotFoundException: javax.ws.rs.core.NoContentException + * at org.apache.catalina.loader.WebappClassLoaderBase.loadClass(WebappClassLoaderBase.java:1338) + * at org.apache.catalina.loader.WebappClassLoaderBase.loadClass(WebappClassLoaderBase.java:1150) + */ +public class NoContentException extends IOException { + private static final long serialVersionUID = -3082577759787473245L; + + public NoContentException(String message) { + super(message); + } + + public NoContentException(String message, Throwable cause) { + super(message, cause); + } + + public NoContentException(Throwable cause) { + super(cause); + } +} diff --git a/ranger-audit-server/ranger-audit-server-service/src/main/java/org/apache/ranger/audit/rest/AuditREST.java b/ranger-audit-server/ranger-audit-server-service/src/main/java/org/apache/ranger/audit/rest/AuditREST.java index 0ae167e116..b68d8c052c 100644 --- a/ranger-audit-server/ranger-audit-server-service/src/main/java/org/apache/ranger/audit/rest/AuditREST.java +++ b/ranger-audit-server/ranger-audit-server-service/src/main/java/org/apache/ranger/audit/rest/AuditREST.java @@ -19,8 +19,6 @@ package org.apache.ranger.audit.rest; -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.security.authentication.util.KerberosName; import org.apache.ranger.audit.model.AuthzAuditEvent; @@ -55,7 +53,6 @@ @Scope("request") public class AuditREST { private static final Logger LOG = LoggerFactory.getLogger(AuditREST.class); - private static final TypeReference> AUDIT_EVENT_LIST_TYPE = new TypeReference>() {}; private static final Set allowedServiceUsers; static { @@ -152,17 +149,16 @@ public Response getStatus() { /** * Access Audits producer endpoint. * @param serviceName Required query parameter to identify the source service (hdfs, hive, kafka, solr, etc.) - * @param accessAudits JSON payload containing audit events * @param request HTTP request to extract authenticated user */ @POST @Path("/access") @Consumes("application/json") @Produces("application/json") - public Response accessAudit(@QueryParam("serviceName") String serviceName, String accessAudits, @Context HttpServletRequest request) { + public Response accessAudit(@QueryParam("serviceName") String serviceName, List accessAudits, @Context HttpServletRequest request) { String authenticatedUser = getAuthenticatedUser(request); - LOG.debug("==> AuditREST.accessAudit(): received JSON payload from service: {}, authenticatedUser: {}", StringUtils.isNotEmpty(serviceName) ? serviceName : "unknown", authenticatedUser); + LOG.debug("==> AuditREST.accessAudit(): received {} audit events from service: {}, authenticatedUser: {}", accessAudits != null ? accessAudits.size() : 0, StringUtils.isNotEmpty(serviceName) ? serviceName : "unknown", authenticatedUser); Response ret; @@ -198,7 +194,7 @@ public Response accessAudit(@QueryParam("serviceName") String serviceName, Strin return ret; } - if (accessAudits == null || accessAudits.trim().isEmpty()) { + if (accessAudits == null || accessAudits.isEmpty()) { LOG.warn("Empty or null audit events batch received from service: {}, user: {}", serviceName, authenticatedUser); ret = Response.status(Response.Status.BAD_REQUEST) .entity(buildErrorResponse("Audit events cannot be empty")) @@ -210,17 +206,14 @@ public Response accessAudit(@QueryParam("serviceName") String serviceName, Strin .build(); } else { try { - ObjectMapper mapper = MiscUtil.getMapper(); - List events = mapper.readValue(accessAudits, AUDIT_EVENT_LIST_TYPE); + LOG.debug("Processing {} audit events from service: {}", accessAudits.size(), serviceName); - LOG.debug("Successfully deserialized {} audit events from service: {}", events.size(), serviceName); - - for (AuthzAuditEvent event : events) { + for (AuthzAuditEvent event : accessAudits) { auditDestinationMgr.log(event); } Map response = new HashMap<>(); - response.put("total", events.size()); + response.put("total", accessAudits.size()); response.put("timestamp", System.currentTimeMillis()); if (StringUtils.isNotEmpty(serviceName)) { response.put("serviceName", serviceName); diff --git a/ranger-audit-server/ranger-audit-server-service/src/main/webapp/WEB-INF/web.xml b/ranger-audit-server/ranger-audit-server-service/src/main/webapp/WEB-INF/web.xml index 06ff7bb7ec..612d595471 100644 --- a/ranger-audit-server/ranger-audit-server-service/src/main/webapp/WEB-INF/web.xml +++ b/ranger-audit-server/ranger-audit-server-service/src/main/webapp/WEB-INF/web.xml @@ -57,6 +57,11 @@ org.apache.ranger.audit.rest + + com.sun.jersey.api.json.POJOMappingFeature + true + + 1 From 24cebf3e1a8d455f6db6c767a91d292adee41e40 Mon Sep 17 00:00:00 2001 From: Ramesh Mani Date: Tue, 24 Feb 2026 11:48:13 -0800 Subject: [PATCH 08/12] RANGER-5482:Create Ranger Audit Server with SOLR and HDFS as audit consumer - Fix review comments set #2 --- .../ranger/plugin/policyengine/RangerAccessResult.java | 8 +------- .../main/java/org/apache/ranger/audit/rest/AuditREST.java | 2 +- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerAccessResult.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerAccessResult.java index 32a0aaab7a..8088b022f9 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerAccessResult.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerAccessResult.java @@ -260,13 +260,7 @@ public int getServiceType() { } public String getServiceTypeName() { - String ret = null; - - if (serviceDef != null && serviceDef.getName() != null) { - ret = serviceDef.getName(); - } - - return ret; + return serviceDef != null ? serviceDef.getName() : null; } public Map getAdditionalInfo() { diff --git a/ranger-audit-server/ranger-audit-server-service/src/main/java/org/apache/ranger/audit/rest/AuditREST.java b/ranger-audit-server/ranger-audit-server-service/src/main/java/org/apache/ranger/audit/rest/AuditREST.java index b68d8c052c..4d9aeec672 100644 --- a/ranger-audit-server/ranger-audit-server-service/src/main/java/org/apache/ranger/audit/rest/AuditREST.java +++ b/ranger-audit-server/ranger-audit-server-service/src/main/java/org/apache/ranger/audit/rest/AuditREST.java @@ -155,7 +155,7 @@ public Response getStatus() { @Path("/access") @Consumes("application/json") @Produces("application/json") - public Response accessAudit(@QueryParam("serviceName") String serviceName, List accessAudits, @Context HttpServletRequest request) { + public Response logAccessAudit(@QueryParam("serviceName") String serviceName, List accessAudits, @Context HttpServletRequest request) { String authenticatedUser = getAuthenticatedUser(request); LOG.debug("==> AuditREST.accessAudit(): received {} audit events from service: {}, authenticatedUser: {}", accessAudits != null ? accessAudits.size() : 0, StringUtils.isNotEmpty(serviceName) ? serviceName : "unknown", authenticatedUser); From 87d9c0c131ea94dc95bd251db4b17bf0fde4ef7b Mon Sep 17 00:00:00 2001 From: Ramesh Mani Date: Wed, 25 Feb 2026 12:18:42 -0800 Subject: [PATCH 09/12] RANGER-5482:Create Ranger Audit Server with SOLR and HDFS as audit consumer - Audit Batch processing and failure reprocessing improvement --- .../ranger/audit/provider/AuditHandler.java | 2 + .../audit/provider/BaseAuditHandler.java | 5 + .../audit/provider/DummyAuditProvider.java | 5 + .../RangerAuditServerDestination.java | 16 +++ .../plugin/service/RangerBasePlugin.java | 7 +- .../audit/producer/AuditDestinationMgr.java | 35 ++++++ .../producer/kafka/AuditMessageQueue.java | 116 +++++++++++++++++- .../audit/producer/kafka/AuditProducer.java | 106 ++++++++++++++++ .../apache/ranger/audit/rest/AuditREST.java | 52 ++++---- 9 files changed, 320 insertions(+), 24 deletions(-) diff --git a/agents-audit/core/src/main/java/org/apache/ranger/audit/provider/AuditHandler.java b/agents-audit/core/src/main/java/org/apache/ranger/audit/provider/AuditHandler.java index 8c51539d51..b95a5152aa 100644 --- a/agents-audit/core/src/main/java/org/apache/ranger/audit/provider/AuditHandler.java +++ b/agents-audit/core/src/main/java/org/apache/ranger/audit/provider/AuditHandler.java @@ -29,6 +29,8 @@ public interface AuditHandler { boolean log(Collection events); + boolean log(Collection events, String batchKey); + boolean logJSON(String event); boolean logJSON(Collection events); diff --git a/agents-audit/core/src/main/java/org/apache/ranger/audit/provider/BaseAuditHandler.java b/agents-audit/core/src/main/java/org/apache/ranger/audit/provider/BaseAuditHandler.java index 4dc35cb458..ab1c54f2c1 100644 --- a/agents-audit/core/src/main/java/org/apache/ranger/audit/provider/BaseAuditHandler.java +++ b/agents-audit/core/src/main/java/org/apache/ranger/audit/provider/BaseAuditHandler.java @@ -107,6 +107,11 @@ public boolean log(AuditEventBase event) { return log(Collections.singletonList(event)); } + @Override + public boolean log(Collection events, String batchKey) { + return log(events); + } + /* * (non-Javadoc) * diff --git a/agents-audit/core/src/main/java/org/apache/ranger/audit/provider/DummyAuditProvider.java b/agents-audit/core/src/main/java/org/apache/ranger/audit/provider/DummyAuditProvider.java index d9af9ce5cd..9a3e3798ec 100644 --- a/agents-audit/core/src/main/java/org/apache/ranger/audit/provider/DummyAuditProvider.java +++ b/agents-audit/core/src/main/java/org/apache/ranger/audit/provider/DummyAuditProvider.java @@ -40,6 +40,11 @@ public boolean log(Collection events) { return true; } + @Override + public boolean log(Collection events, String batchKey) { + return log(events); + } + @Override public boolean logJSON(String event) { AuditEventBase eventObj = MiscUtil.fromJson(event, AuthzAuditEvent.class); diff --git a/agents-audit/dest-auditserver/src/main/java/org/apache/ranger/audit/destination/RangerAuditServerDestination.java b/agents-audit/dest-auditserver/src/main/java/org/apache/ranger/audit/destination/RangerAuditServerDestination.java index 625651cfcf..23cfea49c4 100644 --- a/agents-audit/dest-auditserver/src/main/java/org/apache/ranger/audit/destination/RangerAuditServerDestination.java +++ b/agents-audit/dest-auditserver/src/main/java/org/apache/ranger/audit/destination/RangerAuditServerDestination.java @@ -55,8 +55,10 @@ public class RangerAuditServerDestination extends AuditDestination { public static final String PROP_AUDITSERVER_MAX_RETRY_ATTEMPTS = "xasecure.audit.destination.auditserver.max.retry.attempts"; public static final String PROP_AUDITSERVER_RETRY_INTERVAL_MS = "xasecure.audit.destination.auditserver.retry.interval.ms"; public static final String PROP_SERVICE_TYPE = "ranger.plugin.audit.service.type"; + public static final String PROP_APP_ID = "ranger.plugin.audit.app.id"; public static final String REST_RELATIVE_PATH_POST = "/api/audit/access"; public static final String QUERY_PARAM_SERVICE_NAME = "serviceName"; + public static final String QUERY_PARAM_APP_ID = "appId"; // Authentication types public static final String AUTH_TYPE_KERBEROS = "kerberos"; @@ -67,6 +69,7 @@ public class RangerAuditServerDestination extends AuditDestination { private RangerRESTClient restClient; private String authType; private String serviceType; + private String appId; @Override public void init(Properties props, String propPrefix) { @@ -84,6 +87,7 @@ public void init(Properties props, String propPrefix) { this.authType = MiscUtil.getStringProperty(props, PROP_AUDITSERVER_AUTH_TYPE); this.serviceType = MiscUtil.getStringProperty(props, PROP_SERVICE_TYPE); + this.appId = MiscUtil.getStringProperty(props, PROP_APP_ID); if (StringUtils.isEmpty(authType)) { // Authentication priority: JWT → Kerberos → Basic @@ -110,6 +114,12 @@ public void init(Properties props, String propPrefix) { LOG.info("Audit destination configured for service type: {}", this.serviceType); } + if (StringUtils.isNotEmpty(this.appId)) { + LOG.info("Audit destination configured with appId: {}", this.appId); + } else { + LOG.warn("appId not available in audit properties. Batch processing may not be optimal."); + } + if (AUTH_TYPE_KERBEROS.equalsIgnoreCase(authType)) { preAuthenticateKerberos(); } @@ -267,6 +277,12 @@ private boolean sendBatch(Collection events) { return false; } + // Add appId to query parameters for batch processing + if (StringUtils.isNotEmpty(appId)) { + queryParams.put(QUERY_PARAM_APP_ID, appId); + LOG.debug("Adding appId={} to audit request for batch processing", appId); + } + try { final UserGroupInformation user = MiscUtil.getUGILoginUser(); final boolean isSecureMode = isKerberosAuthenticated(); diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java b/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java index a1b49db0ea..c3d1fef960 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java @@ -407,7 +407,12 @@ public void init() { auditProps.setProperty("ranger.plugin.audit.service.type", serviceType); LOG.info("Added serviceType={} to audit properties for audit destination", serviceType); } - providerFactory.init(auditProps, getAppId()); + String appId = getAppId(); + if (StringUtils.isNotEmpty(appId)) { + auditProps.setProperty("ranger.plugin.audit.app.id", appId); + LOG.info("Added appId={} to audit properties for audit destination", appId); + } + providerFactory.init(auditProps, appId); } else { LOG.error("Audit subsystem is not initialized correctly. Please check audit configuration. "); LOG.error("No authorization audits will be generated. "); diff --git a/ranger-audit-server/ranger-audit-server-service/src/main/java/org/apache/ranger/audit/producer/AuditDestinationMgr.java b/ranger-audit-server/ranger-audit-server-service/src/main/java/org/apache/ranger/audit/producer/AuditDestinationMgr.java index 73cdc0428c..90aa0b4ab9 100644 --- a/ranger-audit-server/ranger-audit-server-service/src/main/java/org/apache/ranger/audit/producer/AuditDestinationMgr.java +++ b/ranger-audit-server/ranger-audit-server-service/src/main/java/org/apache/ranger/audit/producer/AuditDestinationMgr.java @@ -19,6 +19,7 @@ package org.apache.ranger.audit.producer; +import org.apache.ranger.audit.model.AuditEventBase; import org.apache.ranger.audit.model.AuthzAuditEvent; import org.apache.ranger.audit.producer.kafka.AuditMessageQueue; import org.apache.ranger.audit.provider.AuditHandler; @@ -34,6 +35,9 @@ import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; import java.util.Properties; @Component @@ -83,6 +87,37 @@ public boolean log(AuthzAuditEvent authzAuditEvent) throws Exception { return ret; } + /** + * @param events List of audit events to process as a batch + * @param appId The application ID for this batch (used as Kafka partition key) + * @return true if batch was processed successfully, false otherwise + * @throws Exception if processing fails + */ + public boolean logBatch(List events, String appId) throws Exception { + boolean ret = false; + + if (auditHandler == null) { + init(); + } + + if (events == null || events.isEmpty()) { + LOG.warn("Empty event list provided to logBatch"); + return true; + } + + LOG.debug("Processing batch of {} events with appId: {}", events.size(), appId); + + Collection baseEvents = new ArrayList<>(events); + + ret = auditHandler.log(baseEvents, appId); + + if (!ret) { + LOG.error("Batch processing failed for {} events with appId: {}. Events have been spooled to recovery system.", events.size(), appId); + } + + return ret; + } + @PreDestroy public void shutdown() { LOG.info("==> AuditDestinationMgr.shutdown()"); diff --git a/ranger-audit-server/ranger-audit-server-service/src/main/java/org/apache/ranger/audit/producer/kafka/AuditMessageQueue.java b/ranger-audit-server/ranger-audit-server-service/src/main/java/org/apache/ranger/audit/producer/kafka/AuditMessageQueue.java index a80ac77978..cbc0df7fda 100644 --- a/ranger-audit-server/ranger-audit-server-service/src/main/java/org/apache/ranger/audit/producer/kafka/AuditMessageQueue.java +++ b/ranger-audit-server/ranger-audit-server-service/src/main/java/org/apache/ranger/audit/producer/kafka/AuditMessageQueue.java @@ -29,7 +29,9 @@ import org.slf4j.LoggerFactory; import java.security.PrivilegedExceptionAction; +import java.util.ArrayList; import java.util.Collection; +import java.util.List; import java.util.Properties; /** @@ -178,8 +180,118 @@ Partition key is agentId (aka plugin ID). AuditPartitioner allocates configured } @Override - public boolean log(final Collection events) { - return false; + public synchronized boolean log(final Collection events) { + return log(events, null); + } + + @Override + public synchronized boolean log(final Collection events, String batchKey) { + if (events == null || events.isEmpty()) { + return true; + } + + LOG.debug("==> AuditMessageQueue.log(Collection, batchKey): Processing batch of {} events with explicit batchKey: {}", events.size(), batchKey); + + boolean allSuccess = true; + int successCount = 0; + int failCount = 0; + + // Prepare batch data - all events use the SAME appId key for batch commit + List authzEvents = new ArrayList<>(); + List messages = new ArrayList<>(); + + for (AuditEventBase event : events) { + if (event instanceof AuthzAuditEvent) { + AuthzAuditEvent authzEvent = (AuthzAuditEvent) event; + + if (authzEvent.getAgentHostname() == null) { + authzEvent.setAgentHostname(MiscUtil.getHostname()); + } + + if (authzEvent.getLogType() == null) { + authzEvent.setLogType("RangerAudit"); + } + + if (authzEvent.getEventId() == null) { + authzEvent.setEventId(MiscUtil.generateUniqueId()); + } + + // If batchKey not provided, use the first event's agentId as the batch key + if (batchKey == null) { + batchKey = authzEvent.getAgentId(); + LOG.debug("Using first event's agentId as batch key: {}", batchKey); + } + + authzEvents.add(authzEvent); + messages.add(MiscUtil.stringify(event)); + } + } + + if (authzEvents.isEmpty()) { + LOG.warn("No valid AuthzAuditEvent found in batch"); + return false; + } + + if (batchKey == null || batchKey.isEmpty()) { + LOG.warn("Batch key (appId) is null or empty. Using default key."); + batchKey = "unknown-appId"; + } + + LOG.debug("Batch of {} events will be committed with batch key (appId): {}", authzEvents.size(), batchKey); + + try { + if (topicName == null || kafkaProducer == null) { + init(props, propPrefix); + } + + if (kafkaProducer != null) { + // Send entire batch to Kafka with same batch key (appId) for all events + // With custom partitioning: events distributed round-robin for load balancing + // Without custom partitioning: all events to same partition for ordering + final String finalBatchKey = batchKey; + MiscUtil.executePrivilegedAction((PrivilegedExceptionAction) () -> { + AuditProducer.sendBatch(kafkaProducer, topicName, finalBatchKey, messages); + return null; + }); + successCount = authzEvents.size(); + allSuccess = true; + LOG.debug("ranger-audit-server/ranger-audit-server-service/src/main/java/org/apache/ranger/audit/producer/kafka/AuditMessageQueue.javaSuccessfully sent batch of {} events to Kafka topic: {} with key: {}", successCount, topicName, finalBatchKey); + } else { + LOG.warn("Kafka producer not available, spooling batch of {} messages to recovery", authzEvents.size()); + for (String message : messages) { + spoolToRecovery(batchKey, message); + } + failCount = authzEvents.size(); + allSuccess = false; + } + } catch (AuditProducer.BatchSendException bse) { + // Partial failure - only retry the failed messages to avoid duplicates + List failedMessages = bse.getFailedMessages(); + int failedCount = failedMessages.size(); + int succeededCount = authzEvents.size() - failedCount; + + LOG.error("Partial batch failure: {}/{} events failed, {} succeeded. Spooling only failed events to recovery.", failedCount, authzEvents.size(), succeededCount); + + for (String message : failedMessages) { + spoolToRecovery(batchKey, message); + } + + successCount = succeededCount; + failCount = failedCount; + allSuccess = false; + } catch (Throwable t) { + // Complete failure (timeout, connection error, etc.) - retry entire batch + LOG.error("Complete batch failure for {} events. Spooling entire batch to recovery. Error: {}", authzEvents.size(), t.getMessage()); + for (String message : messages) { + spoolToRecovery(batchKey, message); + } + failCount = authzEvents.size(); + allSuccess = false; + } + + LOG.debug("<== AuditMessageQueue.log(Collection, batchKey): successCount={}, failCount={}", successCount, failCount); + + return allSuccess; } private void startRangerAuditRecoveryThread() { diff --git a/ranger-audit-server/ranger-audit-server-service/src/main/java/org/apache/ranger/audit/producer/kafka/AuditProducer.java b/ranger-audit-server/ranger-audit-server-service/src/main/java/org/apache/ranger/audit/producer/kafka/AuditProducer.java index 879424a6e1..25abbb9b27 100644 --- a/ranger-audit-server/ranger-audit-server-service/src/main/java/org/apache/ranger/audit/producer/kafka/AuditProducer.java +++ b/ranger-audit-server/ranger-audit-server-service/src/main/java/org/apache/ranger/audit/producer/kafka/AuditProducer.java @@ -32,7 +32,12 @@ import org.slf4j.LoggerFactory; import java.security.PrivilegedExceptionAction; +import java.util.ArrayList; +import java.util.List; import java.util.Properties; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; public class AuditProducer implements Runnable { private static final Logger LOG = LoggerFactory.getLogger(AuditProducer.class); @@ -150,4 +155,105 @@ public void onCompletion(RecordMetadata metadata, Exception e) { throw new Exception(e); } } + + /** + * Send a batch of audit events to Kafka efficiently using a batch key for all events. + * + * @param producer Kafka producer instance + * @param topic Topic to send to + * @param batchKey Single key (appId) to use for ALL events in the batch + * @param values List of serialized event messages + * @throws Exception if batch send fails + */ + public static void sendBatch(KafkaProducer producer, String topic, String batchKey, List values) throws Exception { + int batchSize = values.size(); + LOG.debug("==> AuditProducer.sendBatch(): Sending batch of {} events to topic: {} with single key: {}", batchSize, topic, batchKey); + + final CountDownLatch latch = new CountDownLatch(batchSize); + final AtomicInteger errorCount = new AtomicInteger(0); + final List errors = new ArrayList<>(); + final List failedIndices = new ArrayList<>(); + final List failedValues = new ArrayList<>(); + + // Send all records asynchronously with the same batch key + // With custom partitioning: events distributed round-robin across partition range + // Without custom partitioning: all events go to same partition (hash-based) + for (int i = 0; i < batchSize; i++) { + final String value = values.get(i); + final int index = i; + + ProducerRecord record = new ProducerRecord<>(topic, batchKey, value); + + try { + producer.send(record, new Callback() { + @Override + public void onCompletion(RecordMetadata metadata, Exception e) { + if (e != null) { + errorCount.incrementAndGet(); + synchronized (errors) { + errors.add(e); + failedIndices.add(index); + failedValues.add(value); + } + LOG.error("Error sending audit event {} in batch to Kafka: {}", index, e.getMessage()); + } else { + LOG.debug("Batch event {} sent to Topic: {} Partition: {} Offset: {}", index, metadata.topic(), metadata.partition(), metadata.offset()); + } + latch.countDown(); + } + }); + } catch (Exception e) { + errorCount.incrementAndGet(); + synchronized (errors) { + errors.add(e); + failedIndices.add(i); + failedValues.add(value); + } + LOG.error("Failed to queue audit event {} for sending: {}", i, e.getMessage()); + latch.countDown(); + } + } + + // max wait time before timeout + boolean completed = latch.await(30, TimeUnit.SECONDS); + + if (!completed) { + String errorMsg = String.format("Batch send timed out after 30 seconds. %d/%d events still pending", latch.getCount(), batchSize); + LOG.error(errorMsg); + throw new Exception(errorMsg); + } + + if (errorCount.get() > 0) { + int successCount = batchSize - errorCount.get(); + String errorMsg = String.format("Batch send had %d/%d failures, %d succeeded", errorCount.get(), batchSize, successCount); + LOG.error(errorMsg); + LOG.error("Failed event indices: {}", failedIndices); + + // Create exception with failed values for selective retry + BatchSendException batchException = new BatchSendException(errorMsg, failedValues); + if (!errors.isEmpty()) { + batchException.initCause(errors.get(0)); + } + throw batchException; + } + + LOG.debug("<== AuditProducer.sendBatch(): Successfully sent batch of {} events to Kafka topic: {}, key: {}", batchSize, topic, batchKey); + } + + /** + * Custom exception that carries the list of failed messages for selective retry. + * This prevents duplicates by only retrying the messages that actually failed. + */ + public static class BatchSendException extends Exception { + private final List failedMessages; + + public BatchSendException(String message, List failedMessages) { + super(message); + this.failedMessages = new ArrayList<>(failedMessages); + } + + public List getFailedMessages() { + return failedMessages; + } + } } diff --git a/ranger-audit-server/ranger-audit-server-service/src/main/java/org/apache/ranger/audit/rest/AuditREST.java b/ranger-audit-server/ranger-audit-server-service/src/main/java/org/apache/ranger/audit/rest/AuditREST.java index 4d9aeec672..6815815831 100644 --- a/ranger-audit-server/ranger-audit-server-service/src/main/java/org/apache/ranger/audit/rest/AuditREST.java +++ b/ranger-audit-server/ranger-audit-server-service/src/main/java/org/apache/ranger/audit/rest/AuditREST.java @@ -149,16 +149,18 @@ public Response getStatus() { /** * Access Audits producer endpoint. * @param serviceName Required query parameter to identify the source service (hdfs, hive, kafka, solr, etc.) + * @param appId Optional query parameter for batch processing - identifies the application instance + * @param accessAudits List of audit events to process * @param request HTTP request to extract authenticated user */ @POST @Path("/access") @Consumes("application/json") @Produces("application/json") - public Response logAccessAudit(@QueryParam("serviceName") String serviceName, List accessAudits, @Context HttpServletRequest request) { + public Response logAccessAudit(@QueryParam("serviceName") String serviceName, @QueryParam("appId") String appId, List accessAudits, @Context HttpServletRequest request) { String authenticatedUser = getAuthenticatedUser(request); - LOG.debug("==> AuditREST.accessAudit(): received {} audit events from service: {}, authenticatedUser: {}", accessAudits != null ? accessAudits.size() : 0, StringUtils.isNotEmpty(serviceName) ? serviceName : "unknown", authenticatedUser); + LOG.debug("==> AuditREST.accessAudit(): received {} audit events from service: {}, appId: {}, authenticatedUser: {}", accessAudits != null ? accessAudits.size() : 0, StringUtils.isNotEmpty(serviceName) ? serviceName : "unknown", StringUtils.isNotEmpty(appId) ? appId : "none", authenticatedUser); Response ret; @@ -206,27 +208,35 @@ public Response logAccessAudit(@QueryParam("serviceName") String serviceName, Li .build(); } else { try { - LOG.debug("Processing {} audit events from service: {}", accessAudits.size(), serviceName); - - for (AuthzAuditEvent event : accessAudits) { - auditDestinationMgr.log(event); - } - - Map response = new HashMap<>(); - response.put("total", accessAudits.size()); - response.put("timestamp", System.currentTimeMillis()); - if (StringUtils.isNotEmpty(serviceName)) { - response.put("serviceName", serviceName); + LOG.debug("Processing {} audit events from service: {}, appId: {}", accessAudits.size(), serviceName, appId); + + boolean success = auditDestinationMgr.logBatch(accessAudits, appId); + + if (success) { + Map response = new HashMap<>(); + response.put("total", accessAudits.size()); + response.put("timestamp", System.currentTimeMillis()); + if (StringUtils.isNotEmpty(serviceName)) { + response.put("serviceName", serviceName); + } + if (StringUtils.isNotEmpty(appId)) { + response.put("appId", appId); + } + if (StringUtils.isNotEmpty(authenticatedUser)) { + response.put("authenticatedUser", authenticatedUser); + } + String jsonString = buildResponse(response); + ret = Response.status(Response.Status.OK) + .entity(jsonString) + .build(); + } else { + LOG.warn("Batch processing failed for {} events from service: {}, appId: {}. Events spooled to recovery.", accessAudits.size(), serviceName, appId); + ret = Response.status(Response.Status.ACCEPTED) + .entity(buildErrorResponse("Batch processing failed. Events have been queued for retry.")) + .build(); } - if (StringUtils.isNotEmpty(authenticatedUser)) { - response.put("authenticatedUser", authenticatedUser); - } - String jsonString = buildResponse(response); - ret = Response.status(Response.Status.OK) - .entity(jsonString) - .build(); } catch (Exception e) { - LOG.error("Error processing access audits batch from service: {}", serviceName, e); + LOG.error("Error processing access audits batch from service: {}, appId: {}", serviceName, appId, e); ret = Response.status(Response.Status.BAD_REQUEST) .entity(buildErrorResponse("Failed to process audit events: " + e.getMessage())) .build(); From 63c94056493f3231e4f9035bf35e15740b289f37 Mon Sep 17 00:00:00 2001 From: Ramesh Mani Date: Wed, 25 Feb 2026 16:36:52 -0800 Subject: [PATCH 10/12] RANGER-5482:Create Ranger Audit Server with SOLR and HDFS as audit consumer - Fix duplicate dependency error in the pom for sl4j --- ranger-audit-server/pom.xml | 6 ------ ranger-audit-server/ranger-audit-server-service/pom.xml | 6 ------ 2 files changed, 12 deletions(-) diff --git a/ranger-audit-server/pom.xml b/ranger-audit-server/pom.xml index 1652dc11a7..1683ede6a6 100644 --- a/ranger-audit-server/pom.xml +++ b/ranger-audit-server/pom.xml @@ -351,12 +351,6 @@ ${slf4j.version} - - org.slf4j - log4j-over-slf4j - ${slf4j.version} - - org.slf4j slf4j-api diff --git a/ranger-audit-server/ranger-audit-server-service/pom.xml b/ranger-audit-server/ranger-audit-server-service/pom.xml index dc1d77c438..202df8161e 100644 --- a/ranger-audit-server/ranger-audit-server-service/pom.xml +++ b/ranger-audit-server/ranger-audit-server-service/pom.xml @@ -336,12 +336,6 @@ ${tomcat.embed.version} - - org.slf4j - log4j-over-slf4j - ${slf4j.version} - - org.slf4j From 4536899d60b25215f7e917532c34a2674d3249e2 Mon Sep 17 00:00:00 2001 From: Ramesh Mani Date: Fri, 27 Feb 2026 12:43:01 -0800 Subject: [PATCH 11/12] RANGER-5482:Create Ranger Audit Server with SOLR and HDFS as audit consumer - Fix ubuntu audit ranger module war file creation failure --- distro/pom.xml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/distro/pom.xml b/distro/pom.xml index cb8277021a..edae8c9684 100644 --- a/distro/pom.xml +++ b/distro/pom.xml @@ -495,13 +495,13 @@ src/main/assembly/plugin-sqoop.xml src/main/assembly/plugin-kylin.xml src/main/assembly/plugin-elasticsearch.xml + src/main/assembly/ranger-audit-server-service.xml + src/main/assembly/ranger-audit-consumer-solr.xml + src/main/assembly/ranger-audit-consumer-hdfs.xml src/main/assembly/plugin-schema-registry.xml src/main/assembly/plugin-presto.xml src/main/assembly/plugin-trino.xml src/main/assembly/sample-client.xml - src/main/assembly/ranger-audit-server-service.xml - src/main/assembly/ranger-audit-consumer-solr.xml - src/main/assembly/ranger-audit-consumer-hdfs.xml @@ -1083,8 +1083,13 @@ src/main/assembly/plugin-sqoop.xml src/main/assembly/plugin-kylin.xml src/main/assembly/plugin-elasticsearch.xml + src/main/assembly/ranger-audit-server-service.xml + src/main/assembly/ranger-audit-consumer-solr.xml + src/main/assembly/ranger-audit-consumer-hdfs.xml src/main/assembly/plugin-schema-registry.xml src/main/assembly/plugin-presto.xml + src/main/assembly/plugin-trino.xml + src/main/assembly/sample-client.xml From 01a1ec8272315336f585ea5481726959acb35a0f Mon Sep 17 00:00:00 2001 From: Ramesh Mani Date: Thu, 5 Mar 2026 18:46:45 -0800 Subject: [PATCH 12/12] RANGER-5482:Create Ranger Audit Server with SOLR and HDFS as audit consumer - Fix Review comments set #3 --- agents-audit/core/pom.xml | 5 -- .../ranger/audit/model/AuthzAuditEvent.java | 6 -- .../ranger/audit/provider/AuditHandler.java | 4 +- .../audit/provider/BaseAuditHandler.java | 5 -- .../audit/provider/DummyAuditProvider.java | 5 -- agents-audit/dest-auditserver/pom.xml | 4 -- .../RangerAuditServerDestination.java | 69 ++++++++++++++----- .../audit/RangerDefaultAuditHandler.java | 2 +- .../plugin/service/RangerBasePlugin.java | 6 +- .../audit/TestRangerDefaultAuditHandler.java | 2 +- .../Dockerfile.ranger-audit-server | 2 +- .../scripts/admin/create-ranger-services.py | 3 +- .../audit/server/AuditServerConstants.java | 1 - .../audit/consumer/kafka/AuditRouterHDFS.java | 7 +- .../apache/ranger/audit/rest/AuditREST.java | 36 ++++------ 15 files changed, 82 insertions(+), 75 deletions(-) diff --git a/agents-audit/core/pom.xml b/agents-audit/core/pom.xml index e6c8c37926..d11b3bf7c8 100644 --- a/agents-audit/core/pom.xml +++ b/agents-audit/core/pom.xml @@ -63,11 +63,6 @@ - - javax.xml.bind - jaxb-api - ${jaxb.api.version} - joda-time joda-time diff --git a/agents-audit/core/src/main/java/org/apache/ranger/audit/model/AuthzAuditEvent.java b/agents-audit/core/src/main/java/org/apache/ranger/audit/model/AuthzAuditEvent.java index e27a45de0b..de91ba1644 100644 --- a/agents-audit/core/src/main/java/org/apache/ranger/audit/model/AuthzAuditEvent.java +++ b/agents-audit/core/src/main/java/org/apache/ranger/audit/model/AuthzAuditEvent.java @@ -27,10 +27,6 @@ import com.fasterxml.jackson.databind.annotation.JsonSerialize; import org.apache.commons.lang3.StringUtils; -import javax.xml.bind.annotation.XmlAccessType; -import javax.xml.bind.annotation.XmlAccessorType; -import javax.xml.bind.annotation.XmlRootElement; - import java.util.Date; import java.util.HashSet; import java.util.Set; @@ -39,8 +35,6 @@ @JsonInclude(JsonInclude.Include.NON_EMPTY) @JsonIgnoreProperties(ignoreUnknown = true) @JsonSerialize -@XmlRootElement -@XmlAccessorType(XmlAccessType.FIELD) public class AuthzAuditEvent extends AuditEventBase { protected static final int MAX_ACTION_FIELD_SIZE = 1800; protected static final int MAX_REQUEST_DATA_FIELD_SIZE = 1800; diff --git a/agents-audit/core/src/main/java/org/apache/ranger/audit/provider/AuditHandler.java b/agents-audit/core/src/main/java/org/apache/ranger/audit/provider/AuditHandler.java index b95a5152aa..c748054957 100644 --- a/agents-audit/core/src/main/java/org/apache/ranger/audit/provider/AuditHandler.java +++ b/agents-audit/core/src/main/java/org/apache/ranger/audit/provider/AuditHandler.java @@ -29,7 +29,9 @@ public interface AuditHandler { boolean log(Collection events); - boolean log(Collection events, String batchKey); + default boolean log(Collection events, String batchKey) { + return log(events); + } boolean logJSON(String event); diff --git a/agents-audit/core/src/main/java/org/apache/ranger/audit/provider/BaseAuditHandler.java b/agents-audit/core/src/main/java/org/apache/ranger/audit/provider/BaseAuditHandler.java index ab1c54f2c1..4dc35cb458 100644 --- a/agents-audit/core/src/main/java/org/apache/ranger/audit/provider/BaseAuditHandler.java +++ b/agents-audit/core/src/main/java/org/apache/ranger/audit/provider/BaseAuditHandler.java @@ -107,11 +107,6 @@ public boolean log(AuditEventBase event) { return log(Collections.singletonList(event)); } - @Override - public boolean log(Collection events, String batchKey) { - return log(events); - } - /* * (non-Javadoc) * diff --git a/agents-audit/core/src/main/java/org/apache/ranger/audit/provider/DummyAuditProvider.java b/agents-audit/core/src/main/java/org/apache/ranger/audit/provider/DummyAuditProvider.java index 9a3e3798ec..d9af9ce5cd 100644 --- a/agents-audit/core/src/main/java/org/apache/ranger/audit/provider/DummyAuditProvider.java +++ b/agents-audit/core/src/main/java/org/apache/ranger/audit/provider/DummyAuditProvider.java @@ -40,11 +40,6 @@ public boolean log(Collection events) { return true; } - @Override - public boolean log(Collection events, String batchKey) { - return log(events); - } - @Override public boolean logJSON(String event) { AuditEventBase eventObj = MiscUtil.fromJson(event, AuthzAuditEvent.class); diff --git a/agents-audit/dest-auditserver/pom.xml b/agents-audit/dest-auditserver/pom.xml index 5fa95e1845..75e9b73538 100644 --- a/agents-audit/dest-auditserver/pom.xml +++ b/agents-audit/dest-auditserver/pom.xml @@ -37,10 +37,6 @@ jackson-jaxrs-json-provider ${fasterxml.jackson.version} - - com.google.code.gson - gson - org.apache.ranger ranger-audit-core diff --git a/agents-audit/dest-auditserver/src/main/java/org/apache/ranger/audit/destination/RangerAuditServerDestination.java b/agents-audit/dest-auditserver/src/main/java/org/apache/ranger/audit/destination/RangerAuditServerDestination.java index 23cfea49c4..61a8360940 100644 --- a/agents-audit/dest-auditserver/src/main/java/org/apache/ranger/audit/destination/RangerAuditServerDestination.java +++ b/agents-audit/dest-auditserver/src/main/java/org/apache/ranger/audit/destination/RangerAuditServerDestination.java @@ -21,12 +21,15 @@ import com.sun.jersey.api.client.ClientResponse; import com.sun.jersey.api.client.WebResource; +import org.apache.commons.collections.MapUtils; import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; import org.apache.ranger.audit.model.AuditEventBase; +import org.apache.ranger.audit.model.AuthzAuditEvent; import org.apache.ranger.audit.provider.MiscUtil; +import org.apache.ranger.authorization.utils.JsonUtils; import org.apache.ranger.plugin.util.RangerRESTClient; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -39,6 +42,7 @@ import java.security.PrivilegedExceptionAction; import java.util.Collection; import java.util.HashMap; +import java.util.Iterator; import java.util.Map; import java.util.Properties; @@ -54,10 +58,8 @@ public class RangerAuditServerDestination extends AuditDestination { public static final String PROP_AUDITSERVER_SSL_CONFIG_FILE = "xasecure.audit.destination.auditserver.ssl.config.file"; public static final String PROP_AUDITSERVER_MAX_RETRY_ATTEMPTS = "xasecure.audit.destination.auditserver.max.retry.attempts"; public static final String PROP_AUDITSERVER_RETRY_INTERVAL_MS = "xasecure.audit.destination.auditserver.retry.interval.ms"; - public static final String PROP_SERVICE_TYPE = "ranger.plugin.audit.service.type"; - public static final String PROP_APP_ID = "ranger.plugin.audit.app.id"; public static final String REST_RELATIVE_PATH_POST = "/api/audit/access"; - public static final String QUERY_PARAM_SERVICE_NAME = "serviceName"; + public static final String QUERY_PARAM_SERVICE_TYPE = "serviceType"; public static final String QUERY_PARAM_APP_ID = "appId"; // Authentication types @@ -86,8 +88,6 @@ public void init(Properties props, String propPrefix) { int retryIntervalMs = MiscUtil.getIntProperty(props, PROP_AUDITSERVER_RETRY_INTERVAL_MS, 1000); this.authType = MiscUtil.getStringProperty(props, PROP_AUDITSERVER_AUTH_TYPE); - this.serviceType = MiscUtil.getStringProperty(props, PROP_SERVICE_TYPE); - this.appId = MiscUtil.getStringProperty(props, PROP_APP_ID); if (StringUtils.isEmpty(authType)) { // Authentication priority: JWT → Kerberos → Basic @@ -121,7 +121,7 @@ public void init(Properties props, String propPrefix) { } if (AUTH_TYPE_KERBEROS.equalsIgnoreCase(authType)) { - preAuthenticateKerberos(); + initKerberos(); } Configuration config = createConfigurationFromProperties(props, authType, userName, password); @@ -171,33 +171,33 @@ private String readJwtTokenFromFile(String tokenFile) throws IOException { } } - private void preAuthenticateKerberos() { - LOG.info("==> RangerAuditServerDestination:preAuthenticateKerberos()"); + private void initKerberos() { + LOG.info("==> RangerAuditServerDestination:initKerberos()"); try { UserGroupInformation ugi = UserGroupInformation.getLoginUser(); if (ugi == null) { - LOG.warn("No UserGroupInformation available for Kerberos pre-authentication"); + LOG.warn("No UserGroupInformation available for Kerberos authentication"); return; } if (!ugi.hasKerberosCredentials()) { - LOG.warn("User {} does not have Kerberos credentials for pre-authentication", ugi.getUserName()); + LOG.warn("User {} does not have Kerberos credentials for authentication", ugi.getUserName()); return; } - LOG.info("Pre-authenticating Kerberos for user: {}, authMethod: {}", ugi.getUserName(), ugi.getAuthenticationMethod()); + LOG.info("Kerberos authentication for user: {}, authMethod: {}", ugi.getUserName(), ugi.getAuthenticationMethod()); ugi.checkTGTAndReloginFromKeytab(); LOG.debug("TGT verified and refreshed if needed for user: {}", ugi.getUserName()); - LOG.info("Kerberos pre-authentication completed successfully"); + LOG.info("Kerberos authentication completed successfully"); } catch (Exception e) { - LOG.warn("Kerberos pre-authentication failed. First request will retry authentication", e); + LOG.warn("Kerberos authentication failed. First request will retry authentication", e); } - LOG.info("<== RangerAuditServerDestination:preAuthenticateKerberos()"); + LOG.info("<== RangerAuditServerDestination:initKerberos()"); } @Override @@ -267,20 +267,24 @@ private boolean sendBatch(Collection events) { boolean ret = false; Map queryParams = new HashMap<>(); - // Add serviceName to query parameters + // Add serviceType to query parameters + serviceType = fetchServiceType(events); if (StringUtils.isNotEmpty(serviceType)) { - queryParams.put(QUERY_PARAM_SERVICE_NAME, serviceType); - LOG.debug("Adding serviceName={} to audit request", serviceType); + queryParams.put(QUERY_PARAM_SERVICE_TYPE, serviceType); + LOG.debug("Adding serviceType={} to audit request", serviceType); + LOG.info("Adding serviceType={} to audit request", serviceType); } else { LOG.error("Cannot send audit batch: serviceType is not set. This indicates a configuration error."); - LOG.error("Audit server requires serviceName parameter. Please ensure RangerBasePlugin is properly initialized."); + LOG.error("Audit server requires serviceType parameter. Please ensure RangerBasePlugin is properly initialized."); return false; } // Add appId to query parameters for batch processing + appId = fetchAppId(events); if (StringUtils.isNotEmpty(appId)) { queryParams.put(QUERY_PARAM_APP_ID, appId); LOG.debug("Adding appId={} to audit request for batch processing", appId); + LOG.info("Adding appId={} to audit request for batch processing", appId); } try { @@ -344,6 +348,35 @@ private boolean sendBatch(Collection events) { return ret; } + private String fetchServiceType(Collection events) { + String ret = null; + + Iterator auditEventInterator = events.iterator(); + if (auditEventInterator.hasNext()) { + AuthzAuditEvent auditEvent = (AuthzAuditEvent) auditEventInterator.next(); + if (auditEvent != null) { + String additionalInfo = auditEvent.getAdditionalInfo(); + Map addInfoMap = JsonUtils.jsonToMapStringString(additionalInfo); + if (MapUtils.isNotEmpty(addInfoMap)) { + ret = addInfoMap.get("serviceType"); + } + } + } + + return ret; + } + + private String fetchAppId(Collection events) { + String ret = null; + + AuthzAuditEvent auditEvent = (AuthzAuditEvent) events.iterator().next(); + if (auditEvent != null) { + ret = auditEvent.getAgentId(); + } + + return ret; + } + private ClientResponse postAuditEvents(String relativeUrl, Map params, Collection events) throws Exception { LOG.debug("Posting {} audit events to {}", events.size(), relativeUrl); diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/audit/RangerDefaultAuditHandler.java b/agents-common/src/main/java/org/apache/ranger/plugin/audit/RangerDefaultAuditHandler.java index 6e20ebe7ab..a517310056 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/audit/RangerDefaultAuditHandler.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/audit/RangerDefaultAuditHandler.java @@ -232,7 +232,7 @@ public final Set getDatasetIds(RangerAccessRequest request) { } public String getAdditionalInfo(RangerAccessRequest request, RangerAccessResult result) { - String ret = org.apache.commons.lang3.StringUtils.EMPTY; + String ret = null; Map addInfomap = new HashMap<>(); if (!CollectionUtils.isEmpty(request.getForwardedAddresses())) { diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java b/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java index c3d1fef960..e2267189e6 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java @@ -401,8 +401,10 @@ public void init() { if (!providerFactory.isInitDone()) { if (pluginConfig.getProperties() != null) { - Properties auditProps = pluginConfig.getProperties(); - String serviceType = getServiceType(); + Properties baseProps = pluginConfig.getProperties(); + Properties auditProps = new Properties(); + auditProps.putAll(baseProps); + String serviceType = getServiceType(); if (StringUtils.isNotEmpty(serviceType)) { auditProps.setProperty("ranger.plugin.audit.service.type", serviceType); LOG.info("Added serviceType={} to audit properties for audit destination", serviceType); diff --git a/agents-common/src/test/java/org/apache/ranger/plugin/audit/TestRangerDefaultAuditHandler.java b/agents-common/src/test/java/org/apache/ranger/plugin/audit/TestRangerDefaultAuditHandler.java index 7fbe23909d..c7fcd613ba 100644 --- a/agents-common/src/test/java/org/apache/ranger/plugin/audit/TestRangerDefaultAuditHandler.java +++ b/agents-common/src/test/java/org/apache/ranger/plugin/audit/TestRangerDefaultAuditHandler.java @@ -292,7 +292,7 @@ public void test08_getAdditionalInfo_nullAndNonNull() { Assertions.assertTrue(handler.getAdditionalInfo(req, res).isEmpty()); RangerAccessRequestImpl req2 = new RangerAccessRequestImpl(); - RangerAccessResult res2 = new RangerAccessResult(0, "svc", null, req); + RangerAccessResult res2 = new RangerAccessResult(0, "svc", null, req2); req2.setRemoteIPAddress("10.1.1.1"); List fwd = new ArrayList<>(Arrays.asList("1.1.1.1")); req2.setForwardedAddresses(fwd); diff --git a/dev-support/ranger-docker/Dockerfile.ranger-audit-server b/dev-support/ranger-docker/Dockerfile.ranger-audit-server index e95dc5c7c5..5035e83b82 100644 --- a/dev-support/ranger-docker/Dockerfile.ranger-audit-server +++ b/dev-support/ranger-docker/Dockerfile.ranger-audit-server @@ -83,4 +83,4 @@ USER ranger # Start the audit server using the custom startup script WORKDIR /opt/ranger-audit-server -ENTRYPOINT ["/home/ranger/scripts/ranger-audit-server.sh"] +ENTRYPOINT ${RANGER_SCRIPTS}/ranger-audit-server.sh diff --git a/dev-support/ranger-docker/scripts/admin/create-ranger-services.py b/dev-support/ranger-docker/scripts/admin/create-ranger-services.py index e3dd0aa213..9a078b5a09 100644 --- a/dev-support/ranger-docker/scripts/admin/create-ranger-services.py +++ b/dev-support/ranger-docker/scripts/admin/create-ranger-services.py @@ -28,7 +28,8 @@ def service_not_exists(service): 'default-policy.1.policyItem.1.users': 'hive', 'default-policy.1.policyItem.1.accessTypes': 'read,write,execute', 'default-policy.2.name': 'ranger-audit-path', - 'default-policy.2.resource.path': '/ranger/audit/*', + 'default-policy.2.resource.path': '/ranger/audit/*,/ranger/audit', + 'default-policy.2.resource.path.is-recursive': 'true', 'default-policy.2.policyItem.1.users': 'rangerauditserver', 'default-policy.2.policyItem.1.accessTypes': 'read,write,execute', 'ranger.plugin.hdfs.policy.refresh.synchronous':'true'}}) diff --git a/ranger-audit-server/ranger-audit-common/src/main/java/org/apache/ranger/audit/server/AuditServerConstants.java b/ranger-audit-server/ranger-audit-common/src/main/java/org/apache/ranger/audit/server/AuditServerConstants.java index b930a246ed..b1a6ce5ea3 100644 --- a/ranger-audit-server/ranger-audit-common/src/main/java/org/apache/ranger/audit/server/AuditServerConstants.java +++ b/ranger-audit-server/ranger-audit-common/src/main/java/org/apache/ranger/audit/server/AuditServerConstants.java @@ -27,7 +27,6 @@ private AuditServerConstants() {} public static final String PROP_ALLOWED_USERS = "ranger.audit.service.allowed.users"; public static final String PROP_AUTH_TO_LOCAL = "ranger.audit.service.auth.to.local"; - public static final String DEFAULT_ALLOWED_USERS = "hdfs,hive,hbase,kafka,yarn,solr,knox,storm,atlas,nifi,ozone,kudu,presto,trino"; public static final String JAAS_KRB5_MODULE = "com.sun.security.auth.module.Krb5LoginModule required"; public static final String JAAS_USE_KEYTAB = "useKeyTab=true"; diff --git a/ranger-audit-server/ranger-audit-consumer-hdfs/src/main/java/org/apache/ranger/audit/consumer/kafka/AuditRouterHDFS.java b/ranger-audit-server/ranger-audit-consumer-hdfs/src/main/java/org/apache/ranger/audit/consumer/kafka/AuditRouterHDFS.java index 061894bdd5..b4b9612a7f 100644 --- a/ranger-audit-server/ranger-audit-consumer-hdfs/src/main/java/org/apache/ranger/audit/consumer/kafka/AuditRouterHDFS.java +++ b/ranger-audit-server/ranger-audit-consumer-hdfs/src/main/java/org/apache/ranger/audit/consumer/kafka/AuditRouterHDFS.java @@ -195,8 +195,11 @@ private HDFSAuditDestination createHDFSDestination(String appId, String serviceT String writerImpl = getWriterImplementation(fileType); // Configure directory properties - String baseDir = MiscUtil.getStringProperty(props, hdfsPropPrefix + ".dir", "/ranger/audit/" + serviceType); - String subDir = MiscUtil.getStringProperty(props, hdfsPropPrefix + ".subdir", appId + "/%time:yyyyMMdd%/"); + String baseDir = MiscUtil.getStringProperty(props, hdfsPropPrefix + ".dir", "/ranger/audit"); + + // Default: {serviceType}/{appId}/%time:yyyyMMdd%/ + String defaultSubDir = (serviceType != null && !serviceType.isEmpty()) ? serviceType + "/" + appId + "/%time:yyyyMMdd%/" : appId + "/%time:yyyyMMdd%/"; + String subDir = MiscUtil.getStringProperty(props, hdfsPropPrefix + ".subdir", defaultSubDir); // Set file extension based on file type String fileExtension = getFileExtension(fileType); diff --git a/ranger-audit-server/ranger-audit-server-service/src/main/java/org/apache/ranger/audit/rest/AuditREST.java b/ranger-audit-server/ranger-audit-server-service/src/main/java/org/apache/ranger/audit/rest/AuditREST.java index 6815815831..d589081142 100644 --- a/ranger-audit-server/ranger-audit-server-service/src/main/java/org/apache/ranger/audit/rest/AuditREST.java +++ b/ranger-audit-server/ranger-audit-server-service/src/main/java/org/apache/ranger/audit/rest/AuditREST.java @@ -148,7 +148,7 @@ public Response getStatus() { /** * Access Audits producer endpoint. - * @param serviceName Required query parameter to identify the source service (hdfs, hive, kafka, solr, etc.) + * @param serviceType Required query parameter to identify the source service (hdfs, hive, kafka, solr, etc.) * @param appId Optional query parameter for batch processing - identifies the application instance * @param accessAudits List of audit events to process * @param request HTTP request to extract authenticated user @@ -157,15 +157,15 @@ public Response getStatus() { @Path("/access") @Consumes("application/json") @Produces("application/json") - public Response logAccessAudit(@QueryParam("serviceName") String serviceName, @QueryParam("appId") String appId, List accessAudits, @Context HttpServletRequest request) { + public Response logAccessAudit(@QueryParam("serviceType") String serviceType, @QueryParam("appId") String appId, List accessAudits, @Context HttpServletRequest request) { String authenticatedUser = getAuthenticatedUser(request); - LOG.debug("==> AuditREST.accessAudit(): received {} audit events from service: {}, appId: {}, authenticatedUser: {}", accessAudits != null ? accessAudits.size() : 0, StringUtils.isNotEmpty(serviceName) ? serviceName : "unknown", StringUtils.isNotEmpty(appId) ? appId : "none", authenticatedUser); + LOG.debug("==> AuditREST.accessAudit(): received {} audit events from serviceType: {}, appId: {}, authenticatedUser: {}", accessAudits != null ? accessAudits.size() : 0, StringUtils.isNotEmpty(serviceType) ? serviceType : "unknown", StringUtils.isNotEmpty(appId) ? appId : "none", authenticatedUser); Response ret; - if (StringUtils.isEmpty(serviceName)) { - LOG.error("serviceName query parameter is required. Rejecting audit request."); + if (StringUtils.isEmpty(serviceType)) { + LOG.error("serviceType query parameter is required. Rejecting audit request."); ret = Response.status(Response.Status.BAD_REQUEST) .entity(buildErrorResponse("serviceName query parameter is required")) .build(); @@ -173,21 +173,13 @@ public Response logAccessAudit(@QueryParam("serviceName") String serviceName, @Q } if (StringUtils.isEmpty(authenticatedUser)) { - LOG.error("No authenticated user found in request for service: {}. Rejecting audit request.", serviceName); + LOG.error("No authenticated user found in request for serviceType: {}. Rejecting audit request.", serviceType); ret = Response.status(Response.Status.UNAUTHORIZED) .entity(buildErrorResponse("Authentication required to send audit events")) .build(); return ret; } - if (!serviceName.equals(authenticatedUser)) { - LOG.error("Authentication mismatch: serviceName={} but authenticatedUser={}. Rejecting audit request.", serviceName, authenticatedUser); - ret = Response.status(Response.Status.FORBIDDEN) - .entity(buildErrorResponse("Service name does not match authenticated user")) - .build(); - return ret; - } - if (!isAllowedServiceUser(authenticatedUser)) { LOG.error("Unauthorized user: authenticatedUser={} is not in the allowed service users list. Rejecting audit request.", authenticatedUser); ret = Response.status(Response.Status.FORBIDDEN) @@ -197,7 +189,7 @@ public Response logAccessAudit(@QueryParam("serviceName") String serviceName, @Q } if (accessAudits == null || accessAudits.isEmpty()) { - LOG.warn("Empty or null audit events batch received from service: {}, user: {}", serviceName, authenticatedUser); + LOG.warn("Empty or null audit events batch received from serviceType: {}, user: {}", serviceType, authenticatedUser); ret = Response.status(Response.Status.BAD_REQUEST) .entity(buildErrorResponse("Audit events cannot be empty")) .build(); @@ -208,7 +200,7 @@ public Response logAccessAudit(@QueryParam("serviceName") String serviceName, @Q .build(); } else { try { - LOG.debug("Processing {} audit events from service: {}, appId: {}", accessAudits.size(), serviceName, appId); + LOG.debug("Processing {} audit events from service: {}, appId: {}", accessAudits.size(), serviceType, appId); boolean success = auditDestinationMgr.logBatch(accessAudits, appId); @@ -216,8 +208,8 @@ public Response logAccessAudit(@QueryParam("serviceName") String serviceName, @Q Map response = new HashMap<>(); response.put("total", accessAudits.size()); response.put("timestamp", System.currentTimeMillis()); - if (StringUtils.isNotEmpty(serviceName)) { - response.put("serviceName", serviceName); + if (StringUtils.isNotEmpty(serviceType)) { + response.put("serviceType", serviceType); } if (StringUtils.isNotEmpty(appId)) { response.put("appId", appId); @@ -230,20 +222,20 @@ public Response logAccessAudit(@QueryParam("serviceName") String serviceName, @Q .entity(jsonString) .build(); } else { - LOG.warn("Batch processing failed for {} events from service: {}, appId: {}. Events spooled to recovery.", accessAudits.size(), serviceName, appId); + LOG.warn("Batch processing failed for {} events from serviceType: {}, appId: {}. Events spooled to recovery.", accessAudits.size(), serviceType, appId); ret = Response.status(Response.Status.ACCEPTED) .entity(buildErrorResponse("Batch processing failed. Events have been queued for retry.")) .build(); } } catch (Exception e) { - LOG.error("Error processing access audits batch from service: {}, appId: {}", serviceName, appId, e); + LOG.error("Error processing access audits batch from serviceType: {}, appId: {}", serviceType, appId, e); ret = Response.status(Response.Status.BAD_REQUEST) .entity(buildErrorResponse("Failed to process audit events: " + e.getMessage())) .build(); } } - LOG.debug("<== AuditREST.accessAudit(): HttpStatus {} for service: {}, user: {}", ret.getStatus(), serviceName, authenticatedUser); + LOG.debug("<== AuditREST.accessAudit(): HttpStatus {} for serviceType: {}, user: {}", ret.getStatus(), serviceType, authenticatedUser); return ret; } @@ -355,7 +347,7 @@ private static Set initializeAllowedUsers() { Set ret = new HashSet<>(); AuditServerConfig config = AuditServerConfig.getInstance(); - String allowedUsersStr = config.get(AuditServerConstants.PROP_ALLOWED_USERS, AuditServerConstants.DEFAULT_ALLOWED_USERS); + String allowedUsersStr = config.get(AuditServerConstants.PROP_ALLOWED_USERS); if (StringUtils.isNotEmpty(allowedUsersStr)) { String[] users = allowedUsersStr.split(","); for (String user : users) {