Skip to content

Commit 89ce14a

Browse files
rameeshmRamesh Manimneethiraj
authored
RANGER-5482:Create Ranger Audit Server with SOLR and HDFS as audit consumer (apache#847)
* RANGER-5482:Create Ranger Audit Server with SOLR and HDFS as audit consumer * RANGER-5482:Create Ranger Audit Server with SOLR and HDFS as audit consumer - fix failing testing * RANGER-5482:Create Ranger Audit Server with SOLR and HDFS as audit consumer - fix pmd issue * RANGER-5482:Create Ranger Audit Server with SOLR and HDFS as audit consumer - Fix audit commit failure propagation and recovery in the consumers * RANGER-5482:Create Ranger Audit Server with SOLR and HDFS as audit consumer - audit server partition management enhancement * RANGER-5482:Create Ranger Audit Server with SOLR and HDFS as audit consumer - Fix review comments * RANGER-5482:Create Ranger Audit Server with SOLR and HDFS as audit consumer - PojoMappingFeature for AuditEvent Object for serialization * RANGER-5482:Create Ranger Audit Server with SOLR and HDFS as audit consumer - Fix review comments set #2 * RANGER-5482:Create Ranger Audit Server with SOLR and HDFS as audit consumer - Audit Batch processing and failure reprocessing improvement * RANGER-5482:Create Ranger Audit Server with SOLR and HDFS as audit consumer - Fix duplicate dependency error in the pom for sl4j * RANGER-5482:Create Ranger Audit Server with SOLR and HDFS as audit consumer - Fix ubuntu audit ranger module war file creation failure * RANGER-5482:Create Ranger Audit Server with SOLR and HDFS as audit consumer - Fix Review comments set apache#3 * RANGER-5482:Create Ranger Audit Server with SOLR and HDFS as audit consumer - Fix failing test * RANGER-5482: addressed review comments/suggestions * RANGER-5482: support configuration to specify authorized users per service, instead of a global list of users * addressed review comments * cleanup in AuditConsumer implementations, RangerAuditServerDestination * added NoContentException to resolve failure in instantiating RangerJsonProvider --------- Co-authored-by: Ramesh Mani <rmani@apache.org> Co-authored-by: Madhan Neethiraj <madhan@apache.org>
1 parent 313ff26 commit 89ce14a

125 files changed

Lines changed: 14527 additions & 122 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.gitignore‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,3 +19,9 @@ winpkg/target
1919
.python-version
2020
/security-admin/src/main/webapp/react-webapp/node_modules
2121
**/target
22+
23+
# Runtime logs and process files
24+
logs/
25+
*.log
26+
*.pid
27+
catalina.out

‎agents-audit/core/src/main/java/org/apache/ranger/audit/model/AuthzAuditEvent.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,10 @@
1919

2020
package org.apache.ranger.audit.model;
2121

22+
import com.fasterxml.jackson.annotation.JsonAutoDetect;
2223
import com.fasterxml.jackson.annotation.JsonIgnore;
24+
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
25+
import com.fasterxml.jackson.annotation.JsonInclude;
2326
import com.fasterxml.jackson.annotation.JsonProperty;
2427
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
2528
import org.apache.commons.lang3.StringUtils;
@@ -28,6 +31,9 @@
2831
import java.util.HashSet;
2932
import java.util.Set;
3033

34+
@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.ANY)
35+
@JsonInclude(JsonInclude.Include.NON_EMPTY)
36+
@JsonIgnoreProperties(ignoreUnknown = true)
3137
@JsonSerialize
3238
public class AuthzAuditEvent extends AuditEventBase {
3339
protected static final int MAX_ACTION_FIELD_SIZE = 1800;

‎agents-audit/core/src/main/java/org/apache/ranger/audit/provider/AuditHandler.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,10 @@ public interface AuditHandler {
2929

3030
boolean log(Collection<AuditEventBase> events);
3131

32+
default boolean log(Collection<AuditEventBase> events, String batchKey) {
33+
return log(events);
34+
}
35+
3236
boolean logJSON(String event);
3337

3438
boolean logJSON(Collection<String> events);

‎agents-audit/core/src/main/java/org/apache/ranger/audit/provider/AuditProviderFactory.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -439,6 +439,8 @@ private AuditHandler getProviderFromConfig(Properties props, String propPrefix,
439439
provider = createDestination("org.apache.ranger.audit.provider.kafka.KafkaAuditProvider");
440440
} else if (providerName.equalsIgnoreCase("log4j")) {
441441
provider = createDestination("org.apache.ranger.audit.destination.Log4JAuditDestination");
442+
} else if (providerName.equalsIgnoreCase("auditserver")) {
443+
provider = createDestination("org.apache.ranger.audit.destination.RangerAuditServerDestination");
442444
} else if (providerName.equalsIgnoreCase("batch")) {
443445
provider = getAuditProvider(props, propPrefix, consumer);
444446
} else if (providerName.equalsIgnoreCase("async")) {

‎agents-audit/core/src/main/java/org/apache/ranger/audit/provider/BaseAuditHandler.java‎

Lines changed: 50 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -72,18 +72,19 @@ public abstract class BaseAuditHandler implements AuditHandler {
7272
protected Map<String, String> configProps = new HashMap<>();
7373
protected Properties props;
7474

75-
int errorLogIntervalMS = 30 * 1000; // Every 30 seconds
76-
long lastErrorLogMS;
77-
long totalCount;
78-
long totalSuccessCount;
79-
long totalFailedCount;
80-
long totalStashedCount;
81-
long totalDeferredCount;
82-
long lastIntervalCount;
83-
long lastIntervalSuccessCount;
84-
long lastIntervalFailedCount;
85-
long lastStashedCount;
86-
long lastDeferredCount;
75+
int errorLogIntervalMS = 30 * 1000; // Every 30 seconds
76+
long lastErrorLogMS;
77+
long lastIntervalCount;
78+
long lastIntervalSuccessCount;
79+
long lastIntervalFailedCount;
80+
long lastStashedCount;
81+
long lastDeferredCount;
82+
AtomicLong totalCount = new AtomicLong(0);
83+
AtomicLong totalSuccessCount = new AtomicLong(0);
84+
AtomicLong totalFailedCount = new AtomicLong(0);
85+
AtomicLong totalStashedCount = new AtomicLong(0);
86+
AtomicLong totalDeferredCount = new AtomicLong(0);
87+
8788
boolean statusLogEnabled = DEFAULT_AUDIT_LOG_STATUS_LOG_ENABLED;
8889
long statusLogIntervalMS = DEFAULT_AUDIT_LOG_STATUS_LOG_INTERVAL_SEC * 1000;
8990
long lastStatusLogTime = System.currentTimeMillis();
@@ -237,57 +238,47 @@ public String getFinalPath() {
237238
}
238239

239240
public long addTotalCount(int count) {
240-
totalCount += count;
241-
242-
return totalCount;
241+
return totalCount.addAndGet(count);
243242
}
244243

245244
public long addSuccessCount(int count) {
246-
totalSuccessCount += count;
247-
248-
return totalSuccessCount;
245+
return totalSuccessCount.addAndGet(count);
249246
}
250247

251248
public long addFailedCount(int count) {
252-
totalFailedCount += count;
253-
254-
return totalFailedCount;
249+
return totalFailedCount.addAndGet(count);
255250
}
256251

257252
public long addStashedCount(int count) {
258-
totalStashedCount += count;
259-
260-
return totalStashedCount;
253+
return totalStashedCount.addAndGet(count);
261254
}
262255

263256
public long addDeferredCount(int count) {
264-
totalDeferredCount += count;
265-
266-
return totalDeferredCount;
257+
return totalDeferredCount.addAndGet(count);
267258
}
268259

269260
public long getTotalCount() {
270-
return totalCount;
261+
return totalCount.get();
271262
}
272263

273264
public long getTotalSuccessCount() {
274-
return totalSuccessCount;
265+
return totalSuccessCount.get();
275266
}
276267

277268
public long getTotalFailedCount() {
278-
return totalFailedCount;
269+
return totalFailedCount.get();
279270
}
280271

281272
public long getTotalStashedCount() {
282-
return totalStashedCount;
273+
return totalStashedCount.get();
283274
}
284275

285276
public long getLastStashedCount() {
286277
return lastStashedCount;
287278
}
288279

289280
public long getTotalDeferredCount() {
290-
return totalDeferredCount;
281+
return totalDeferredCount.get();
291282
}
292283

293284
public long getLastDeferredCount() {
@@ -312,21 +303,27 @@ public void logStatus() {
312303
lastStatusLogTime = currTime;
313304
nextStatusLogTime = currTime + statusLogIntervalMS;
314305

315-
long diffCount = totalCount - lastIntervalCount;
316-
long diffSuccess = totalSuccessCount - lastIntervalSuccessCount;
317-
long diffFailed = totalFailedCount - lastIntervalFailedCount;
318-
long diffStashed = totalStashedCount - lastStashedCount;
319-
long diffDeferred = totalDeferredCount - lastDeferredCount;
306+
long currentTotalCount = totalCount.get();
307+
long currentSuccessCount = totalSuccessCount.get();
308+
long currentFailedCount = totalFailedCount.get();
309+
long currentStashedCount = totalStashedCount.get();
310+
long currentDeferredCount = totalDeferredCount.get();
311+
312+
long diffCount = currentTotalCount - lastIntervalCount;
313+
long diffSuccess = currentSuccessCount - lastIntervalSuccessCount;
314+
long diffFailed = currentFailedCount - lastIntervalFailedCount;
315+
long diffStashed = currentStashedCount - lastStashedCount;
316+
long diffDeferred = currentDeferredCount - lastDeferredCount;
320317

321318
if (diffCount == 0 && diffSuccess == 0 && diffFailed == 0 && diffStashed == 0 && diffDeferred == 0) {
322319
return;
323320
}
324321

325-
lastIntervalCount = totalCount;
326-
lastIntervalSuccessCount = totalSuccessCount;
327-
lastIntervalFailedCount = totalFailedCount;
328-
lastStashedCount = totalStashedCount;
329-
lastDeferredCount = totalDeferredCount;
322+
lastIntervalCount = currentTotalCount;
323+
lastIntervalSuccessCount = currentSuccessCount;
324+
lastIntervalFailedCount = currentFailedCount;
325+
lastStashedCount = currentStashedCount;
326+
lastDeferredCount = currentDeferredCount;
330327

331328
if (statusLogEnabled) {
332329
String finalPath = "";
@@ -475,6 +472,12 @@ public void logFailedEventJSON(Collection<String> events, Throwable excp) {
475472
}
476473

477474
private void logAuditStatus(long diffTime, long diffCount, long diffSuccess, long diffFailed, long diffStashed, long diffDeferred, String finalPath) {
475+
long currentTotalCount = totalCount.get();
476+
long currentTotalSuccessCount = totalSuccessCount.get();
477+
long currentTotalFailedCount = totalFailedCount.get();
478+
long currentTotalStashedCount = totalStashedCount.get();
479+
long currentTotalDeferredCount = totalDeferredCount.get();
480+
478481
String msg = "Audit Status Log: name="
479482
+ getName()
480483
+ finalPath
@@ -489,14 +492,14 @@ private void logAuditStatus(long diffTime, long diffCount, long diffSuccess, lon
489492
+ (diffDeferred > 0 ? (", deferredCount=" + diffDeferred)
490493
: "")
491494
+ ", totalEvents="
492-
+ totalCount
493-
+ (totalSuccessCount > 0 ? (", totalSuccessCount=" + totalSuccessCount)
495+
+ currentTotalCount
496+
+ (currentTotalSuccessCount > 0 ? (", totalSuccessCount=" + currentTotalSuccessCount)
494497
: "")
495-
+ (totalFailedCount > 0 ? (", totalFailedCount=" + totalFailedCount)
498+
+ (currentTotalFailedCount > 0 ? (", totalFailedCount=" + currentTotalFailedCount)
496499
: "")
497-
+ (totalStashedCount > 0 ? (", totalStashedCount=" + totalStashedCount)
500+
+ (currentTotalStashedCount > 0 ? (", totalStashedCount=" + currentTotalStashedCount)
498501
: "")
499-
+ (totalDeferredCount > 0 ? (", totalDeferredCount=" + totalDeferredCount)
502+
+ (currentTotalDeferredCount > 0 ? (", totalDeferredCount=" + currentTotalDeferredCount)
500503
: "");
501504
LOG.info(msg);
502505
}

‎agents-audit/core/src/main/java/org/apache/ranger/audit/utils/AbstractRangerAuditWriter.java‎

Lines changed: 13 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -133,13 +133,15 @@ public void createFileSystemFolders() throws Exception {
133133

134134
String defaultPath = fullPath;
135135

136+
fileSystemScheme = getFileSystemScheme();
137+
136138
conf = createConfiguration();
137139

138140
URI uri = URI.create(fullPath);
139141

140-
fileSystem = FileSystem.get(uri, conf);
141-
auditPath = new Path(fullPath);
142-
fileSystemScheme = getFileSystemScheme();
142+
fileSystem = FileSystem.get(uri, conf);
143+
144+
auditPath = new Path(fullPath);
143145

144146
logger.info("Checking whether log file exists. {} Path={}, UGI={}", fileSystemScheme, fullPath, MiscUtil.getUGILoginUser());
145147

@@ -195,6 +197,9 @@ public void createParents(Path pathLogfile, FileSystem fileSystem) throws Except
195197

196198
if (parentPath != null && fileSystem != null && !fileSystem.exists(parentPath)) {
197199
fileSystem.mkdirs(parentPath);
200+
logger.info("Successfully created parent folder: {}", parentPath);
201+
} else {
202+
logger.info("Parent folder already exists or not required: {}", parentPath);
198203
}
199204
}
200205

@@ -308,14 +313,17 @@ public PrintWriter createWriter() throws Exception {
308313

309314
if (!appendMode) {
310315
// Create the file to write
311-
logger.info("Creating new log file. auditPath = {}", fullPath);
312-
313316
createFileSystemFolders();
314317

318+
logger.info("Creating new log file. fullPath = {}", fullPath);
319+
315320
ostream = fileSystem.create(auditPath);
321+
logger.info("Successfully created {} output stream for file: {}", fileSystemScheme, fullPath);
316322
}
317323
logWriter = new PrintWriter(ostream);
318324
isHFlushCapableStream = ostream.hasCapability(StreamCapabilities.HFLUSH);
325+
326+
logger.info("{} audit writer initialized successfully. File: {}, HFlush capable: {}", fileSystemScheme, fullPath, isHFlushCapableStream);
319327
}
320328

321329
logger.debug("<== AbstractRangerAuditWriter.createWriter()");

‎agents-audit/core/src/main/java/org/apache/ranger/audit/utils/RangerJSONAuditWriter.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -102,9 +102,13 @@ public synchronized boolean logJSON(final Collection<String> events) throws Exce
102102
} else {
103103
out1 = getLogFileStream();
104104

105+
logger.debug("Writing {} audit events to HDFS file: {}", events.size(), currentFileName);
106+
105107
for (String event : events) {
106108
out1.println(event);
107109
}
110+
111+
logger.debug("Successfully wrote {} audit events to HDFS", events.size());
108112
}
109113

110114
return out1;

‎agents-audit/dest-auditserver/pom.xml‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<!--
3+
Licensed to the Apache Software Foundation (ASF) under one or more
4+
contributor license agreements. See the NOTICE file distributed with
5+
this work for additional information regarding copyright ownership.
6+
The ASF licenses this file to You under the Apache License, Version 2.0
7+
(the "License"); you may not use this file except in compliance with
8+
the License. You may obtain a copy of the License at
9+
10+
http://www.apache.org/licenses/LICENSE-2.0
11+
12+
Unless required by applicable law or agreed to in writing, software
13+
distributed under the License is distributed on an "AS IS" BASIS,
14+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
See the License for the specific language governing permissions and
16+
limitations under the License.
17+
-->
18+
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
19+
<modelVersion>4.0.0</modelVersion>
20+
<parent>
21+
<groupId>org.apache.ranger</groupId>
22+
<artifactId>ranger</artifactId>
23+
<version>3.0.0-SNAPSHOT</version>
24+
<relativePath>../..</relativePath>
25+
</parent>
26+
<artifactId>ranger-audit-dest-auditserver</artifactId>
27+
<packaging>jar</packaging>
28+
<name>Ranger Audit Destination - auditserver</name>
29+
<description>Ranger Audit Destination - auditserver</description>
30+
<properties>
31+
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
32+
<securesm.version>1.2</securesm.version>
33+
</properties>
34+
<dependencies>
35+
<dependency>
36+
<groupId>com.fasterxml.jackson.jaxrs</groupId>
37+
<artifactId>jackson-jaxrs-json-provider</artifactId>
38+
<version>${fasterxml.jackson.version}</version>
39+
</dependency>
40+
<dependency>
41+
<groupId>org.apache.ranger</groupId>
42+
<artifactId>ranger-audit-core</artifactId>
43+
<version>${project.version}</version>
44+
</dependency>
45+
<dependency>
46+
<groupId>org.apache.ranger</groupId>
47+
<artifactId>ranger-plugins-common</artifactId>
48+
<version>${project.version}</version>
49+
</dependency>
50+
<dependency>
51+
<groupId>org.slf4j</groupId>
52+
<artifactId>slf4j-api</artifactId>
53+
<version>${slf4j.version}</version>
54+
</dependency>
55+
56+
<!-- Test -->
57+
<dependency>
58+
<groupId>org.slf4j</groupId>
59+
<artifactId>log4j-over-slf4j</artifactId>
60+
<version>${slf4j.version}</version>
61+
<scope>test</scope>
62+
</dependency>
63+
<dependency>
64+
<groupId>org.testng</groupId>
65+
<artifactId>testng</artifactId>
66+
<scope>test</scope>
67+
</dependency>
68+
</dependencies>
69+
</project>

0 commit comments

Comments
 (0)