Skip to content

Commit ce40430

Browse files
mneethirajmneethiraj
committed
Merge branch 'RANGER-5309' into RANGER-5312
2 parents 6f86107 + f0b8c4e commit ce40430

56 files changed

Lines changed: 17545 additions & 2284 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerServiceDefHelper.java‎

Lines changed: 35 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,11 @@
4848
public class RangerServiceDefHelper {
4949
private static final Logger LOG = LoggerFactory.getLogger(RangerServiceDefHelper.class);
5050

51+
public static final String RRN_RESOURCE_PREFIX = "{";
52+
public static final String RRN_RESOURCE_SUFFIX = "}";
53+
public static final String RRN_RESOURCE_SEP = ".";
54+
public static final String RRN_PATH_RESOURCE_SEP = "/";
55+
5156
static final Map<String, Delegate> cache = new ConcurrentHashMap<>();
5257
final Delegate delegate;
5358

@@ -492,21 +497,10 @@ public Delegate(RangerServiceDef serviceDef, boolean checkForCycles) {
492497
orderedResourceNames = buildSortedResourceNames();
493498

494499
for (RangerResourceDef resourceDef : serviceDef.getResources()) {
495-
// when rrnTemplate is not specified, create a default one using the full path from root to this resource
496500
if (StringUtils.isBlank(resourceDef.getRrnTemplate())) {
497-
List<String> path = new ArrayList<>();
498-
499-
for (RangerResourceDef resource = resourceDef; resource != null; resource = getResourceDef(resource.getParent(), RangerPolicy.POLICY_TYPE_ACCESS)) {
500-
path.add(resource.getName());
501-
}
502-
503-
Collections.reverse(path);
501+
resourceDef.setRrnTemplate(getDefaultRrnTemplate(resourceDef));
504502

505-
String rrnTemplate = "{" + StringUtils.join(path, "}.{") + "}";
506-
507-
LOG.debug("Setting rrnTemplate for resource {}.{} to: {}", serviceName, resourceDef.getName(), rrnTemplate);
508-
509-
resourceDef.setRrnTemplate(rrnTemplate);
503+
LOG.debug("No rrnTemplate was defined for resource {}.{}. It is now set to default: {}", serviceName, resourceDef.getName(), resourceDef.getRrnTemplate());
510504
}
511505

512506
this.rrnTemplates.put(resourceDef.getName(), resourceDef.getRrnTemplate());
@@ -905,6 +899,34 @@ public int compareTo(ResourceNameLevel other) {
905899
return Integer.compare(this.level, other.level);
906900
}
907901
}
902+
903+
// create default resource-name template for the resource-def, like:
904+
// database:{database}
905+
// table:{database}.{table}
906+
// column:{database}.{table}.{column}
907+
// path:{bucket}/{path}
908+
// key:{volume}.{bucket}/{key}
909+
private String getDefaultRrnTemplate(RangerResourceDef resourceDef) {
910+
List<RangerResourceDef> path = new ArrayList<>();
911+
912+
for (RangerResourceDef resource = resourceDef; resource != null; resource = getResourceDef(resource.getParent(), RangerPolicy.POLICY_TYPE_ACCESS)) {
913+
path.add(0, resource);
914+
}
915+
916+
StringBuilder sb = new StringBuilder();
917+
918+
for (int i = 0; i < path.size(); i++) {
919+
RangerResourceDef res = path.get(i);
920+
921+
if (i > 0) {
922+
sb.append(StringUtils.equalsIgnoreCase(res.getType(), "path") ? RRN_PATH_RESOURCE_SEP : RRN_RESOURCE_SEP);
923+
}
924+
925+
sb.append(RRN_RESOURCE_PREFIX).append(res.getName()).append(RRN_RESOURCE_SUFFIX);
926+
}
927+
928+
return sb.toString();
929+
}
908930
}
909931

910932
/**

‎agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java‎

Lines changed: 35 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -104,6 +104,7 @@ public class RangerBasePlugin {
104104
private RangerRoles roles;
105105
private boolean isUserStoreEnricherAddedImplcitly;
106106
private Map<String, String> serviceConfigs;
107+
private boolean synchronousPolicyRefresh;
107108

108109
public RangerBasePlugin(String serviceType, String appId) {
109110
this(new RangerPluginConfig(serviceType, null, appId, null, null, null));
@@ -660,6 +661,10 @@ public Collection<RangerAccessResult> isAccessAllowed(Collection<RangerAccessReq
660661
}
661662

662663
public RangerAccessResult isAccessAllowed(RangerAccessRequest request, RangerAccessResultProcessor resultProcessor) {
664+
if (this.synchronousPolicyRefresh) {
665+
refreshPoliciesAndTags();
666+
}
667+
663668
RangerAccessResult ret = null;
664669
RangerPolicyEngine policyEngine = this.policyEngine;
665670

@@ -701,6 +706,10 @@ public RangerAccessResult isAccessAllowed(RangerAccessRequest request, RangerAcc
701706
}
702707

703708
public Collection<RangerAccessResult> isAccessAllowed(Collection<RangerAccessRequest> requests, RangerAccessResultProcessor resultProcessor) {
709+
if (this.synchronousPolicyRefresh) {
710+
refreshPoliciesAndTags();
711+
}
712+
704713
Collection<RangerAccessResult> ret = null;
705714
RangerPolicyEngine policyEngine = this.policyEngine;
706715

@@ -742,6 +751,10 @@ public Collection<RangerAccessResult> isAccessAllowed(Collection<RangerAccessReq
742751
}
743752

744753
public RangerAccessResult evalDataMaskPolicies(RangerAccessRequest request, RangerAccessResultProcessor resultProcessor) {
754+
if (this.synchronousPolicyRefresh) {
755+
refreshPoliciesAndTags();
756+
}
757+
745758
RangerPolicyEngine policyEngine = this.policyEngine;
746759
RangerAccessResult ret = null;
747760

@@ -772,6 +785,10 @@ public RangerAccessResult evalDataMaskPolicies(RangerAccessRequest request, Rang
772785
}
773786

774787
public RangerAccessResult evalRowFilterPolicies(RangerAccessRequest request, RangerAccessResultProcessor resultProcessor) {
788+
if (this.synchronousPolicyRefresh) {
789+
refreshPoliciesAndTags();
790+
}
791+
775792
RangerPolicyEngine policyEngine = this.policyEngine;
776793
RangerAccessResult ret = null;
777794

@@ -802,6 +819,10 @@ public RangerAccessResult evalRowFilterPolicies(RangerAccessRequest request, Ran
802819
}
803820

804821
public void evalAuditPolicies(RangerAccessResult result) {
822+
if (this.synchronousPolicyRefresh) {
823+
refreshPoliciesAndTags();
824+
}
825+
805826
RangerPolicyEngine policyEngine = this.policyEngine;
806827

807828
if (policyEngine != null) {
@@ -824,6 +845,10 @@ public RangerResourceACLs getResourceACLs(RangerAccessRequest request) {
824845
}
825846

826847
public RangerResourceACLs getResourceACLs(RangerAccessRequest request, Integer policyType) {
848+
if (this.synchronousPolicyRefresh) {
849+
refreshPoliciesAndTags();
850+
}
851+
827852
RangerResourceACLs ret = null;
828853
RangerPolicyEngine policyEngine = this.policyEngine;
829854

@@ -1065,7 +1090,7 @@ public void registerAuthContextEventListener(RangerAuthContextListener authConte
10651090
}
10661091

10671092
public void refreshPoliciesAndTags() {
1068-
LOG.debug("==> refreshPoliciesAndTags()");
1093+
LOG.debug("==> refreshPoliciesAndTags(): synchronousPolicyRefresh={}", synchronousPolicyRefresh);
10691094

10701095
try {
10711096
long oldPolicyVersion = getPoliciesVersion();
@@ -1193,7 +1218,7 @@ public GdsPolicyEngine getGdsPolicyEngine() {
11931218
}
11941219

11951220
public Map<String, String> getServiceConfigs() {
1196-
return serviceConfigs;
1221+
return (serviceConfigs == null) ? Collections.emptyMap() : serviceConfigs;
11971222
}
11981223

11991224
public Long getPolicyVersion() {
@@ -1216,6 +1241,14 @@ private void setServiceConfigs(Map<String, String> serviceConfigs) {
12161241
if (authContext != null && !Objects.equals(oldServiceConfigs, this.serviceConfigs)) {
12171242
authContext.onServiceConfigsUpdate(this.serviceConfigs);
12181243
}
1244+
1245+
String isSyncPolicyRefresh = this.pluginConfig == null ? null : this.serviceConfigs.get(this.pluginConfig.getPropertyPrefix() + ".policy.refresh.synchronous");
1246+
1247+
this.synchronousPolicyRefresh = Boolean.parseBoolean(isSyncPolicyRefresh);
1248+
1249+
if (this.synchronousPolicyRefresh) {
1250+
LOG.info("synchronousPolicyRefresh = {}", this.synchronousPolicyRefresh);
1251+
}
12191252
}
12201253

12211254
private void auditGrantRevoke(GrantRevokeRequest request, String action, boolean isSuccess, RangerAccessResultProcessor resultProcessor) {

‎agents-common/src/test/java/org/apache/ranger/plugin/model/validation/TestRangerServiceDefHelper.java‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -377,7 +377,7 @@ public void test_invalid_resourceHierarchies_with_leaf_specification() {
377377
}
378378

379379
@Test
380-
public void testRrnTemplate() {
380+
public void testRrnTemplateHive() {
381381
InputStreamReader reader = new InputStreamReader(requireNonNull(this.getClass().getResourceAsStream("/admin/service-defs/test-hive-servicedef.json")));
382382
RangerServiceDef svcDef = JsonUtils.jsonToObject(reader, RangerServiceDef.class);
383383
RangerServiceDefHelper svcDefHelper = new RangerServiceDefHelper(svcDef);
@@ -397,6 +397,19 @@ public void testRrnTemplate() {
397397
assertNull(rrnUnknown);
398398
}
399399

400+
@Test
401+
public void testRrnTemplateS3() {
402+
InputStreamReader reader = new InputStreamReader(requireNonNull(this.getClass().getResourceAsStream("/admin/service-defs/test-s3-servicedef.json")));
403+
RangerServiceDef svcDef = JsonUtils.jsonToObject(reader, RangerServiceDef.class);
404+
RangerServiceDefHelper svcDefHelper = new RangerServiceDefHelper(svcDef);
405+
406+
String rrnBucket = svcDefHelper.getRrnTemplate("bucket");
407+
String rrnPath = svcDefHelper.getRrnTemplate("path");
408+
409+
assertEquals("{bucket}", rrnBucket);
410+
assertEquals("{bucket}/{path}", rrnPath);
411+
}
412+
400413
RangerResourceDef createResourceDef(String name, String parent) {
401414
return createResourceDef(name, parent, null);
402415
}

‎agents-common/src/test/resources/admin/service-defs/test-s3-servicedef.json‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
{
2+
"id": 1, "name": "s3", "label": "AWS S3", "description": "AWS S3", "implClass": "org.apache.ranger.services.s3.RangerServiceS3",
3+
"resources":
4+
[
5+
{
6+
"itemId": 1, "name": "bucket", "type": "string", "parent": "", "level": 10, "label": "S3 Bucket", "description": "S3 Bucket",
7+
"mandatory": true, "lookupSupported": false, "recursiveSupported": false, "excludesSupported": true,
8+
"matcher": "org.apache.ranger.plugin.resourcematcher.RangerDefaultResourceMatcher", "matcherOptions": { "wildCard": true, "ignoreCase": true }
9+
},
10+
{
11+
"itemId": 2, "name": "path", "type": "path", "parent": "bucket", "level": 20, "label": "Path", "description": "Path inside Bucket",
12+
"mandatory": true, "lookupSupported": false, "recursiveSupported": true, "excludesSupported": true,
13+
"matcher": "org.apache.ranger.plugin.resourcematcher.RangerPathResourceMatcher", "matcherOptions": { "wildCard": true, "ignoreCase": true }
14+
}
15+
],
16+
17+
"accessTypes":
18+
[
19+
{ "itemId": 1, "name": "read", "label": "Read" },
20+
{ "itemId": 2, "name": "write", "label": "Write" }
21+
]
22+
}

‎dev-support/ranger-docker/scripts/create-ranger-services.py‎

Lines changed: 17 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -17,24 +17,29 @@ def service_not_exists(service):
1717
'configs': {'username': 'hdfs', 'password': 'hdfs',
1818
'fs.default.name': 'hdfs://ranger-hadoop:9000',
1919
'hadoop.security.authentication': 'simple',
20-
'hadoop.security.authorization': 'true'}})
20+
'hadoop.security.authorization': 'true',
21+
'ranger.plugin.hdfs.policy.refresh.synchronous':'true'}})
2122

2223
hive = RangerService({'name': 'dev_hive', 'type': 'hive',
2324
'configs': {'username': 'hive', 'password': 'hive',
2425
'jdbc.driverClassName': 'org.apache.hive.jdbc.HiveDriver',
2526
'jdbc.url': 'jdbc:hive2://ranger-hive:10000',
26-
'hadoop.security.authorization': 'true'}})
27+
'hadoop.security.authorization': 'true',
28+
'ranger.plugin.hive.policy.refresh.synchronous':'true'}})
2729

2830
kafka = RangerService({'name': 'dev_kafka', 'type': 'kafka',
2931
'configs': {'username': 'kafka', 'password': 'kafka',
30-
'zookeeper.connect': 'ranger-zk.example.com:2181'}})
32+
'zookeeper.connect': 'ranger-zk.example.com:2181',
33+
'ranger.plugin.kafka.policy.refresh.synchronous':'true'}})
3134

3235
knox = RangerService({'name': 'dev_knox', 'type': 'knox',
33-
'configs': {'username': 'knox', 'password': 'knox', 'knox.url': 'https://ranger-knox:8443'}})
36+
'configs': {'username': 'knox', 'password': 'knox', 'knox.url': 'https://ranger-knox:8443',
37+
'ranger.plugin.knox.policy.refresh.synchronous':'true'}})
3438

3539
yarn = RangerService({'name': 'dev_yarn', 'type': 'yarn',
3640
'configs': {'username': 'yarn', 'password': 'yarn',
37-
'yarn.url': 'http://ranger-hadoop:8088'}})
41+
'yarn.url': 'http://ranger-hadoop:8088',
42+
'ranger.plugin.yarn.policy.refresh.synchronous':'true'}})
3843

3944
hbase = RangerService({'name': 'dev_hbase', 'type': 'hbase',
4045
'configs': {'username': 'hbase', 'password': 'hbase',
@@ -43,11 +48,13 @@ def service_not_exists(service):
4348
'hadoop.security.authorization': 'true',
4449
'hbase.zookeeper.property.clientPort': '2181',
4550
'hbase.zookeeper.quorum': 'ranger-zk',
46-
'zookeeper.znode.parent': '/hbase'}})
51+
'zookeeper.znode.parent': '/hbase',
52+
'ranger.plugin.hbase.policy.refresh.synchronous':'true'}})
4753

4854
kms = RangerService({'name': 'dev_kms', 'type': 'kms',
4955
'configs': {'username': 'keyadmin', 'password': 'rangerR0cks!',
50-
'provider': 'http://ranger-kms:9292'}})
56+
'provider': 'http://ranger-kms:9292',
57+
'ranger.plugin.kms.policy.refresh.synchronous':'true'}})
5158

5259
trino = RangerService({'name': 'dev_trino',
5360
'type': 'trino',
@@ -56,14 +63,16 @@ def service_not_exists(service):
5663
'password': 'trino',
5764
'jdbc.driverClassName': 'io.trino.jdbc.TrinoDriver',
5865
'jdbc.url': 'jdbc:trino://ranger-trino:8080',
66+
'ranger.plugin.trino.policy.refresh.synchronous':'true'
5967
}})
6068

6169
ozone = RangerService({'name': 'dev_ozone',
6270
'type': 'ozone',
6371
'displayName': 'dev_ozone',
6472
'configs': {'username': 'hdfs', 'password': 'hdfs',
6573
'ozone.om.http-address': 'http://om:9874',
66-
'hadoop.security.authentication': 'simple'}})
74+
'hadoop.security.authentication': 'simple',
75+
'ranger.plugin.ozone.policy.refresh.synchronous':'true'}})
6776

6877
services = [hdfs, yarn, hive, hbase, kafka, knox, kms, trino, ozone]
6978
for service in services:

‎security-admin/src/main/java/org/apache/ranger/AccessAuditsService.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,7 @@ public AccessAuditsService() {
7171
*/
7272
searchFields.add(new SearchField("-repoType", "-repoType", SearchField.DATA_TYPE.INTEGER, SearchField.SEARCH_TYPE.FULL));
7373
searchFields.add(new SearchField("-requestUser", "-reqUser", SearchField.DATA_TYPE.STRING, SearchField.SEARCH_TYPE.FULL));
74+
searchFields.add(new SearchField("excludeResourceName", "-resource", SearchField.DATA_TYPE.STRING, SearchField.SEARCH_TYPE.PARTIAL));
7475
searchFields.add(new SearchField("resourceType", "resType", SearchField.DATA_TYPE.STRING, SearchField.SEARCH_TYPE.FULL));
7576
searchFields.add(new SearchField("reason", "reason", SearchField.DATA_TYPE.STRING, SearchField.SEARCH_TYPE.FULL));
7677
searchFields.add(new SearchField("action", "action", SearchField.DATA_TYPE.STRING, SearchField.SEARCH_TYPE.FULL));

‎security-admin/src/main/java/org/apache/ranger/rest/AssetREST.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -552,6 +552,7 @@ public VXAccessAuditList getAccessLogs(@Context HttpServletRequest request, @Que
552552
searchUtil.extractStringList(request, searchCriteria, "excludeUser", "Exclude Users", "-requestUser", null, StringUtil.VALIDATION_TEXT);
553553
searchUtil.extractString(request, searchCriteria, "requestData", "Request Data", StringUtil.VALIDATION_TEXT);
554554
searchUtil.extractString(request, searchCriteria, "resourcePath", "Resource Name", StringUtil.VALIDATION_TEXT);
555+
searchUtil.extractString(request, searchCriteria, "excludeResourceName", "Exclude Resource Name", StringUtil.VALIDATION_TEXT);
555556
searchUtil.extractString(request, searchCriteria, "clientIP", "Client IP", StringUtil.VALIDATION_TEXT);
556557
searchUtil.extractString(request, searchCriteria, "resourceType", "Resource Type", StringUtil.VALIDATION_TEXT);
557558
searchUtil.extractString(request, searchCriteria, "excludeServiceUser", "Exclude Service User", StringUtil.VALIDATION_TEXT);

‎security-admin/src/main/webapp/react-webapp/src/views/AuditEvent/AccessLogs.jsx‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1009,6 +1009,12 @@ function Access() {
10091009
urlLabel: "zoneName",
10101010
type: "textoptions",
10111011
options: getZones
1012+
},
1013+
{
1014+
category: "excludeResourceName",
1015+
label: "Exclude Resource Name",
1016+
urlLabel: "excludeResourceName",
1017+
type: "text"
10121018
}
10131019
];
10141020

0 commit comments

Comments
 (0)