From 5cee194b98340c9466804f0e55d3c18f8beb2645 Mon Sep 17 00:00:00 2001 From: Abhay Kulkarni Date: Wed, 21 Jun 2017 14:09:28 -0700 Subject: [PATCH 001/151] RANGER-1494: Policy engine updates to support tag-based masking policies - update datamask/rowfilter resources --- .../plugin/store/AbstractServiceStore.java | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/store/AbstractServiceStore.java b/agents-common/src/main/java/org/apache/ranger/plugin/store/AbstractServiceStore.java index 2ee786ca1a..5842b60ac4 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/store/AbstractServiceStore.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/store/AbstractServiceStore.java @@ -28,6 +28,7 @@ import org.apache.ranger.plugin.model.RangerService; import org.apache.ranger.plugin.model.RangerServiceDef; import org.apache.ranger.plugin.util.SearchFilter; +import org.apache.ranger.services.tag.RangerServiceTag; import java.util.ArrayList; import java.util.Collection; @@ -303,6 +304,10 @@ private void updateTagServiceDefForUpdatingAccessTypes(RangerServiceDef serviceD updateNeeded = true; } + boolean resourceUpdated = updateResourceInTagServiceDef(tagServiceDef); + + updateNeeded = updateNeeded || resourceUpdated; + if (updateNeeded) { try { updateServiceDef(tagServiceDef); @@ -346,6 +351,8 @@ private void updateTagServiceDefForDeletingAccessTypes(String serviceDefName) th updateTagServiceDefForDeletingDataMaskDef(tagServiceDef, serviceDefName); updateTagServiceDefForDeletingRowFilterDef(tagServiceDef, serviceDefName); + updateResourceInTagServiceDef(tagServiceDef); + try { updateServiceDef(tagServiceDef); LOG.info("AbstractServiceStore.updateTagServiceDefForDeletingAccessTypes -- updated TAG service def with " + serviceDefName + " access types"); @@ -548,4 +555,53 @@ private void updateTagServiceDefForDeletingRowFilterDef(RangerServiceDef tagServ LOG.debug("<== AbstractServiceStore.updateTagServiceDefForDeletingRowFilterDef(" + serviceDefName + ")"); } } + + private boolean updateResourceInTagServiceDef(RangerServiceDef tagServiceDef) throws Exception { + if (LOG.isDebugEnabled()) { + LOG.debug("==> AbstractServiceStore.updateResourceInTagServiceDef(" + tagServiceDef + ")"); + } + boolean ret = false; + + RangerServiceDef.RangerResourceDef tagResource = new RangerServiceDef.RangerResourceDef(); + tagResource.setName(RangerServiceTag.TAG_RESOURCE_NAME); + List resources = new ArrayList<>(); + resources.add(tagResource); + + RangerServiceDef.RangerDataMaskDef dataMaskDef = tagServiceDef.getDataMaskDef(); + + if (dataMaskDef != null) { + if (CollectionUtils.isNotEmpty(dataMaskDef.getAccessTypes())) { + if (CollectionUtils.isEmpty(dataMaskDef.getResources())) { + dataMaskDef.setResources(resources); + ret = true; + } + } else { + if (CollectionUtils.isNotEmpty(dataMaskDef.getResources())) { + dataMaskDef.setResources(null); + ret = true; + } + } + } + + RangerServiceDef.RangerRowFilterDef rowFilterDef = tagServiceDef.getRowFilterDef(); + + if (rowFilterDef != null) { + if (CollectionUtils.isNotEmpty(rowFilterDef.getAccessTypes())) { + if (CollectionUtils.isEmpty(rowFilterDef.getResources())) { + rowFilterDef.setResources(resources); + ret = true; + } + } else { + if (CollectionUtils.isNotEmpty(rowFilterDef.getResources())) { + rowFilterDef.setResources(null); + ret = true; + } + } + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== AbstractServiceStore.updateResourceInTagServiceDef(" + tagServiceDef + ") : " + ret); + } + return ret; + } } \ No newline at end of file From 3e504e8b02b09c6a7ccc2960cdb191b865ca8b5f Mon Sep 17 00:00:00 2001 From: Abhay Kulkarni Date: Thu, 22 Jun 2017 15:34:25 -0700 Subject: [PATCH 002/151] RANGER-1494: Policy engine updates to support tag-based masking policies - disable rowfilterdef --- .../plugin/store/AbstractServiceStore.java | 50 ++++++++++++------- 1 file changed, 31 insertions(+), 19 deletions(-) diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/store/AbstractServiceStore.java b/agents-common/src/main/java/org/apache/ranger/plugin/store/AbstractServiceStore.java index 5842b60ac4..a75ca59e87 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/store/AbstractServiceStore.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/store/AbstractServiceStore.java @@ -23,6 +23,7 @@ import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.ranger.authorization.hadoop.config.RangerConfiguration; import org.apache.ranger.plugin.model.RangerBaseModelObject; import org.apache.ranger.plugin.model.RangerPolicy; import org.apache.ranger.plugin.model.RangerService; @@ -41,6 +42,10 @@ public abstract class AbstractServiceStore implements ServiceStore { public static final String COMPONENT_ACCESSTYPE_SEPARATOR = ":"; + private static final String AUTOPROPAGATE_ROWFILTERDEF_TO_TAG_PROP = "ranger.servicedef.autopropagate.rowfilterdef.to.tag"; + + private static final boolean AUTOPROPAGATE_ROWFILTERDEF_TO_TAG_PROP_DEFAULT = false; + private static final int MAX_ACCESS_TYPES_IN_SERVICE_DEF = 1000; // when a service-def is updated, the updated service-def should be made available to plugins @@ -349,6 +354,7 @@ private void updateTagServiceDefForDeletingAccessTypes(String serviceDefName) th tagServiceDef.getAccessTypes().removeAll(accessTypes); updateTagServiceDefForDeletingDataMaskDef(tagServiceDef, serviceDefName); + updateTagServiceDefForDeletingRowFilterDef(tagServiceDef, serviceDefName); updateResourceInTagServiceDef(tagServiceDef); @@ -509,19 +515,22 @@ private boolean updateTagServiceDefForUpdatingRowFilterDef(RangerServiceDef tagS } boolean ret = false; - RangerServiceDef.RangerRowFilterDef svcRowFilterDef = serviceDef.getRowFilterDef(); - RangerServiceDef.RangerRowFilterDef tagRowFilterDef = tagServiceDef.getRowFilterDef(); + boolean autopropagateRowfilterdefToTag = RangerConfiguration.getInstance().getBoolean(AUTOPROPAGATE_ROWFILTERDEF_TO_TAG_PROP, AUTOPROPAGATE_ROWFILTERDEF_TO_TAG_PROP_DEFAULT); - List svcDefAccessTypes = svcRowFilterDef.getAccessTypes(); - List tagDefAccessTypes = tagRowFilterDef.getAccessTypes(); + if (autopropagateRowfilterdefToTag) { + RangerServiceDef.RangerRowFilterDef svcRowFilterDef = serviceDef.getRowFilterDef(); + RangerServiceDef.RangerRowFilterDef tagRowFilterDef = tagServiceDef.getRowFilterDef(); - boolean tagRowFilterAccessTypesUpdated = updateTagAccessTypeDefs(svcDefAccessTypes, tagDefAccessTypes, itemIdOffset, prefix); + List svcDefAccessTypes = svcRowFilterDef.getAccessTypes(); + List tagDefAccessTypes = tagRowFilterDef.getAccessTypes(); - if (tagRowFilterAccessTypesUpdated) { - tagRowFilterDef.setAccessTypes(tagDefAccessTypes); - ret = true; - } + boolean tagRowFilterAccessTypesUpdated = updateTagAccessTypeDefs(svcDefAccessTypes, tagDefAccessTypes, itemIdOffset, prefix); + if (tagRowFilterAccessTypesUpdated) { + tagRowFilterDef.setAccessTypes(tagDefAccessTypes); + ret = true; + } + } if (LOG.isDebugEnabled()) { LOG.debug("<== AbstractServiceStore.updateTagServiceDefForUpdatingRowFilterDef(" + serviceDef.getName() + ") : " + ret); } @@ -586,15 +595,18 @@ private boolean updateResourceInTagServiceDef(RangerServiceDef tagServiceDef) th RangerServiceDef.RangerRowFilterDef rowFilterDef = tagServiceDef.getRowFilterDef(); if (rowFilterDef != null) { - if (CollectionUtils.isNotEmpty(rowFilterDef.getAccessTypes())) { - if (CollectionUtils.isEmpty(rowFilterDef.getResources())) { - rowFilterDef.setResources(resources); - ret = true; - } - } else { - if (CollectionUtils.isNotEmpty(rowFilterDef.getResources())) { - rowFilterDef.setResources(null); - ret = true; + boolean autopropagateRowfilterdefToTag = RangerConfiguration.getInstance().getBoolean(AUTOPROPAGATE_ROWFILTERDEF_TO_TAG_PROP, AUTOPROPAGATE_ROWFILTERDEF_TO_TAG_PROP_DEFAULT); + if (autopropagateRowfilterdefToTag) { + if (CollectionUtils.isNotEmpty(rowFilterDef.getAccessTypes())) { + if (CollectionUtils.isEmpty(rowFilterDef.getResources())) { + rowFilterDef.setResources(resources); + ret = true; + } + } else { + if (CollectionUtils.isNotEmpty(rowFilterDef.getResources())) { + rowFilterDef.setResources(null); + ret = true; + } } } } @@ -604,4 +616,4 @@ private boolean updateResourceInTagServiceDef(RangerServiceDef tagServiceDef) th } return ret; } -} \ No newline at end of file +} From a219604a4ba44c64e401a70abbc25a1ada0cdb7e Mon Sep 17 00:00:00 2001 From: Abhay Kulkarni Date: Fri, 23 Jun 2017 15:46:16 -0700 Subject: [PATCH 003/151] RANGER-1665: provide a way to get list of policies associated with given resource --- .../contextenricher/RangerTagEnricher.java | 4 + .../policyengine/RangerPolicyEngine.java | 2 + .../policyengine/RangerPolicyEngineCache.java | 29 +-- ...ngerPolicyEngineCacheForEngineOptions.java | 62 ++++++ .../policyengine/RangerPolicyEngineImpl.java | 76 ++++++- .../RangerPolicyEngineOptions.java | 98 ++++++++- .../policyengine/RangerPolicyRepository.java | 43 ++-- .../ranger/plugin/store/AbstractTagStore.java | 5 + .../apache/ranger/plugin/store/TagStore.java | 1 + .../org/apache/ranger/biz/TagDBStore.java | 7 + .../ranger/common/RangerAdminTagEnricher.java | 112 ++++++++++ .../org/apache/ranger/rest/PublicAPIsv2.java | 9 + .../org/apache/ranger/rest/ServiceREST.java | 203 +++++++++++++++--- 13 files changed, 568 insertions(+), 83 deletions(-) create mode 100644 agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineCacheForEngineOptions.java create mode 100644 security-admin/src/main/java/org/apache/ranger/common/RangerAdminTagEnricher.java diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java index 9a577197d1..5f0a422dc7 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java @@ -199,6 +199,10 @@ public void setServiceTags(final ServiceTags serviceTags) { } } + protected Long getServiceTagsVersion() { + return enrichedServiceTags != null ? enrichedServiceTags.getServiceTags().getTagVersion() : null; + } + @Override public boolean preCleanup() { boolean ret = true; diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngine.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngine.java index b758d693d8..d9b0298a12 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngine.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngine.java @@ -75,6 +75,8 @@ public interface RangerPolicyEngine { List getAllowedPolicies(String user, Set userGroups, String accessType); + List getMatchingPolicies(RangerAccessResource resource); + RangerResourceAccessInfo getResourceAccessInfo(RangerAccessRequest request); void reorderPolicyEvaluators(); diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineCache.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineCache.java index 51f2142e32..58fbffd7ec 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineCache.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineCache.java @@ -19,7 +19,6 @@ package org.apache.ranger.plugin.policyengine; -import java.util.Collections; import java.util.HashMap; import java.util.Map; @@ -28,20 +27,12 @@ import org.apache.ranger.plugin.store.ServiceStore; import org.apache.ranger.plugin.util.ServicePolicies; -public class RangerPolicyEngineCache { +class RangerPolicyEngineCache { private static final Log LOG = LogFactory.getLog(RangerPolicyEngineCache.class); - private static final RangerPolicyEngineCache sInstance = new RangerPolicyEngineCache(); + private final Map policyEngineCache = new HashMap(); - private final Map policyEngineCache = Collections.synchronizedMap(new HashMap()); - - private RangerPolicyEngineOptions options = null; - - public static RangerPolicyEngineCache getInstance() { - return sInstance; - } - - public RangerPolicyEngine getPolicyEngine(String serviceName, ServiceStore svcStore) { + synchronized final RangerPolicyEngine getPolicyEngine(String serviceName, ServiceStore svcStore, RangerPolicyEngineOptions options) { RangerPolicyEngine ret = null; if(serviceName != null) { @@ -55,9 +46,9 @@ public RangerPolicyEngine getPolicyEngine(String serviceName, ServiceStore svcSt if(policies != null) { if(ret == null) { - ret = addPolicyEngine(policies); + ret = addPolicyEngine(policies, options); } else if(policies.getPolicyVersion() != null && !policies.getPolicyVersion().equals(policyVersion)) { - ret = addPolicyEngine(policies); + ret = addPolicyEngine(policies, options); } } } catch(Exception excp) { @@ -69,15 +60,7 @@ public RangerPolicyEngine getPolicyEngine(String serviceName, ServiceStore svcSt return ret; } - public RangerPolicyEngineOptions getPolicyEngineOptions() { - return options; - } - - public void setPolicyEngineOptions(RangerPolicyEngineOptions options) { - this.options = options; - } - - private RangerPolicyEngine addPolicyEngine(ServicePolicies policies) { + private RangerPolicyEngine addPolicyEngine(ServicePolicies policies, RangerPolicyEngineOptions options) { RangerPolicyEngine ret = new RangerPolicyEngineImpl("ranger-admin", policies, options); policyEngineCache.put(policies.getServiceName(), ret); diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineCacheForEngineOptions.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineCacheForEngineOptions.java new file mode 100644 index 0000000000..ca6a2a3956 --- /dev/null +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineCacheForEngineOptions.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.plugin.policyengine; + +import org.apache.ranger.plugin.store.ServiceStore; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +public class RangerPolicyEngineCacheForEngineOptions { + + private static volatile RangerPolicyEngineCacheForEngineOptions sInstance = null; + + private final Map policyEngineCacheForEngineOptions = Collections.synchronizedMap(new HashMap()); + + public static RangerPolicyEngineCacheForEngineOptions getInstance() { + RangerPolicyEngineCacheForEngineOptions ret = sInstance; + if (ret == null) { + synchronized (RangerPolicyEngineCacheForEngineOptions.class) { + ret = sInstance; + if (ret == null) { + sInstance = new RangerPolicyEngineCacheForEngineOptions(); + ret = sInstance; + } + } + } + return ret; + } + + public final RangerPolicyEngine getPolicyEngine(String serviceName, ServiceStore svcStore, RangerPolicyEngineOptions options) { + + RangerPolicyEngineCache policyEngineCache; + + synchronized (this) { + policyEngineCache = policyEngineCacheForEngineOptions.get(options); + if (policyEngineCache == null) { + policyEngineCache = new RangerPolicyEngineCache(); + policyEngineCacheForEngineOptions.put(options, policyEngineCache); + } + } + return policyEngineCache.getPolicyEngine(serviceName, svcStore, options); + } +} + diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineImpl.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineImpl.java index a359d010fe..34ae4169ee 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineImpl.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineImpl.java @@ -417,7 +417,7 @@ public boolean isAccessAllowed(RangerAccessResource resource, String user, Set getAllowedPolicies(String user, Set userGroups return ret; } + @Override + public List getMatchingPolicies(RangerAccessResource resource) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerPolicyEngineImpl.getMatchingPolicies(" + resource + ")"); + } + + List ret = new ArrayList<>(); + + RangerAccessRequestImpl request = new RangerAccessRequestImpl(resource, RangerPolicyEngine.ANY_ACCESS, null, null); + + preProcess(request); + + if (hasTagPolicies()) { + Set tags = RangerAccessRequestUtil.getRequestTagsFromContext(request.getContext()); + + if (CollectionUtils.isNotEmpty(tags)) { + for (RangerTagForEval tag : tags) { + RangerAccessRequest tagEvalRequest = new RangerTagAccessRequest(tag, tagPolicyRepository.getServiceDef(), request); + RangerAccessResource tagResource = tagEvalRequest.getResource(); + List accessPolicyEvaluators = tagPolicyRepository.getLikelyMatchPolicyEvaluators(tagResource); + List dataMaskPolicyEvaluators = tagPolicyRepository.getLikelyMatchDataMaskPolicyEvaluators(tagResource); + List rowFilterPolicyEvaluators = tagPolicyRepository.getLikelyMatchRowFilterPolicyEvaluators(tagResource); + + List[] likelyEvaluators = new List[] { accessPolicyEvaluators, dataMaskPolicyEvaluators, rowFilterPolicyEvaluators }; + + for (List evaluators : likelyEvaluators) { + for (RangerPolicyEvaluator evaluator : evaluators) { + if (evaluator.isMatch(tagResource, null)) { + ret.add(evaluator.getPolicy()); + } + } + } + } + } + } + + if (hasResourcePolicies()) { + List accessPolicyEvaluators = policyRepository.getLikelyMatchPolicyEvaluators(resource); + List dataMaskPolicyEvaluators = policyRepository.getLikelyMatchDataMaskPolicyEvaluators(resource); + List rowFilterPolicyEvaluators = policyRepository.getLikelyMatchRowFilterPolicyEvaluators(resource); + + List[] likelyEvaluators = new List[] { accessPolicyEvaluators, dataMaskPolicyEvaluators, rowFilterPolicyEvaluators }; + + for (List evaluators : likelyEvaluators) { + for (RangerPolicyEvaluator evaluator : evaluators) { + if (evaluator.isMatch(resource, null)) { + ret.add(evaluator.getPolicy()); + } + } + } + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerPolicyEngineImpl.getMatchingPolicies(" + resource + ") : " + ret.size()); + } + return ret; + } + @Override public RangerResourceAccessInfo getResourceAccessInfo(RangerAccessRequest request) { if(LOG.isDebugEnabled()) { @@ -558,7 +616,7 @@ public RangerResourceAccessInfo getResourceAccessInfo(RangerAccessRequest reques for (RangerTagForEval tag : tags) { RangerAccessRequest tagEvalRequest = new RangerTagAccessRequest(tag, tagPolicyRepository.getServiceDef(), request); - List evaluators = tagPolicyRepository.getPolicyEvaluators(tagEvalRequest.getResource()); + List evaluators = tagPolicyRepository.getLikelyMatchPolicyEvaluators(tagEvalRequest.getResource()); for (RangerPolicyEvaluator evaluator : evaluators) { evaluator.getResourceAccessInfo(tagEvalRequest, ret); @@ -567,7 +625,7 @@ public RangerResourceAccessInfo getResourceAccessInfo(RangerAccessRequest reques } } - List resPolicyEvaluators = policyRepository.getPolicyEvaluators(request.getResource()); + List resPolicyEvaluators = policyRepository.getLikelyMatchPolicyEvaluators(request.getResource()); if(CollectionUtils.isNotEmpty(resPolicyEvaluators)) { for (RangerPolicyEvaluator evaluator : resPolicyEvaluators) { @@ -615,7 +673,7 @@ protected RangerAccessResult isAccessAllowedNoAudit(RangerAccessRequest request) ret.setIsAccessDetermined(false); // discard allowed result by tag-policies, to evaluate resource policies for possible deny } - List evaluators = policyRepository.getPolicyEvaluators(request.getResource()); + List evaluators = policyRepository.getLikelyMatchPolicyEvaluators(request.getResource()); for (RangerPolicyEvaluator evaluator : evaluators) { ret.incrementEvaluatedPoliciesCount(); evaluator.evaluate(request, ret); @@ -673,7 +731,7 @@ protected void isAccessAllowedForTagPolicies(final RangerAccessRequest request, } tagEvalResult.setAuditResultFrom(result); - List evaluators = tagPolicyRepository.getPolicyEvaluators(tagEvalRequest.getResource()); + List evaluators = tagPolicyRepository.getLikelyMatchPolicyEvaluators(tagEvalRequest.getResource()); for (RangerPolicyEvaluator evaluator : evaluators) { result.incrementEvaluatedPoliciesCount(); @@ -751,7 +809,7 @@ RangerDataMaskResult evalDataMaskPoliciesNoAudit(RangerAccessRequest request) { if (evaluateResourcePolicies) { boolean findAuditByResource = !ret.getIsAuditedDetermined(); boolean foundInCache = findAuditByResource && policyRepository.setAuditEnabledFromCache(request, ret); - List evaluators = policyRepository.getDataMaskPolicyEvaluators(request.getResource()); + List evaluators = policyRepository.getLikelyMatchDataMaskPolicyEvaluators(request.getResource()); for (RangerPolicyEvaluator evaluator : evaluators) { ret.incrementEvaluatedPoliciesCount(); @@ -789,7 +847,7 @@ protected void evalDataMaskPoliciesForTagPolicies(final RangerAccessRequest requ if (CollectionUtils.isNotEmpty(tagEvaluators)) { Set tags = RangerAccessRequestUtil.getRequestTagsFromContext(request.getContext()); - List dataMaskEvaluators = tagPolicyRepository.getDataMaskPolicyEvaluators(tags); + List dataMaskEvaluators = tagPolicyRepository.getLikelyMatchDataMaskPolicyEvaluators(tags); if (CollectionUtils.isNotEmpty(dataMaskEvaluators)) { for (PolicyEvaluatorForTag dataMaskEvaluator : dataMaskEvaluators) { @@ -862,7 +920,7 @@ RangerRowFilterResult evalRowFilterPoliciesNoAudit(RangerAccessRequest request) if (evaluateResourcePolicies) { boolean findAuditByResource = !ret.getIsAuditedDetermined(); boolean foundInCache = findAuditByResource && policyRepository.setAuditEnabledFromCache(request, ret); - List evaluators = policyRepository.getRowFilterPolicyEvaluators(request.getResource()); + List evaluators = policyRepository.getLikelyMatchRowFilterPolicyEvaluators(request.getResource()); for (RangerPolicyEvaluator evaluator : evaluators) { ret.incrementEvaluatedPoliciesCount(); @@ -894,7 +952,7 @@ protected void evalRowFilterPoliciesForTagPolicies(final RangerAccessRequest req if (CollectionUtils.isNotEmpty(tagEvaluators)) { Set tags = RangerAccessRequestUtil.getRequestTagsFromContext(request.getContext()); - List rowFilterEvaluators = tagPolicyRepository.getRowFilterPolicyEvaluators(tags); + List rowFilterEvaluators = tagPolicyRepository.getLikelyMatchRowFilterPolicyEvaluators(tags); if (CollectionUtils.isNotEmpty(rowFilterEvaluators)) { for (PolicyEvaluatorForTag rowFilterEvaluator : rowFilterEvaluators) { diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineOptions.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineOptions.java index 2b2cf9b375..35056439d5 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineOptions.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineOptions.java @@ -19,15 +19,105 @@ package org.apache.ranger.plugin.policyengine; +import org.apache.hadoop.conf.Configuration; import org.apache.ranger.plugin.policyevaluator.RangerPolicyEvaluator; public class RangerPolicyEngineOptions { - public String evaluatorType = RangerPolicyEvaluator.EVALUATOR_TYPE_AUTO; - public boolean cacheAuditResults = true; + public String evaluatorType = RangerPolicyEvaluator.EVALUATOR_TYPE_AUTO; + public boolean disableContextEnrichers = false; public boolean disableCustomConditions = false; - public boolean disableTagPolicyEvaluation = true; - public boolean evaluateDelegateAdminOnly = false; + public boolean disableTagPolicyEvaluation = false; public boolean disableTrieLookupPrefilter = false; + public boolean cacheAuditResults = true; + public boolean evaluateDelegateAdminOnly = false; + public boolean enableTagEnricherWithLocalRefresher = false; + + public void configureForPlugin(Configuration conf, String propertyPrefix) { + disableContextEnrichers = conf.getBoolean(propertyPrefix + ".policyengine.option.disable.context.enrichers", false); + disableCustomConditions = conf.getBoolean(propertyPrefix + ".policyengine.option.disable.custom.conditions", false); + disableTagPolicyEvaluation = conf.getBoolean(propertyPrefix + ".policyengine.option.disable.tagpolicy.evaluation", false); + disableTrieLookupPrefilter = conf.getBoolean(propertyPrefix + ".policyengine.option.disable.trie.lookup.prefilter", false); + + cacheAuditResults = conf.getBoolean(propertyPrefix + ".policyengine.option.cache.audit.results", true); + + evaluateDelegateAdminOnly = false; + enableTagEnricherWithLocalRefresher = false; + } + + public void configureDefaultRangerAdmin(Configuration conf, String propertyPrefix) { + disableContextEnrichers = conf.getBoolean(propertyPrefix + ".policyengine.option.disable.context.enrichers", true); + disableCustomConditions = conf.getBoolean(propertyPrefix + ".policyengine.option.disable.custom.conditions", true); + disableTagPolicyEvaluation = conf.getBoolean(propertyPrefix + ".policyengine.option.disable.tagpolicy.evaluation", true); + disableTrieLookupPrefilter = conf.getBoolean(propertyPrefix + ".policyengine.option.disable.trie.lookup.prefilter", false); + + cacheAuditResults = false; + evaluateDelegateAdminOnly = false; + enableTagEnricherWithLocalRefresher = false; + } + + public void configureDelegateAdmin(Configuration conf, String propertyPrefix) { + disableContextEnrichers = conf.getBoolean(propertyPrefix + ".policyengine.option.disable.context.enrichers", true); + disableCustomConditions = conf.getBoolean(propertyPrefix + ".policyengine.option.disable.custom.conditions", true); + disableTagPolicyEvaluation = conf.getBoolean(propertyPrefix + ".policyengine.option.disable.tagpolicy.evaluation", true); + disableTrieLookupPrefilter = conf.getBoolean(propertyPrefix + ".policyengine.option.disable.trie.lookup.prefilter", false); + + cacheAuditResults = false; + evaluateDelegateAdminOnly = true; + enableTagEnricherWithLocalRefresher = false; + + } + + public void configureRangerAdminForPolicySearch(Configuration conf, String propertyPrefix) { + disableContextEnrichers = conf.getBoolean(propertyPrefix + ".policyengine.option.disable.context.enrichers", true); + disableCustomConditions = conf.getBoolean(propertyPrefix + ".policyengine.option.disable.custom.conditions", true); + disableTagPolicyEvaluation = conf.getBoolean(propertyPrefix + ".policyengine.option.disable.tagpolicy.evaluation", false); + disableTrieLookupPrefilter = conf.getBoolean(propertyPrefix + ".policyengine.option.disable.trie.lookup.prefilter", false); + + cacheAuditResults = false; + evaluateDelegateAdminOnly = false; + enableTagEnricherWithLocalRefresher = true; + } + + /* + * There is no need to implement these, as the options are predefined in a component ServiceREST and hence + * guaranteed to be unique objects. That implies that the default equals and hashCode should suffice. + */ + + @Override + public boolean equals(Object other) { + boolean ret = false; + if (other instanceof RangerPolicyEngineOptions) { + RangerPolicyEngineOptions that = (RangerPolicyEngineOptions) other; + ret = this.disableContextEnrichers == that.disableContextEnrichers + && this.disableCustomConditions == that.disableCustomConditions + && this.disableTagPolicyEvaluation == that.disableTagPolicyEvaluation + && this.disableTrieLookupPrefilter == that.disableTrieLookupPrefilter + && this.cacheAuditResults == that.cacheAuditResults + && this.evaluateDelegateAdminOnly == that.evaluateDelegateAdminOnly + && this.enableTagEnricherWithLocalRefresher == that.enableTagEnricherWithLocalRefresher; + } + return ret; + } + + @Override + public int hashCode() { + int ret = 0; + ret += disableContextEnrichers ? 1 : 0; + ret *= 2; + ret += disableCustomConditions ? 1 : 0; + ret *= 2; + ret += disableTagPolicyEvaluation ? 1 : 0; + ret *= 2; + ret += disableTrieLookupPrefilter ? 1 : 0; + ret *= 2; + ret += cacheAuditResults ? 1 : 0; + ret *= 2; + ret += evaluateDelegateAdminOnly ? 1 : 0; + ret *= 2; + ret += enableTagEnricherWithLocalRefresher ? 1 : 0; + ret *= 2; + return ret; + } } diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java index bdbdd13f14..5631973470 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java @@ -25,6 +25,7 @@ import org.apache.commons.logging.LogFactory; import org.apache.ranger.authorization.hadoop.config.RangerConfiguration; import org.apache.ranger.plugin.contextenricher.RangerContextEnricher; +import org.apache.ranger.plugin.contextenricher.RangerTagEnricher; import org.apache.ranger.plugin.contextenricher.RangerTagForEval; import org.apache.ranger.plugin.model.RangerPolicy; import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyItemDataMaskInfo; @@ -216,23 +217,23 @@ List getPolicyEvaluators() { return policyEvaluators; } - List getPolicyEvaluators(RangerAccessResource resource) { + List getLikelyMatchPolicyEvaluators(RangerAccessResource resource) { String resourceStr = resource == null ? null : resource.getAsString(); - return policyResourceTrie == null || StringUtils.isEmpty(resourceStr) ? getPolicyEvaluators() : getPolicyEvaluators(policyResourceTrie, resource); + return policyResourceTrie == null || StringUtils.isEmpty(resourceStr) ? getPolicyEvaluators() : getLikelyMatchPolicyEvaluators(policyResourceTrie, resource); } List getDataMaskPolicyEvaluators() { return dataMaskPolicyEvaluators; } - List getDataMaskPolicyEvaluators(RangerAccessResource resource) { + List getLikelyMatchDataMaskPolicyEvaluators(RangerAccessResource resource) { String resourceStr = resource == null ? null : resource.getAsString(); - return dataMaskResourceTrie == null || StringUtils.isEmpty(resourceStr) ? getDataMaskPolicyEvaluators() : getPolicyEvaluators(dataMaskResourceTrie, resource); + return dataMaskResourceTrie == null || StringUtils.isEmpty(resourceStr) ? getDataMaskPolicyEvaluators() : getLikelyMatchPolicyEvaluators(dataMaskResourceTrie, resource); } - List getDataMaskPolicyEvaluators(Set tags) { + List getLikelyMatchDataMaskPolicyEvaluators(Set tags) { return getSortedPolicyEvaluatorsForTags(tags, RangerPolicy.POLICY_TYPE_DATAMASK); } @@ -240,19 +241,19 @@ List getRowFilterPolicyEvaluators() { return rowFilterPolicyEvaluators; } - List getRowFilterPolicyEvaluators(RangerAccessResource resource) { + List getLikelyMatchRowFilterPolicyEvaluators(RangerAccessResource resource) { String resourceStr = resource == null ? null : resource.getAsString(); - return rowFilterResourceTrie == null || StringUtils.isEmpty(resourceStr) ? getRowFilterPolicyEvaluators() : getPolicyEvaluators(rowFilterResourceTrie, resource); + return rowFilterResourceTrie == null || StringUtils.isEmpty(resourceStr) ? getRowFilterPolicyEvaluators() : getLikelyMatchPolicyEvaluators(rowFilterResourceTrie, resource); } - List getRowFilterPolicyEvaluators(Set tags) { + List getLikelyMatchRowFilterPolicyEvaluators(Set tags) { return getSortedPolicyEvaluatorsForTags(tags, RangerPolicy.POLICY_TYPE_ROWFILTER); } AuditModeEnum getAuditModeEnum() { return auditModeEnum; } - private List getPolicyEvaluators(Map resourceTrie, RangerAccessResource resource) { + private List getLikelyMatchPolicyEvaluators(Map resourceTrie, RangerAccessResource resource) { List ret = null; Set resourceKeys = resource == null ? null : resource.getKeys(); @@ -305,7 +306,7 @@ private List getPolicyEvaluators(Map RangerAdminTagEnricher.init()"); + } + super.init(); + + ServiceStore svcStore = tagStore != null ? tagStore.getServiceStore() : null; + + if (tagStore == null || svcStore == null) { + LOG.error("ServiceDBStore/TagDBStore is not initialized!! Internal Error!"); + } else { + try { + RangerService service = svcStore.getServiceByName(serviceName); + serviceId = service.getId(); + } catch (Exception e) { + LOG.error("Cannot find service with name:[" + serviceName + "]", e); + LOG.error("This will cause tag-enricher in Ranger-Admin to fail!!"); + } + } + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerAdminTagEnricher.init()"); + } + } + + @Override + public void enrich(RangerAccessRequest request) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerAdminTagEnricher.enrich(" + request + ")"); + } + + refreshTagsIfNeeded(); + super.enrich(request); + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerAdminTagEnricher.enrich(" + request + ")"); + } + } + + private void refreshTagsIfNeeded() { + ServiceTags serviceTags = null; + try { + serviceTags = RangerServiceTagsCache.getInstance().getServiceTags(serviceName, serviceId, tagStore); + } catch (Exception e) { + LOG.error("Could not get cached service-tags, continue to use old ones..", e); + } + + if (serviceTags != null) { + Long enrichedServiceTagsVersion = getServiceTagsVersion(); + + if (enrichedServiceTagsVersion == null || !enrichedServiceTagsVersion.equals(serviceTags.getTagVersion())) { + synchronized(this) { + enrichedServiceTagsVersion = getServiceTagsVersion(); + + if (enrichedServiceTagsVersion == null || !enrichedServiceTagsVersion.equals(serviceTags.getTagVersion())) { + setServiceTags(serviceTags); + } + } + } + } + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("RangerAdminTagEnricher={serviceName=").append(serviceName).append(", "); + sb.append("serviceId=").append(serviceId).append("}"); + return sb.toString(); + } +} diff --git a/security-admin/src/main/java/org/apache/ranger/rest/PublicAPIsv2.java b/security-admin/src/main/java/org/apache/ranger/rest/PublicAPIsv2.java index dbb34bdff5..fa3c68ed69 100644 --- a/security-admin/src/main/java/org/apache/ranger/rest/PublicAPIsv2.java +++ b/security-admin/src/main/java/org/apache/ranger/rest/PublicAPIsv2.java @@ -339,6 +339,15 @@ public List searchPolicies(@PathParam("servicename") String servic return serviceREST.getServicePoliciesByName(serviceName, request).getPolicies(); } + @GET + @Path("/api/policies/{serviceDefName}/for-resource/") + @Produces({ "application/json", "application/xml" }) + public List getPoliciesForResource(@PathParam("serviceDefName") String serviceDefName, + @DefaultValue("") @QueryParam("serviceName") String serviceName, + @Context HttpServletRequest request) { + return serviceREST.getPoliciesForResource(serviceDefName, serviceName, request); + } + @POST @Path("/api/policy/") @Produces({ "application/json", "application/xml" }) diff --git a/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java b/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java index c33d044e39..239081d140 100644 --- a/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java +++ b/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java @@ -32,6 +32,7 @@ import java.util.Set; import java.util.TreeMap; +import javax.annotation.PostConstruct; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import javax.ws.rs.Consumes; @@ -51,6 +52,7 @@ import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.MapUtils; import org.apache.commons.io.IOUtils; +import org.apache.commons.lang.ArrayUtils; import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -86,16 +88,16 @@ import org.apache.ranger.plugin.model.RangerService; import org.apache.ranger.plugin.model.RangerServiceDef; import org.apache.ranger.plugin.model.validation.RangerPolicyValidator; +import org.apache.ranger.plugin.model.validation.RangerServiceDefHelper; import org.apache.ranger.plugin.model.validation.RangerServiceDefValidator; import org.apache.ranger.plugin.model.validation.RangerServiceValidator; import org.apache.ranger.plugin.model.validation.RangerValidator.Action; import org.apache.ranger.plugin.policyengine.RangerAccessResource; import org.apache.ranger.plugin.policyengine.RangerAccessResourceImpl; import org.apache.ranger.plugin.policyengine.RangerPolicyEngine; -import org.apache.ranger.plugin.policyengine.RangerPolicyEngineCache; +import org.apache.ranger.plugin.policyengine.RangerPolicyEngineCacheForEngineOptions; import org.apache.ranger.plugin.policyengine.RangerPolicyEngineImpl; import org.apache.ranger.plugin.policyengine.RangerPolicyEngineOptions; -import org.apache.ranger.plugin.policyevaluator.RangerPolicyEvaluator; import org.apache.ranger.plugin.service.ResourceLookupContext; import org.apache.ranger.plugin.store.PList; import org.apache.ranger.plugin.store.EmbeddedServiceDefsUtil; @@ -201,9 +203,21 @@ public class ServiceREST { @Autowired JSONUtil jsonUtil; + private RangerPolicyEngineOptions delegateAdminOptions; + private RangerPolicyEngineOptions policySearchAdminOptions; + private RangerPolicyEngineOptions defaultAdminOptions; + public ServiceREST() { } + @PostConstruct + public void initStore() { + tagStore.setServiceStore(svcStore); + delegateAdminOptions = getDelegatedAdminPolicyEngineOptions(); + policySearchAdminOptions = getPolicySearchRangerAdminPolicyEngineOptions(); + defaultAdminOptions = getDefaultRangerAdminPolicyEngineOptions(); + } + @POST @Path("/definitions") @Produces({ "application/json", "application/xml" }) @@ -472,7 +486,130 @@ public RangerServiceDefList getServiceDefs(@Context HttpServletRequest request) } return ret; } - + + @GET + @Path("/policies/{serviceDefName}/for-resource") + @Produces({ "application/json", "application/xml" }) + public List getPoliciesForResource(@PathParam("serviceDefName") String serviceDefName, + @DefaultValue("") @QueryParam("serviceName") String serviceName, + @Context HttpServletRequest request) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> ServiceREST.getPoliciesForResource(service-type=" + serviceDefName + ", service-name=" + serviceName + ")"); + } + + List ret = new ArrayList<>(); + + List services = new ArrayList<>(); + Map resource = new HashMap<>(); + + String validationMessage = validateResourcePoliciesRequest(serviceDefName, serviceName, request, services, resource); + + if (StringUtils.isNotEmpty(validationMessage)) { + LOG.error("Invalid request: [" + validationMessage + "]"); + throw restErrorUtil.createRESTException(validationMessage, + MessageEnums.INVALID_INPUT_DATA); + } else { + RangerService service = services.get(0); + if (LOG.isDebugEnabled()) { + LOG.debug("getServicePolicies with service-name=" + service.getName()); + } + + RangerPolicyEngine engine = null; + + try { + engine = getPolicySearchPolicyEngine(service.getName()); + } catch (Exception e) { + LOG.error("Cannot initialize Policy-Engine", e); + throw restErrorUtil.createRESTException("Cannot initialize Policy Engine", + MessageEnums.ERROR_SYSTEM); + } + + if (engine != null) { + ret = engine.getMatchingPolicies(new RangerAccessResourceImpl(resource)); + } + + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== ServiceREST.getPoliciesForResource(service-type=" + serviceDefName + ", service-name=" + serviceName + ") : " + ret.toString()); + } + return ret; + } + + private String validateResourcePoliciesRequest(String serviceDefName, String serviceName, HttpServletRequest request, List services, Map resource) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> ServiceREST.validatePoliciesForResourceRequest(service-type=" + serviceDefName + ", service-name=" + serviceName + ")"); + } + final String ret; + + if (MapUtils.isNotEmpty(request.getParameterMap())) { + for (Map.Entry e : request.getParameterMap().entrySet()) { + String name = e.getKey(); + String[] values = e.getValue(); + + if (!StringUtils.isEmpty(name) && !ArrayUtils.isEmpty(values) + && name.startsWith(SearchFilter.RESOURCE_PREFIX)) { + resource.put(name.substring(SearchFilter.RESOURCE_PREFIX.length()), values[0]); + } + } + } + if (MapUtils.isEmpty(resource)) { + ret = "No resource specified"; + } else { + RangerServiceDef serviceDef = null; + try { + serviceDef = svcStore.getServiceDefByName(serviceDefName); + } catch (Exception e) { + LOG.error("Invalid service-type:[" + serviceDefName + "]", e); + } + if (serviceDef == null) { + ret = "Invalid service-type:[" + serviceDefName + "]"; + } else { + Set resourceDefNames = resource.keySet(); + RangerServiceDefHelper serviceDefHelper = new RangerServiceDefHelper(serviceDef); + Set> resourceHierarchies = serviceDefHelper.getResourceHierarchies(RangerPolicy.POLICY_TYPE_ACCESS, resourceDefNames); + if (CollectionUtils.isEmpty(resourceHierarchies)) { + ret = "Invalid resource specified: resource-names:" + resourceDefNames +" are not part of any valid resource hierarchy for service-type:[" + serviceDefName + "]"; + } else { + if (StringUtils.isNotBlank(serviceName)) { + RangerService service = null; + try { + service = svcStore.getServiceByName(serviceName); + } catch (Exception e) { + LOG.error("Invalid service-name:[" + serviceName + "]"); + } + if (service == null || !StringUtils.equals(service.getType(), serviceDefName)) { + ret = "Invalid service-name:[" + serviceName + "] or service-name is not of service-type:[" + serviceDefName + "]"; + } else { + services.add(service); + ret = StringUtils.EMPTY; + } + } else { + SearchFilter filter = new SearchFilter(); + filter.setParam(SearchFilter.SERVICE_TYPE, serviceDefName); + List serviceList = null; + try { + serviceList = svcStore.getServices(filter); + } catch (Exception e) { + LOG.error("Cannot find service of service-type:[" + serviceDefName + "]"); + } + if (CollectionUtils.isEmpty(serviceList) || serviceList.size() != 1) { + ret = "Either 0 or more than 1 services found for service-type :[" + serviceDefName + "]"; + } else { + services.add(serviceList.get(0)); + ret = StringUtils.EMPTY; + } + } + } + } + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== ServiceREST.validatePoliciesForResourceRequest(service-type=" + serviceDefName + ", service-name=" + serviceName + ") : " + ret); + } + return ret; + } + @POST @Path("/services") @Produces({ "application/json", "application/xml" }) @@ -2821,6 +2958,34 @@ void ensureAdminAccess(String serviceName, Map res } } + private RangerPolicyEngineOptions getDelegatedAdminPolicyEngineOptions() { + RangerPolicyEngineOptions opts = new RangerPolicyEngineOptions(); + + final String propertyPrefix = "ranger.admin"; + + opts.configureDelegateAdmin(RangerConfiguration.getInstance(), propertyPrefix); + + return opts; + } + + private RangerPolicyEngineOptions getPolicySearchRangerAdminPolicyEngineOptions() { + RangerPolicyEngineOptions opts = new RangerPolicyEngineOptions(); + + final String propertyPrefix = "ranger.admin"; + + opts.configureRangerAdminForPolicySearch(RangerConfiguration.getInstance(), propertyPrefix); + return opts; + } + + private RangerPolicyEngineOptions getDefaultRangerAdminPolicyEngineOptions() { + RangerPolicyEngineOptions opts = new RangerPolicyEngineOptions(); + + final String propertyPrefix = "ranger.admin"; + + opts.configureDefaultRangerAdmin(RangerConfiguration.getInstance(), propertyPrefix); + return opts; + } + private boolean hasAdminAccess(String serviceName, String userName, Set userGroups, Map resources) { boolean isAllowed = false; @@ -2846,40 +3011,18 @@ private boolean hasAdminAccess(String serviceName, String userName, Set } private RangerPolicyEngine getDelegatedAdminPolicyEngine(String serviceName) { - if(RangerPolicyEngineCache.getInstance().getPolicyEngineOptions() == null) { - RangerPolicyEngineOptions options = new RangerPolicyEngineOptions(); - - String propertyPrefix = "ranger.admin"; - - options.evaluatorType = RangerPolicyEvaluator.EVALUATOR_TYPE_OPTIMIZED; - options.cacheAuditResults = RangerConfiguration.getInstance().getBoolean(propertyPrefix + ".policyengine.option.cache.audit.results", false); - options.disableContextEnrichers = RangerConfiguration.getInstance().getBoolean(propertyPrefix + ".policyengine.option.disable.context.enrichers", true); - options.disableCustomConditions = RangerConfiguration.getInstance().getBoolean(propertyPrefix + ".policyengine.option.disable.custom.conditions", true); - options.evaluateDelegateAdminOnly = RangerConfiguration.getInstance().getBoolean(propertyPrefix + ".policyengine.option.evaluate.delegateadmin.only", true); - - RangerPolicyEngineCache.getInstance().setPolicyEngineOptions(options); - } - - RangerPolicyEngine ret = RangerPolicyEngineCache.getInstance().getPolicyEngine(serviceName, svcStore); + return RangerPolicyEngineCacheForEngineOptions.getInstance().getPolicyEngine(serviceName, svcStore, delegateAdminOptions); + } - return ret; + private RangerPolicyEngine getPolicySearchPolicyEngine(String serviceName) throws Exception { + return RangerPolicyEngineCacheForEngineOptions.getInstance().getPolicyEngine(serviceName, svcStore, policySearchAdminOptions); } private RangerPolicyEngine getPolicyEngine(String serviceName) throws Exception { - RangerPolicyEngineOptions options = new RangerPolicyEngineOptions(); - - String propertyPrefix = "ranger.admin"; - - options.evaluatorType = RangerPolicyEvaluator.EVALUATOR_TYPE_OPTIMIZED; - options.cacheAuditResults = RangerConfiguration.getInstance().getBoolean(propertyPrefix + ".policyengine.option.cache.audit.results", false); - options.disableContextEnrichers = RangerConfiguration.getInstance().getBoolean(propertyPrefix + ".policyengine.option.disable.context.enrichers", true); - options.disableCustomConditions = RangerConfiguration.getInstance().getBoolean(propertyPrefix + ".policyengine.option.disable.custom.conditions", true); - options.evaluateDelegateAdminOnly = false; - options.disableTrieLookupPrefilter = RangerConfiguration.getInstance().getBoolean(propertyPrefix + ".policyengine.option.disable.trie.lookup.prefilter", false); ServicePolicies policies = svcStore.getServicePoliciesIfUpdated(serviceName, -1L); - RangerPolicyEngine ret = new RangerPolicyEngineImpl("ranger-admin", policies, options); + RangerPolicyEngine ret = new RangerPolicyEngineImpl("ranger-admin", policies, defaultAdminOptions); return ret; } From 6a953dbd6de0468143afae8e5b5546646559ac13 Mon Sep 17 00:00:00 2001 From: Mehul Parikh Date: Tue, 20 Jun 2017 17:54:34 +0530 Subject: [PATCH 004/151] RANGER-1651 : Improve Ranger and Ranger KMS REST Api documentation --- docs/src/site/site.xml | 2 ++ enunciate.xml | 37 +++++++++++++++++++++++++++++++++++++ kms/pom.xml | 15 +++++++++++++++ pom.xml | 27 +++++++++++++++++++++++++++ security-admin/pom.xml | 15 +++++++++++++++ 5 files changed, 96 insertions(+) create mode 100644 enunciate.xml diff --git a/docs/src/site/site.xml b/docs/src/site/site.xml index fc4b037fd8..e729499805 100644 --- a/docs/src/site/site.xml +++ b/docs/src/site/site.xml @@ -46,6 +46,8 @@ under the License. + + diff --git a/enunciate.xml b/enunciate.xml new file mode 100644 index 0000000000..c91d1969bf --- /dev/null +++ b/enunciate.xml @@ -0,0 +1,37 @@ + + + Ranger REST API + Apache Ranger is a framework to enable, monitor and manage comprehensive data security across the Hadoop platform. Apache Ranger currently provides a centralized security adminstration, fine grain access control and detailed auditing for user access within Apache Hadoop, Apache Hive, Apache HBase and other Apache components + + + + + + + + + + + + + + + + diff --git a/kms/pom.xml b/kms/pom.xml index dd3f0c5f36..54fa01cd80 100644 --- a/kms/pom.xml +++ b/kms/pom.xml @@ -429,6 +429,10 @@ ${derby.version} test + + com.webcohesion.enunciate + enunciate-core-annotations + @@ -438,6 +442,17 @@ maven-war-plugin 2.6 + + com.webcohesion.enunciate + enunciate-maven-plugin + + enunciate.xml + + docs/target/kms/ + 1.8 + 1.8 + + diff --git a/pom.xml b/pom.xml index 91e4434ed8..80de97e8bf 100644 --- a/pom.xml +++ b/pom.xml @@ -353,6 +353,11 @@ gson ${gson.version} + + com.webcohesion.enunciate + enunciate-core-annotations + 2.8.0 + @@ -475,6 +480,26 @@ maven-release-plugin 2.5.2 + + com.webcohesion.enunciate + enunciate-maven-plugin + 2.8.0 + + enunciate.xml + + docs/src/site/ + 1.8 + 1.8 + + + + + docs + + package + + + @@ -576,6 +601,8 @@ **/.pydevproject **/derby.log **/*.jaas + **/target/apidocs/** + **/target/kms/apidocs/** diff --git a/security-admin/pom.xml b/security-admin/pom.xml index 90b8cab802..f863d746ec 100644 --- a/security-admin/pom.xml +++ b/security-admin/pom.xml @@ -434,6 +434,10 @@ guice 3.0 + + com.webcohesion.enunciate + enunciate-core-annotations + @@ -516,6 +520,17 @@ + + com.webcohesion.enunciate + enunciate-maven-plugin + + enunciate.xml + + docs/target/ + 1.8 + 1.8 + + From 8675a2495001404ca5ef5678923cf35999ab2211 Mon Sep 17 00:00:00 2001 From: Ankita Sinha Date: Mon, 19 Jun 2017 17:15:09 +0530 Subject: [PATCH 005/151] RANGER-1653 : Proxying Ranger UI does not work with Ranger-KnoxSSO --- .../filter/RangerSSOAuthenticationFilter.java | 52 +++++++++++++++++-- 1 file changed, 48 insertions(+), 4 deletions(-) diff --git a/security-admin/src/main/java/org/apache/ranger/security/web/filter/RangerSSOAuthenticationFilter.java b/security-admin/src/main/java/org/apache/ranger/security/web/filter/RangerSSOAuthenticationFilter.java index 2f0b3c16b9..c987c18670 100644 --- a/security-admin/src/main/java/org/apache/ranger/security/web/filter/RangerSSOAuthenticationFilter.java +++ b/security-admin/src/main/java/org/apache/ranger/security/web/filter/RangerSSOAuthenticationFilter.java @@ -52,8 +52,10 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Date; +import java.util.Enumeration; import java.util.List; +import org.apache.commons.lang.StringUtils; import org.apache.ranger.biz.UserMgr; import org.apache.ranger.common.PropertiesUtil; import org.apache.ranger.common.UserSessionBase; @@ -78,6 +80,7 @@ public class RangerSSOAuthenticationFilter implements Filter { public static final String JWT_ORIGINAL_URL_QUERY_PARAM_DEFAULT = "originalUrl"; public static final String LOCAL_LOGIN_URL = "locallogin"; public static final String DEFAULT_BROWSER_USERAGENT = "ranger.default.browser-useragents"; + public static final String PROXY_RANGER_URL_PATH = "/ranger"; private SSOAuthenticationProperties jwtProperties; @@ -116,6 +119,9 @@ public void init(FilterConfig filterConfig) throws ServletException { public void doFilter(ServletRequest servletRequest, ServletResponse servletResponse, FilterChain filterChain)throws IOException, ServletException { HttpServletRequest httpRequest = (HttpServletRequest)servletRequest; + + String xForwardedURL = constructForwardableURL(httpRequest); + if (httpRequest.getRequestedSessionId() != null && !httpRequest.isRequestedSessionIdValid()){ synchronized(httpRequest.getServletContext()){ if(httpRequest.getServletContext().getAttribute(httpRequest.getRequestedSessionId()) != null && httpRequest.getServletContext().getAttribute(httpRequest.getRequestedSessionId()).toString().equals("locallogin")){ @@ -178,7 +184,7 @@ public void doFilter(ServletRequest servletRequest, ServletResponse servletRespo // if the token is not valid then redirect to knox sso else { if (isWebUserAgent(userAgent)) { - String ssourl = constructLoginURL(httpRequest); + String ssourl = constructLoginURL(httpRequest, xForwardedURL); if (LOG.isDebugEnabled()) { LOG.debug("SSO URL = " + ssourl); } @@ -194,7 +200,7 @@ public void doFilter(ServletRequest servletRequest, ServletResponse servletRespo // if the jwt token is not available then redirect it to knox sso else { if (isWebUserAgent(userAgent)) { - String ssourl = constructLoginURL(httpRequest); + String ssourl = constructLoginURL(httpRequest, xForwardedURL); if (LOG.isDebugEnabled()) { LOG.debug("SSO URL = " + ssourl); } @@ -222,6 +228,39 @@ public void doFilter(ServletRequest servletRequest, ServletResponse servletRespo } } + private String constructForwardableURL(HttpServletRequest httpRequest){ + String xForwardedProto = null; + String xForwardedHost = null; + String xForwardedContext = null; + Enumeration names = httpRequest.getHeaderNames(); + while (names.hasMoreElements()) { + String name = (String) names.nextElement(); + Enumeration values = httpRequest.getHeaders(name); + String value = null; + if (values != null) { + while (values.hasMoreElements()) { + value = (String) values.nextElement(); + } + } + if (StringUtils.trimToNull(name) != null && StringUtils.trimToNull(value) != null) { + if (name.equalsIgnoreCase("x-forwarded-proto")) { + xForwardedProto = value; + } else if (name.equalsIgnoreCase("x-forwarded-host")) { + xForwardedHost = value; + } else if (name.equalsIgnoreCase("x-forwarded-context")) { + xForwardedContext = value; + } + } + } + String xForwardedURL = null; + if (StringUtils.trimToNull(xForwardedProto) != null && StringUtils.trimToNull(xForwardedHost) != null && StringUtils.trimToNull(xForwardedContext) != null) { + xForwardedURL = xForwardedProto + "://" + xForwardedHost + + xForwardedContext + PROXY_RANGER_URL_PATH + + httpRequest.getRequestURI(); + } + return xForwardedURL; + } + private Authentication getGrantedAuthority(Authentication authentication) { UsernamePasswordAuthenticationToken result=null; if(authentication!=null && authentication.isAuthenticated()){ @@ -326,12 +365,17 @@ protected String getJWTFromCookie(HttpServletRequest req) { * for getting the original request URL * @return url to use as login url for redirect */ - protected String constructLoginURL(HttpServletRequest request) { + protected String constructLoginURL(HttpServletRequest request, String xForwardedURL) { String delimiter = "?"; if (authenticationProviderUrl.contains("?")) { delimiter = "&"; } - String loginURL = authenticationProviderUrl + delimiter + originalUrlQueryParam + "=" + request.getRequestURL().append(getOriginalQueryString(request)); + String loginURL = authenticationProviderUrl + delimiter + originalUrlQueryParam + "="; + if (StringUtils.trimToNull(xForwardedURL) != null) { + loginURL += xForwardedURL + getOriginalQueryString(request); + } else { + loginURL += request.getRequestURL().append(getOriginalQueryString(request)); + } return loginURL; } From 98da92207ba78a640d4ddce8f02cb2de8657040f Mon Sep 17 00:00:00 2001 From: Gautam Borad Date: Fri, 23 Jun 2017 13:50:18 +0530 Subject: [PATCH 006/151] RANGER-1638 : Improve the password validation from Ranger API --- .../java/org/apache/ranger/biz/XUserMgr.java | 27 ++++++++++++++++--- .../org/apache/ranger/biz/TestXUserMgr.java | 3 ++- 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/security-admin/src/main/java/org/apache/ranger/biz/XUserMgr.java b/security-admin/src/main/java/org/apache/ranger/biz/XUserMgr.java index cd1de9ffdd..b973b9a362 100644 --- a/security-admin/src/main/java/org/apache/ranger/biz/XUserMgr.java +++ b/security-admin/src/main/java/org/apache/ranger/biz/XUserMgr.java @@ -180,6 +180,7 @@ public VXGroup getGroupByGroupName(String groupName) { public VXUser createXUser(VXUser vXUser) { checkAdminAccess(); + validatePassword(vXUser); String userName = vXUser.getName(); if (userName == null || "null".equalsIgnoreCase(userName) || userName.trim().isEmpty()) { @@ -382,8 +383,10 @@ public VXUser updateXUser(VXUser vXUser) { && password.equals(hiddenPasswordString)) { vXPortalUser.setPassword(oldUserProfile.getPassword()); } - vXPortalUser.setPassword(password); - + else if(password != null){ + validatePassword(vXUser); + vXPortalUser.setPassword(password); + } Collection groupIdList = vXUser.getGroupIdList(); XXPortalUser xXPortalUser = new XXPortalUser(); xXPortalUser = userMgr.updateUserWithPass(vXPortalUser); @@ -514,6 +517,9 @@ public VXUser updateXUser(VXUser vXUser) { public VXUserGroupInfo createXUserGroupFromMap( VXUserGroupInfo vXUserGroupInfo) { checkAdminAccess(); + if(vXUserGroupInfo.getXuserInfo() != null) { + validatePassword(vXUserGroupInfo.getXuserInfo()); + } VXUserGroupInfo vxUGInfo = new VXUserGroupInfo(); VXUser vXUser = vXUserGroupInfo.getXuserInfo(); @@ -614,6 +620,7 @@ public VXGroupUserInfo getXGroupUserFromMap( public VXUser createXUserWithOutLogin(VXUser vXUser) { checkAdminAccess(); + validatePassword(vXUser); return xUserService.createXUserWithOutLogin(vXUser); } @@ -2145,5 +2152,19 @@ public VXUser createServiceConfigUser(String userName){ } } return createdXUser; -} + } + private void validatePassword(VXUser vXUser) { + if (vXUser.getPassword() != null && !vXUser.getPassword().isEmpty()) { + boolean checkPassword = false; + String pattern = "(?=.*[0-9])(?=.*[a-zA-Z]).{8,}"; + checkPassword = vXUser.getPassword().trim().matches(pattern); + if (!checkPassword) { + logger.warn("validatePassword(). Password should be minimum 8 characters with min one alphabet and one numeric."); + throw restErrorUtil.createRESTException("serverMsg.xuserMgrValidatePassword", MessageEnums.INVALID_PASSWORD, null, "Password should be minimum 8 characters with min one alphabet and one numeric", null); + } + } else { + logger.warn("validatePassword(). Password cannot be blank/null."); + throw restErrorUtil.createRESTException("serverMsg.xuserMgrValidatePassword", MessageEnums.INVALID_PASSWORD, null, "Password cannot be blank/null", null); + } + } } diff --git a/security-admin/src/test/java/org/apache/ranger/biz/TestXUserMgr.java b/security-admin/src/test/java/org/apache/ranger/biz/TestXUserMgr.java index 3323f11f63..2542f91b52 100644 --- a/security-admin/src/test/java/org/apache/ranger/biz/TestXUserMgr.java +++ b/security-admin/src/test/java/org/apache/ranger/biz/TestXUserMgr.java @@ -188,6 +188,7 @@ private VXUser vxUser() { vxUser.setName("grouptest"); vxUser.setUserRoleList(userRoleList); vxUser.setGroupNameList(groupNameList); + vxUser.setPassword("usertest123"); return vxUser; } @@ -835,7 +836,7 @@ public void test30CreateVXUserGroupInfo() { VXUser vXUser = new VXUser(); vXUser.setName("user1"); vXUser.setDescription("testuser1 -added for unit testing"); - + vXUser.setPassword("usertest123"); List vXGroupUserList = new ArrayList(); List vXGroupList = new ArrayList(); From 50ef5f0fd94f29debb81bffb15d2aefe6dad4381 Mon Sep 17 00:00:00 2001 From: Ankita Sinha Date: Fri, 23 Jun 2017 10:44:51 +0530 Subject: [PATCH 007/151] RANGER-1639 : Ranger KMS should validate key name before importing into DB --- .../hadoop/crypto/key/RangerKeyStore.java | 20 ++- .../crypto/key/kms/TestRangerKeyStore.java | 156 ++++++++++++++++++ 2 files changed, 173 insertions(+), 3 deletions(-) create mode 100644 kms/src/test/java/org/apache/hadoop/crypto/key/kms/TestRangerKeyStore.java diff --git a/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKeyStore.java b/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKeyStore.java index a001c0876f..4b1b9bb069 100644 --- a/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKeyStore.java +++ b/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKeyStore.java @@ -42,11 +42,12 @@ import java.security.UnrecoverableKeyException; import java.security.cert.Certificate; import java.security.cert.CertificateException; -import java.util.ArrayList; import java.util.Date; import java.util.Enumeration; import java.util.Hashtable; import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import javax.crypto.SealedObject; import javax.xml.bind.DatatypeConverter; @@ -64,6 +65,8 @@ public class RangerKeyStore extends KeyStoreSpi { static final Logger logger = Logger.getLogger(RangerKeyStore.class); + private static final String KEY_NAME_VALIDATION = "[a-z,A-Z,0-9](?!.*--)(?!.*__)(?!.*-_)(?!.*_-)[\\w\\-\\_]*"; + private static final Pattern pattern = Pattern.compile(KEY_NAME_VALIDATION); private DaoManager daoManager; @@ -89,7 +92,7 @@ private static final class SecretKeyEntry { RangerKeyStore() { } - RangerKeyStore(DaoManager daoManager) { + public RangerKeyStore(DaoManager daoManager) { this.daoManager = daoManager; } @@ -526,6 +529,7 @@ public void engineLoadKeyStoreFile(InputStream stream, char[] storePass, char[] entry.cipher_field = k.getAlgorithm(); } String keyName = alias.split("@")[0]; + validateKeyName(keyName); entry.attributes = "{\"key.acl.name\":\"" + keyName + "\"}"; Class c = null; Object o = null; @@ -581,7 +585,17 @@ public void engineLoadToKeyStoreFile(OutputStream stream, char[] storePass, char } } } - + + private void validateKeyName(String name) { + Matcher matcher = pattern.matcher(name); + if (!matcher.matches()) { + throw new IllegalArgumentException( + "Key Name : " + + name + + ", should start with alpha/numeric letters and can have special characters - (hypen) or _ (underscore)"); + } + } + public void clearDeltaEntires(){ deltaEntries.clear(); } diff --git a/kms/src/test/java/org/apache/hadoop/crypto/key/kms/TestRangerKeyStore.java b/kms/src/test/java/org/apache/hadoop/crypto/key/kms/TestRangerKeyStore.java new file mode 100644 index 0000000000..f366964829 --- /dev/null +++ b/kms/src/test/java/org/apache/hadoop/crypto/key/kms/TestRangerKeyStore.java @@ -0,0 +1,156 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.crypto.key.kms; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.security.Key; +import java.security.KeyStore; +import java.security.KeyStoreException; +import java.security.NoSuchAlgorithmException; +import java.security.cert.CertificateException; + +import javax.crypto.KeyGenerator; + +import org.apache.hadoop.crypto.key.RangerKeyStore; +import org.apache.ranger.kms.dao.DaoManager; +import org.junit.After; +import org.junit.Before; +import org.junit.FixMethodOrder; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.junit.runner.RunWith; +import org.junit.runners.MethodSorters; +import org.mockito.Mockito; +import org.mockito.runners.MockitoJUnitRunner; + +@RunWith(MockitoJUnitRunner.class) +@FixMethodOrder(MethodSorters.NAME_ASCENDING) +public class TestRangerKeyStore { + + String fileFormat = "jceks"; + String keyStoreFileName = "KmsKeyStoreFile"; + char[] storePass = "none".toCharArray(); + char[] keyPass = "none".toCharArray(); + char[] masterKey = "MasterPassword".toCharArray(); + + @Rule + public ExpectedException thrown = ExpectedException.none(); + + @Before + public void checkFileIfExists() { + deleteKeyStoreFile(); + } + + @After + public void cleanKeystoreFile() { + deleteKeyStoreFile(); + } + + @Test(expected=IOException.class) + public void testInvalidKey1() throws NoSuchAlgorithmException, + CertificateException, IOException, KeyStoreException { + + DaoManager daoManager = Mockito.mock(DaoManager.class); + RangerKeyStore rangerKeyStore = new RangerKeyStore(daoManager); + String keyValue = "enckey:1"; + rangerKeyStore.engineLoadKeyStoreFile(generateKeyStoreFile(keyValue),storePass, keyPass, masterKey, fileFormat); + } + + @Test(expected=IOException.class) + public void testInvalidKey2() throws NoSuchAlgorithmException, + CertificateException, IOException, KeyStoreException { + + DaoManager daoManager = Mockito.mock(DaoManager.class); + RangerKeyStore rangerKeyStore = new RangerKeyStore(daoManager); + String keyValue = "1%enckey"; + rangerKeyStore.engineLoadKeyStoreFile(generateKeyStoreFile(keyValue),storePass, keyPass, masterKey, fileFormat); + } + + @Test(expected=IOException.class) + public void testInvalidKey3() throws NoSuchAlgorithmException, + CertificateException, IOException, KeyStoreException { + + DaoManager daoManager = Mockito.mock(DaoManager.class); + RangerKeyStore rangerKeyStore = new RangerKeyStore(daoManager); + String keyValue = "1 enckey"; + rangerKeyStore.engineLoadKeyStoreFile(generateKeyStoreFile(keyValue),storePass, keyPass, masterKey, fileFormat); + } + + @Test(expected=IOException.class) + public void testInvalidKey4() throws NoSuchAlgorithmException, + CertificateException, IOException, KeyStoreException { + + DaoManager daoManager = Mockito.mock(DaoManager.class); + RangerKeyStore rangerKeyStore = new RangerKeyStore(daoManager); + String keyValue = "_1-enckey"; + rangerKeyStore.engineLoadKeyStoreFile(generateKeyStoreFile(keyValue),storePass, keyPass, masterKey, fileFormat); + } + + @Test + public void testValidKey1() throws NoSuchAlgorithmException, + CertificateException, IOException, KeyStoreException { + + DaoManager daoManager = Mockito.mock(DaoManager.class); + RangerKeyStore rangerKeyStore = new RangerKeyStore(daoManager); + String keyValue = "enckey_1-test"; + rangerKeyStore.engineLoadKeyStoreFile(generateKeyStoreFile(keyValue),storePass, keyPass, masterKey, fileFormat); + } + + @Test + public void testValidKey2() throws NoSuchAlgorithmException, + CertificateException, IOException, KeyStoreException { + + DaoManager daoManager = Mockito.mock(DaoManager.class); + RangerKeyStore rangerKeyStore = new RangerKeyStore(daoManager); + String keyValue = "1-enckey_test"; + rangerKeyStore.engineLoadKeyStoreFile(generateKeyStoreFile(keyValue),storePass, keyPass, masterKey, fileFormat); + } + + private InputStream generateKeyStoreFile(String keyValue) throws KeyStoreException, NoSuchAlgorithmException, CertificateException, IOException { + FileOutputStream stream = new FileOutputStream(new File(keyStoreFileName)); + KeyStore ks; + try { + ks = KeyStore.getInstance(fileFormat); + if (ks != null) { + ks.load(null, storePass); + String alias = keyValue; + + KeyGenerator kg = KeyGenerator.getInstance("AES"); + kg.init(256); + Key key = kg.generateKey(); + ks.setKeyEntry(alias, key, keyPass, null); + ks.store(stream, storePass); + } + return new FileInputStream(new File(keyStoreFileName)); + } catch (Throwable t) { + throw new IOException(t); + } + } + + private void deleteKeyStoreFile() { + File f = new File(keyStoreFileName); + if (f.exists()) { + f.delete(); + } + } +} From 496fc23b1cc1f829c00e7ec3f3ab594d83317e2a Mon Sep 17 00:00:00 2001 From: Nitin Galave Date: Mon, 3 Jul 2017 15:06:38 +0530 Subject: [PATCH 008/151] RANGER-1666-Ranger UI should consider recursiveSupported attribute value at each resource level to Store the Policy --- .../webapp/scripts/modules/XAOverrides.js | 67 +++++++++++++------ .../views/policies/RangerPolicyForm.js | 26 ------- security-admin/src/main/webapp/styles/xa.css | 25 +++++-- 3 files changed, 67 insertions(+), 51 deletions(-) diff --git a/security-admin/src/main/webapp/scripts/modules/XAOverrides.js b/security-admin/src/main/webapp/scripts/modules/XAOverrides.js index 7d7a9d133d..5810d5d147 100644 --- a/security-admin/src/main/webapp/scripts/modules/XAOverrides.js +++ b/security-admin/src/main/webapp/scripts/modules/XAOverrides.js @@ -490,13 +490,13 @@ this.resourceOpts = {}; _.extend(this, _.pick(this.schema,'excludeSupport','recursiveSupport','resourceOpts','resourcesAtSameLevel','sameLevelOpts', 'initilializePathPlugin', 'validators','name','formView')); - //(edit mode)set values for sameLevel if first option is not selected - if(!_.isNull(this.value) && !_.isUndefined(this.value) + //(edit mode)set values for sameLevel if first option is not selected + if(!_.isNull(this.value) && !_.isUndefined(this.value) && !_.isUndefined(this.value.resourceType)){ - var def = _.findWhere(this.form.rangerServiceDefModel.get('resources'), {'name': this.value.resourceType }); - this.recursiveSupport = def.recursiveSupported; - this.excludeSupport = def.excludesSupported; - } + var def = _.findWhere(this.form.rangerServiceDefModel.get('resources'), {'name': this.value.resourceType }); + this.recursiveSupport = def.recursiveSupported; + this.excludeSupport = def.excludesSupported; + } this.template = this.getTemplate(); }, initializeElements : function() { @@ -574,7 +574,21 @@ this.value.isRecursive = _.isUndefined(this.value.isRecursive) ? true : this.value.isRecursive; isRecursive = this.value.isRecursive; } - } + this.$recursiveSupport.show(); + this.$recursiveSupport.removeClass('recursive-toggle-1 recursive-toggle-2'); + this.$recursiveSupport.addClass(this.excludeSupport ? 'recursive-toggle-2' : 'recursive-toggle-1') + this.$recursiveSupport.toggles({ + on: isRecursive, + text : {on : 'recursive', off : 'non-recursive' }, + width: 120, + }).on('toggle', function (e, active) { + that.value.isRecursive = active; + XAUtil.checkDirtyFieldForToggle($(e.currentTarget)) + }); + } else { + this.$recursiveSupport.hide(); + } + }, renderSameLevelResource : function() { var that = this, dirtyFieldValue = null; @@ -600,15 +614,15 @@ that.$el.parents('.control-group').attr('data-name', 'field-'+this.value); that.formView.trigger('policyForm:parentChildHideShow',true); if(!_.isUndefined(this.value) - && ( XAUtil.capitaliseFirstLetter(this.value) === XAEnums.ResourceType.RESOURCE_UDF.label) ){ + && ( XAUtil.capitaliseFirstLetter(this.value) === XAEnums.ResourceType.RESOURCE_UDF.label) ){ XAUtil.alertPopup({ msg :localization.tt('msg.udfPolicyViolation') }); } - //set flags for newly selected resource and re-render - var def = _.findWhere(that.form.rangerServiceDefModel.get('resources'), {'name': this.value}); - that.recursiveSupport = def.recursiveSupported; - if(that.recursiveSupport) that.value.isRecursive = true; - that.excludeSupport = def.excludesSupported; - that.renderToggles(); + //set flags for newly selected resource and re-render + var def = _.findWhere(that.form.rangerServiceDefModel.get('resources'), {'name': this.value}); + that.recursiveSupport = def.recursiveSupported; + if(that.recursiveSupport) that.value.isRecursive = true; + that.excludeSupport = def.excludesSupported; + that.renderToggles(); }); } @@ -667,16 +681,29 @@ } }, getTemplate : function() { + var that = this , resourcesType ; var optionsHtml="", selectTemplate = '',excludeSupportToggleDiv='', recursiveSupportToggleDiv=''; - this.preserveResourceValues = {}; - if(this.resourcesAtSameLevel){ - _.each(this.sameLevelOpts, function(option){ return optionsHtml += ""; },this); + this.preserveResourceValues = {},klass = ''; + if(this.resourcesAtSameLevel){ + _.each(this.sameLevelOpts, function(option){ return optionsHtml += ""; },this); selectTemplate = ''; - } - excludeSupportToggleDiv = '
'; - return _.template(selectTemplate+''+ + } + excludeSupportToggleDiv = '
'; + _.each(this.form.rangerServiceDefModel.get('resources') , function(m){ + if(that.name === m.name){ + resourcesType = m.type ; + } + }) + if(resourcesType == "path"){ + klass = (!this.excludeSupport) ? "recursive-toggle-hdfs-1" : "recursive-toggle-hdfs-2"; + }else{ + klass = (!this.excludeSupport) ? "recursive-toggle-1" : "recursive-toggle-2"; + } + recursiveSupportToggleDiv = '
'; + + return _.template(selectTemplate+''+ excludeSupportToggleDiv+''+recursiveSupportToggleDiv); }, }); diff --git a/security-admin/src/main/webapp/scripts/views/policies/RangerPolicyForm.js b/security-admin/src/main/webapp/scripts/views/policies/RangerPolicyForm.js index 9145b88020..6f27d5db06 100644 --- a/security-admin/src/main/webapp/scripts/views/policies/RangerPolicyForm.js +++ b/security-admin/src/main/webapp/scripts/views/policies/RangerPolicyForm.js @@ -118,22 +118,6 @@ define(function(require){ var attr1 = _.pick(_.result(this.model,'schemaBase'),basicSchema); var attr2 = _.pick(_.result(this.model,'schemaBase'),schemaNames); - var arr = {}; - - _.each(attrs,function(resourceObject,resourceName){ - if(resourceObject.hasOwnProperty('recursiveSupport')) { - if(resourceObject.recursiveSupport) { - var recursiveAttrSchema = _.pick(_.result(that.model,'schemaBase'),'recursive'); - if(!_.isUndefined(that.model.get('id'))) { - recursiveAttrSchema.recursive.switchOn=(that.model.get(resourceName)).isRecursive; - } - arr[resourceName] = resourceObject; - _.extend(arr,recursiveAttrSchema); - } - } - }); - _.extend(attrs,arr); - return _.extend(attr1,_.extend(attrs,attr2)); }, /** on render callback */ @@ -356,16 +340,6 @@ define(function(require){ var that = this, resources = []; var resources = {}; - //set 'isRecursive' attribute of resource object to value of field recursive - var recursiveValue = ''; - if(!_.isUndefined(this.model.get('recursive'))){ - recursiveValue = that.model.get('recursive'); - } - _.each(this.model.attributes,function(val) { - if(_.isObject(val) && !_.isUndefined(val.isRecursive)) { - val.isRecursive = recursiveValue; - } - });// 'isRecursive' attribute of model is updated //set sameLevel fieldAttr value with resource name _.each(this.model.attributes, function(val, key) { if(key.indexOf("sameLevel") >= 0 && !_.isNull(val)){ diff --git a/security-admin/src/main/webapp/styles/xa.css b/security-admin/src/main/webapp/styles/xa.css index c6f50d3376..4fb905734c 100644 --- a/security-admin/src/main/webapp/styles/xa.css +++ b/security-admin/src/main/webapp/styles/xa.css @@ -1791,10 +1791,7 @@ margin-bottom: 5px; .include-toggle{ margin-left: 264px; margin-top: -29px; -} -.recursive-toggle{ - margin-left: 363px; - margin-top: -20px; + width:80px; } .recursive-toggle-path { margin-left: 347px; @@ -2195,4 +2192,22 @@ td.subgrid-custom-cell{ text-align: center; font-weight: bold; font-style: italic; -} \ No newline at end of file +} +.recursive-toggle-1 { + margin-left: 264px; + margin-top: -29px; + width:110px; +} +.recursive-toggle-2 { + margin-left: 363px; + margin-top: -20px; + width:110px; +} +.recursive-toggle-hdfs-1{ + margin-left: 348px; + margin-top: -27px; +} +.recursive-toggle-hdfs-2{ + margin-left: 388px; + margin-top: -27px; +} From 97b80aa0fc72066b0e429601dcafed79a8b4e003 Mon Sep 17 00:00:00 2001 From: Nitin Galave Date: Fri, 7 Jul 2017 18:22:02 +0530 Subject: [PATCH 009/151] RANGER-1679 : Export Policy not working when Knox proxy is Enabled --- security-admin/src/main/webapp/scripts/utils/XAUtils.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/security-admin/src/main/webapp/scripts/utils/XAUtils.js b/security-admin/src/main/webapp/scripts/utils/XAUtils.js index e294353001..1a86177a4b 100644 --- a/security-admin/src/main/webapp/scripts/utils/XAUtils.js +++ b/security-admin/src/main/webapp/scripts/utils/XAUtils.js @@ -1248,7 +1248,7 @@ define(function(require) { } return window.location.origin + window.location.pathname.substring(window.location.pathname - .indexOf('/', 2) + 1, 0); + .lastIndexOf('/') + 1, 0); }; XAUtils.isMaskingPolicy = function(type){ From 87b78f9b37c39dabf004bed0eaa0b304e10aab28 Mon Sep 17 00:00:00 2001 From: srsiva Date: Mon, 10 Jul 2017 23:24:40 -0700 Subject: [PATCH 010/151] RANGER-1689: Enabling recursive policy only for relativepath Signed-off-by: rmani --- .../src/main/resources/service-defs/ranger-servicedef-wasb.json | 1 + 1 file changed, 1 insertion(+) diff --git a/agents-common/src/main/resources/service-defs/ranger-servicedef-wasb.json b/agents-common/src/main/resources/service-defs/ranger-servicedef-wasb.json index 9b3eafe338..946b662ad2 100644 --- a/agents-common/src/main/resources/service-defs/ranger-servicedef-wasb.json +++ b/agents-common/src/main/resources/service-defs/ranger-servicedef-wasb.json @@ -48,6 +48,7 @@ "level":30, "mandatory": true, "lookupSupported": true, + "recursiveSupported": true, "excludesSupported": false, "matcher": "org.apache.ranger.plugin.resourcematcher.RangerPathResourceMatcher", "matcherOptions": {"wildCard":true, "ignoreCase":false}, From 61d352fc4c2a14527ccb9f5cd216b8711096c346 Mon Sep 17 00:00:00 2001 From: Ankita Sinha Date: Sat, 15 Jul 2017 17:11:24 +0530 Subject: [PATCH 011/151] RANGER-1638 : Improve the password validation from Ranger API --- .../src/main/java/org/apache/ranger/biz/XUserMgr.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/security-admin/src/main/java/org/apache/ranger/biz/XUserMgr.java b/security-admin/src/main/java/org/apache/ranger/biz/XUserMgr.java index b973b9a362..ca06805389 100644 --- a/security-admin/src/main/java/org/apache/ranger/biz/XUserMgr.java +++ b/security-admin/src/main/java/org/apache/ranger/biz/XUserMgr.java @@ -517,9 +517,6 @@ else if(password != null){ public VXUserGroupInfo createXUserGroupFromMap( VXUserGroupInfo vXUserGroupInfo) { checkAdminAccess(); - if(vXUserGroupInfo.getXuserInfo() != null) { - validatePassword(vXUserGroupInfo.getXuserInfo()); - } VXUserGroupInfo vxUGInfo = new VXUserGroupInfo(); VXUser vXUser = vXUserGroupInfo.getXuserInfo(); From 992e318cbdd24832fd0857a3b1e2f29f0e9ccb13 Mon Sep 17 00:00:00 2001 From: Abhay Kulkarni Date: Mon, 24 Jul 2017 19:13:30 -0700 Subject: [PATCH 012/151] RANGER-1696: Request to get all policies for hive or hbase service-type does not include policies that apply to specific child resource --- .../policyengine/RangerPolicyEngineImpl.java | 7 ++++-- .../RangerPathResourceMatcher.java | 17 +++++++++----- .../plugin/util/StringTokenReplacer.java | 6 ++++- .../test_resourcematcher_dynamic.json | 23 ++++++++++++++++--- ...sourcematcher_wildcards_as_delimiters.json | 6 +++-- 5 files changed, 45 insertions(+), 14 deletions(-) diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineImpl.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineImpl.java index 34ae4169ee..18a7d3faee 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineImpl.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineImpl.java @@ -31,6 +31,7 @@ import org.apache.ranger.plugin.model.RangerServiceDef; import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyResource; import org.apache.ranger.plugin.policyevaluator.RangerPolicyEvaluator; +import org.apache.ranger.plugin.policyresourcematcher.RangerPolicyResourceMatcher; import org.apache.ranger.plugin.util.RangerAccessRequestUtil; import org.apache.ranger.plugin.util.RangerPerfTracer; import org.apache.ranger.plugin.util.ServicePolicies; @@ -569,7 +570,8 @@ public List getMatchingPolicies(RangerAccessResource resource) { for (List evaluators : likelyEvaluators) { for (RangerPolicyEvaluator evaluator : evaluators) { - if (evaluator.isMatch(tagResource, null)) { + RangerPolicyResourceMatcher matcher = evaluator.getPolicyResourceMatcher(); + if (matcher != null && matcher.isMatch(tagResource, RangerPolicyResourceMatcher.MatchScope.SELF_OR_ANCESTOR_OR_DESCENDANT, null)) { ret.add(evaluator.getPolicy()); } } @@ -587,7 +589,8 @@ public List getMatchingPolicies(RangerAccessResource resource) { for (List evaluators : likelyEvaluators) { for (RangerPolicyEvaluator evaluator : evaluators) { - if (evaluator.isMatch(resource, null)) { + RangerPolicyResourceMatcher matcher = evaluator.getPolicyResourceMatcher(); + if (matcher != null && matcher.isMatch(resource, RangerPolicyResourceMatcher.MatchScope.SELF_OR_ANCESTOR_OR_DESCENDANT, null)) { ret.add(evaluator.getPolicy()); } } diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/resourcematcher/RangerPathResourceMatcher.java b/agents-common/src/main/java/org/apache/ranger/plugin/resourcematcher/RangerPathResourceMatcher.java index 98c3c42714..90c132f35a 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/resourcematcher/RangerPathResourceMatcher.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/resourcematcher/RangerPathResourceMatcher.java @@ -227,6 +227,9 @@ abstract class RecursiveMatcher extends ResourceMatcher { } String getStringToCompare(String policyValue) { + if (policyValue == null) { + return null; + } return (policyValue.lastIndexOf(levelSeparatorChar) == policyValue.length()-1) ? policyValue.substring(0, policyValue.length()-1) : policyValue; } @@ -242,9 +245,10 @@ boolean isMatch(String resourceValue, Map evalContext) { final String noSeparator; if (getNeedsDynamicEval()) { - noSeparator = getStringToCompare(getExpandedValue(evalContext)); + String expandedPolicyValue = getExpandedValue(evalContext); + noSeparator = expandedPolicyValue != null ? getStringToCompare(expandedPolicyValue) : null; } else { - if (valueWithoutSeparator == null) { + if (valueWithoutSeparator == null && value != null) { valueWithoutSeparator = getStringToCompare(value); valueWithSeparator = valueWithoutSeparator + Character.toString(levelSeparatorChar); } @@ -253,7 +257,7 @@ boolean isMatch(String resourceValue, Map evalContext) { boolean ret = StringUtils.equals(resourceValue, noSeparator); - if (!ret) { + if (!ret && noSeparator != null) { final String withSeparator = getNeedsDynamicEval() ? noSeparator + Character.toString(levelSeparatorChar) : valueWithSeparator; ret = StringUtils.startsWith(resourceValue, withSeparator); } @@ -273,9 +277,10 @@ boolean isMatch(String resourceValue, Map evalContext) { final String noSeparator; if (getNeedsDynamicEval()) { - noSeparator = getStringToCompare(getExpandedValue(evalContext)); + String expandedPolicyValue = getExpandedValue(evalContext); + noSeparator = expandedPolicyValue != null ? getStringToCompare(expandedPolicyValue) : null; } else { - if (valueWithoutSeparator == null) { + if (valueWithoutSeparator == null && value != null) { valueWithoutSeparator = getStringToCompare(value); valueWithSeparator = valueWithoutSeparator + Character.toString(levelSeparatorChar); } @@ -284,7 +289,7 @@ boolean isMatch(String resourceValue, Map evalContext) { boolean ret = StringUtils.equalsIgnoreCase(resourceValue, noSeparator); - if (!ret) { + if (!ret && noSeparator != null) { final String withSeparator = getNeedsDynamicEval() ? noSeparator + Character.toString(levelSeparatorChar) : valueWithSeparator; ret = StringUtils.startsWithIgnoreCase(resourceValue, withSeparator); } diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/StringTokenReplacer.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/StringTokenReplacer.java index 4ec1595080..2ec809ceed 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/util/StringTokenReplacer.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/StringTokenReplacer.java @@ -72,6 +72,10 @@ public String replaceTokens(String value, Map tokens) { Object replaced = RangerAccessRequestUtil.getTokenFromContext(tokens, rawToken.substring(tokenPrefix.length())); if (replaced != null) { ret.append(replaced.toString()); + } else { + ret = null; + token = null; + break; } } else { ret.append(startChar).append(token).append(endChar); @@ -87,6 +91,6 @@ public String replaceTokens(String value, Map tokens) { ret.append(startChar).append(token); } - return ret.toString(); + return ret != null ? ret.toString() : null; } } diff --git a/agents-common/src/test/resources/resourcematcher/test_resourcematcher_dynamic.json b/agents-common/src/test/resources/resourcematcher/test_resourcematcher_dynamic.json index 168a50f4aa..5237d4773e 100644 --- a/agents-common/src/test/resources/resourcematcher/test_resourcematcher_dynamic.json +++ b/agents-common/src/test/resources/resourcematcher/test_resourcematcher_dynamic.json @@ -7,13 +7,15 @@ "matcherOptions":{"wildCard":true, "ignoreCase":true, "replaceTokens":true, "tokenDelimiterStart":"%", "tokenDelimiterEnd":"%", "tokenDelimiterEscape":"@" } }, "policyResource":{ - "values": ["/abc%xyz%w", "/xyz%somestuff%z", "/abc@%xyz@w", "/mad@@%xyy%"], + "values": ["/abc%xyz%w", "/xyz%somestuff%z", "/abc@%xyz@w", "/mad@@%xyy%","/tmp/tmpdir4/%FILENAME%", "/tmp/tmpdir5/%BASE_FILENAME%.txt"], "isRecursive":false }, "tests":[ { "name":"exact-path","input":"/mad@new", "evalContext": {"token:xyy": "new"}, "result":true} , - { "name":"exact-path","input":"/abcw", "evalContext": {"token:somestuff": "somethingelse"}, "result":true} + { "name":"exact-path","input":"/abcw", "evalContext": {"token:somestuff": "somethingelse"}, "result":false} + , + { "name":"exact-path","input":"/abc%xyz%w", "evalContext": {"token:somestuff": "somethingelse"}, "result":false} , { "name":"exact-path","input":"/abc%xyz%w", "evalContext": {"token:somestuff": "somethingelse", "token:xyz":"abcd"}, "result":false} , @@ -26,7 +28,22 @@ { "name":"exact-path","input":"/abc%xyzw", "evalContext": {"token:somestuff": "somethingelse"}, "result":true} , { "name":"exact-path","input":"/abcabcdw", "evalContext": {"token:somestuff": "somethingelse", "xyz":"abcd"}, "result":false} - + , + { "name":"hdfs-agent-test-10","input":"/tmp/tmpdir4/data-file", "evalContext": {"token:FILENAME": "data-file"}, "result":true} + , + { "name":"hdfs-agent-test-11","input":"/tmp/tmpdir4/data-file", "evalContext": {"token:FILENAME": ""}, "result":false} + , + { "name":"hdfs-agent-test-12","input":"/tmp/tmpdir4/data-file", "evalContext": {"token:USER": "admin"}, "result":false} + , + { "name":"hdfs-agent-test-13","input":"/tmp/tmpdir4", "evalContext": {}, "result":false} + , + { "name":"hdfs-agent-test-11","input":"/tmp/tmpdir4/", "evalContext": {"token:FILENAME": ""}, "result":true} + , + { "name":"hdfs-agent-test-12","input":"/tmp/tmpdir4/%FILENAME%", "evalContext": {"token:USER": "admin"}, "result":false} + , + { "name":"hdfs-agent-test-13","input":"/tmp/tmpdir5/data-file.txt", "evalContext": {"token:BASE_FILENAME": "data-file"}, "result":true} + , + { "name":"hdfs-agent-test-14","input":"/tmp/tmpdir5/txt", "evalContext": {"token:BASE_FILENAME": ""}, "result":false} ] } ] diff --git a/agents-common/src/test/resources/resourcematcher/test_resourcematcher_wildcards_as_delimiters.json b/agents-common/src/test/resources/resourcematcher/test_resourcematcher_wildcards_as_delimiters.json index f896745c7f..c907f414ee 100644 --- a/agents-common/src/test/resources/resourcematcher/test_resourcematcher_wildcards_as_delimiters.json +++ b/agents-common/src/test/resources/resourcematcher/test_resourcematcher_wildcards_as_delimiters.json @@ -15,9 +15,11 @@ , { "name":"exact-path","input":"/mad@new", "evalContext": {"token:xyy": "new"}, "result":true} , - { "name":"exact-path","input":"/abcw", "evalContext": {"token:somestuff": "somethingelse"}, "result":true} + { "name":"exact-path","input":"/abcw", "evalContext": {"token:somestuff": "somethingelse"}, "result":false} , - { "name":"exact-path","input":"/abc*xyz?w", "evalContext": {"token:somestuff": "somethingelse", "token:xyz":"abcd"}, "result":false} + { "name":"exact-path","input":"/abc*xyz?w", "evalContext": {"token:somestuff": "somethingelse"}, "result":false} + , + { "name":"exact-path","input":"/abcabcdw", "evalContext": {"token:somestuff": "somethingelse", "token:xyz":"abcd"}, "result":true} , { "name":"exact-path","input":"/xyzsomethingelsez", "evalContext": {"token:somestuff": "somethingelse"}, "result":true} , From 5c8b45cec1851442edecf02712b5bc9a86677ec9 Mon Sep 17 00:00:00 2001 From: Abhay Kulkarni Date: Wed, 26 Jul 2017 11:31:04 -0700 Subject: [PATCH 013/151] RANGER-1695:Optimize Ranger code for authorization of HDFS 'getContentSummary' and 'delete' commands --- .../constants/RangerHadoopConstants.java | 2 + .../policyengine/RangerPolicyEngineImpl.java | 4 +- .../RangerPolicyEngineOptions.java | 15 ++ .../plugin/service/RangerBasePlugin.java | 2 + .../util/RangerPerfCollectorTracer.java | 5 +- .../hbase/RangerAuthorizationCoprocessor.java | 40 +++-- .../hadoop/RangerHdfsAuthorizer.java | 141 +++++++++++++++++- .../ranger/services/hdfs/HDFSRangerTest.java | 55 +++++++ .../src/test/resources/hdfs-policies.json | 27 ++++ .../src/test/resources/log4j.properties | 34 +++++ .../test/resources/ranger-hdfs-security.xml | 8 + .../hive/authorizer/RangerHiveAuthorizer.java | 28 +++- .../knox/RangerPDPKnoxFilter.java | 11 ++ .../authorizer/RangerAtlasAuthorizer.java | 10 ++ .../authorizer/RangerKafkaAuthorizer.java | 9 ++ .../kms/authorizer/RangerKmsAuthorizer.java | 11 +- .../solr/authorizer/RangerSolrAuthorizer.java | 9 ++ .../yarn/authorizer/RangerYarnAuthorizer.java | 20 +++ ranger-tools/conf/log4j.properties | 1 + .../RangerPolicyenginePerfTester.java | 1 + .../authorizer/RangerStormAuthorizer.java | 22 ++- 21 files changed, 428 insertions(+), 27 deletions(-) create mode 100644 hdfs-agent/src/test/resources/log4j.properties diff --git a/agents-common/src/main/java/org/apache/ranger/authorization/hadoop/constants/RangerHadoopConstants.java b/agents-common/src/main/java/org/apache/ranger/authorization/hadoop/constants/RangerHadoopConstants.java index 83f720a41f..6d9fe26a47 100644 --- a/agents-common/src/main/java/org/apache/ranger/authorization/hadoop/constants/RangerHadoopConstants.java +++ b/agents-common/src/main/java/org/apache/ranger/authorization/hadoop/constants/RangerHadoopConstants.java @@ -21,7 +21,9 @@ public class RangerHadoopConstants { public static final String RANGER_ADD_HDFS_PERMISSION_PROP = "xasecure.add-hadoop-authorization"; + public static final String RANGER_OPTIMIZE_SUBACCESS_AUTHORIZATION_PROP = "ranger.optimize-subaccess-authorization" ; public static final boolean RANGER_ADD_HDFS_PERMISSION_DEFAULT = false; + public static final boolean RANGER_OPTIMIZE_SUBACCESS_AUTHORIZATION_DEFAULT = false ; public static final String READ_ACCCESS_TYPE = "read"; public static final String WRITE_ACCCESS_TYPE = "write"; public static final String EXECUTE_ACCCESS_TYPE = "execute"; diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineImpl.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineImpl.java index 18a7d3faee..ab99b051cf 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineImpl.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineImpl.java @@ -297,7 +297,9 @@ public RangerAccessResult isAccessAllowed(RangerAccessRequest request, RangerAcc RangerPerfTracer perf = null; if(RangerPerfTracer.isPerfTraceEnabled(PERF_POLICYENGINE_REQUEST_LOG)) { - perf = RangerPerfTracer.getPerfTracer(PERF_POLICYENGINE_REQUEST_LOG, "RangerPolicyEngine.isAccessAllowed(requestHashCode=" + Integer.toHexString(System.identityHashCode(request)) + ")"); + String requestHashCode = Integer.toHexString(System.identityHashCode(request)); + perf = RangerPerfTracer.getPerfTracer(PERF_POLICYENGINE_REQUEST_LOG, "RangerPolicyEngine.isAccessAllowed(requestHashCode=" + requestHashCode + ")"); + LOG.info("RangerPolicyEngineImpl.isAccessAllowed(" + requestHashCode + ", " + request + ")"); } RangerAccessResult ret = isAccessAllowedNoAudit(request); diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineOptions.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineOptions.java index 35056439d5..22a63fd520 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineOptions.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineOptions.java @@ -42,6 +42,9 @@ public void configureForPlugin(Configuration conf, String propertyPrefix) { cacheAuditResults = conf.getBoolean(propertyPrefix + ".policyengine.option.cache.audit.results", true); + if (!disableTrieLookupPrefilter) { + cacheAuditResults = false; + } evaluateDelegateAdminOnly = false; enableTagEnricherWithLocalRefresher = false; } @@ -120,4 +123,16 @@ public int hashCode() { ret *= 2; return ret; } + + @Override + public String toString() { + return "PolicyEngineOptions: {" + + " evaluatorType: " + evaluatorType + + ", cacheAuditResult: " + cacheAuditResults + + ", disableContextEnrichers: " + disableContextEnrichers + + ", disableCustomConditions: " + disableContextEnrichers + + ", disableTrieLookupPrefilter: " + disableTrieLookupPrefilter + + " }"; + + } } diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java b/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java index 927dcc2b7c..7a33429895 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java @@ -148,6 +148,8 @@ public void init() { policyEngineOptions.disableTagPolicyEvaluation = RangerConfiguration.getInstance().getBoolean(propertyPrefix + ".policyengine.option.disable.tagpolicy.evaluation", false); policyEngineOptions.disableTrieLookupPrefilter = RangerConfiguration.getInstance().getBoolean(propertyPrefix + ".policyengine.option.disable.trie.lookup.prefilter", false); + LOG.info(policyEngineOptions); + RangerAdminClient admin = createAdminClient(serviceName, appId, propertyPrefix); refresher = new PolicyRefresher(this, serviceType, appId, serviceName, admin, pollingIntervalMs, cacheDir); diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPerfCollectorTracer.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPerfCollectorTracer.java index d899c6f5dd..353f7daec6 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPerfCollectorTracer.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPerfCollectorTracer.java @@ -22,13 +22,16 @@ import org.apache.commons.logging.Log; public class RangerPerfCollectorTracer extends RangerPerfTracer { + private final long startTimeNanos; public RangerPerfCollectorTracer(Log logger, String tag, String data) { super(logger, tag, data); + startTimeNanos = System.nanoTime(); } @Override public void log() { - PerfDataRecorder.recordStatistic(tag, getElapsedTime()); + // Collect elapsed time in microseconds + PerfDataRecorder.recordStatistic(tag, ((System.nanoTime() - startTimeNanos) + 500) / 1000); } } diff --git a/hbase-agent/src/main/java/org/apache/ranger/authorization/hbase/RangerAuthorizationCoprocessor.java b/hbase-agent/src/main/java/org/apache/ranger/authorization/hbase/RangerAuthorizationCoprocessor.java index cf2ffcf150..fc1db4693b 100644 --- a/hbase-agent/src/main/java/org/apache/ranger/authorization/hbase/RangerAuthorizationCoprocessor.java +++ b/hbase-agent/src/main/java/org/apache/ranger/authorization/hbase/RangerAuthorizationCoprocessor.java @@ -111,9 +111,11 @@ import com.google.protobuf.RpcCallback; import com.google.protobuf.RpcController; import com.google.protobuf.Service; +import org.apache.ranger.plugin.util.RangerPerfTracer; public class RangerAuthorizationCoprocessor extends RangerAuthorizationCoprocessorBase implements AccessControlService.Interface, CoprocessorService { private static final Log LOG = LogFactory.getLog(RangerAuthorizationCoprocessor.class.getName()); + private static final Log PERF_HBASEAUTH_REQUEST_LOG = RangerPerfTracer.getPerfLogger("hbaseauth.request"); private static boolean UpdateRangerPoliciesOnGrantRevoke = RangerHadoopConstants.HBASE_UPDATE_RANGER_POLICIES_ON_GRANT_REVOKE_DEFAULT_VALUE; private static final String GROUP_PREFIX = "@"; @@ -334,7 +336,7 @@ ColumnFamilyAccessResult evaluateAccess(String operation, Action action, final R } return result; } - + // let's create a session that would be reused. Set things on it that won't change. HbaseAuditHandler auditHandler = _factory.getAuditHandler(); AuthorizationSession session = new AuthorizationSession(hbasePlugin) @@ -505,7 +507,11 @@ Filter authorizeAccess(String operation, Action action, final RegionCoprocessorE if (LOG.isDebugEnabled()) { LOG.debug("==> authorizeAccess"); } + RangerPerfTracer perf = null; + try { + perf = RangerPerfTracer.getPerfTracer(PERF_HBASEAUTH_REQUEST_LOG, "RangerAuthorizationCoprocessor.authorizeAccess(request=Operation[" + operation + "]"); + ColumnFamilyAccessResult accessResult = evaluateAccess(operation, action, env, familyMap); RangerDefaultAuditHandler auditHandler = new RangerDefaultAuditHandler(); if (accessResult._everythingIsAccessible) { @@ -525,6 +531,7 @@ Filter authorizeAccess(String operation, Action action, final RegionCoprocessorE throw new AccessDeniedException(accessResult._denialReason); } } finally { + RangerPerfTracer.log(perf); if (LOG.isDebugEnabled()) { LOG.debug("<== authorizeAccess"); } @@ -542,17 +549,26 @@ Filter combineFilters(Filter filter, Filter existingFilter) { void requirePermission(final String operation, final Action action, final RegionCoprocessorEnvironment regionServerEnv, final Map> familyMap) throws AccessDeniedException { - ColumnFamilyAccessResult accessResult = evaluateAccess(operation, action, regionServerEnv, familyMap); - RangerDefaultAuditHandler auditHandler = new RangerDefaultAuditHandler(); - if (accessResult._everythingIsAccessible) { - auditHandler.logAuthzAudits(accessResult._accessAllowedEvents); - auditHandler.logAuthzAudits(accessResult._familyLevelAccessEvents); - LOG.debug("requirePermission: exiting: all access was allowed"); - return; - } else { - auditHandler.logAuthzAudit(accessResult._accessDeniedEvent); - LOG.debug("requirePermission: exiting: throwing exception as everything wasn't accessible"); - throw new AccessDeniedException(accessResult._denialReason); + RangerPerfTracer perf = null; + + try { + if (RangerPerfTracer.isPerfTraceEnabled(PERF_HBASEAUTH_REQUEST_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_HBASEAUTH_REQUEST_LOG, "RangerAuthorizationCoprocessor.requirePermission(request=Operation[" + operation + "]"); + } + ColumnFamilyAccessResult accessResult = evaluateAccess(operation, action, regionServerEnv, familyMap); + RangerDefaultAuditHandler auditHandler = new RangerDefaultAuditHandler(); + if (accessResult._everythingIsAccessible) { + auditHandler.logAuthzAudits(accessResult._accessAllowedEvents); + auditHandler.logAuthzAudits(accessResult._familyLevelAccessEvents); + LOG.debug("requirePermission: exiting: all access was allowed"); + return; + } else { + auditHandler.logAuthzAudit(accessResult._accessDeniedEvent); + LOG.debug("requirePermission: exiting: throwing exception as everything wasn't accessible"); + throw new AccessDeniedException(accessResult._denialReason); + } + } finally { + RangerPerfTracer.log(perf); } } diff --git a/hdfs-agent/src/main/java/org/apache/ranger/authorization/hadoop/RangerHdfsAuthorizer.java b/hdfs-agent/src/main/java/org/apache/ranger/authorization/hadoop/RangerHdfsAuthorizer.java index d28685a481..f82fd57c55 100644 --- a/hdfs-agent/src/main/java/org/apache/ranger/authorization/hadoop/RangerHdfsAuthorizer.java +++ b/hdfs-agent/src/main/java/org/apache/ranger/authorization/hadoop/RangerHdfsAuthorizer.java @@ -28,10 +28,12 @@ import java.util.HashMap; import java.util.HashSet; import java.util.Map; +import java.util.Random; import java.util.Set; import java.util.Stack; import org.apache.commons.lang.ArrayUtils; +import org.apache.commons.lang.RandomStringUtils; import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -54,7 +56,9 @@ import org.apache.ranger.plugin.policyengine.RangerAccessResource; import org.apache.ranger.plugin.policyengine.RangerAccessResourceImpl; import org.apache.ranger.plugin.policyengine.RangerAccessResult; +import org.apache.ranger.plugin.resourcematcher.RangerPathResourceMatcher; import org.apache.ranger.plugin.service.RangerBasePlugin; +import org.apache.ranger.plugin.util.RangerPerfTracer; import com.google.common.collect.Sets; @@ -70,6 +74,7 @@ public class RangerHdfsAuthorizer extends INodeAttributeProvider { public static final String RANGER_FILENAME_EXTENSION_SEPARATOR_PROP = "ranger.plugin.hdfs.filename.extension.separator"; private static final Log LOG = LogFactory.getLog(RangerHdfsAuthorizer.class); + private static final Log PERF_HDFSAUTH_REQUEST_LOG = RangerPerfTracer.getPerfLogger("hdfsauth.request"); private RangerHdfsPlugin rangerPlugin = null; private Map> access2ActionListMapper = new HashMap>(); @@ -92,6 +97,10 @@ public void start() { RangerHdfsPlugin plugin = new RangerHdfsPlugin(); plugin.init(); + if (plugin.isOptimizeSubAccessAuthEnabled()) { + LOG.info(RangerHadoopConstants.RANGER_OPTIMIZE_SUBACCESS_AUTHORIZATION_PROP + " is enabled"); + } + access2ActionListMapper.put(FsAction.NONE, new HashSet()); access2ActionListMapper.put(FsAction.ALL, Sets.newHashSet(READ_ACCCESS_TYPE, WRITE_ACCCESS_TYPE, EXECUTE_ACCCESS_TYPE)); access2ActionListMapper.put(FsAction.READ, Sets.newHashSet(READ_ACCCESS_TYPE)); @@ -208,6 +217,12 @@ public void checkPermission(String fsOwner, String superGroup, UserGroupInformat + ", access=" + access + ", subAccess=" + subAccess + ", ignoreEmptyDir=" + ignoreEmptyDir + ")"); } + RangerPerfTracer perf = null; + + if(RangerPerfTracer.isPerfTraceEnabled(PERF_HDFSAUTH_REQUEST_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_HDFSAUTH_REQUEST_LOG, "RangerHdfsAuthorizer.checkPermission(path=" + path + ")"); + } + try { boolean isTraverseOnlyCheck = access == null && parentAccess == null && ancestorAccess == null && subAccess == null; INode ancestor = null; @@ -311,19 +326,37 @@ public void checkPermission(String fsOwner, String superGroup, UserGroupInformat if(authzStatus != AuthzStatus.ALLOW) { break; } - } - for(INode child : cList) { - if (child.isDirectory()) { - directories.push(child.asDirectory()); + AuthzStatus subDirAuthStatus = AuthzStatus.NOT_DETERMINED; + + boolean optimizeSubAccessAuthEnabled = RangerHdfsPlugin.isOptimizeSubAccessAuthEnabled(); + + if (optimizeSubAccessAuthEnabled) { + subDirAuthStatus = isAccessAllowedForHierarchy(dir, dirAttribs, subAccess, user, groups, plugin); + } + + if (subDirAuthStatus != AuthzStatus.ALLOW) { + for(INode child : cList) { + if (child.isDirectory()) { + directories.push(child.asDirectory()); + } + } } } } if (authzStatus == AuthzStatus.NOT_DETERMINED) { + RangerPerfTracer hadoopAuthPerf = null; + + if(RangerPerfTracer.isPerfTraceEnabled(PERF_HDFSAUTH_REQUEST_LOG)) { + hadoopAuthPerf = RangerPerfTracer.getPerfTracer(PERF_HDFSAUTH_REQUEST_LOG, "defaultEnforcer.checkPermission(path=" + path + ")"); + } + authzStatus = checkDefaultEnforcer(fsOwner, superGroup, ugi, inodeAttrs, inodes, pathByNameArr, snapshotId, path, ancestorIndex, doCheckOwner, FsAction.NONE, FsAction.NONE, FsAction.NONE, subAccess, ignoreEmptyDir, isTraverseOnlyCheck, ancestor, parent, inode, auditHandler); + + RangerPerfTracer.log(hadoopAuthPerf); } } @@ -363,6 +396,8 @@ public void checkPermission(String fsOwner, String superGroup, UserGroupInformat auditHandler.flushAudit(); } + RangerPerfTracer.log(perf); + if(LOG.isDebugEnabled()) { LOG.debug("<== RangerAccessControlEnforcer.checkPermission(" + path + ", " + access + ", user=" + user + ") : " + authzStatus); } @@ -379,6 +414,7 @@ private AuthzStatus checkDefaultEnforcer(String fsOwner, String superGroup, User ) throws AccessControlException { AuthzStatus authzStatus = AuthzStatus.NOT_DETERMINED; if(RangerHdfsPlugin.isHadoopAuthEnabled() && defaultEnforcer != null) { + try { defaultEnforcer.checkPermission(fsOwner, superGroup, ugi, inodeAttrs, inodes, pathByNameArr, snapshotId, path, ancestorIndex, doCheckOwner, @@ -475,6 +511,70 @@ private AuthzStatus isAccessAllowed(INode inode, INodeAttributes inodeAttribs, F return ret; } + + private AuthzStatus isAccessAllowedForHierarchy(INode inode, INodeAttributes inodeAttribs, FsAction access, String user, Set groups, RangerHdfsPlugin plugin) { + AuthzStatus ret = null; + String path = inode != null ? inode.getFullPathName() : null; + String pathOwner = inodeAttribs != null ? inodeAttribs.getUserName() : null; + String clusterName = plugin.getClusterName(); + + if (pathOwner == null && inode != null) { + pathOwner = inode.getUserName(); + } + + if (RangerHadoopConstants.HDFS_ROOT_FOLDER_PATH_ALT.equals(path)) { + path = RangerHadoopConstants.HDFS_ROOT_FOLDER_PATH; + } + + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerAccessControlEnforcer.isAccessAllowedForHierarchy(" + path + ", " + access + ", " + user + ")"); + } + + if (path != null) { + + Set accessTypes = access2ActionListMapper.get(access); + + if (accessTypes == null) { + LOG.warn("RangerAccessControlEnforcer.isAccessAllowedForHierarchy(" + path + ", " + access + ", " + user + "): no Ranger accessType found for " + access); + + accessTypes = access2ActionListMapper.get(FsAction.NONE); + } + + String subDirPath = path; + if (subDirPath.charAt(subDirPath.length() - 1) != org.apache.hadoop.fs.Path.SEPARATOR_CHAR) { + subDirPath = subDirPath + Character.toString(org.apache.hadoop.fs.Path.SEPARATOR_CHAR); + } + subDirPath = subDirPath + RangerHdfsPlugin.getRandomizedWildcardPathName(); + + for (String accessType : accessTypes) { + RangerHdfsAccessRequest request = new RangerHdfsAccessRequest(null, subDirPath, pathOwner, access, accessType, user, groups, clusterName); + + RangerAccessResult result = plugin.isAccessAllowed(request, null); + + if (result == null || !result.getIsAccessDetermined()) { + ret = AuthzStatus.NOT_DETERMINED; + // don't break yet; subsequent accessType could be denied + } else if(! result.getIsAllowed()) { // explicit deny + ret = AuthzStatus.DENY; + break; + } else { // allowed + if(!AuthzStatus.NOT_DETERMINED.equals(ret)) { // set to ALLOW only if there was no NOT_DETERMINED earlier + ret = AuthzStatus.ALLOW; + } + } + } + } + + if(ret == null) { + ret = AuthzStatus.NOT_DETERMINED; + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerAccessControlEnforcer.isAccessAllowedForHierarchy(" + path + ", " + access + ", " + user + "): " + ret); + } + + return ret; + } } } @@ -482,7 +582,8 @@ private AuthzStatus isAccessAllowed(INode inode, INodeAttributes inodeAttribs, F class RangerHdfsPlugin extends RangerBasePlugin { private static boolean hadoopAuthEnabled = RangerHadoopConstants.RANGER_ADD_HDFS_PERMISSION_DEFAULT; private static String fileNameExtensionSeparator; - + private static boolean optimizeSubAccessAuthEnabled = RangerHadoopConstants.RANGER_OPTIMIZE_SUBACCESS_AUTHORIZATION_DEFAULT; + private static String randomizedWildcardPathName; public RangerHdfsPlugin() { super("hdfs", "hdfs"); @@ -493,6 +594,17 @@ public void init() { RangerHdfsPlugin.hadoopAuthEnabled = RangerConfiguration.getInstance().getBoolean(RangerHadoopConstants.RANGER_ADD_HDFS_PERMISSION_PROP, RangerHadoopConstants.RANGER_ADD_HDFS_PERMISSION_DEFAULT); RangerHdfsPlugin.fileNameExtensionSeparator = RangerConfiguration.getInstance().get(RangerHdfsAuthorizer.RANGER_FILENAME_EXTENSION_SEPARATOR_PROP, RangerHdfsAuthorizer.DEFAULT_FILENAME_EXTENSION_SEPARATOR); + RangerHdfsPlugin.optimizeSubAccessAuthEnabled = RangerConfiguration.getInstance().getBoolean(RangerHadoopConstants.RANGER_OPTIMIZE_SUBACCESS_AUTHORIZATION_PROP, RangerHadoopConstants.RANGER_OPTIMIZE_SUBACCESS_AUTHORIZATION_DEFAULT); + + // Build random string of random length + byte[] bytes = new byte[1]; + new Random().nextBytes(bytes); + int count = bytes[0]; + count = count < 56 ? 56 : count; + count = count > 112 ? 112 : count; + + String random = RandomStringUtils.random(count, "^&#@!%()-_+=@:;'<>`~abcdefghijklmnopqrstuvwxyz01234567890"); + randomizedWildcardPathName = RangerPathResourceMatcher.WILDCARD_ASTERISK + random + RangerPathResourceMatcher.WILDCARD_ASTERISK; } public static boolean isHadoopAuthEnabled() { @@ -501,6 +613,12 @@ public static boolean isHadoopAuthEnabled() { public static String getFileNameExtensionSeparator() { return RangerHdfsPlugin.fileNameExtensionSeparator; } + public static boolean isOptimizeSubAccessAuthEnabled() { + return RangerHdfsPlugin.optimizeSubAccessAuthEnabled; + } + public static String getRandomizedWildcardPathName() { + return RangerHdfsPlugin.randomizedWildcardPathName; + } } class RangerHdfsResource extends RangerAccessResourceImpl { @@ -589,17 +707,28 @@ public void processResult(RangerAccessResult result) { isAuditEnabled = true; } - auditEvent = super.getAuthzEvents(result); + if (auditEvent == null) { + auditEvent = super.getAuthzEvents(result); + } if (auditEvent != null) { RangerAccessRequest request = result.getAccessRequest(); RangerAccessResource resource = request.getResource(); String resourcePath = resource != null ? resource.getAsString() : null; + // Overwrite fields in original auditEvent auditEvent.setEventTime(request.getAccessTime()); auditEvent.setAccessType(request.getAction()); auditEvent.setResourcePath(this.pathToBeValidated); auditEvent.setResultReason(resourcePath); + + auditEvent.setAccessResult((short) (result.getIsAllowed() ? 1 : 0)); + auditEvent.setPolicyId(result.getPolicyId()); + + Set tags = getTags(request); + if (tags != null) { + auditEvent.setTags(tags); + } } if(LOG.isDebugEnabled()) { diff --git a/hdfs-agent/src/test/java/org/apache/ranger/services/hdfs/HDFSRangerTest.java b/hdfs-agent/src/test/java/org/apache/ranger/services/hdfs/HDFSRangerTest.java index f5e8cb7e0a..c5b7a75b7a 100644 --- a/hdfs-agent/src/test/java/org/apache/ranger/services/hdfs/HDFSRangerTest.java +++ b/hdfs-agent/src/test/java/org/apache/ranger/services/hdfs/HDFSRangerTest.java @@ -23,6 +23,7 @@ import org.apache.commons.io.IOUtils; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.ContentSummary; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; @@ -243,6 +244,11 @@ public void HDFSBaseFileNameTokenReadTest() throws Exception { HDFSReadFailTest("/tmp/tmpdir5/t/data-file.txt"); } + @org.junit.Test + public void HDFSContentSummaryTest() throws Exception { + HDFSGetContentSummary("/tmp/get-content-summary"); + } + void HDFSReadTest(String fileName) throws Exception { FileSystem fileSystem = hdfsCluster.getFileSystem(); @@ -413,4 +419,53 @@ public Void run() throws Exception { } }); } + + void HDFSGetContentSummary(final String dirName) throws Exception { + + String subdirName = dirName + "/tmpdir"; + + createFile(subdirName, 1); + createFile(subdirName, 2); + + UserGroupInformation ugi = UserGroupInformation.createUserForTesting("bob", new String[] {}); + ugi.doAs(new PrivilegedExceptionAction() { + + public Void run() throws Exception { + Configuration conf = new Configuration(); + conf.set("fs.defaultFS", defaultFs); + + FileSystem fs = FileSystem.get(conf); + + try { + // GetContentSummary on the directory dirName + ContentSummary contentSummary = fs.getContentSummary(new Path(dirName)); + + long directoryCount = contentSummary.getDirectoryCount(); + Assert.assertTrue("Found unexpected number of directories; expected-count=3, actual-count=" + directoryCount, directoryCount == 3); + } catch (Exception e) { + Assert.fail("Failed to getContentSummary, exception=" + e); + } + fs.close(); + return null; + } + }); + } + + void createFile(String baseDir, Integer index) throws Exception { + FileSystem fileSystem = hdfsCluster.getFileSystem(); + + // Write a file - the AccessControlEnforcer won't be invoked as we are the "superuser" + String dirName = baseDir + (index != null ? String.valueOf(index) : ""); + String fileName = dirName + "/dummy-data"; + final Path file = new Path(fileName); + FSDataOutputStream out = fileSystem.create(file); + for (int i = 0; i < 1024; ++i) { + out.write(("data" + i + "\n").getBytes("UTF-8")); + out.flush(); + } + out.close(); + + // Change permissions to read-only + fileSystem.setPermission(file, new FsPermission(FsAction.READ, FsAction.NONE, FsAction.NONE)); + } } diff --git a/hdfs-agent/src/test/resources/hdfs-policies.json b/hdfs-agent/src/test/resources/hdfs-policies.json index 4565b8b64b..52a2cc1608 100644 --- a/hdfs-agent/src/test/resources/hdfs-policies.json +++ b/hdfs-agent/src/test/resources/hdfs-policies.json @@ -233,6 +233,33 @@ "createdBy": "Admin", "updatedBy": "Admin", "version": 1 + }, + { + "service": "cl1_hadoop", + "name": "/tmp/get-content-summary", + "policyType": 0, + "description": "", + "isAuditEnabled": true, + "resources": { + "path": {"values": ["/tmp/get-content-summary", "/tmp/get-content-summary/tmpdir1", "/tmp/get-content-summary/tmpdir2"], "isExcludes": false, "isRecursive": false} + }, + "policyItems": [ + { + "accesses": [{"type": "read","isAllowed": true}, {"type": "execute","isAllowed": true}], + "users": ["bob"], + "groups": ["IT"], + "conditions": [], + "delegateAdmin": false + } + ], + "denyPolicyItems": [], + "allowExceptions": [], + "denyExceptions": [], + "dataMaskPolicyItems": [], + "rowFilterPolicyItems": [], + "id": 40, + "isEnabled": true, + "version": 1 } ], "serviceDef": { diff --git a/hdfs-agent/src/test/resources/log4j.properties b/hdfs-agent/src/test/resources/log4j.properties new file mode 100644 index 0000000000..f7ab2bad53 --- /dev/null +++ b/hdfs-agent/src/test/resources/log4j.properties @@ -0,0 +1,34 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +##-- To prevent junits from cluttering the build run by default all test runs send output to null appender +log4j.appender.devnull=org.apache.log4j.varia.NullAppender +ranger.root.logger=FATAL,devnull + +##-- uncomment the following line during during development/debugging so see debug messages during test run to be emitted to console +# ranger.root.logger=DEBUG,console +log4j.rootLogger=${ranger.root.logger} + +# Logging Threshold +log4j.threshold=ALL + +# +# console +# Add "console" to rootlogger above if you want to use this +# +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.target=System.err +log4j.appender.console.layout=org.apache.log4j.PatternLayout +log4j.appender.console.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2}: %m%n diff --git a/hdfs-agent/src/test/resources/ranger-hdfs-security.xml b/hdfs-agent/src/test/resources/ranger-hdfs-security.xml index 3062108d50..57de6dbaaf 100644 --- a/hdfs-agent/src/test/resources/ranger-hdfs-security.xml +++ b/hdfs-agent/src/test/resources/ranger-hdfs-security.xml @@ -42,4 +42,12 @@ + + ranger.optimize-subaccess-authorization + true + + Enable skipping subdirectories if proper policy exists for getContentSummary and delete commands + + + diff --git a/hive-agent/src/main/java/org/apache/ranger/authorization/hive/authorizer/RangerHiveAuthorizer.java b/hive-agent/src/main/java/org/apache/ranger/authorization/hive/authorizer/RangerHiveAuthorizer.java index 56ef1872e0..85a865ad58 100644 --- a/hive-agent/src/main/java/org/apache/ranger/authorization/hive/authorizer/RangerHiveAuthorizer.java +++ b/hive-agent/src/main/java/org/apache/ranger/authorization/hive/authorizer/RangerHiveAuthorizer.java @@ -66,11 +66,14 @@ import com.google.common.collect.Sets; +import org.apache.ranger.plugin.util.RangerPerfTracer; import org.apache.ranger.plugin.util.RangerRequestedResources; public class RangerHiveAuthorizer extends RangerHiveAuthorizerBase { private static final Log LOG = LogFactory.getLog(RangerHiveAuthorizer.class); + private static final Log PERF_HIVEAUTH_REQUEST_LOG = RangerPerfTracer.getPerfLogger("hiveauth.request"); + private static final char COLUMN_SEP = ','; private static final String HIVE_CONF_VAR_QUERY_STRING = "hive.query.string"; @@ -221,6 +224,8 @@ public void checkPrivileges(HiveOperationType hiveOpType, RangerHiveAuditHandler auditHandler = new RangerHiveAuditHandler(); + RangerPerfTracer perf = null; + try { HiveAuthzSessionContext sessionContext = getHiveAuthzSessionContext(); String user = ugi.getShortUserName(); @@ -237,6 +242,10 @@ public void checkPrivileges(HiveOperationType hiveOpType, return; } + if(RangerPerfTracer.isPerfTraceEnabled(PERF_HIVEAUTH_REQUEST_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_HIVEAUTH_REQUEST_LOG, "RangerHiveAuthorizer.checkPrivileges(hiveOpType=" + hiveOpType + ")"); + } + List requests = new ArrayList(); if(!CollectionUtils.isEmpty(inputHObjs)) { @@ -420,6 +429,7 @@ public void checkPrivileges(HiveOperationType hiveOpType, } } finally { auditHandler.flushAudit(); + RangerPerfTracer.log(perf); } } @@ -439,7 +449,13 @@ public List filterListCmdObjects(List if (LOG.isDebugEnabled()) { LOG.debug(String.format("==> filterListCmdObjects(%s, %s)", objs, context)); } - + + RangerPerfTracer perf = null; + + if(RangerPerfTracer.isPerfTraceEnabled(PERF_HIVEAUTH_REQUEST_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_HIVEAUTH_REQUEST_LOG, "RangerHiveAuthorizer.filterListCmdObjects()"); + } + List ret = null; // bail out early if nothing is there to validate! @@ -509,6 +525,8 @@ public List filterListCmdObjects(List } } + RangerPerfTracer.log(perf); + if (LOG.isDebugEnabled()) { int count = ret == null ? 0 : ret.size(); LOG.debug(String.format("<== filterListCmdObjects: count[%d], ret[%s]", count, ret)); @@ -524,6 +542,12 @@ public List applyRowFilterAndColumnMasking(HiveAuthzContext LOG.debug("==> applyRowFilterAndColumnMasking(" + queryContext + ", objCount=" + hiveObjs.size() + ")"); } + RangerPerfTracer perf = null; + + if(RangerPerfTracer.isPerfTraceEnabled(PERF_HIVEAUTH_REQUEST_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_HIVEAUTH_REQUEST_LOG, "RangerHiveAuthorizer.applyRowFilterAndColumnMasking()"); + } + if(CollectionUtils.isNotEmpty(hiveObjs)) { for (HivePrivilegeObject hiveObj : hiveObjs) { HivePrivilegeObjectType hiveObjType = hiveObj.getType(); @@ -576,6 +600,8 @@ public List applyRowFilterAndColumnMasking(HiveAuthzContext } } + RangerPerfTracer.log(perf); + if(LOG.isDebugEnabled()) { LOG.debug("<== applyRowFilterAndColumnMasking(" + queryContext + ", objCount=" + hiveObjs.size() + "): retCount=" + ret.size()); } diff --git a/knox-agent/src/main/java/org/apache/ranger/authorization/knox/RangerPDPKnoxFilter.java b/knox-agent/src/main/java/org/apache/ranger/authorization/knox/RangerPDPKnoxFilter.java index 1d58b210e3..24e8702e14 100644 --- a/knox-agent/src/main/java/org/apache/ranger/authorization/knox/RangerPDPKnoxFilter.java +++ b/knox-agent/src/main/java/org/apache/ranger/authorization/knox/RangerPDPKnoxFilter.java @@ -42,11 +42,14 @@ import org.apache.ranger.audit.provider.MiscUtil; import org.apache.ranger.plugin.policyengine.RangerAccessRequest; import org.apache.ranger.plugin.policyengine.RangerAccessResult; +import org.apache.ranger.plugin.util.RangerPerfTracer; public class RangerPDPKnoxFilter implements Filter { private static final Log LOG = LogFactory.getLog(RangerPDPKnoxFilter.class); + private static final Log PERF_KNOXAUTH_REQUEST_LOG = RangerPerfTracer.getPerfLogger("knoxauth.request"); + private static final String KNOX_GATEWAY_JASS_CONFIG_SECTION = "com.sun.security.jgss.initiate"; private String resourceRole = null; @@ -93,6 +96,12 @@ public void doFilter(ServletRequest request, ServletResponse response, String topologyName = getTopologyName(sourceUrl); String serviceName = getServiceName(); + RangerPerfTracer perf = null; + + if(RangerPerfTracer.isPerfTraceEnabled(PERF_KNOXAUTH_REQUEST_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_KNOXAUTH_REQUEST_LOG, "RangerPDPKnoxFilter.doFilter(url=" + sourceUrl + ", topologyName=" + topologyName + ")"); + } + Subject subject = Subject.getSubject(AccessController.getContext()); Principal primaryPrincipal = (Principal) subject.getPrincipals( @@ -151,6 +160,8 @@ public void doFilter(ServletRequest request, ServletResponse response, LOG.debug("Access allowed: " + accessAllowed); } + RangerPerfTracer.log(perf); + if (accessAllowed) { chain.doFilter(request, response); } else { diff --git a/plugin-atlas/src/main/java/org/apache/ranger/authorization/atlas/authorizer/RangerAtlasAuthorizer.java b/plugin-atlas/src/main/java/org/apache/ranger/authorization/atlas/authorizer/RangerAtlasAuthorizer.java index 9712f95c71..acd11115ce 100644 --- a/plugin-atlas/src/main/java/org/apache/ranger/authorization/atlas/authorizer/RangerAtlasAuthorizer.java +++ b/plugin-atlas/src/main/java/org/apache/ranger/authorization/atlas/authorizer/RangerAtlasAuthorizer.java @@ -26,15 +26,18 @@ import org.apache.atlas.authorize.AtlasAuthorizationException; import org.apache.atlas.authorize.AtlasAuthorizer; import org.apache.atlas.authorize.AtlasResourceTypes; +import org.apache.commons.logging.Log; import org.apache.ranger.plugin.audit.RangerDefaultAuditHandler; import org.apache.ranger.plugin.policyengine.RangerAccessRequestImpl; import org.apache.ranger.plugin.policyengine.RangerAccessResult; import org.apache.ranger.plugin.service.RangerBasePlugin; +import org.apache.ranger.plugin.util.RangerPerfTracer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; public class RangerAtlasAuthorizer implements AtlasAuthorizer { private static final Logger LOG = LoggerFactory.getLogger(RangerAtlasAuthorizer.class); + private static final Log PERF_ATLASAUTH_REQUEST_LOG = RangerPerfTracer.getPerfLogger("atlasauth.request"); private static boolean isDebugEnabled = LOG.isDebugEnabled(); private static volatile RangerBasePlugin atlasPlugin = null; @@ -70,6 +73,11 @@ public boolean isAccessAllowed(AtlasAccessRequest request) throws AtlasAuthoriza if (isDebugEnabled) { LOG.debug("==> isAccessAllowed( " + request + " )"); } + RangerPerfTracer perf = null; + + if(RangerPerfTracer.isPerfTraceEnabled(PERF_ATLASAUTH_REQUEST_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_ATLASAUTH_REQUEST_LOG, "RangerAtlasAuthorizer.isAccessAllowed(request=" + request + ")"); + } String resource = request.getResource(); String user = request.getUser(); @@ -93,6 +101,8 @@ public boolean isAccessAllowed(AtlasAccessRequest request) throws AtlasAuthoriza } } + RangerPerfTracer.log(perf); + if (isDebugEnabled) { LOG.debug("<== isAccessAllowed Returning value :: " + isAccessAllowed); } diff --git a/plugin-kafka/src/main/java/org/apache/ranger/authorization/kafka/authorizer/RangerKafkaAuthorizer.java b/plugin-kafka/src/main/java/org/apache/ranger/authorization/kafka/authorizer/RangerKafkaAuthorizer.java index ec7f88774d..8425fd393e 100644 --- a/plugin-kafka/src/main/java/org/apache/ranger/authorization/kafka/authorizer/RangerKafkaAuthorizer.java +++ b/plugin-kafka/src/main/java/org/apache/ranger/authorization/kafka/authorizer/RangerKafkaAuthorizer.java @@ -42,12 +42,14 @@ import org.apache.ranger.plugin.policyengine.RangerAccessResult; import org.apache.ranger.plugin.service.RangerBasePlugin; +import org.apache.ranger.plugin.util.RangerPerfTracer; import scala.collection.immutable.HashSet; import scala.collection.immutable.Set; public class RangerKafkaAuthorizer implements Authorizer { private static final Log logger = LogFactory .getLog(RangerKafkaAuthorizer.class); + private static final Log PERF_KAFKAAUTH_REQUEST_LOG = RangerPerfTracer.getPerfLogger("kafkaauth.request"); public static final String KEY_TOPIC = "topic"; public static final String KEY_CLUSTER = "cluster"; @@ -130,6 +132,11 @@ public boolean authorize(Session session, Operation operation, return true; } + RangerPerfTracer perf = null; + + if(RangerPerfTracer.isPerfTraceEnabled(PERF_KAFKAAUTH_REQUEST_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_KAFKAAUTH_REQUEST_LOG, "RangerKafkaAuthorizer.authorize(resource=" + resource + ")"); + } String userName = null; if (session.principal() != null) { userName = session.principal().getName(); @@ -206,6 +213,8 @@ public boolean authorize(Session session, Operation operation, + rangerRequest, t); } } + RangerPerfTracer.log(perf); + if (logger.isDebugEnabled()) { logger.debug("rangerRequest=" + rangerRequest + ", return=" + returnValue); diff --git a/plugin-kms/src/main/java/org/apache/ranger/authorization/kms/authorizer/RangerKmsAuthorizer.java b/plugin-kms/src/main/java/org/apache/ranger/authorization/kms/authorizer/RangerKmsAuthorizer.java index 4cda8fa686..c3d75a14d1 100755 --- a/plugin-kms/src/main/java/org/apache/ranger/authorization/kms/authorizer/RangerKmsAuthorizer.java +++ b/plugin-kms/src/main/java/org/apache/ranger/authorization/kms/authorizer/RangerKmsAuthorizer.java @@ -27,6 +27,8 @@ import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; + +import org.apache.commons.logging.Log; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.crypto.key.kms.server.KMSACLsType; import org.apache.hadoop.crypto.key.kms.server.KMSConfiguration; @@ -46,6 +48,7 @@ import org.apache.ranger.plugin.policyengine.RangerAccessResourceImpl; import org.apache.ranger.plugin.policyengine.RangerAccessResult; import org.apache.ranger.plugin.service.RangerBasePlugin; +import org.apache.ranger.plugin.util.RangerPerfTracer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -53,6 +56,7 @@ public class RangerKmsAuthorizer implements Runnable, KeyACLs { private static final Logger LOG = LoggerFactory.getLogger(RangerKmsAuthorizer.class); + private static final Log PERF_KMSAUTH_REQUEST_LOG = RangerPerfTracer.getPerfLogger("kmsauth.request"); private static final String KMS_USER_PRINCIPAL = "ranger.ks.kerberos.principal"; private static final String KMS_USER_KEYTAB = "ranger.ks.kerberos.keytab"; @@ -200,6 +204,11 @@ public boolean hasAccess(Type type, UserGroupInformation ugi, String clientIp) { if(LOG.isDebugEnabled()) { LOG.debug("==> RangerKmsAuthorizer.hasAccess(" + type + ", " + ugi + ")"); } + RangerPerfTracer perf = null; + + if(RangerPerfTracer.isPerfTraceEnabled(PERF_KMSAUTH_REQUEST_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_KMSAUTH_REQUEST_LOG, "RangerKmsAuthorizer.hasAccess(type=" + type + ")"); + } boolean ret = false; RangerKMSPlugin plugin = kmsPlugin; String rangerAccessType = getRangerAccessType(type); @@ -215,7 +224,7 @@ public boolean hasAccess(Type type, UserGroupInformation ugi, String clientIp) { RangerAccessResult result = plugin.isAccessAllowed(request); ret = result == null ? false : result.getIsAllowed(); } - + RangerPerfTracer.log(perf); if(LOG.isDebugEnabled()) { LOG.debug("<== RangerkmsAuthorizer.hasAccess(" + type + ", " + ugi + "): " + ret); } diff --git a/plugin-solr/src/main/java/org/apache/ranger/authorization/solr/authorizer/RangerSolrAuthorizer.java b/plugin-solr/src/main/java/org/apache/ranger/authorization/solr/authorizer/RangerSolrAuthorizer.java index 6ac0a1f038..0c32eb18aa 100644 --- a/plugin-solr/src/main/java/org/apache/ranger/authorization/solr/authorizer/RangerSolrAuthorizer.java +++ b/plugin-solr/src/main/java/org/apache/ranger/authorization/solr/authorizer/RangerSolrAuthorizer.java @@ -39,6 +39,7 @@ import org.apache.ranger.plugin.policyengine.RangerAccessResourceImpl; import org.apache.ranger.plugin.policyengine.RangerAccessResult; import org.apache.ranger.plugin.service.RangerBasePlugin; +import org.apache.ranger.plugin.util.RangerPerfTracer; import org.apache.solr.security.AuthorizationContext.RequestType; import org.apache.solr.security.AuthorizationPlugin; import org.apache.solr.security.AuthorizationResponse; @@ -48,6 +49,7 @@ public class RangerSolrAuthorizer implements AuthorizationPlugin { private static final Log logger = LogFactory .getLog(RangerSolrAuthorizer.class); + private static final Log PERF_SOLRAUTH_REQUEST_LOG = RangerPerfTracer.getPerfLogger("solrauth.request"); public static final String PROP_USE_PROXY_IP = "xasecure.solr.use_proxy_ip"; public static final String PROP_PROXY_IP_HEADER = "xasecure.solr.proxy_ip_header"; @@ -167,6 +169,12 @@ public AuthorizationResponse authorize(AuthorizationContext context) { RangerMultiResourceAuditHandler auditHandler = new RangerMultiResourceAuditHandler(); + RangerPerfTracer perf = null; + + if(RangerPerfTracer.isPerfTraceEnabled(PERF_SOLRAUTH_REQUEST_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_SOLRAUTH_REQUEST_LOG, "RangerSolrAuthorizer.authorize()"); + } + String userName = getUserName(context); Set userGroups = getGroupsForUser(userName); String ip = null; @@ -213,6 +221,7 @@ public AuthorizationResponse authorize(AuthorizationContext context) { } } finally { auditHandler.flushAudit(); + RangerPerfTracer.log(perf); } } catch (Throwable t) { isDenied = true; diff --git a/plugin-yarn/src/main/java/org/apache/ranger/authorization/yarn/authorizer/RangerYarnAuthorizer.java b/plugin-yarn/src/main/java/org/apache/ranger/authorization/yarn/authorizer/RangerYarnAuthorizer.java index c589060ebf..b4488304f7 100644 --- a/plugin-yarn/src/main/java/org/apache/ranger/authorization/yarn/authorizer/RangerYarnAuthorizer.java +++ b/plugin-yarn/src/main/java/org/apache/ranger/authorization/yarn/authorizer/RangerYarnAuthorizer.java @@ -44,6 +44,7 @@ import org.apache.ranger.plugin.service.RangerBasePlugin; import com.google.common.collect.Sets; +import org.apache.ranger.plugin.util.RangerPerfTracer; public class RangerYarnAuthorizer extends YarnAuthorizationProvider { public static final String ACCESS_TYPE_ADMIN_QUEUE = "admin-queue"; @@ -56,6 +57,8 @@ public class RangerYarnAuthorizer extends YarnAuthorizationProvider { private static final Log LOG = LogFactory.getLog(RangerYarnAuthorizer.class); + private static final Log PERF_YARNAUTH_REQUEST_LOG = RangerPerfTracer.getPerfLogger("yarnauth.request"); + private static volatile RangerYarnPlugin yarnPlugin = null; private AccessControlList admins = null; @@ -101,7 +104,15 @@ public boolean checkPermission(AccessType accessType, PrivilegedEntity entity, U RangerAccessResult result = null; String clusterName = yarnPlugin.getClusterName(); + RangerPerfTracer perf = null; + RangerPerfTracer yarnAclPerf = null; + if(plugin != null) { + + if(RangerPerfTracer.isPerfTraceEnabled(PERF_YARNAUTH_REQUEST_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_YARNAUTH_REQUEST_LOG, "RangerYarnAuthorizer.checkPermission(entity=" + entity + ")"); + } + RangerYarnAccessRequest request = new RangerYarnAccessRequest(entity, getRangerAccessType(accessType), accessType.name(), ugi, clusterName); auditHandler = new RangerYarnAuditHandler(); @@ -110,6 +121,11 @@ public boolean checkPermission(AccessType accessType, PrivilegedEntity entity, U } if(RangerYarnAuthorizer.yarnAuthEnabled && (result == null || !result.getIsAccessDetermined())) { + + if(RangerPerfTracer.isPerfTraceEnabled(PERF_YARNAUTH_REQUEST_LOG)) { + yarnAclPerf = RangerPerfTracer.getPerfTracer(PERF_YARNAUTH_REQUEST_LOG, "RangerYarnNativeAuthorizer.isAllowedByYarnAcl(entity=" + entity + ")"); + } + ret = isAllowedByYarnAcl(accessType, entity, ugi, auditHandler); } else { ret = result == null ? false : result.getIsAllowed(); @@ -119,6 +135,10 @@ public boolean checkPermission(AccessType accessType, PrivilegedEntity entity, U auditHandler.flushAudit(); } + RangerPerfTracer.log(yarnAclPerf); + + RangerPerfTracer.log(perf); + if(LOG.isDebugEnabled()) { LOG.debug("<== RangerYarnAuthorizer.checkPermission(" + accessType + ", " + toString(entity) + ", " + ugi + "): " + ret); } diff --git a/ranger-tools/conf/log4j.properties b/ranger-tools/conf/log4j.properties index 4ead8023d5..e95a6c8ea3 100644 --- a/ranger-tools/conf/log4j.properties +++ b/ranger-tools/conf/log4j.properties @@ -31,6 +31,7 @@ log4j.appender.console.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2}: %m% ranger.perf.logger=DEBUG,PERF ranger.perf.log.file=ranger-perf-test.log +log4j.logger.org.apache.ranger.plugin.util.PerfDataRecorder=${ranger.perf.logger} log4j.logger.org.apache.ranger.perf=${ranger.perf.logger} log4j.additivity.org.apache.ranger.perf=false diff --git a/ranger-tools/src/main/java/org/apache/ranger/policyengine/RangerPolicyenginePerfTester.java b/ranger-tools/src/main/java/org/apache/ranger/policyengine/RangerPolicyenginePerfTester.java index 056c548703..78cbe02cf8 100644 --- a/ranger-tools/src/main/java/org/apache/ranger/policyengine/RangerPolicyenginePerfTester.java +++ b/ranger-tools/src/main/java/org/apache/ranger/policyengine/RangerPolicyenginePerfTester.java @@ -59,6 +59,7 @@ public static void main(String[] args) { RangerPolicyEngineOptions policyEngineOptions = new RangerPolicyEngineOptions(); policyEngineOptions.disableTagPolicyEvaluation = false; policyEngineOptions.evaluatorType = RangerPolicyEvaluator.EVALUATOR_TYPE_OPTIMIZED; + policyEngineOptions.cacheAuditResults = false; policyEngineOptions.disableTrieLookupPrefilter = perfTestOptions.getIsTrieLookupPrefixDisabled(); PerfTestEngine perfTestEngine = new PerfTestEngine(servicePoliciesFileURL, policyEngineOptions, perfTestOptions.getIsDynamicReorderingDisabled()); diff --git a/storm-agent/src/main/java/org/apache/ranger/authorization/storm/authorizer/RangerStormAuthorizer.java b/storm-agent/src/main/java/org/apache/ranger/authorization/storm/authorizer/RangerStormAuthorizer.java index 23c2b5f214..0fe658e243 100644 --- a/storm-agent/src/main/java/org/apache/ranger/authorization/storm/authorizer/RangerStormAuthorizer.java +++ b/storm-agent/src/main/java/org/apache/ranger/authorization/storm/authorizer/RangerStormAuthorizer.java @@ -23,12 +23,14 @@ import java.util.Map; import java.util.Set; +import org.apache.commons.logging.Log; import org.apache.hadoop.security.UserGroupInformation; import org.apache.ranger.audit.provider.MiscUtil; import org.apache.ranger.authorization.storm.StormRangerPlugin; import org.apache.ranger.authorization.utils.StringUtil; import org.apache.ranger.plugin.policyengine.RangerAccessRequest; import org.apache.ranger.plugin.policyengine.RangerAccessResult; +import org.apache.ranger.plugin.util.RangerPerfTracer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -43,6 +45,8 @@ public class RangerStormAuthorizer implements IAuthorizer { private static final Logger LOG = LoggerFactory.getLogger(RangerStormAuthorizer.class); + private static final Log PERF_STORMAUTH_REQUEST_LOG = RangerPerfTracer.getPerfLogger("stormauth.request"); + private static final String STORM_CLIENT_JASS_CONFIG_SECTION = "StormClient"; private static volatile StormRangerPlugin plugin = null; @@ -51,9 +55,9 @@ public class RangerStormAuthorizer implements IAuthorizer { /** * permit() method is invoked for each incoming Thrift request. - * @param context request context includes info about - * @param operation operation name - * @param topology_storm configuration of targeted topology + * @param aRequestContext request context includes info about + * @param aOperationName operation name + * @param aTopologyConfigMap configuration of targeted topology * @return true if the request is authorized, false if reject */ @@ -64,8 +68,15 @@ public boolean permit(ReqContext aRequestContext, String aOperationName, Map aTo boolean isAuditEnabled = false; String topologyName = null; - + + RangerPerfTracer perf = null; + try { + + if(RangerPerfTracer.isPerfTraceEnabled(PERF_STORMAUTH_REQUEST_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_STORMAUTH_REQUEST_LOG, "RangerStormAuthorizer.permit()"); + } + topologyName = (aTopologyConfigMap == null ? "" : (String)aTopologyConfigMap.get(Config.TOPOLOGY_NAME)); if (LOG.isDebugEnabled()) { @@ -130,6 +141,7 @@ public boolean permit(ReqContext aRequestContext, String aOperationName, Map aTo LOG.error("RangerStormAuthorizer found this exception", t); } finally { + RangerPerfTracer.log(perf); if (LOG.isDebugEnabled()) { LOG.debug("[req "+ aRequestContext.requestID()+ "] Access " + " from: [" + aRequestContext.remoteAddress() + "]" @@ -144,7 +156,7 @@ public boolean permit(ReqContext aRequestContext, String aOperationName, Map aTo /** * Invoked once immediately after construction - * @param conf Storm configuration + * @param aStormConfigMap Storm configuration */ @Override From b84428968dabdf7f64363a0c1cd9a4eaf63c5790 Mon Sep 17 00:00:00 2001 From: Abhay Kulkarni Date: Fri, 28 Jul 2017 12:20:18 -0700 Subject: [PATCH 014/151] RANGER-1714:Disable dynamic sorting of policies when trie pre-filter is enabled --- .../RangerServiceResourceMatcher.java | 11 ++++ .../policyengine/RangerPolicyRepository.java | 26 ++------ .../RangerPolicyEvaluator.java | 5 ++ .../RangerPolicyResourceEvaluator.java | 9 --- .../plugin/service/RangerBasePlugin.java | 5 +- .../plugin/util/RangerResourceTrie.java | 65 ++++--------------- 6 files changed, 36 insertions(+), 85 deletions(-) diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerServiceResourceMatcher.java b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerServiceResourceMatcher.java index ecddf75e2e..f9bbb12b14 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerServiceResourceMatcher.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerServiceResourceMatcher.java @@ -28,9 +28,13 @@ import org.apache.ranger.plugin.resourcematcher.RangerResourceMatcher; import org.apache.ranger.plugin.util.ServiceDefUtil; +import java.io.Serializable; +import java.util.Comparator; import java.util.Map; public class RangerServiceResourceMatcher implements RangerPolicyResourceEvaluator { + public static final Comparator ID_COMPARATOR = new IdComparator(); + private final RangerServiceResource serviceResource; private final RangerPolicyResourceMatcher policyResourceMatcher; private final Integer leafResourceLevel; @@ -74,4 +78,11 @@ public RangerPolicyResourceMatcher.MatchType getMatchType(RangerAccessResource r RangerServiceDef getServiceDef() { return policyResourceMatcher != null ? policyResourceMatcher.getServiceDef() : null; } + + static class IdComparator implements Comparator, Serializable { + @Override + public int compare(RangerServiceResourceMatcher me, RangerServiceResourceMatcher other) { + return Long.compare(me.getId(), other.getId()); + } + } } diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java index 5631973470..f7302bb7f7 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java @@ -794,21 +794,15 @@ void reorderPolicyEvaluators() { LOG.debug("==> reorderEvaluators()"); } - if(policyResourceTrie != null) { - reorderPolicyEvaluators(policyResourceTrie); - } else { + if(policyResourceTrie == null) { policyEvaluators = getReorderedPolicyEvaluators(policyEvaluators); } - if(dataMaskResourceTrie != null) { - reorderPolicyEvaluators(dataMaskResourceTrie); - } else { + if(dataMaskResourceTrie == null) { dataMaskPolicyEvaluators = getReorderedPolicyEvaluators(dataMaskPolicyEvaluators); } - if(rowFilterResourceTrie != null) { - reorderPolicyEvaluators(rowFilterResourceTrie); - } else { + if(rowFilterResourceTrie == null) { rowFilterPolicyEvaluators = getReorderedPolicyEvaluators(rowFilterPolicyEvaluators); } @@ -817,18 +811,6 @@ void reorderPolicyEvaluators() { } } - private void reorderPolicyEvaluators(Map trieMap) { - if(trieMap != null) { - for(Map.Entry entry : trieMap.entrySet()) { - RangerResourceTrie trie = entry.getValue(); - - if(trie != null) { - trie.reorderEvaluators(); - } - } - } - } - private List getReorderedPolicyEvaluators(List evaluators) { List ret = evaluators; @@ -849,7 +831,7 @@ private Map createResourceTrieMap(List(); for (RangerServiceDef.RangerResourceDef resourceDef : serviceDef.getResources()) { - ret.put(resourceDef.getName(), new RangerResourceTrie(resourceDef, evaluators)); + ret.put(resourceDef.getName(), new RangerResourceTrie(resourceDef, evaluators, RangerPolicyEvaluator.EVAL_ORDER_COMPARATOR)); } } else { ret = null; diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerPolicyEvaluator.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerPolicyEvaluator.java index 191ad98c86..7165594fb8 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerPolicyEvaluator.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerPolicyEvaluator.java @@ -39,6 +39,8 @@ public interface RangerPolicyEvaluator extends RangerPolicyResourceEvaluator { + Comparator EVAL_ORDER_COMPARATOR = new RangerPolicyEvaluator.PolicyEvalOrderComparator(); + String EVALUATOR_TYPE_AUTO = "auto"; String EVALUATOR_TYPE_OPTIMIZED = "optimized"; String EVALUATOR_TYPE_CACHED = "cached"; @@ -96,11 +98,14 @@ public int compare(RangerPolicyEvaluator me, RangerPolicyEvaluator other) { result = 1; } else { result = Long.compare(other.getUsageCount(), me.getUsageCount()); + if (result == 0) { result = Integer.compare(me.getEvalOrder(), other.getEvalOrder()); } } + return result; } } + } diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyresourcematcher/RangerPolicyResourceEvaluator.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyresourcematcher/RangerPolicyResourceEvaluator.java index 181863c38c..7d43b4b8c5 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyresourcematcher/RangerPolicyResourceEvaluator.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyresourcematcher/RangerPolicyResourceEvaluator.java @@ -23,8 +23,6 @@ import org.apache.ranger.plugin.model.RangerPolicy; import org.apache.ranger.plugin.resourcematcher.RangerResourceMatcher; -import java.io.Serializable; -import java.util.Comparator; import java.util.Map; public interface RangerPolicyResourceEvaluator { @@ -37,11 +35,4 @@ public interface RangerPolicyResourceEvaluator { RangerResourceMatcher getResourceMatcher(String resourceName); Integer getLeafResourceLevel(); - - class IdComparator implements Comparator, Serializable { - @Override - public int compare(RangerPolicyResourceEvaluator me, RangerPolicyResourceEvaluator other) { - return Long.compare(me.getId(), other.getId()); - } - } } diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java b/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java index 7a33429895..87ceeec99a 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java @@ -165,7 +165,7 @@ public void init() { LOG.debug(propertyPrefix + ".policy.policyReorderInterval:" + policyReorderIntervalMs); } - if (policyReorderIntervalMs > 0) { + if (policyEngineOptions.disableTrieLookupPrefilter && policyReorderIntervalMs > 0) { policyEngineRefreshTimer = new Timer("PolicyEngineRefreshTimer", true); try { policyEngineRefreshTimer.schedule(new PolicyEngineRefresher(this), policyReorderIntervalMs, policyReorderIntervalMs); @@ -178,8 +178,7 @@ public void init() { policyEngineRefreshTimer = null; } } else { - LOG.info("Policies will NOT be reordered based on number of evaluations because " - + propertyPrefix + ".policy.policyReorderInterval is set to a negative number[" + policyReorderIntervalMs +"]"); + LOG.info("Policies will NOT be reordered based on number of evaluations"); } } diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerResourceTrie.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerResourceTrie.java index f2c4c3a401..189a72b29c 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerResourceTrie.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerResourceTrie.java @@ -31,11 +31,11 @@ import java.util.ArrayList; import java.util.Collections; +import java.util.Comparator; import java.util.HashMap; import java.util.List; import java.util.Map; - public class RangerResourceTrie { private static final Log LOG = LogFactory.getLog(RangerResourceTrie.class); @@ -48,6 +48,10 @@ public class RangerResourceTrie { private final TrieNode root; public RangerResourceTrie(RangerServiceDef.RangerResourceDef resourceDef, List evaluators) { + this(resourceDef, evaluators, null); + } + + public RangerResourceTrie(RangerServiceDef.RangerResourceDef resourceDef, List evaluators, Comparator comparator) { if(LOG.isDebugEnabled()) { LOG.debug("==> RangerResourceTrie(" + resourceDef.getName() + ", evaluatorCount=" + evaluators.size() + ")"); } @@ -103,7 +107,7 @@ public RangerResourceTrie(RangerServiceDef.RangerResourceDef resourceDef, List parentWildcardEvaluators) { + void postSetup(List parentWildcardEvaluators, Comparator comparator) { // finalize wildcard-evaluators list by including parent's wildcard evaluators if(parentWildcardEvaluators != null) { if(CollectionUtils.isEmpty(this.wildcardEvaluators)) { @@ -380,44 +380,21 @@ void postSetup(List parentWildcardEvaluators) { } } - RangerPolicyResourceEvaluator.IdComparator comparator = new RangerPolicyResourceEvaluator.IdComparator(); - if(!isSharingParentWildcardEvaluators && CollectionUtils.isNotEmpty(wildcardEvaluators)) { - Collections.sort(wildcardEvaluators, comparator); - } - - if(evaluators != wildcardEvaluators && CollectionUtils.isNotEmpty(evaluators)) { - Collections.sort(evaluators, comparator); - } - - if(children != null) { - for(Map.Entry entry : children.entrySet()) { - TrieNode child = entry.getValue(); - - child.postSetup(wildcardEvaluators); + if (comparator != null) { + if (!isSharingParentWildcardEvaluators && CollectionUtils.isNotEmpty(wildcardEvaluators)) { + Collections.sort(wildcardEvaluators, comparator); } - } - } - void reorderEvaluators(List parentWildcardEvaluators) { - boolean isEvaluatorsSameAsWildcardEvaluators = evaluators == wildcardEvaluators; - - if(isSharingParentWildcardEvaluators) { - wildcardEvaluators = parentWildcardEvaluators; - } else { - wildcardEvaluators = getSortedCopy(wildcardEvaluators); - } - - if(isEvaluatorsSameAsWildcardEvaluators) { - evaluators = wildcardEvaluators; - } else { - evaluators = getSortedCopy(evaluators); + if (evaluators != wildcardEvaluators && CollectionUtils.isNotEmpty(evaluators)) { + Collections.sort(evaluators, comparator); + } } if(children != null) { for(Map.Entry entry : children.entrySet()) { TrieNode child = entry.getValue(); - child.reorderEvaluators(wildcardEvaluators); + child.postSetup(wildcardEvaluators, comparator); } } } @@ -462,18 +439,4 @@ public void clear() { evaluators = null; wildcardEvaluators = null; } - - private List getSortedCopy(List evaluators) { - final List ret; - - if(CollectionUtils.isNotEmpty(evaluators)) { - ret = new ArrayList(evaluators); - - Collections.sort(ret, new RangerPolicyResourceEvaluator.IdComparator()); - } else { - ret = evaluators; - } - - return ret; - } } From 694ff57f15ed108bbfe428ea97be960f8937c5df Mon Sep 17 00:00:00 2001 From: Nitin Galave Date: Mon, 24 Jul 2017 11:17:14 +0530 Subject: [PATCH 015/151] RANGER-1708 : Remove tag services from service type and service name filters under Access Audit. --- .../scripts/views/reports/AuditLayout.js | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/security-admin/src/main/webapp/scripts/views/reports/AuditLayout.js b/security-admin/src/main/webapp/scripts/views/reports/AuditLayout.js index 40f56dd5f4..68be81e59d 100644 --- a/security-admin/src/main/webapp/scripts/views/reports/AuditLayout.js +++ b/security-admin/src/main/webapp/scripts/views/reports/AuditLayout.js @@ -320,15 +320,25 @@ define(function(require) { switch (facet) { case 'Service Name': - var serviceList = new RangerServiceList(); + var serviceList = new RangerServiceList() , serviceNameVal = []; serviceList.setPageSize(100); serviceList.fetch().done(function(){ - callback(serviceList.map(function(model){return model.get('name');})); + serviceList.each(function(m){ + if(m.get('type') !== XAEnums.ServiceType.SERVICE_TAG.label){ + serviceNameVal.push(m.get('name')); + }; + }); + callback(serviceNameVal); }); break; case 'Service Type': - var serviceList = that.serviceDefList.map(function(serviceDef){ return {'label' : serviceDef.get('name').toUpperCase(), 'value' : serviceDef.get('name').toUpperCase()}; }) - callback(serviceList); + var serviveDefs = []; + that.serviceDefList.each(function(m){ + if(m.get('name').toUpperCase() != (XAEnums.ServiceType.SERVICE_TAG.label).toUpperCase()){ + serviveDefs.push({ 'label' : m.get('name').toUpperCase(), 'value' : m.get('name').toUpperCase() }); + } + }); + callback(serviveDefs); break; case 'Result': callback(XAUtils.hackForVSLabelValuePairs(XAEnums.AccessResult)); From 99abbcfa99667b60ae5f217db4bce44ac01bfdce Mon Sep 17 00:00:00 2001 From: Bhavik Patel Date: Tue, 8 Aug 2017 10:59:54 +0530 Subject: [PATCH 016/151] RANGER-1491:Automatically map group of external users to Administrator Role Signed-off-by: Gautam Borad --- .../java/org/apache/ranger/biz/UserMgr.java | 63 +++++- .../java/org/apache/ranger/biz/XUserMgr.java | 87 +++++--- .../apache/ranger/service/XUserService.java | 7 +- .../java/org/apache/ranger/view/VXUser.java | 1 + .../org/apache/ranger/biz/TestUserMgr.java | 4 +- .../org/apache/ranger/biz/TestXUserMgr.java | 45 +++- .../LdapPolicyMgrUserGroupBuilder.java | 123 ++++++++++- .../config/UserGroupSyncConfig.java | 41 ++++ .../ranger/unixusersync/model/XUserInfo.java | 20 +- .../process/PolicyMgrUserGroupBuilder.java | 201 +++++++++++++++++- unixauthservice/scripts/install.properties | 15 ++ unixauthservice/scripts/setup.py | 17 ++ .../templates/installprop2xml.properties | 4 + .../templates/ranger-ugsync-template.xml | 16 ++ 14 files changed, 588 insertions(+), 56 deletions(-) diff --git a/security-admin/src/main/java/org/apache/ranger/biz/UserMgr.java b/security-admin/src/main/java/org/apache/ranger/biz/UserMgr.java index be16f75864..f27bfc1fe2 100644 --- a/security-admin/src/main/java/org/apache/ranger/biz/UserMgr.java +++ b/security-admin/src/main/java/org/apache/ranger/biz/UserMgr.java @@ -142,6 +142,7 @@ public XXPortalUser createUser(VXPortalUser userProfile, int userStatus, Collection userRoleList) { XXPortalUser user = mapVXPortalUserToXXPortalUser(userProfile); checkAdminAccess(); + xUserMgr.checkAccessRoles((List) userRoleList); user = createUser(user, userStatus, userRoleList); return user; @@ -175,7 +176,11 @@ public XXPortalUser createUser(VXPortalUser userProfile, int userStatus) { Collection reqRoleList = userProfile.getUserRoleList(); if (reqRoleList != null && reqRoleList.size() > 0) { for (String role : reqRoleList) { - roleList.add(role); + if (role != null) { + roleList.add(role); + } else { + roleList.add(RangerConstants.ROLE_USER); + } } } else { roleList.add(RangerConstants.ROLE_USER); @@ -1109,6 +1114,8 @@ public VXPortalUser createDefaultAccountUser(VXPortalUser userProfile) { checkAdminAccess(); logger.info("create:" + userProfile.getLoginId()); XXPortalUser xXPortalUser = null; + Collection existingRoleList = null; + Collection reqRoleList = null; String loginId = userProfile.getLoginId(); String emailAddress = userProfile.getEmailAddress(); @@ -1143,13 +1150,59 @@ public VXPortalUser createDefaultAccountUser(VXPortalUser userProfile) { */ } } + VXPortalUser userProfileRes = null; if (xXPortalUser != null) { - return mapXXPortalUserToVXPortalUserForDefaultAccount(xXPortalUser); - } else { - return null; - } + userProfileRes = mapXXPortalUserToVXPortalUserForDefaultAccount(xXPortalUser); + if (userProfile.getUserRoleList() != null + && userProfile.getUserRoleList().size() > 0 + && ((List) userProfile.getUserRoleList()).get(0) != null) { + reqRoleList = userProfile.getUserRoleList(); + existingRoleList = this.getRolesByLoginId(loginId); + XXPortalUser xxPortalUser = daoManager.getXXPortalUser() + .findByLoginId(userProfile.getLoginId()); + if (xxPortalUser != null && xxPortalUser.getUserSource() == RangerCommonEnums.USER_EXTERNAL) { + userProfileRes = updateRoleForExternalUsers(reqRoleList, existingRoleList, userProfileRes); + } + } + } + return userProfileRes; } + protected VXPortalUser updateRoleForExternalUsers(Collection reqRoleList, Collection existingRoleList, VXPortalUser userProfileRes) { + UserSessionBase session = ContextUtil.getCurrentUserSession(); + if ("rangerusersync".equals(session.getXXPortalUser().getLoginId()) + && reqRoleList != null && !reqRoleList.isEmpty() + && existingRoleList != null && !existingRoleList.isEmpty()) { + if (!reqRoleList.equals(existingRoleList)) { + userProfileRes.setUserRoleList(reqRoleList); + userProfileRes.setUserSource(RangerCommonEnums.USER_EXTERNAL); + List xuserPermissionList = daoManager.getXXUserPermission().findByUserPermissionId(userProfileRes.getId()); + + if (xuserPermissionList!=null && xuserPermissionList.size()>0){ + + for (XXUserPermission xXUserPermission : xuserPermissionList) { + if (xXUserPermission != null) { + try { + xUserPermissionService.deleteResource(xXUserPermission.getId()); + } catch (Exception e) { + logger.error(e.getMessage()); + } + } + + } + } + updateUser(userProfileRes); + } + } else { + if (logger.isDebugEnabled()) { + logger.debug("Permission" + " denied. LoggedInUser=" + + (session != null ? session.getXXPortalUser().getId() : "") + + " isn't permitted to perform the action."); + } + } + return userProfileRes; + } + protected VXPortalUser mapXXPortalUserToVXPortalUserForDefaultAccount( XXPortalUser user) { diff --git a/security-admin/src/main/java/org/apache/ranger/biz/XUserMgr.java b/security-admin/src/main/java/org/apache/ranger/biz/XUserMgr.java index ca06805389..676b1e3224 100644 --- a/security-admin/src/main/java/org/apache/ranger/biz/XUserMgr.java +++ b/security-admin/src/main/java/org/apache/ranger/biz/XUserMgr.java @@ -156,6 +156,9 @@ public class XUserMgr extends XUserMgrBase { @Autowired GUIDUtil guidUtil; + @Autowired + UserMgr userManager; + static final Logger logger = Logger.getLogger(XUserMgr.class); @@ -520,7 +523,13 @@ public VXUserGroupInfo createXUserGroupFromMap( VXUserGroupInfo vxUGInfo = new VXUserGroupInfo(); VXUser vXUser = vXUserGroupInfo.getXuserInfo(); - + VXPortalUser vXPortalUser = userMgr.getUserProfileByLoginId(vXUser.getName()); + XXPortalUser xxPortalUser = daoManager.getXXPortalUser().findByLoginId(vXUser.getName()); + Collection reqRoleList = vXUser.getUserRoleList(); + List existingRole = daoManager.getXXPortalUserRole().findXPortalUserRolebyXPortalUserId(xxPortalUser.getId()); + if (xxPortalUser.getUserSource() == RangerCommonEnums.USER_EXTERNAL) { + vXPortalUser = userManager.updateRoleForExternalUsers(reqRoleList,existingRole, vXPortalUser); + } vXUser = xUserService.createXUserWithOutLogin(vXUser); vxUGInfo.setXuserInfo(vXUser); @@ -536,9 +545,7 @@ public VXUserGroupInfo createXUserGroupFromMap( vXGroupUser = xGroupUserService .createXGroupUserWithOutLogin(vXGroupUser); } - VXPortalUser vXPortalUser = userMgr.getUserProfileByLoginId(vXUser - .getName()); - if(vXPortalUser!=null){ + if (vXPortalUser != null) { assignPermissionToUser(vXPortalUser, true); } vxUGInfo.setXgroupInfo(vxg); @@ -562,17 +569,37 @@ public VXGroupUserInfo createXGroupUserFromMap( List vxu = new ArrayList(); for (VXUser vXUser : vXGroupUserInfo.getXuserInfo()) { - XXUser xUser = daoManager.getXXUser().findByUserName(vXUser.getName()); + XXUser xUser = daoManager.getXXUser().findByUserName( + vXUser.getName()); + XXPortalUser xXPortalUser = daoManager.getXXPortalUser() + .findByLoginId(vXUser.getName()); if (xUser != null) { - // Add or update group user mapping only if the user already exists in x_user table. + // Add or update group user mapping only if the user already + // exists in x_user table. vXGroup = xGroupService.createXGroupWithOutLogin(vXGroup); vxGUInfo.setXgroupInfo(vXGroup); vxu.add(vXUser); VXGroupUser vXGroupUser = new VXGroupUser(); vXGroupUser.setUserId(xUser.getId()); vXGroupUser.setName(vXGroup.getName()); - vXGroupUser = xGroupUserService - .createXGroupUserWithOutLogin(vXGroupUser); + if (xXPortalUser.getUserSource() == RangerCommonEnums.USER_EXTERNAL) { + vXGroupUser = xGroupUserService + .createXGroupUserWithOutLogin(vXGroupUser); + } + Collection reqRoleList = vXUser.getUserRoleList(); + + XXPortalUser xxPortalUser = daoManager.getXXPortalUser() + .findByLoginId(vXUser.getName()); + List existingRole = daoManager.getXXPortalUserRole() + .findXPortalUserRolebyXPortalUserId( + xxPortalUser.getId()); + VXPortalUser vxPortalUser = userManager + .mapXXPortalUserToVXPortalUserForDefaultAccount(xxPortalUser); + if (xxPortalUser.getUserSource() == RangerCommonEnums.USER_EXTERNAL) { + vxPortalUser = userManager.updateRoleForExternalUsers( + reqRoleList, existingRole, vxPortalUser); + assignPermissionToUser(vxPortalUser, true); + } } } @@ -1271,30 +1298,42 @@ private void populatePageList(List auditMapList, int startIndex, int public void checkAccessRoles(List stringRolesList) { UserSessionBase session = ContextUtil.getCurrentUserSession(); - if (session != null && stringRolesList!=null) { + if (session != null && stringRolesList != null) { if (!session.isUserAdmin() && !session.isKeyAdmin()) { throw restErrorUtil.create403RESTException("Permission" + " denied. LoggedInUser=" + (session != null ? session.getXXPortalUser().getId() : "Not Logged In") + " ,isn't permitted to perform the action."); - }else{ - if (session.isUserAdmin() && stringRolesList.contains(RangerConstants.ROLE_KEY_ADMIN)) { - throw restErrorUtil.create403RESTException("Permission" - + " denied. LoggedInUser=" - + (session != null ? session.getXXPortalUser().getId() - : "") - + " isn't permitted to perform the action."); - } - if (session.isKeyAdmin() && stringRolesList.contains(RangerConstants.ROLE_SYS_ADMIN)) { - throw restErrorUtil.create403RESTException("Permission" - + " denied. LoggedInUser=" - + (session != null ? session.getXXPortalUser().getId() - : "") - + " isn't permitted to perform the action."); + } else { + if (!"rangerusersync".equals(session.getXXPortalUser() + .getLoginId())) {// new logic for rangerusersync user + if (session.isUserAdmin() + && stringRolesList + .contains(RangerConstants.ROLE_KEY_ADMIN)) { + throw restErrorUtil.create403RESTException("Permission" + + " denied. LoggedInUser=" + + (session != null ? session.getXXPortalUser() + .getId() : "") + + " isn't permitted to perform the action."); + } + if (session.isKeyAdmin() + && stringRolesList + .contains(RangerConstants.ROLE_SYS_ADMIN)) { + throw restErrorUtil.create403RESTException("Permission" + + " denied. LoggedInUser=" + + (session != null ? session.getXXPortalUser() + .getId() : "") + + " isn't permitted to perform the action."); + } + } else { + logger.info("LoggedInUser=" + + (session != null ? session.getXXPortalUser() + .getId() + : " is permitted to perform the action")); } } - }else{ + } else { VXResponse vXResponse = new VXResponse(); vXResponse.setStatusCode(HttpServletResponse.SC_UNAUTHORIZED); vXResponse.setMsgDesc("Bad Credentials"); diff --git a/security-admin/src/main/java/org/apache/ranger/service/XUserService.java b/security-admin/src/main/java/org/apache/ranger/service/XUserService.java index 0d07982bbf..b2b06ff8c5 100644 --- a/security-admin/src/main/java/org/apache/ranger/service/XUserService.java +++ b/security-admin/src/main/java/org/apache/ranger/service/XUserService.java @@ -49,7 +49,7 @@ import org.springframework.context.annotation.Scope; import org.springframework.stereotype.Service; import org.springframework.util.CollectionUtils; - +import org.apache.ranger.common.RangerCommonEnums; @Service @Scope("singleton") public class XUserService extends XUserServiceBase { @@ -168,7 +168,10 @@ public VXUser createXUserWithOutLogin(VXUser vxUser) { xxUser = new XXUser(); userExists = false; } - + XXPortalUser xxPortalUser = daoManager.getXXPortalUser().findByLoginId(vxUser.getName()); + if (xxPortalUser != null && xxPortalUser.getUserSource() == RangerCommonEnums.USER_EXTERNAL) { + vxUser.setIsVisible(xxUser.getIsVisible()); + } xxUser = mapViewToEntityBean(vxUser, xxUser, 0); XXPortalUser xXPortalUser = daoManager.getXXPortalUser().getById(createdByUserId); if (xXPortalUser != null) { diff --git a/security-admin/src/main/java/org/apache/ranger/view/VXUser.java b/security-admin/src/main/java/org/apache/ranger/view/VXUser.java index ecfd1ac6fe..6e1d2996db 100644 --- a/security-admin/src/main/java/org/apache/ranger/view/VXUser.java +++ b/security-admin/src/main/java/org/apache/ranger/view/VXUser.java @@ -300,6 +300,7 @@ public String toString( ) { str += "isVisible={" + isVisible + "} "; str += "groupIdList={" + groupIdList + "} "; str += "groupNameList={" + groupNameList + "} "; + str += "roleList={" + userRoleList + "} "; str += "}"; return str; } diff --git a/security-admin/src/test/java/org/apache/ranger/biz/TestUserMgr.java b/security-admin/src/test/java/org/apache/ranger/biz/TestUserMgr.java index 60837780bc..6dc483d8df 100644 --- a/security-admin/src/test/java/org/apache/ranger/biz/TestUserMgr.java +++ b/security-admin/src/test/java/org/apache/ranger/biz/TestUserMgr.java @@ -774,8 +774,8 @@ public void test22CreateDefaultAccountUser() { dbVXPortalUser.getEmailAddress()); Assert.assertEquals(user.getPassword(), dbVXPortalUser.getPassword()); - Mockito.verify(daoManager).getXXPortalUser(); - Mockito.verify(daoManager).getXXPortalUserRole(); + Mockito.verify(daoManager, Mockito.atLeast(1)).getXXPortalUser(); + Mockito.verify(daoManager, Mockito.atLeast(1)).getXXPortalUserRole(); } @Test diff --git a/security-admin/src/test/java/org/apache/ranger/biz/TestXUserMgr.java b/security-admin/src/test/java/org/apache/ranger/biz/TestXUserMgr.java index 2542f91b52..6e6be7246e 100644 --- a/security-admin/src/test/java/org/apache/ranger/biz/TestXUserMgr.java +++ b/security-admin/src/test/java/org/apache/ranger/biz/TestXUserMgr.java @@ -24,7 +24,8 @@ import java.util.List; import java.util.Map; import java.util.Set; - +import org.apache.ranger.common.RangerCommonEnums; +import org.apache.ranger.common.RangerConstants; import org.apache.ranger.common.ContextUtil; import org.apache.ranger.common.RESTErrorUtil; import org.apache.ranger.common.SearchCriteria; @@ -175,6 +176,10 @@ public void setup() { UserSessionBase currentUserSession = ContextUtil .getCurrentUserSession(); currentUserSession.setUserAdmin(true); + XXPortalUser gjUser = new XXPortalUser(); + gjUser.setLoginId("test"); + gjUser.setId(1L); + currentUserSession.setXXPortalUser(gjUser); } private VXUser vxUser() { @@ -628,14 +633,16 @@ public void test24GetXUserByUserName() { Mockito.when(xUserService.getXUserByUserName(userName)).thenReturn( vxUser); - + XXModuleDefDao xxModuleDefDao = Mockito.mock(XXModuleDefDao.class); + Mockito.when(daoManager.getXXModuleDef()).thenReturn(xxModuleDefDao); VXUser dbVXUser = xUserMgr.getXUserByUserName(userName); Assert.assertNotNull(dbVXUser); userId = dbVXUser.getId(); Assert.assertEquals(userId, dbVXUser.getId()); Assert.assertEquals(dbVXUser.getName(), vxUser.getName()); Assert.assertEquals(dbVXUser.getOwner(), vxUser.getOwner()); - Mockito.verify(xUserService).getXUserByUserName(userName); + Mockito.verify(xUserService, Mockito.atLeast(2)).getXUserByUserName( + userName); } @Test @@ -873,6 +880,20 @@ public void test30CreateVXUserGroupInfo() { Mockito.when( xGroupUserService.createXGroupUserWithOutLogin(vXGroupUser2)) .thenReturn(vXGroupUser2); + XXPortalUserDao portalUser = Mockito.mock(XXPortalUserDao.class); + Mockito.when(daoManager.getXXPortalUser()).thenReturn(portalUser); + XXPortalUser user = new XXPortalUser(); + user.setId(1L); + user.setUserSource(RangerCommonEnums.USER_APP); + Mockito.when(portalUser.findByLoginId(vXUser.getName())).thenReturn( + user); + XXPortalUserRoleDao userDao = Mockito.mock(XXPortalUserRoleDao.class); + Mockito.when(daoManager.getXXPortalUserRole()).thenReturn(userDao); + List lstRole = new ArrayList(); + lstRole.add(RangerConstants.ROLE_SYS_ADMIN); + Mockito.when( + userDao.findXPortalUserRolebyXPortalUserId(Mockito.anyLong())) + .thenReturn(lstRole); VXUserGroupInfo vxUserGroupTest = xUserMgr .createXUserGroupFromMap(vXUserGroupInfo); @@ -882,6 +903,11 @@ public void test30CreateVXUserGroupInfo() { expected.add(vXGroup1); expected.add(vXGroup2); Assert.assertTrue(result.containsAll(expected)); + Mockito.verify(daoManager).getXXPortalUser(); + Mockito.verify(portalUser).findByLoginId(vXUser.getName()); + Mockito.verify(daoManager).getXXPortalUserRole(); + Mockito.verify(userDao).findXPortalUserRolebyXPortalUserId( + Mockito.anyLong()); } // Module permission @@ -1312,9 +1338,20 @@ public void test44getGroupsForUser() { String userName = "test"; Mockito.when(xUserService.getXUserByUserName(userName)).thenReturn( vxUser); + XXModuleDefDao modDef = Mockito.mock(XXModuleDefDao.class); + Mockito.when(daoManager.getXXModuleDef()).thenReturn(modDef); + List lstModule = new ArrayList(); + lstModule.add(RangerConstants.MODULE_USER_GROUPS); + Mockito.when( + modDef.findAccessibleModulesByUserId(Mockito.anyLong(), + Mockito.anyLong())).thenReturn(lstModule); Set list = xUserMgr.getGroupsForUser(userName); Assert.assertNotNull(list); - Mockito.verify(xUserService).getXUserByUserName(userName); + Mockito.verify(xUserService, Mockito.atLeast(2)).getXUserByUserName( + userName); + Mockito.verify(daoManager).getXXModuleDef(); + Mockito.verify(modDef).findAccessibleModulesByUserId(Mockito.anyLong(), + Mockito.anyLong()); } @Test diff --git a/ugsync/src/main/java/org/apache/ranger/ldapusersync/process/LdapPolicyMgrUserGroupBuilder.java b/ugsync/src/main/java/org/apache/ranger/ldapusersync/process/LdapPolicyMgrUserGroupBuilder.java index 428ad30203..9548ed42c5 100644 --- a/ugsync/src/main/java/org/apache/ranger/ldapusersync/process/LdapPolicyMgrUserGroupBuilder.java +++ b/ugsync/src/main/java/org/apache/ranger/ldapusersync/process/LdapPolicyMgrUserGroupBuilder.java @@ -65,6 +65,10 @@ import com.sun.jersey.api.client.config.DefaultClientConfig; import com.sun.jersey.api.client.filter.HTTPBasicAuthFilter; import com.sun.jersey.client.urlconnection.HTTPSProperties; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.HashMap; +import java.util.StringTokenizer; public class LdapPolicyMgrUserGroupBuilder implements UserGroupSink { @@ -100,7 +104,8 @@ public class LdapPolicyMgrUserGroupBuilder implements UserGroupSink { private UserGroupInfo usergroupInfo = new UserGroupInfo(); private GroupUserInfo groupuserInfo = new GroupUserInfo(); - + Map userMap = new LinkedHashMap(); + Map groupMap = new LinkedHashMap(); Table groupsUsersTable; private String keyStoreFile = null; @@ -147,7 +152,10 @@ synchronized public void init() throws Throwable { } keytab = config.getProperty(KEYTAB,""); nameRules = config.getProperty(NAME_RULE,"DEFAULT"); - + String userGroupRoles = config.getGroupRoleRules(); + if (userGroupRoles != null && !userGroupRoles.isEmpty()) { + getRoleForUserGroups(userGroupRoles); + } } @Override @@ -331,7 +339,11 @@ private XUserInfo addXUserInfo(String aUserName) { xuserInfo.setName(aUserName); xuserInfo.setDescription(aUserName + " - add from Unix box"); - + if (userMap.containsKey(aUserName)) { + List roleList = new ArrayList(); + roleList.add(userMap.get(aUserName)); + xuserInfo.setUserRoleList(roleList); + } usergroupInfo.setXuserInfo(xuserInfo); return xuserInfo; @@ -414,9 +426,11 @@ public GroupUserInfo run() { } List oldUsers = new ArrayList(); + Map > oldUserMap = new HashMap>(); if (groupUserInfo != null && groupUserInfo.getXuserInfo() != null) { for (XUserInfo xUserInfo : groupUserInfo.getXuserInfo()) { oldUsers.add(xUserInfo.getName()); + oldUserMap.put(xUserInfo.getName(), xUserInfo.getUserRoleList()); } LOG.debug("Returned users for group " + groupUserInfo.getXgroupInfo().getName() + " are: " + oldUsers); } @@ -433,7 +447,7 @@ public GroupUserInfo run() { addUsers = users; } else { for (String user : users) { - if (!oldUsers.contains(user)) { + if (!oldUsers.contains(user)|| !(oldUserMap.get(user).equals(groupMap.get(groupName)))) { addUsers.add(user); } } @@ -569,7 +583,30 @@ private GroupUserInfo getGroupUserInfo(GroupUserInfo ret) { WebResource r = c.resource(getURL(PM_ADD_GROUP_USER_INFO_URI)); Gson gson = new GsonBuilder().create(); - + if (groupuserInfo != null + && groupuserInfo.getXgroupInfo() != null + && groupuserInfo.getXuserInfo() != null + && groupMap + .containsKey(groupuserInfo.getXgroupInfo().getName()) + && groupuserInfo.getXuserInfo().size() > 0) { + List userRoleList = new ArrayList(); + userRoleList.add(groupMap.get(groupuserInfo.getXgroupInfo() + .getName())); + int i = groupuserInfo.getXuserInfo().size(); + for (int j = 0; j < i; j++) { + if (userMap.containsKey(groupuserInfo.getXuserInfo().get(j) + .getName())) { + List userRole = new ArrayList(); + userRole.add(userMap.get(groupuserInfo.getXuserInfo() + .get(j).getName())); + groupuserInfo.getXuserInfo().get(j) + .setUserRoleList(userRole); + } else { + groupuserInfo.getXuserInfo().get(j) + .setUserRoleList(userRoleList); + } + } + } String jsonString = gson.toJson(groupuserInfo); LOG.debug("GROUP USER MAPPING" + jsonString); @@ -591,7 +628,11 @@ private MUserInfo addMUser(String aUserName) { userInfo.setLoginId(aUserName); userInfo.setFirstName(aUserName); userInfo.setLastName(aUserName); - + String str[] = new String[1]; + if (userMap.containsKey(aUserName)) { + str[0] = userMap.get(aUserName); + } + userInfo.setUserRoleList(str); if (authenticationType != null && AUTH_KERBEROS.equalsIgnoreCase(authenticationType) && SecureClientLogin.isKerberosCredentialExists(principal, keytab)) { try { Subject sub = SecureClientLogin.loginUserFromKeytab(principal, keytab, nameRules); @@ -804,4 +845,74 @@ private InputStream getFileInputStream(String path) throws FileNotFoundException return ret; } + private void getRoleForUserGroups(String userGroupRolesData) { + String roleDelimiter = config.getRoleDelimiter(); + String userGroupDelimiter = config.getUserGroupDelimiter(); + String userNameDelimiter = config.getUserGroupNameDelimiter(); + if (roleDelimiter == null || roleDelimiter.isEmpty()) { + roleDelimiter = "&"; + } + if (userGroupDelimiter == null || userGroupDelimiter.isEmpty()) { + userGroupDelimiter = ":"; + } + if (userNameDelimiter == null || userNameDelimiter.isEmpty()) { + userNameDelimiter = ","; + } + StringTokenizer str = new StringTokenizer(userGroupRolesData, + roleDelimiter); + int flag = 0; + String userGroupCheck = null; + String roleName = null; + while (str.hasMoreTokens()) { + flag = 0; + String tokens = str.nextToken(); + if (tokens != null && !tokens.isEmpty()) { + StringTokenizer userGroupRoles = new StringTokenizer(tokens, + userGroupDelimiter); + if (userGroupRoles != null) { + while (userGroupRoles.hasMoreElements()) { + String userGroupRolesTokens = userGroupRoles + .nextToken(); + if (userGroupRolesTokens != null + && !userGroupRolesTokens.isEmpty()) { + flag++; + switch (flag) { + case 1: + roleName = userGroupRolesTokens; + break; + case 2: + userGroupCheck = userGroupRolesTokens; + break; + case 3: + StringTokenizer userGroupNames = new StringTokenizer( + userGroupRolesTokens, userNameDelimiter); + if (userGroupNames != null) { + while (userGroupNames.hasMoreElements()) { + String userGroup = userGroupNames + .nextToken(); + if (userGroup != null + && !userGroup.isEmpty()) { + if (userGroupCheck + .equalsIgnoreCase("u")) { + userMap.put(userGroup.trim(), roleName.trim()); + } else if (userGroupCheck + .equalsIgnoreCase("g")) { + groupMap.put(userGroup.trim(), + roleName.trim()); + } + } + } + } + break; + default: + userMap.clear(); + groupMap.clear(); + break; + } + } + } + } + } + } + } } diff --git a/ugsync/src/main/java/org/apache/ranger/unixusersync/config/UserGroupSyncConfig.java b/ugsync/src/main/java/org/apache/ranger/unixusersync/config/UserGroupSyncConfig.java index fc239af8cb..df16043267 100644 --- a/ugsync/src/main/java/org/apache/ranger/unixusersync/config/UserGroupSyncConfig.java +++ b/ugsync/src/main/java/org/apache/ranger/unixusersync/config/UserGroupSyncConfig.java @@ -235,6 +235,11 @@ public class UserGroupSyncConfig { private static final String SYNC_MAPPING_GROUPNAME_HANDLER = "ranger.usersync.mapping.groupname.handler"; private static final String DEFAULT_SYNC_MAPPING_GROUPNAME_HANDLER = "org.apache.ranger.usergroupsync.RegEx"; + private static final String ROLE_ASSIGNMENT_LIST_DELIMITER = "ranger.usersync.role.assignment.list.delimiter"; + private static final String USERS_GROUPS_ASSIGNMENT_LIST_DELIMITER = "ranger.usersync.users.groups.assignment.list.delimiter"; + private static final String USERNAME_GROUPNAME_ASSIGNMENT_LIST_DELIMITER = "ranger.usersync.username.groupname.assignment.list.delimiter"; + private static final String GROUP_BASED_ROLE_ASSIGNMENT_RULES = "ranger.usersync.group.based.role.assignment.rules"; + private Properties prop = new Properties(); private static volatile UserGroupSyncConfig me = null; @@ -1063,4 +1068,40 @@ public void setGroupObjectClass(String groupObjectClass) { public void setDeltaSync(boolean deltaSyncEnabled) { prop.setProperty(LGSYNC_LDAP_DELTASYNC_ENABLED, String.valueOf(deltaSyncEnabled)); } + public String getGroupRoleRules() { + if(prop != null && prop.containsKey(GROUP_BASED_ROLE_ASSIGNMENT_RULES)) { + String GroupRoleRules = prop.getProperty(GROUP_BASED_ROLE_ASSIGNMENT_RULES); + if(GroupRoleRules != null && !GroupRoleRules.isEmpty()) { + return GroupRoleRules.trim(); + } + } + return null; + } + public String getUserGroupDelimiter() { + if(prop != null && prop.containsKey(USERS_GROUPS_ASSIGNMENT_LIST_DELIMITER)) { + String UserGroupDelimiter = prop.getProperty(USERS_GROUPS_ASSIGNMENT_LIST_DELIMITER); + if(UserGroupDelimiter != null && !UserGroupDelimiter.isEmpty()) { + return UserGroupDelimiter; + } + } + return null; + } + public String getUserGroupNameDelimiter() { + if(prop != null && prop.containsKey(USERNAME_GROUPNAME_ASSIGNMENT_LIST_DELIMITER)) { + String UserGroupNameDelimiter = prop.getProperty(USERNAME_GROUPNAME_ASSIGNMENT_LIST_DELIMITER); + if(UserGroupNameDelimiter != null && !UserGroupNameDelimiter.isEmpty()) { + return UserGroupNameDelimiter; + } + } + return null; + } + public String getRoleDelimiter() { + if(prop != null && prop.containsKey(ROLE_ASSIGNMENT_LIST_DELIMITER)) { + String roleDelimiter = prop.getProperty(ROLE_ASSIGNMENT_LIST_DELIMITER); + if(roleDelimiter != null && !roleDelimiter.isEmpty()) { + return roleDelimiter; + } + } + return null; + } } diff --git a/ugsync/src/main/java/org/apache/ranger/unixusersync/model/XUserInfo.java b/ugsync/src/main/java/org/apache/ranger/unixusersync/model/XUserInfo.java index 7d636fdb86..b21468b0d4 100644 --- a/ugsync/src/main/java/org/apache/ranger/unixusersync/model/XUserInfo.java +++ b/ugsync/src/main/java/org/apache/ranger/unixusersync/model/XUserInfo.java @@ -26,8 +26,8 @@ public class XUserInfo { private String id; private String name; private String description; - - private List groupNameList = new ArrayList(); + private List groupNameList = new ArrayList(); + private List userRoleList = new ArrayList(); public String getId() { return id; @@ -59,5 +59,19 @@ public List getGroupNameList() { public List getGroups() { return groupNameList; } - + + public List getUserRoleList() { + return userRoleList; + } + + public void setUserRoleList(List userRoleList) { + this.userRoleList = userRoleList; + } + + @Override + public String toString() { + return "XUserInfo [id=" + id + ", name=" + name + ", description=" + + description + ", groupNameList=" + groupNameList + + ", userRoleList=" + userRoleList + "]"; + } } diff --git a/ugsync/src/main/java/org/apache/ranger/unixusersync/process/PolicyMgrUserGroupBuilder.java b/ugsync/src/main/java/org/apache/ranger/unixusersync/process/PolicyMgrUserGroupBuilder.java index 070a39b6b7..87b48832fb 100644 --- a/ugsync/src/main/java/org/apache/ranger/unixusersync/process/PolicyMgrUserGroupBuilder.java +++ b/ugsync/src/main/java/org/apache/ranger/unixusersync/process/PolicyMgrUserGroupBuilder.java @@ -68,7 +68,9 @@ import org.apache.ranger.unixusersync.model.UserGroupInfo; import org.apache.ranger.usergroupsync.UserGroupSink; import org.apache.ranger.usersync.util.UserSyncUtil; - +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.StringTokenizer; public class PolicyMgrUserGroupBuilder implements UserGroupSink { private static final Logger LOG = Logger.getLogger(PolicyMgrUserGroupBuilder.class); @@ -121,7 +123,8 @@ public class PolicyMgrUserGroupBuilder implements UserGroupSink { String principal; String keytab; String nameRules; - + Map userMap = new LinkedHashMap(); + Map groupMap = new LinkedHashMap(); static { try { LOCAL_HOSTNAME = java.net.InetAddress.getLocalHost().getCanonicalHostName(); @@ -160,6 +163,10 @@ synchronized public void init() throws Throwable { } keytab = config.getProperty(KEYTAB,""); nameRules = config.getProperty(NAME_RULE,"DEFAULT"); + String userGroupRoles = config.getGroupRoleRules(); + if (userGroupRoles != null && !userGroupRoles.isEmpty()) { + getRoleForUserGroups(userGroupRoles); + } buildUserGroupInfo(); } @@ -366,7 +373,28 @@ public void addOrUpdateUser(String userName, List groups) throws Throwab } if (! isMockRun) { if (!addGroups.isEmpty()){ - ugInfo.setXuserInfo(addXUserInfo(userName)); + XUserInfo obj = addXUserInfo(userName); + if (obj != null) { + for (int i = 0; i < addGroups.size(); i++) { + if (groupMap.containsKey(addGroups.get(i))) { + List userRoleList = new ArrayList(); + userRoleList + .add(groupMap.get(addGroups.get(i))); + if (userMap.containsKey(obj.getName())) { + List userRole = new ArrayList(); + userRole.add(userMap.get(obj.getName())); + if (!obj.getUserRoleList().equals(userRole)) { + obj.setUserRoleList(userRole); + + } + } else if (!obj.getUserRoleList().equals( + userRoleList)) { + obj.setUserRoleList(userRoleList); + } + } + } + } + ugInfo.setXuserInfo(obj); ugInfo.setXgroupInfo(getXGroupInfoList(addGroups)); try{ // If the rest call to ranger admin fails, @@ -393,7 +421,27 @@ public void addOrUpdateUser(String userName, List groups) throws Throwab } if (! isMockRun) { if (!updateGroups.isEmpty()){ - ugInfo.setXuserInfo(addXUserInfo(userName)); + XUserInfo obj = addXUserInfo(userName); + if (obj != null) { + for (int i = 0; i < updateGroups.size(); i++) { + if (groupMap.containsKey(updateGroups.get(i))) { + List userRoleList = new ArrayList(); + userRoleList.add(groupMap.get(updateGroups + .get(i))); + if (userMap.containsKey(obj.getName())) { + List userRole = new ArrayList(); + userRole.add(userMap.get(obj.getName())); + if (!obj.getUserRoleList().equals(userRole)) { + obj.setUserRoleList(userRole); + } + } else if (!obj.getUserRoleList().equals( + userRoleList)) { + obj.setUserRoleList(userRoleList); + } + } + } + } + ugInfo.setXuserInfo(obj); ugInfo.setXgroupInfo(getXGroupInfoList(updateGroups)); try{ // If the rest call to ranger admin fails, @@ -409,8 +457,53 @@ public void addOrUpdateUser(String userName, List groups) throws Throwab } } } - } - } + if (!isMockRun) { + XUserInfo obj = addXUserInfo(userName); + boolean roleFlag = false; + if (obj != null && updateGroups.isEmpty() + && addGroups.isEmpty()) { + if (userMap.containsKey(obj.getName())) { + List userRole = new ArrayList(); + userRole.add(userMap.get(obj.getName())); + if (!obj.getUserRoleList().equals(userRole)) { + obj.setUserRoleList(userRole); + roleFlag = true; + } + } else { + for (int i = 0; i < groups.size(); i++) { + if (groupMap.containsKey(groups.get(i))) { + List userRoleList = new ArrayList(); + userRoleList.add(groupMap.get(groups.get(i))); + if (!obj.getUserRoleList().equals(userRoleList)) { + obj.setUserRoleList(userRoleList); + roleFlag = true; + } + } + } + + } + ugInfo.setXuserInfo(obj); + ugInfo.setXgroupInfo(getXGroupInfoList(groups)); + } + if (roleFlag) { + try { + // If the rest call to ranger admin fails, + // propagate the failure to the caller for retry in next + // sync cycle. + if (addUserGroupInfo(ugInfo) == null) { + String msg = "Failed to add user group info"; + LOG.error(msg); + throw new Exception(msg); + } + } catch (Throwable t) { + LOG.error("PolicyMgrUserGroupBuilder.addUserGroupInfo failed with exception: " + + t.getMessage() + + ", for user-group entry: " + + ugInfo); + } + } + } + } } private void buildGroupList() { if (LOG.isDebugEnabled()) { @@ -530,6 +623,23 @@ private UserGroupInfo addUserGroupInfo(String userName, List groups){ if (! isMockRun) { user = addXUserInfo(userName); } + if (!groups.isEmpty() && user != null) { + for (int i = 0; i < groups.size(); i++) { + if (groupMap.containsKey(groups.get(i))) { + List userRoleList = new ArrayList(); + userRoleList.add(groupMap.get(groups.get(i))); + if (userMap.containsKey(user.getName())) { + List userRole = new ArrayList(); + userRole.add(userMap.get(user.getName())); + user.setUserRoleList(userRole); + } else { + user.setUserRoleList(userRoleList); + } + } + } + } + usergroupInfo.setXuserInfo(user); + for(String g : groups) { LOG.debug("INFO: addPMXAGroupToUser(" + userName + "," + g + ")" ); @@ -809,7 +919,11 @@ private MUserInfo addMUser(String aUserName) { userInfo.setLoginId(aUserName); userInfo.setFirstName(aUserName); userInfo.setLastName(aUserName); - + String str[] = new String[1]; + if (userMap.containsKey(aUserName)) { + str[0] = userMap.get(aUserName); + } + userInfo.setUserRoleList(str); if (authenticationType != null && AUTH_KERBEROS.equalsIgnoreCase(authenticationType) && SecureClientLogin.isKerberosCredentialExists(principal, keytab)) { try { Subject sub = SecureClientLogin.loginUserFromKeytab(principal, keytab, nameRules); @@ -1080,6 +1194,73 @@ public void addOrUpdateGroup(String group, List users) throws Throwable // TODO Auto-generated method stub } - - -} + private void getRoleForUserGroups(String userGroupRolesData) { + + String roleDelimiter = config.getRoleDelimiter(); + String userGroupDelimiter = config.getUserGroupDelimiter(); + String userNameDelimiter = config.getUserGroupNameDelimiter(); + if (roleDelimiter == null || roleDelimiter.isEmpty()) { + roleDelimiter = "&"; + } + if (userGroupDelimiter == null || userGroupDelimiter.isEmpty()) { + userGroupDelimiter = ":"; + } + if (userNameDelimiter == null || userNameDelimiter.isEmpty()) { + userNameDelimiter = ","; + } + StringTokenizer str = new StringTokenizer(userGroupRolesData, + roleDelimiter); + int flag = 0; + String userGroupCheck = null; + String roleName = null; + while (str.hasMoreTokens()) { + flag = 0; + String tokens = str.nextToken(); + if (tokens != null && !tokens.isEmpty()) { + StringTokenizer userGroupRoles = new StringTokenizer(tokens, + userGroupDelimiter); + if (userGroupRoles != null) { + while (userGroupRoles.hasMoreElements()) { + String userGroupRolesTokens = userGroupRoles + .nextToken(); + if (userGroupRolesTokens != null + && !userGroupRolesTokens.isEmpty()) { + flag++; + switch (flag) { + case 1: + roleName = userGroupRolesTokens; + break; + case 2: + userGroupCheck = userGroupRolesTokens; + break; + case 3: + StringTokenizer userGroupNames = new StringTokenizer( + userGroupRolesTokens, userNameDelimiter); + if (userGroupNames != null) { + while (userGroupNames.hasMoreElements()) { + String userGroup = userGroupNames + .nextToken(); + if (userGroup != null + && !userGroup.isEmpty()) { + if (userGroupCheck.trim().equalsIgnoreCase("u")) { + userMap.put(userGroup.trim(), roleName.trim()); + } else if (userGroupCheck.trim().equalsIgnoreCase("g")) { + groupMap.put(userGroup.trim(), + roleName.trim()); + } + } + } + } + break; + default: + userMap.clear(); + groupMap.clear(); + break; + } + } + } + } + } + } + } + } diff --git a/unixauthservice/scripts/install.properties b/unixauthservice/scripts/install.properties index 13ae1e55f0..0be2c8f974 100644 --- a/unixauthservice/scripts/install.properties +++ b/unixauthservice/scripts/install.properties @@ -64,6 +64,21 @@ AUTH_SSL_TRUSTSTORE_PASSWORD= # --------------------------------------------------------------- # The following properties are relevant only if SYNC_SOURCE = ldap # --------------------------------------------------------------- +# The below properties ROLE_ASSIGNMENT_LIST_DELIMITER, USERS_GROUPS_ASSIGNMENT_LIST_DELIMITER, USERNAME_GROUPNAME_ASSIGNMENT_LIST_DELIMITER, +#and GROUP_BASED_ROLE_ASSIGNMENT_RULES can be used to assign role to LDAP synced users and groups +#NOTE all the delimiters should have different values and the delimiters should not contain characters that are allowed in userName or GroupName + +# default value ROLE_ASSIGNMENT_LIST_DELIMITER = & +ROLE_ASSIGNMENT_LIST_DELIMITER = & + +#default value USERS_GROUPS_ASSIGNMENT_LIST_DELIMITER = : +USERS_GROUPS_ASSIGNMENT_LIST_DELIMITER = : + +#default value USERNAME_GROUPNAME_ASSIGNMENT_LIST_DELIMITER = , +USERNAME_GROUPNAME_ASSIGNMENT_LIST_DELIMITER = , + +# with above mentioned delimiters a sample value would be &ROLE_SYS_ADMIN:u:userName1,userName2&ROLE_SYS_ADMIN:g:groupName1,groupName2&ROLE_KEY_ADMIN:u:userName&ROLE_KEY_ADMIN:g:groupName +GROUP_BASED_ROLE_ASSIGNMENT_RULES = # URL of source ldap # a sample value would be: ldap://ldap.example.com:389 diff --git a/unixauthservice/scripts/setup.py b/unixauthservice/scripts/setup.py index c7aa95903a..211da6488b 100755 --- a/unixauthservice/scripts/setup.py +++ b/unixauthservice/scripts/setup.py @@ -347,6 +347,23 @@ def main(): hadoop_conf = globalDict['hadoop_conf'] pid_dir_path = globalDict['USERSYNC_PID_DIR_PATH'] unix_user = globalDict['unix_user'] + if globalDict['SYNC_SOURCE'].lower() == SYNC_SOURCE_LDAP and globalDict.has_key('ROLE_ASSIGNMENT_LIST_DELIMITER') \ + and globalDict.has_key('USERS_GROUPS_ASSIGNMENT_LIST_DELIMITER') and globalDict.has_key('USERNAME_GROUPNAME_ASSIGNMENT_LIST_DELIMITER'): + roleAssignmentDelimiter = globalDict['ROLE_ASSIGNMENT_LIST_DELIMITER'] + userGroupAssignmentDelimiter= globalDict['USERS_GROUPS_ASSIGNMENT_LIST_DELIMITER'] + userNameGroupNameAssignmentListDelimiter= globalDict['USERNAME_GROUPNAME_ASSIGNMENT_LIST_DELIMITER']; + if roleAssignmentDelimiter != "" : + if roleAssignmentDelimiter == userGroupAssignmentDelimiter or roleAssignmentDelimiter == userNameGroupNameAssignmentListDelimiter : + print "ERROR: All Delimiters ROLE_ASSIGNMENT_LIST_DELIMITER, USERS_GROUPS_ASSIGNMENT_LIST_DELIMITER and USERNAME_GROUPNAME_ASSIGNMENT_LIST_DELIMITER should be different" + sys.exit(1) + if userGroupAssignmentDelimiter != "" : + if roleAssignmentDelimiter == userGroupAssignmentDelimiter or userGroupAssignmentDelimiter == userNameGroupNameAssignmentListDelimiter: + print "ERROR: All Delimiters ROLE_ASSIGNMENT_LIST_DELIMITER, USERS_GROUPS_ASSIGNMENT_LIST_DELIMITER and USERNAME_GROUPNAME_ASSIGNMENT_LIST_DELIMITER should be different" + sys.exit(1) + if userNameGroupNameAssignmentListDelimiter != "": + if roleAssignmentDelimiter == userNameGroupNameAssignmentListDelimiter or userGroupAssignmentDelimiter == userNameGroupNameAssignmentListDelimiter: + print "ERROR: All Delimiters ROLE_ASSIGNMENT_LIST_DELIMITER, USERS_GROUPS_ASSIGNMENT_LIST_DELIMITER and USERNAME_GROUPNAME_ASSIGNMENT_LIST_DELIMITER should be different" + sys.exit(1) if pid_dir_path == "": pid_dir_path = "/var/run/ranger" diff --git a/unixauthservice/scripts/templates/installprop2xml.properties b/unixauthservice/scripts/templates/installprop2xml.properties index 1a9bf36a45..8a889a9b67 100644 --- a/unixauthservice/scripts/templates/installprop2xml.properties +++ b/unixauthservice/scripts/templates/installprop2xml.properties @@ -16,6 +16,10 @@ POLICY_MGR_URL = ranger.usersync.policymanager.baseURL MIN_UNIX_USER_ID_TO_SYNC = ranger.usersync.unix.minUserId SYNC_INTERVAL = ranger.usersync.sleeptimeinmillisbetweensynccycle +ROLE_ASSIGNMENT_LIST_DELIMITER = ranger.usersync.role.assignment.list.delimiter +USERS_GROUPS_ASSIGNMENT_LIST_DELIMITER = ranger.usersync.users.groups.assignment.list.delimiter +USERNAME_GROUPNAME_ASSIGNMENT_LIST_DELIMITER = ranger.usersync.username.groupname.assignment.list.delimiter +GROUP_BASED_ROLE_ASSIGNMENT_RULES = ranger.usersync.group.based.role.assignment.rules SYNC_LDAP_URL = ranger.usersync.ldap.url SYNC_LDAP_BIND_DN = ranger.usersync.ldap.binddn SYNC_LDAP_BIND_PASSWORD = ranger.usersync.ldap.ldapbindpassword diff --git a/unixauthservice/scripts/templates/ranger-ugsync-template.xml b/unixauthservice/scripts/templates/ranger-ugsync-template.xml index 0025dc819b..5a0cf988d7 100644 --- a/unixauthservice/scripts/templates/ranger-ugsync-template.xml +++ b/unixauthservice/scripts/templates/ranger-ugsync-template.xml @@ -205,4 +205,20 @@ ranger.usersync.truststore.password + + ranger.usersync.role.assignment.list.delimiter + + + + ranger.usersync.users.groups.assignment.list.delimiter + + + + ranger.usersync.username.groupname.assignment.list.delimiter + + + + ranger.usersync.group.based.role.assignment.rules + + From 41da4514acdd434bb85305a406f3b2f5378af44d Mon Sep 17 00:00:00 2001 From: Abhay Kulkarni Date: Wed, 16 Aug 2017 13:14:32 -0700 Subject: [PATCH 017/151] RANGER-1737: Fixed RANGER-1181 by providing correct set of parameters to Hdfs Native Authorizer in case of fall-back --- .../hadoop/RangerHdfsAuthorizer.java | 110 +++++++++++------- 1 file changed, 65 insertions(+), 45 deletions(-) diff --git a/hdfs-agent/src/main/java/org/apache/ranger/authorization/hadoop/RangerHdfsAuthorizer.java b/hdfs-agent/src/main/java/org/apache/ranger/authorization/hadoop/RangerHdfsAuthorizer.java index f82fd57c55..97fd5cd5a1 100644 --- a/hdfs-agent/src/main/java/org/apache/ranger/authorization/hadoop/RangerHdfsAuthorizer.java +++ b/hdfs-agent/src/main/java/org/apache/ranger/authorization/hadoop/RangerHdfsAuthorizer.java @@ -279,7 +279,7 @@ public void checkPermission(String fsOwner, String superGroup, UserGroupInformat if (authzStatus == AuthzStatus.NOT_DETERMINED) { authzStatus = checkDefaultEnforcer(fsOwner, superGroup, ugi, inodeAttrs, inodes, pathByNameArr, snapshotId, path, ancestorIndex, doCheckOwner, - ancestorAccess, FsAction.NONE, FsAction.NONE, FsAction.NONE, ignoreEmptyDir, + ancestorAccess, null, null, null, ignoreEmptyDir, isTraverseOnlyCheck, ancestor, parent, inode, auditHandler); } } @@ -292,7 +292,7 @@ public void checkPermission(String fsOwner, String superGroup, UserGroupInformat if (authzStatus == AuthzStatus.NOT_DETERMINED) { authzStatus = checkDefaultEnforcer(fsOwner, superGroup, ugi, inodeAttrs, inodes, pathByNameArr, snapshotId, path, ancestorIndex, doCheckOwner, - FsAction.NONE, parentAccess, FsAction.NONE, FsAction.NONE, ignoreEmptyDir, + null, parentAccess, null, null, ignoreEmptyDir, isTraverseOnlyCheck, ancestor, parent, inode, auditHandler); } } @@ -305,7 +305,7 @@ public void checkPermission(String fsOwner, String superGroup, UserGroupInformat if (authzStatus == AuthzStatus.NOT_DETERMINED) { authzStatus = checkDefaultEnforcer(fsOwner, superGroup, ugi, inodeAttrs, inodes, pathByNameArr, snapshotId, path, ancestorIndex, doCheckOwner, - FsAction.NONE, FsAction.NONE, access, FsAction.NONE, ignoreEmptyDir, + null, null, access, null, ignoreEmptyDir, isTraverseOnlyCheck, ancestor, parent, inode, auditHandler); } } @@ -345,18 +345,12 @@ public void checkPermission(String fsOwner, String superGroup, UserGroupInformat } } if (authzStatus == AuthzStatus.NOT_DETERMINED) { - RangerPerfTracer hadoopAuthPerf = null; - - if(RangerPerfTracer.isPerfTraceEnabled(PERF_HDFSAUTH_REQUEST_LOG)) { - hadoopAuthPerf = RangerPerfTracer.getPerfTracer(PERF_HDFSAUTH_REQUEST_LOG, "defaultEnforcer.checkPermission(path=" + path + ")"); - } authzStatus = checkDefaultEnforcer(fsOwner, superGroup, ugi, inodeAttrs, inodes, pathByNameArr, snapshotId, path, ancestorIndex, doCheckOwner, - FsAction.NONE, FsAction.NONE, FsAction.NONE, subAccess, ignoreEmptyDir, + null, null, null, subAccess, ignoreEmptyDir, isTraverseOnlyCheck, ancestor, parent, inode, auditHandler); - RangerPerfTracer.log(hadoopAuthPerf); } } @@ -412,50 +406,76 @@ private AuthzStatus checkDefaultEnforcer(String fsOwner, String superGroup, User boolean isTraverseOnlyCheck, INode ancestor, INode parent, INode inode, RangerHdfsAuditHandler auditHandler ) throws AccessControlException { - AuthzStatus authzStatus = AuthzStatus.NOT_DETERMINED; - if(RangerHdfsPlugin.isHadoopAuthEnabled() && defaultEnforcer != null) { - - try { - defaultEnforcer.checkPermission(fsOwner, superGroup, ugi, inodeAttrs, inodes, - pathByNameArr, snapshotId, path, ancestorIndex, doCheckOwner, - ancestorAccess, parentAccess, access, subAccess, ignoreEmptyDir); - - authzStatus = AuthzStatus.ALLOW; - } finally { - if(auditHandler != null) { - INode nodeChecked = inode; - FsAction action = access; - if(isTraverseOnlyCheck) { - if(nodeChecked == null || nodeChecked.isFile()) { - if(parent != null) { - nodeChecked = parent; - } else if(ancestor != null) { - nodeChecked = ancestor; - } - } + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerAccessControlEnforcer.checkDefaultEnforcer(" + + "fsOwner=" + fsOwner + "; superGroup=" + superGroup + ", inodesCount=" + (inodes != null ? inodes.length : 0) + + ", snapshotId=" + snapshotId + ", path=" + path + ", ancestorIndex=" + ancestorIndex + + ", doCheckOwner=" + doCheckOwner + ", ancestorAccess=" + ancestorAccess + ", parentAccess=" + parentAccess + + ", access=" + access + ", subAccess=" + subAccess + ", ignoreEmptyDir=" + ignoreEmptyDir + + ", isTraverseOnlyCheck=" + isTraverseOnlyCheck + ",ancestor=" + (ancestor == null ? null : ancestor.getFullPathName()) + + ", parent=" + (parent == null ? null : parent.getFullPathName()) + ", inode=" + (inode == null ? null : inode.getFullPathName()) + + ")"); + } + + AuthzStatus authzStatus = AuthzStatus.NOT_DETERMINED; + if(RangerHdfsPlugin.isHadoopAuthEnabled() && defaultEnforcer != null) { + + RangerPerfTracer hadoopAuthPerf = null; + + if(RangerPerfTracer.isPerfTraceEnabled(PERF_HDFSAUTH_REQUEST_LOG)) { + hadoopAuthPerf = RangerPerfTracer.getPerfTracer(PERF_HDFSAUTH_REQUEST_LOG, "RangerAccessControlEnforcer.checkDefaultEnforcer(path=" + path + ")"); + } - action = FsAction.EXECUTE; - } else if(action == null || action == FsAction.NONE) { - if(parentAccess != null && parentAccess != FsAction.NONE ) { + try { + defaultEnforcer.checkPermission(fsOwner, superGroup, ugi, inodeAttrs, inodes, + pathByNameArr, snapshotId, path, ancestorIndex, doCheckOwner, + ancestorAccess, parentAccess, access, subAccess, ignoreEmptyDir); + + authzStatus = AuthzStatus.ALLOW; + } finally { + if (auditHandler != null) { + INode nodeChecked = inode; + FsAction action = access; + if (isTraverseOnlyCheck) { + if (nodeChecked == null || nodeChecked.isFile()) { + if (parent != null) { nodeChecked = parent; - action = parentAccess; - } else if(ancestorAccess != null && ancestorAccess != FsAction.NONE ) { + } else if (ancestor != null) { nodeChecked = ancestor; - action = ancestorAccess; - } else if(subAccess != null && subAccess != FsAction.NONE ) { - action = subAccess; } } - String pathChecked = nodeChecked != null ? nodeChecked.getFullPathName() : path; - - auditHandler.logHadoopEvent(pathChecked, action, authzStatus == AuthzStatus.ALLOW); + action = FsAction.EXECUTE; + } else if (action == null || action == FsAction.NONE) { + if (parentAccess != null && parentAccess != FsAction.NONE) { + nodeChecked = parent; + action = parentAccess; + } else if (ancestorAccess != null && ancestorAccess != FsAction.NONE) { + nodeChecked = ancestor; + action = ancestorAccess; + } else if (subAccess != null && subAccess != FsAction.NONE) { + action = subAccess; + } } + + String pathChecked = nodeChecked != null ? nodeChecked.getFullPathName() : path; + + auditHandler.logHadoopEvent(pathChecked, action, authzStatus == AuthzStatus.ALLOW); } - return authzStatus; + RangerPerfTracer.log(hadoopAuthPerf); } - return authzStatus; - } + } + LOG.debug("<== RangerAccessControlEnforcer.checkDefaultEnforcer(" + + "fsOwner=" + fsOwner + "; superGroup=" + superGroup + ", inodesCount=" + (inodes != null ? inodes.length : 0) + + ", snapshotId=" + snapshotId + ", path=" + path + ", ancestorIndex=" + ancestorIndex + + ", doCheckOwner="+ doCheckOwner + ", ancestorAccess=" + ancestorAccess + ", parentAccess=" + parentAccess + + ", access=" + access + ", subAccess=" + subAccess + ", ignoreEmptyDir=" + ignoreEmptyDir + + ", isTraverseOnlyCheck=" + isTraverseOnlyCheck + ",ancestor=" + (ancestor == null ? null : ancestor.getFullPathName()) + + ", parent=" + (parent == null ? null : parent.getFullPathName()) + ", inode=" + (inode == null ? null : inode.getFullPathName()) + + ") : " + authzStatus ); + + return authzStatus; + } private AuthzStatus isAccessAllowed(INode inode, INodeAttributes inodeAttribs, FsAction access, String user, Set groups, RangerHdfsPlugin plugin, RangerHdfsAuditHandler auditHandler) { AuthzStatus ret = null; From 9beaebfebab097bc3c369b2aae4a6e221a701971 Mon Sep 17 00:00:00 2001 From: Bhavik Patel Date: Fri, 21 Jul 2017 12:20:53 +0530 Subject: [PATCH 018/151] RANGER-1674:IMPORT START audit is not appearing on audit page --- .../src/main/java/org/apache/ranger/rest/ServiceREST.java | 1 + 1 file changed, 1 insertion(+) diff --git a/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java b/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java index 239081d140..5033df7442 100644 --- a/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java +++ b/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java @@ -1882,6 +1882,7 @@ public void importPoliciesFromFile( XXTrxLog xxTrxLog = new XXTrxLog(); xxTrxLog.setAction("IMPORT START"); xxTrxLog.setObjectClassType(AppConstants.CLASS_TYPE_RANGER_POLICY); + xxTrxLog.setPreviousValue("IMPORT START"); trxLogList.add(xxTrxLog); bizUtil.createTrxLog(trxLogList); From 590529752491fda89e1344b8d9d8bcdbbcb269b5 Mon Sep 17 00:00:00 2001 From: ni3galave Date: Fri, 18 Aug 2017 17:54:45 +0530 Subject: [PATCH 019/151] RANGER-1724: On Report listing page for masking/row filter policies show only mask/row filter conditions --- .../scripts/modules/globalize/message/en.js | 3 +- .../scripts/views/common/CustomSubgrid.js | 8 +- .../scripts/views/policies/PermissionList.js | 5 +- .../views/policies/RangerPolicyTableLayout.js | 2 +- .../reports/PlugableServiceDiffDetail.js | 12 +- .../scripts/views/reports/UserAccessLayout.js | 152 ++++++++++++++---- security-admin/src/main/webapp/styles/xa.css | 10 ++ .../common/ServiceManagerLayout_tmpl.html | 3 +- .../PlugableServicePolicyDeleteDiff_tmpl.html | 4 +- .../PlugableServicePolicyDiff_tmpl.html | 4 +- 10 files changed, 162 insertions(+), 41 deletions(-) diff --git a/security-admin/src/main/webapp/scripts/modules/globalize/message/en.js b/security-admin/src/main/webapp/scripts/modules/globalize/message/en.js index 4bac746e5b..422e9526b5 100644 --- a/security-admin/src/main/webapp/scripts/modules/globalize/message/en.js +++ b/security-admin/src/main/webapp/scripts/modules/globalize/message/en.js @@ -242,7 +242,8 @@ define(function(require) { selectAndAddGroup : 'Select and Add Group', download : 'Download', lastUpdate : 'Last Update', - clusterName : 'Cluster Name' + clusterName : 'Cluster Name', + url : 'Hive url.' }, btn : { add : 'Add', diff --git a/security-admin/src/main/webapp/scripts/views/common/CustomSubgrid.js b/security-admin/src/main/webapp/scripts/views/common/CustomSubgrid.js index 0c7e52a899..ad89a1377e 100644 --- a/security-admin/src/main/webapp/scripts/views/common/CustomSubgrid.js +++ b/security-admin/src/main/webapp/scripts/views/common/CustomSubgrid.js @@ -152,7 +152,9 @@ define(function(require){ var labelName = this.column.attributes.label; $(this.el).html(""); if (this.state == "collasped"){ - $(this.el).parent().addClass("warning"); +// $(this.el).parent().addClass("warning"); +// Add warning class to sub grid table + $(this.el).addClass("warning"); this.state = "expanded"; this.subrow = new SubgridCustomRow({ columns: this.column.collection, @@ -164,7 +166,9 @@ define(function(require){ $(this.el).parent("tr").after(this.subrow.render().$el); } else { if( $(this.el).parent().siblings('.warning').length <= 1 ){ - $(this.el).parent().removeClass("warning") +// $(this.el).parent().removeClass("warning") +// Remove warning class from sub grid table + $(this.el).removeClass("warning") } this.state = "collasped"; this.subrow.remove(); diff --git a/security-admin/src/main/webapp/scripts/views/policies/PermissionList.js b/security-admin/src/main/webapp/scripts/views/policies/PermissionList.js index 067bf3b34c..bb64984054 100644 --- a/security-admin/src/main/webapp/scripts/views/policies/PermissionList.js +++ b/security-admin/src/main/webapp/scripts/views/policies/PermissionList.js @@ -485,7 +485,7 @@ define(function(require) { }); this.$el.find('input[data-id="maskTypeCustom"]').on('change', function(e){ if(!_.isUndefined(that.model.get('dataMaskInfo'))){ - that.model.get('dataMaskInfo').valueExpr = e.currentTarget.value; + that.model.get('dataMaskInfo').valueExpr = _.escape(e.currentTarget.value); } }).trigger('change'); if(!this.accessPermSetForTagMasking){ @@ -707,6 +707,7 @@ define(function(require) { $(this).siblings('[data-id="maskTypeCustom"]').css("display",""); }else{ $(this).siblings('[data-id="maskTypeCustom"]').css("display","none"); + $(this).siblings('[data-id="maskTypeCustom"]').val(" ") } $(this).html("" + obj.text + ""); @@ -723,7 +724,7 @@ define(function(require) { }); this.$el.find('input[data-id="maskTypeCustom"]').on('change', function(e){ if(!_.isUndefined(that.model.get('dataMaskInfo'))){ - that.model.get('dataMaskInfo').valueExpr = e.currentTarget.value; + that.model.get('dataMaskInfo').valueExpr = _.escape(e.currentTarget.value); } }).trigger('change'); }, diff --git a/security-admin/src/main/webapp/scripts/views/policies/RangerPolicyTableLayout.js b/security-admin/src/main/webapp/scripts/views/policies/RangerPolicyTableLayout.js index eb88686844..09e2e16699 100644 --- a/security-admin/src/main/webapp/scripts/views/policies/RangerPolicyTableLayout.js +++ b/security-admin/src/main/webapp/scripts/views/policies/RangerPolicyTableLayout.js @@ -354,7 +354,7 @@ define(function(require){ table:localization.tt('lbl.tableName') , tag : localization.tt('h.tagsMsg'), taxonomy:localization.tt('h.taxonomy') ,term: localization.tt('h.term') , topic:localization.tt('h.topic') ,topology:localization.tt('lbl.topologyName'), - type:localization.tt('h.type') ,udf:localization.tt('h.udf') , + type:localization.tt('h.type') ,udf:localization.tt('h.udf') , url:localization.tt('h.url') }; var serverRsrcAttrName = _.map(resourceSearchOpt,function(opt){ return { diff --git a/security-admin/src/main/webapp/scripts/views/reports/PlugableServiceDiffDetail.js b/security-admin/src/main/webapp/scripts/views/reports/PlugableServiceDiffDetail.js index f24fe8767f..914f271354 100644 --- a/security-admin/src/main/webapp/scripts/views/reports/PlugableServiceDiffDetail.js +++ b/security-admin/src/main/webapp/scripts/views/reports/PlugableServiceDiffDetail.js @@ -327,14 +327,20 @@ define(function(require){ if(itemType === 'Masked Policy Items') { // its for new created record for(var i = 0; i < newPolicyItems.length ; i++){ - if(newPolicyItems[i].DataMasklabel){ + if(newPolicyItems[i].DataMasklabel && newPolicyItems[i].DataMasklabel == "Custom"){ var maskingType = newPolicyItems[i].dataMaskInfo.dataMaskType; - newPolicyItems[i].dataMaskInfo.dataMaskType = newPolicyItems[i].DataMasklabel; + newPolicyItems[i].dataMaskInfo.dataMaskType = newPolicyItems[i].DataMasklabel +' : '+newPolicyItems[i].dataMaskInfo.valueExpr; + }else if(newPolicyItems[i].DataMasklabel){ + var maskingType = newPolicyItems[i].dataMaskInfo.dataMaskType; + newPolicyItems[i].dataMaskInfo.dataMaskType = newPolicyItems[i].DataMasklabel; } } for(var i = 0; i < oldPolicyItems.length ; i++){ - if(oldPolicyItems[i].DataMasklabel){ + if(oldPolicyItems[i].DataMasklabel && oldPolicyItems[i].DataMasklabel == "Custom"){ + var maskingType = oldPolicyItems[i].dataMaskInfo.dataMaskType; + oldPolicyItems[i].dataMaskInfo.dataMaskType = oldPolicyItems[i].DataMasklabel +' : '+oldPolicyItems[i].dataMaskInfo.valueExpr; + }else if(oldPolicyItems[i].DataMasklabel){ var maskingType = oldPolicyItems[i].dataMaskInfo.dataMaskType; oldPolicyItems[i].dataMaskInfo.dataMaskType = oldPolicyItems[i].DataMasklabel; } diff --git a/security-admin/src/main/webapp/scripts/views/reports/UserAccessLayout.js b/security-admin/src/main/webapp/scripts/views/reports/UserAccessLayout.js index 197bc84360..c5dc053cc3 100644 --- a/security-admin/src/main/webapp/scripts/views/reports/UserAccessLayout.js +++ b/security-admin/src/main/webapp/scripts/views/reports/UserAccessLayout.js @@ -141,10 +141,11 @@ define(function(require) {'use strict'; this.initializePlugins(); this.setupGroupAutoComplete(); this.renderComponentAndPolicyTypeSelect(); - //Show policies listing for each service and GET policies for each service + var policyType = this.ui.policyType.val(); +// Show policies listing for each service and GET policies for each service _.each(this.policyCollList, function(obj,i){ this.renderTable(obj.collName, obj.serviceDefName); - this.getResourceLists(obj.collName,obj.serviceDefName); + this.getResourceLists(obj.collName,obj.serviceDefName,policyType); },this); this.$el.find('[data-js="policyName"]').focus() var urlString = XAUtil.getBaseUrl(); @@ -161,11 +162,15 @@ define(function(require) {'use strict'; {text :'Search By' , info :localization.tt('msg.searchBy')}, {text :'Resource' , info :localization.tt('msg.resourceMsg')}] }, - getResourceLists: function(collName, serviceDefName){ + getResourceLists: function(collName, serviceDefName , policyType){ var that = this, coll = this[collName]; that.allowDownload = false; coll.queryParams.serviceType = serviceDefName; + if(policyType){ +// to set default value access type in policy type + coll.queryParams.policyType = policyType; + } coll.fetch({ cache : false, reset: true, @@ -178,9 +183,13 @@ define(function(require) {'use strict'; _.each(that[collName].models,function(model,ind){ if (XAUtil.isMaskingPolicy(model.get('policyType'))) { //'Collection' must be same as subgrid custom column name - model.attributes.allowCollection = model.get('dataMaskPolicyItems'); + model.attributes.maskCollection = model.get('dataMaskPolicyItems'); +// Add service type in masking condition + _.each(model.attributes.dataMaskPolicyItems , function(m){ + m.type = model.collection.queryParams.serviceType; + }) } else if (XAUtil.isRowFilterPolicy(model.get('policyType'))) { - model.attributes.allowCollection = model.get('rowFilterPolicyItems'); + model.attributes.rowlvlCollection = model.get('rowFilterPolicyItems'); } else { model.attributes.allowCollection = model.get('policyItems'); } @@ -215,16 +224,15 @@ define(function(require) {'use strict'; label: 'Groups', formatter: _.extend({}, Backgrid.CellFormatter.prototype, { fromRaw: function (rawValue,model, coll) { - var startSpanEle = '',endSpanEle = ''; var group_str = ''; if(_.isEmpty(model.get('groups'))){ return '
--
'; } else { _.each(model.get('groups'),function(group,index){ if(index < 4) { - group_str += '' + _.escape(group) + endSpanEle + " "; + group_str += '' + _.escape(group) + '' + " "; } else { - group_str += ' false + + + ranger.usersync.group.hierarchylevels + 2 + From 1d0e8af417e1bc2c3cc9ff154108531aa3bcd5bb Mon Sep 17 00:00:00 2001 From: Abhay Kulkarni Date: Sun, 10 Sep 2017 22:22:44 -0700 Subject: [PATCH 030/151] RANGER-1771: Improve performance of merging lists of policyEvaluators returned by Trie --- .../policyengine/RangerPolicyRepository.java | 56 +++++++++++-------- 1 file changed, 32 insertions(+), 24 deletions(-) diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java index f7302bb7f7..1580766996 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java @@ -258,7 +258,8 @@ private List getLikelyMatchPolicyEvaluators(Map resourceKeys = resource == null ? null : resource.getKeys(); if(CollectionUtils.isNotEmpty(resourceKeys)) { - boolean isRetModifiable = false; + List> resourceEvaluatorsList = null; + List smallestList = null; for(String resourceName : resourceKeys) { RangerResourceTrie trie = resourceTrie.get(resourceName); @@ -270,34 +271,41 @@ private List getLikelyMatchPolicyEvaluators(Map resourceEvaluators = trie.getEvaluatorsForResource(resource.getValue(resourceName)); if(CollectionUtils.isEmpty(resourceEvaluators)) { // no policies for this resource, bail out - ret = null; - } else if(ret == null) { // initialize ret with policies found for this resource - ret = resourceEvaluators; - } else { // remove policies from ret that are not in resourceEvaluators - if(isRetModifiable) { - ret.retainAll(resourceEvaluators); - } else { - final List shorterList; - final List longerList; - - if (ret.size() < resourceEvaluators.size()) { - shorterList = ret; - longerList = resourceEvaluators; - } else { - shorterList = resourceEvaluators; - longerList = ret; - } + resourceEvaluatorsList = null; + smallestList = null; + break; + } + + if (smallestList == null) { + smallestList = resourceEvaluators; + } else { + if (resourceEvaluatorsList == null) { + resourceEvaluatorsList = new ArrayList<>(); + resourceEvaluatorsList.add(smallestList); + } + resourceEvaluatorsList.add(resourceEvaluators); - ret = new ArrayList<>(shorterList); - ret.retainAll(longerList); - isRetModifiable = true; + if (smallestList.size() > resourceEvaluators.size()) { + smallestList = resourceEvaluators; } } + } - if(CollectionUtils.isEmpty(ret)) { // if no policy exists, bail out and return empty list - ret = null; - break; + if (resourceEvaluatorsList != null) { + ret = new ArrayList<>(smallestList); + for (List resourceEvaluators : resourceEvaluatorsList) { + if (resourceEvaluators != smallestList) { + // remove policies from ret that are not in resourceEvaluators + ret.retainAll(resourceEvaluators); + + if (CollectionUtils.isEmpty(ret)) { // if no policy exists, bail out and return empty list + ret = null; + break; + } + } } + } else { + ret = smallestList; } } From adc3819e4be0ca20c08917043c9629a817cb6f61 Mon Sep 17 00:00:00 2001 From: Nikhil P Date: Wed, 30 Aug 2017 15:18:52 +0530 Subject: [PATCH 031/151] RANGER-1750 : In Nifi default policy is getting created with policyitem without any user/group but permission set for the same Signed-off-by: Gautam Borad --- .../org/apache/ranger/biz/ServiceDBStore.java | 32 ++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java b/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java index 8132357d75..63fdf4f1bc 100644 --- a/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java +++ b/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java @@ -1859,6 +1859,29 @@ public RangerPolicy createPolicy(RangerPolicy policy) throws Exception { return createdPolicy; } + private boolean validatePolicyItem(List policyItems) { + boolean isPolicyItemValid=true; + for (RangerPolicyItem policyItem : policyItems) { + if (policyItem != null) { + if (CollectionUtils.isEmpty(policyItem.getUsers()) + || (policyItem.getUsers() != null) && policyItem.getUsers().contains(null) + || (policyItem.getUsers().contains(""))) { + if (CollectionUtils.isEmpty(policyItem.getGroups()) + || (policyItem.getGroups() != null) && policyItem.getGroups().contains(null) + || (policyItem.getGroups().contains(""))) { + + isPolicyItemValid = false; + } + } + if (CollectionUtils.isEmpty(policyItem.getAccesses()) + || (policyItem.getAccesses() != null) && policyItem.getAccesses().contains(null)) { + isPolicyItemValid = false; + } + } + } + return isPolicyItemValid; + } + @Override public RangerPolicy updatePolicy(RangerPolicy policy) throws Exception { if(LOG.isDebugEnabled()) { @@ -2502,6 +2525,7 @@ void createDefaultPolicies(RangerService createdService) throws Exception { createDefaultPolicyUsersAndGroups(defaultPolicies); for (RangerPolicy defaultPolicy : defaultPolicies) { + List policyItems = defaultPolicy.getPolicyItems(); if (CollectionUtils.isNotEmpty(serviceCheckUsers) && StringUtils.equalsIgnoreCase(defaultPolicy.getService(), createdService.getName())) { @@ -2513,7 +2537,13 @@ void createDefaultPolicies(RangerService createdService) throws Exception { defaultPolicy.getPolicyItems().add(policyItem); } - createPolicy(defaultPolicy); + boolean isPolicyItemValid=validatePolicyItem(policyItems); + if (isPolicyItemValid) { + createPolicy(defaultPolicy); + } else { + LOG.warn("Default policy won't be created,since policyItems not valid-either users/groups not present or access not present in policy."); + } + } } } From a0f43d87a6e2d1de8cb311bc82827f0c836fa21c Mon Sep 17 00:00:00 2001 From: Nikhil P Date: Tue, 5 Sep 2017 16:22:51 +0530 Subject: [PATCH 032/151] RANGER 1697 : Update NiFi service def and handle upgrade scenario Signed-off-by: Gautam Borad --- .../service-defs/ranger-servicedef-nifi.json | 2 +- ...chForNifiResourceUpdateExclude_J10008.java | 145 ++++++++++++++++++ .../org/apache/ranger/rest/ServiceREST.java | 2 +- .../apache/ranger/service/XTrxLogService.java | 10 +- 4 files changed, 152 insertions(+), 7 deletions(-) create mode 100644 security-admin/src/main/java/org/apache/ranger/patch/PatchForNifiResourceUpdateExclude_J10008.java diff --git a/agents-common/src/main/resources/service-defs/ranger-servicedef-nifi.json b/agents-common/src/main/resources/service-defs/ranger-servicedef-nifi.json index b81785d9e3..1d1123225a 100644 --- a/agents-common/src/main/resources/service-defs/ranger-servicedef-nifi.json +++ b/agents-common/src/main/resources/service-defs/ranger-servicedef-nifi.json @@ -14,7 +14,7 @@ "mandatory":true, "lookupSupported":true, "recursiveSupported":false, - "excludesSupported":true, + "excludesSupported":false, "matcher":"org.apache.ranger.plugin.resourcematcher.RangerDefaultResourceMatcher", "matcherOptions":{ "wildCard":true, diff --git a/security-admin/src/main/java/org/apache/ranger/patch/PatchForNifiResourceUpdateExclude_J10008.java b/security-admin/src/main/java/org/apache/ranger/patch/PatchForNifiResourceUpdateExclude_J10008.java new file mode 100644 index 0000000000..634082c049 --- /dev/null +++ b/security-admin/src/main/java/org/apache/ranger/patch/PatchForNifiResourceUpdateExclude_J10008.java @@ -0,0 +1,145 @@ + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ranger.patch; + +import java.util.List; +import org.apache.commons.collections.CollectionUtils; +import org.apache.log4j.Logger; +import org.apache.ranger.biz.ServiceDBStore; +import org.apache.ranger.common.JSONUtil; +import org.apache.ranger.common.RangerValidatorFactory; +import org.apache.ranger.common.StringUtil; +import org.apache.ranger.db.RangerDaoManager; +import org.apache.ranger.entity.XXPolicyResource; +import org.apache.ranger.entity.XXResourceDef; +import org.apache.ranger.plugin.model.RangerPolicy; +import org.apache.ranger.plugin.model.RangerServiceDef; +import org.apache.ranger.plugin.model.RangerServiceDef.RangerResourceDef; +import org.apache.ranger.plugin.model.validation.RangerServiceDefValidator; +import org.apache.ranger.plugin.model.validation.RangerValidator.Action; +import org.apache.ranger.plugin.store.EmbeddedServiceDefsUtil; +import org.apache.ranger.service.RangerPolicyService; +import org.apache.ranger.util.CLIUtil; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +@Component +public class PatchForNifiResourceUpdateExclude_J10008 extends BaseLoader { + private static final Logger logger = Logger.getLogger(PatchForHiveServiceDefUpdate_J10006.class); + @Autowired + RangerDaoManager daoMgr; + + @Autowired + ServiceDBStore svcDBStore; + + @Autowired + JSONUtil jsonUtil; + + @Autowired + StringUtil stringUtil; + + @Autowired + RangerValidatorFactory validatorFactory; + + @Autowired + ServiceDBStore svcStore; + + @Autowired + RangerPolicyService policyService; + + public static void main(String[] args) { + logger.info("main()"); + try { + PatchForNifiResourceUpdateExclude_J10008 loader = (PatchForNifiResourceUpdateExclude_J10008) CLIUtil.getBean(PatchForNifiResourceUpdateExclude_J10008.class); + loader.init(); + while (loader.isMoreToProcess()) { + loader.load(); + } + logger.info("Load complete. Exiting!!!"); + System.exit(0); + } catch (Exception e) { + logger.error("Error loading", e); + System.exit(1); + } + } + + @Override + public void init() throws Exception { + // Do Nothing + } + + @Override + public void execLoad() { + logger.info("==> PatchForNifiResourceUpdateExclude.execLoad()"); + try { + updateNifiServiceDef(); + } catch (Exception e) { + logger.error("Error whille updateNifiServiceDef()data.", e); + } + logger.info("<== PatchForNifiResourceUpdateExclude.execLoad()"); + } + + @Override + public void printStats() { + logger.info("updateNifiServiceDef data "); + } + + private void updateNifiServiceDef(){ + RangerServiceDef ret = null; + RangerServiceDef dbNifiServiceDef = null; + try { + dbNifiServiceDef = svcDBStore.getServiceDefByName(EmbeddedServiceDefsUtil.EMBEDDED_SERVICEDEF_NIFI_NAME); + if (dbNifiServiceDef != null) { + List rRDefList = null; + rRDefList = dbNifiServiceDef.getResources(); + if (CollectionUtils.isNotEmpty(rRDefList)) { + for (RangerResourceDef rRDef : rRDefList) { + + if (rRDef.getExcludesSupported()) { + rRDef.setExcludesSupported(false); + } + + XXResourceDef sdf=daoMgr.getXXResourceDef().findByNameAndServiceDefId(rRDef.getName(), dbNifiServiceDef.getId()); + long ResourceDefId=sdf.getId(); + List RangerPolicyResourceList=daoMgr.getXXPolicyResource().findByResDefId(ResourceDefId); + if (CollectionUtils.isNotEmpty(RangerPolicyResourceList)){ + for(XXPolicyResource RangerPolicyResource : RangerPolicyResourceList){ + if(RangerPolicyResource.getIsexcludes()){ + RangerPolicy rPolicy=svcDBStore.getPolicy(RangerPolicyResource.getPolicyid()); + rPolicy.setIsEnabled(false); + svcStore.updatePolicy(rPolicy); + } + } + } + } + } + RangerServiceDefValidator validator = validatorFactory.getServiceDefValidator(svcStore); + validator.validate(dbNifiServiceDef, Action.UPDATE); + ret = svcStore.updateServiceDef(dbNifiServiceDef); + } + if (ret == null) { + logger.error("Error while updating " + EmbeddedServiceDefsUtil.EMBEDDED_SERVICEDEF_NIFI_NAME+ "service-def"); + System.exit(1); + } + } catch (Exception e) { + logger.error("Error while updating " + EmbeddedServiceDefsUtil.EMBEDDED_SERVICEDEF_NIFI_NAME + "service-def", e); + } + } + +} diff --git a/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java b/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java index 6687e604e5..5fa114d642 100644 --- a/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java +++ b/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java @@ -1425,7 +1425,7 @@ public RangerPolicy createPolicy(RangerPolicy policy, @Context HttpServletReques policy.setName(StringUtils.trim(policyName)); } - if(Boolean.valueOf(updateIfExists)) { + if (updateIfExists != null && Boolean.valueOf(updateIfExists)) { RangerPolicy existingPolicy = null; try { if(StringUtils.isNotEmpty(policy.getGuid())) { diff --git a/security-admin/src/main/java/org/apache/ranger/service/XTrxLogService.java b/security-admin/src/main/java/org/apache/ranger/service/XTrxLogService.java index 6c3034f040..6c56eefd5f 100644 --- a/security-admin/src/main/java/org/apache/ranger/service/XTrxLogService.java +++ b/security-admin/src/main/java/org/apache/ranger/service/XTrxLogService.java @@ -215,18 +215,18 @@ public VXTrxLogList searchXTrxLogs(SearchCriteria searchCriteria) { } List keyAdminTrxLogList = new ArrayList(); - if (session != null && session.isKeyAdmin() && xxServiceDef != null && resultList != null) { + if (session != null && session.isKeyAdmin() && xxServiceDef != null) { List vXTrxLogs = new ArrayList(); for (VXTrxLog xTrxLog : trxLogList) { int parentObjectClassType = xTrxLog.getParentObjectClassType(); Long parentObjectId = xTrxLog.getParentObjectId(); - if (parentObjectClassType == AppConstants.CLASS_TYPE_XA_SERVICE_DEF && parentObjectId == xxServiceDef.getId()) { + if (parentObjectClassType == AppConstants.CLASS_TYPE_XA_SERVICE_DEF && parentObjectId.equals(xxServiceDef.getId())) { vXTrxLogs.add(xTrxLog); - } else if (parentObjectClassType == AppConstants.CLASS_TYPE_XA_SERVICE && parentObjectId != xxServiceDef.getId()) { + } else if (parentObjectClassType == AppConstants.CLASS_TYPE_XA_SERVICE && !(parentObjectId.equals(xxServiceDef.getId()))) { for (VXTrxLog vxTrxLog : trxLogList) { if (parentObjectClassType == vxTrxLog.getObjectClassType() - && parentObjectId == vxTrxLog.getObjectId() - && vxTrxLog.getParentObjectId() == xxServiceDef.getId()) { + && parentObjectId.equals(vxTrxLog.getObjectId()) + && vxTrxLog.getParentObjectId().equals(xxServiceDef.getId())) { vXTrxLogs.add(xTrxLog); break; } From 44b8769df6d89aa3598d20dc48d7c81ca31e5a38 Mon Sep 17 00:00:00 2001 From: "peng.jianhua" Date: Tue, 12 Sep 2017 10:02:56 +0800 Subject: [PATCH 033/151] RANGER-1732 Collection added to itsel Signed-off-by: Colm O hEigeartaigh --- .../ranger/plugin/audit/RangerMultiResourceAuditHandler.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/audit/RangerMultiResourceAuditHandler.java b/agents-common/src/main/java/org/apache/ranger/plugin/audit/RangerMultiResourceAuditHandler.java index 839618e53f..fdf0054fe2 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/audit/RangerMultiResourceAuditHandler.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/audit/RangerMultiResourceAuditHandler.java @@ -43,7 +43,7 @@ public void logAuthzAudit(AuthzAuditEvent auditEvent) { @Override public void logAuthzAudits(Collection auditEvents) { - auditEvents.addAll(auditEvents); + this.auditEvents.addAll(auditEvents); } public void flushAudit() { From 309abeff4118fc373d842297612bad18ebece2a1 Mon Sep 17 00:00:00 2001 From: Bhavik Patel Date: Fri, 8 Sep 2017 09:38:24 +0530 Subject: [PATCH 034/151] RANGER-1765:Add unique key constraint in x_group and x_group_users table --- ...add-unique-constraint-on-table-x_group.sql | 45 ++++++++++++++++++ ...add-unique-constraint-on-table-x_group.sql | 46 +++++++++++++++++++ ...add-unique-constraint-on-table-x_group.sql | 43 +++++++++++++++++ ...add-unique-constraint-on-table-x_group.sql | 44 ++++++++++++++++++ .../current/ranger_core_db_sqlserver.sql | 21 +++++++-- ...add-unique-constraint-on-table-x_group.sql | 40 ++++++++++++++++ 6 files changed, 236 insertions(+), 3 deletions(-) create mode 100644 security-admin/db/mysql/patches/028-add-unique-constraint-on-table-x_group.sql create mode 100644 security-admin/db/oracle/patches/028-add-unique-constraint-on-table-x_group.sql create mode 100644 security-admin/db/postgres/patches/028-add-unique-constraint-on-table-x_group.sql create mode 100644 security-admin/db/sqlanywhere/patches/028-add-unique-constraint-on-table-x_group.sql create mode 100644 security-admin/db/sqlserver/patches/028-add-unique-constraint-on-table-x_group.sql diff --git a/security-admin/db/mysql/patches/028-add-unique-constraint-on-table-x_group.sql b/security-admin/db/mysql/patches/028-add-unique-constraint-on-table-x_group.sql new file mode 100644 index 0000000000..2c5be737ea --- /dev/null +++ b/security-admin/db/mysql/patches/028-add-unique-constraint-on-table-x_group.sql @@ -0,0 +1,45 @@ +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +drop procedure if exists create_unique_constraint_on_groupname; + +delimiter ;; +create procedure create_unique_constraint_on_groupname() begin +DECLARE loginID bigint(20); + /* check tables exist or not */ + if exists (select * from information_schema.columns where table_schema=database() and table_name = 'x_group' and column_name='group_name') then + /* check unique constraint exist on group_name column or not */ + if not exists (select * from information_schema.columns where table_schema=database() and table_name = 'x_group' and column_name='group_name' and column_key='UNI') then + if not exists (select * from information_schema.table_constraints where table_schema=database() and table_name = 'x_group' and constraint_name='x_group_UK_group_name') then + ALTER IGNORE TABLE x_group ADD UNIQUE INDEX x_group_UK_group_name(group_name(767)); +-- ALTER TABLE x_group MODIFY COLUMN group_name varchar(767) NOT NULL, ADD CONSTRAINT x_group_UK_group_name UNIQUE(group_name(767)); + end if; + end if; + end if; + if exists (select * from information_schema.columns where table_schema=database() and table_name = 'x_group_users' and column_name='group_name') then + if exists (select * from information_schema.columns where table_schema=database() and table_name = 'x_group_users' and column_name='user_id') then + /* check unique constraint exist on group_name column or not */ + if not exists (select * from information_schema.table_constraints where table_schema=database() and table_name = 'x_group_users' and constraint_name='x_group_users_UK_uid_gname') then + ALTER IGNORE TABLE x_group_users ADD UNIQUE INDEX x_group_users_UK_uid_gname(user_id,group_name(740)); +-- ALTER TABLE x_group_users MODIFY COLUMN group_name varchar(767), ADD CONSTRAINT x_group_users_UK_uid_gname UNIQUE(user_id,group_name(767)); + end if; + end if; + end if; +end;; + +delimiter ; +call create_unique_constraint_on_groupname(); + +drop procedure if exists create_unique_constraint_on_groupname; diff --git a/security-admin/db/oracle/patches/028-add-unique-constraint-on-table-x_group.sql b/security-admin/db/oracle/patches/028-add-unique-constraint-on-table-x_group.sql new file mode 100644 index 0000000000..d512465a99 --- /dev/null +++ b/security-admin/db/oracle/patches/028-add-unique-constraint-on-table-x_group.sql @@ -0,0 +1,46 @@ +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +DECLARE + v_count number:=0; + gu_count number:=0; +BEGIN + select count(*) into v_count from user_tab_cols where table_name='X_GROUP' and column_name='GROUP_NAME'; + if (v_count = 1) then + v_count:=0; + select count(*) into v_count from user_constraints where table_name='X_GROUP' and constraint_name='X_GROUP_UK_GROUP_NAME' and constraint_type='U'; + if (v_count = 0) then + v_count:=0; + select count(*) into v_count from user_ind_columns WHERE table_name='X_GROUP' and column_name='GROUP_NAME' and index_name='X_GROUP_UK_GROUP_NAME'; + if (v_count = 0) then + execute immediate 'ALTER TABLE x_group MODIFY(group_name VARCHAR(767)) ADD CONSTRAINT x_group_UK_group_name UNIQUE (group_name)'; + end if; + commit; + end if; + end if; + + select count(*) into gu_count from user_tab_cols where table_name='X_GROUP_USERS' and column_name='GROUP_NAME'; + if (gu_count = 1) then + gu_count:=0; + select count(*) into gu_count from user_constraints where table_name='X_GROUP_USERS' and constraint_name='X_GROUP_USERS_UK_UID_GNAME' and constraint_type='U'; + if (gu_count = 0) then + gu_count:=0; + select count(*) into gu_count from user_ind_columns WHERE table_name='X_GROUP_USERS' and column_name='GROUP_NAME' and index_name='X_GROUP_USERS_UK_UID_GNAME'; + if (gu_count = 0) then + execute immediate 'ALTER TABLE x_group_users MODIFY(group_name VARCHAR(767)) ADD CONSTRAINT x_group_users_uk_uid_gname UNIQUE (user_id,group_name)'; + end if; + commit; + end if; + end if; +end;/ diff --git a/security-admin/db/postgres/patches/028-add-unique-constraint-on-table-x_group.sql b/security-admin/db/postgres/patches/028-add-unique-constraint-on-table-x_group.sql new file mode 100644 index 0000000000..501ec2ea98 --- /dev/null +++ b/security-admin/db/postgres/patches/028-add-unique-constraint-on-table-x_group.sql @@ -0,0 +1,43 @@ +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +select 'delimiter start'; +CREATE OR REPLACE FUNCTION create_unique_constraint_on_username() +RETURNS void AS $$ +DECLARE + v_attnum integer := 0; +gu_attnum integer := 0; +BEGIN + select attnum into v_attnum from pg_attribute where attrelid in(select oid from pg_class where relname='x_group') and attname='group_name'; + IF v_attnum > 0 THEN + IF not exists (select * from pg_constraint where conrelid in(select oid from pg_class where relname='x_group') and conname='x_group_uk_group_name' and contype='u') THEN + IF not exists (select * from pg_index where indrelid in(select oid from pg_class where relname='x_group') and indkey[0]=v_attnum) THEN + ALTER TABLE x_group ALTER COLUMN group_name TYPE VARCHAR(767),ADD CONSTRAINT x_group_UK_group_name UNIQUE(group_name); + END IF; + END IF; + END IF; + +select attnum into gu_attnum from pg_attribute where attrelid in(select oid from pg_class where relname='x_group_users') and attname='group_name'; + IF gu_attnum > 0 THEN + IF not exists (select * from pg_constraint where conrelid in(select oid from pg_class where relname='x_group_users') and conname='x_group_users_UK_uid_gname' and contype='u') THEN + IF not exists (select * from pg_index where indrelid in(select oid from pg_class where relname='x_group_users') and indkey[0]=gu_attnum) THEN + ALTER TABLE x_group_users ALTER COLUMN group_name TYPE VARCHAR(767),ADD CONSTRAINT x_group_users_UK_uid_gname UNIQUE(user_id,group_name); + END IF; + END IF; + END IF; + +END; +$$ LANGUAGE plpgsql; +select create_unique_constraint_on_username(); +select 'delimiter end'; diff --git a/security-admin/db/sqlanywhere/patches/028-add-unique-constraint-on-table-x_group.sql b/security-admin/db/sqlanywhere/patches/028-add-unique-constraint-on-table-x_group.sql new file mode 100644 index 0000000000..c39e68d7c6 --- /dev/null +++ b/security-admin/db/sqlanywhere/patches/028-add-unique-constraint-on-table-x_group.sql @@ -0,0 +1,44 @@ +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +BEGIN +DECLARE tableID INT = 0; +DECLARE columnID INT = 0; +DECLARE guTableID INT = 0; +DECLARE guColumnID INT = 0; + IF EXISTS(select * from SYS.SYSCOLUMNS where tname = 'x_group' and cname='group_name') THEN + IF NOT EXISTS(select * from SYS.SYSCONSTRAINT where constraint_name = 'x_group_UK_group_name') THEN + select table_id into tableID from SYS.SYSTAB where table_name = 'x_group'; + select column_id into columnID from SYS.SYSTABCOL where table_id=tableID and column_name = 'group_name'; + IF NOT EXISTS(select * from SYS.SYSIDXCOL where table_id=tableID and column_id=columnID) THEN + ALTER TABLE dbo.x_group ALTER group_name varchar(767) NOT NULL; + ALTER TABLE dbo.x_group ADD CONSTRAINT x_group_UK_group_name UNIQUE NONCLUSTERED (group_name); + END IF; + END IF; + END IF; + IF EXISTS(select * from SYS.SYSCOLUMNS where tname = 'x_group_users' and cname='group_name') THEN + IF NOT EXISTS(select * from SYS.SYSCONSTRAINT where constraint_name = 'x_group_users_UK_uid_gname') THEN + select table_id into guTableID from SYS.SYSTAB where table_name = 'x_group_users'; + select column_id into guColumnID from SYS.SYSTABCOL where table_id=guTableID and column_name = 'group_name'; + IF NOT EXISTS(select * from SYS.SYSIDXCOL where table_id=guTableID and column_id=guColumnID) THEN + ALTER TABLE dbo.x_group_users ALTER group_name varchar(767) NOT NULL; + alter table dbo.x_group_users drop constraint x_group_users_FK_user_id; + ALTER TABLE dbo.x_group_users ALTER user_id bigint NOT NULL; + ALTER TABLE dbo.x_group_users ADD CONSTRAINT x_group_users_FK_user_id FOREIGN KEY(user_id) REFERENCES dbo.x_user (id); + ALTER TABLE dbo.x_group_users ADD CONSTRAINT x_group_users_UK_uid_gname UNIQUE NONCLUSTERED (user_id,group_name); + END IF; + END IF; + END IF; +END +GO diff --git a/security-admin/db/sqlserver/optimized/current/ranger_core_db_sqlserver.sql b/security-admin/db/sqlserver/optimized/current/ranger_core_db_sqlserver.sql index c4710f2ae0..27257e3fa8 100644 --- a/security-admin/db/sqlserver/optimized/current/ranger_core_db_sqlserver.sql +++ b/security-admin/db/sqlserver/optimized/current/ranger_core_db_sqlserver.sql @@ -453,6 +453,10 @@ IF (OBJECT_ID('x_service_version_info_service_id') IS NOT NULL) BEGIN ALTER TABLE [dbo].[x_service_version_info] DROP CONSTRAINT x_service_version_info_service_id END +IF (OBJECT_ID('x_group_UK_group_name') IS NOT NULL) +BEGIN + ALTER TABLE [dbo].[x_group] DROP CONSTRAINT x_group_UK_group_name +END IF (OBJECT_ID('x_plugin_info_UK') IS NOT NULL) BEGIN ALTER TABLE [dbo].[x_plugin_info] DROP CONSTRAINT x_plugin_info_UK @@ -851,7 +855,7 @@ CREATE TABLE [dbo].[x_group]( [update_time] [datetime2] DEFAULT NULL NULL, [added_by_id] [bigint] DEFAULT NULL NULL, [upd_by_id] [bigint] DEFAULT NULL NULL, - [group_name] [varchar](1024) NOT NULL, + [group_name] [varchar](767) NOT NULL, [descr] [varchar](4000) NOT NULL, [status] [int] DEFAULT 0 NOT NULL, [group_type] [int] DEFAULT 0 NOT NULL, @@ -861,6 +865,10 @@ CREATE TABLE [dbo].[x_group]( PRIMARY KEY CLUSTERED ( [id] ASC +)WITH (PAD_INDEX = OFF,STATISTICS_NORECOMPUTE = OFF,IGNORE_DUP_KEY = OFF,ALLOW_ROW_LOCKS = ON,ALLOW_PAGE_LOCKS = ON) ON [PRIMARY], +CONSTRAINT [x_group$x_group_UK_group_name] UNIQUE NONCLUSTERED +( + [group_name] ASC )WITH (PAD_INDEX = OFF,STATISTICS_NORECOMPUTE = OFF,IGNORE_DUP_KEY = OFF,ALLOW_ROW_LOCKS = ON,ALLOW_PAGE_LOCKS = ON) ON [PRIMARY] ) ON [PRIMARY] @@ -915,12 +923,17 @@ CREATE TABLE [dbo].[x_group_users]( [update_time] [datetime2] DEFAULT NULL NULL, [added_by_id] [bigint] DEFAULT NULL NULL, [upd_by_id] [bigint] DEFAULT NULL NULL, - [group_name] [varchar](1024) NOT NULL, + [group_name] [varchar](767) NOT NULL, [p_group_id] [bigint] DEFAULT NULL NULL, [user_id] [bigint] DEFAULT NULL NULL, PRIMARY KEY CLUSTERED ( [id] ASC +)WITH (PAD_INDEX = OFF,STATISTICS_NORECOMPUTE = OFF,IGNORE_DUP_KEY = OFF,ALLOW_ROW_LOCKS = ON,ALLOW_PAGE_LOCKS = ON) ON [PRIMARY], +CONSTRAINT [[x_group_users$x_group_users_UK_uid_gname] UNIQUE NONCLUSTERED +( + [user_id] ASC, + [group_name] ASC )WITH (PAD_INDEX = OFF,STATISTICS_NORECOMPUTE = OFF,IGNORE_DUP_KEY = OFF,ALLOW_ROW_LOCKS = ON,ALLOW_PAGE_LOCKS = ON) ON [PRIMARY] ) ON [PRIMARY] @@ -2975,6 +2988,7 @@ INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active) VALUES ('025',CURRENT_TIMESTAMP,'Ranger 0.7.0',CURRENT_TIMESTAMP,'localhost','Y'); INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active) VALUES ('026',CURRENT_TIMESTAMP,'Ranger 0.7.0',CURRENT_TIMESTAMP,'localhost','Y'); INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active) VALUES ('027',CURRENT_TIMESTAMP,'Ranger 0.7.0',CURRENT_TIMESTAMP,'localhost','Y'); +INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active) VALUES ('028',CURRENT_TIMESTAMP,'Ranger 0.7.0',CURRENT_TIMESTAMP,'localhost','Y'); INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active) VALUES ('DB_PATCHES',CURRENT_TIMESTAMP,'Ranger 0.7.0',CURRENT_TIMESTAMP,'localhost','Y'); INSERT INTO x_user_module_perm (user_id,module_id,create_time,update_time,added_by_id,upd_by_id,is_allowed) VALUES (1,3,CURRENT_TIMESTAMP,CURRENT_TIMESTAMP,1,1,1); INSERT INTO x_user_module_perm (user_id,module_id,create_time,update_time,added_by_id,upd_by_id,is_allowed) VALUES (1,1,CURRENT_TIMESTAMP,CURRENT_TIMESTAMP,1,1,1); @@ -3000,9 +3014,10 @@ INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active) VALUES ('J10004',CURRENT_TIMESTAMP,'Ranger 0.7.0',CURRENT_TIMESTAMP,'localhost','Y'); INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active) VALUES ('J10005',CURRENT_TIMESTAMP,'Ranger 0.7.0',CURRENT_TIMESTAMP,'localhost','Y'); INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active) VALUES ('J10006',CURRENT_TIMESTAMP,'Ranger 0.7.0',CURRENT_TIMESTAMP,'localhost','Y'); +INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active) VALUES ('J10007',CURRENT_TIMESTAMP,'Ranger 0.7.0',CURRENT_TIMESTAMP,'localhost','Y'); INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active) VALUES ('JAVA_PATCHES',CURRENT_TIMESTAMP,'Ranger 0.7.0',CURRENT_TIMESTAMP,'localhost','Y'); GO CREATE VIEW [dbo].[vx_trx_log] AS select x_trx_log.id AS id,x_trx_log.create_time AS create_time,x_trx_log.update_time AS update_time,x_trx_log.added_by_id AS added_by_id,x_trx_log.upd_by_id AS upd_by_id,x_trx_log.class_type AS class_type,x_trx_log.object_id AS object_id,x_trx_log.parent_object_id AS parent_object_id,x_trx_log.parent_object_class_type AS parent_object_class_type,x_trx_log.attr_name AS attr_name,x_trx_log.parent_object_name AS parent_object_name,x_trx_log.object_name AS object_name,x_trx_log.prev_val AS prev_val,x_trx_log.new_val AS new_val,x_trx_log.trx_id AS trx_id,x_trx_log.action AS action,x_trx_log.sess_id AS sess_id,x_trx_log.req_id AS req_id,x_trx_log.sess_type AS sess_type from x_trx_log where id in(select min(x_trx_log.id) from x_trx_log group by x_trx_log.trx_id) -GO \ No newline at end of file +GO diff --git a/security-admin/db/sqlserver/patches/028-add-unique-constraint-on-table-x_group.sql b/security-admin/db/sqlserver/patches/028-add-unique-constraint-on-table-x_group.sql new file mode 100644 index 0000000000..7559976116 --- /dev/null +++ b/security-admin/db/sqlserver/patches/028-add-unique-constraint-on-table-x_group.sql @@ -0,0 +1,40 @@ + +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +IF EXISTS(select * from INFORMATION_SCHEMA.columns where table_name = 'x_group' and column_name = 'group_name') +BEGIN + IF NOT EXISTS(select * from INFORMATION_SCHEMA.CONSTRAINT_COLUMN_USAGE where table_name='x_group' and column_name='group_name' and constraint_name = 'x_group$x_group_UK_group_name') + BEGIN + IF NOT EXISTS(select * from INFORMATION_SCHEMA.TABLE_CONSTRAINTS where table_name='x_group' and constraint_name = 'x_group$x_group_UK_group_name' and CONSTRAINT_TYPE='UNIQUE') + BEGIN + ALTER TABLE [dbo].[x_group] ALTER COLUMN [group_name] [varchar](767) NOT NULL; + ALTER TABLE [dbo].[x_group] ADD CONSTRAINT [x_group$x_group_UK_group_name] UNIQUE ([group_name]); + END + END +END +GO +IF EXISTS(select * from INFORMATION_SCHEMA.columns where table_name = 'x_group_users' and column_name = 'group_name') +BEGIN + IF NOT EXISTS(select * from INFORMATION_SCHEMA.CONSTRAINT_COLUMN_USAGE where table_name='x_group_users' and column_name='group_name' and constraint_name = 'x_group_users$x_group_users_UK_uid_gname') + BEGIN + IF NOT EXISTS(select * from INFORMATION_SCHEMA.TABLE_CONSTRAINTS where table_name='x_group_users' and constraint_name = 'x_group_users$x_group_users_UK_uid_gname' and CONSTRAINT_TYPE='UNIQUE') + BEGIN + ALTER TABLE [dbo].[x_group_users] ALTER COLUMN [group_name] [varchar](767) NOT NULL; + ALTER TABLE [dbo].[x_group_users] ADD CONSTRAINT [x_group_users$x_group_users_UK_uid_gname] UNIQUE (user_id,group_name); + END + END +END +GO +exit From dbe1a3a3f033f0e423e70d14b9937162ad5d4a66 Mon Sep 17 00:00:00 2001 From: Sailaja Polavarapu Date: Wed, 13 Sep 2017 15:57:33 -0700 Subject: [PATCH 035/151] RANGER-1647: Allow Ranger policy conditions to use tag attributes and values in Ranger -- ranger-0.7 branch --- .../RangerScriptConditionEvaluator.java | 8 +++++++- .../RangerScriptExecutionContext.java | 10 +++++----- .../service-defs/ranger-servicedef-tag.json | 8 ++++++++ .../resources/policyengine/resourceTags.json | 2 +- .../policyengine/test_policyengine_owner.json | 10 +++++----- .../policyengine/test_policyengine_tag_hive.json | 14 +++++++------- .../test_policyengine_tag_hive_filebased.json | 16 ++++++++-------- 7 files changed, 41 insertions(+), 27 deletions(-) diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/conditionevaluator/RangerScriptConditionEvaluator.java b/agents-common/src/main/java/org/apache/ranger/plugin/conditionevaluator/RangerScriptConditionEvaluator.java index 48ffc38253..5febf956d9 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/conditionevaluator/RangerScriptConditionEvaluator.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/conditionevaluator/RangerScriptConditionEvaluator.java @@ -24,12 +24,14 @@ import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.ranger.plugin.contextenricher.RangerTagForEval; import org.apache.ranger.plugin.policyengine.RangerAccessRequest; import javax.script.Bindings; import javax.script.ScriptEngine; import javax.script.ScriptEngineManager; import javax.script.ScriptException; +import java.util.Collections; import java.util.List; import java.util.Map; @@ -90,11 +92,15 @@ public boolean isMatched(RangerAccessRequest request) { RangerAccessRequest readOnlyRequest = request.getReadOnlyCopy(); - RangerScriptExecutionContext context = new RangerScriptExecutionContext(readOnlyRequest); + RangerScriptExecutionContext context = new RangerScriptExecutionContext(readOnlyRequest); + RangerTagForEval currentTag = context.getCurrentTag(); + Map tagAttribs = currentTag != null ? currentTag.getAttributes() : Collections.emptyMap(); Bindings bindings = scriptEngine.createBindings(); bindings.put("ctx", context); + bindings.put("tag", currentTag); + bindings.put("tagAttr", tagAttribs); if (LOG.isDebugEnabled()) { LOG.debug("RangerScriptConditionEvaluator.isMatched(): script={" + script + "}"); diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/conditionevaluator/RangerScriptExecutionContext.java b/agents-common/src/main/java/org/apache/ranger/plugin/conditionevaluator/RangerScriptExecutionContext.java index acd96be187..415d7fd542 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/conditionevaluator/RangerScriptExecutionContext.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/conditionevaluator/RangerScriptExecutionContext.java @@ -368,23 +368,23 @@ private Set getAllTags() { return ret; } - public void logDebug(String msg) { + public void logDebug(Object msg) { LOG.debug(msg); } - public void logInfo(String msg) { + public void logInfo(Object msg) { LOG.info(msg); } - public void logWarn(String msg) { + public void logWarn(Object msg) { LOG.warn(msg); } - public void logError(String msg) { + public void logError(Object msg) { LOG.error(msg); } - public void logFatal(String msg) { + public void logFatal(Object msg) { LOG.fatal(msg); } } diff --git a/agents-common/src/main/resources/service-defs/ranger-servicedef-tag.json b/agents-common/src/main/resources/service-defs/ranger-servicedef-tag.json index 3bad2222f5..c17b750706 100644 --- a/agents-common/src/main/resources/service-defs/ranger-servicedef-tag.json +++ b/agents-common/src/main/resources/service-defs/ranger-servicedef-tag.json @@ -69,6 +69,14 @@ "uiHint": "{ \"singleValue\":true }", "label":"Accessed after expiry_date (yes/no)?", "description": "Accessed after expiry_date? (yes/no)" + }, + { + "itemId":2, + "name":"expression", + "evaluator": "org.apache.ranger.plugin.conditionevaluator.RangerScriptConditionEvaluator", + "evaluatorOptions" : {"engineName":"JavaScript", "ui.isMultiline":"true"}, + "label":"Enter boolean expression", + "description": "Boolean expression" } ] } diff --git a/agents-common/src/test/resources/policyengine/resourceTags.json b/agents-common/src/test/resources/policyengine/resourceTags.json index 9523ca06f9..c564673b39 100644 --- a/agents-common/src/test/resources/policyengine/resourceTags.json +++ b/agents-common/src/test/resources/policyengine/resourceTags.json @@ -49,7 +49,7 @@ }, "3": { "type": "RESTRICTED", - "attributes": { "activation_date": "2015/08/10" }, + "attributes": { "activation_date": "2015/08/10", "score": "2" }, "id": 3, "guid": "tag-restricted-3-guid" }, diff --git a/agents-common/src/test/resources/policyengine/test_policyengine_owner.json b/agents-common/src/test/resources/policyengine/test_policyengine_owner.json index 82a663248c..223a0c617a 100644 --- a/agents-common/src/test/resources/policyengine/test_policyengine_owner.json +++ b/agents-common/src/test/resources/policyengine/test_policyengine_owner.json @@ -16,11 +16,11 @@ "policyConditions": [ { "itemId":1, - "name":"ScriptConditionEvaluator", - "evaluator": "org.apache.ranger.plugin.conditionevaluator.RangerScriptConditionEvaluator", - "evaluatorOptions" : {"engineName":"JavaScript"}, - "label":"Script", - "description": "Script to execute" + "name":"expression", + "evaluator": "org.apache.ranger.plugin.conditionevaluator.RangerScriptConditionEvaluator", + "evaluatorOptions" : {"engineName":"JavaScript", "ui.isMultiline":"true"}, + "label":"Enter boolean expression", + "description": "Boolean expression" } ] }, diff --git a/agents-common/src/test/resources/policyengine/test_policyengine_tag_hive.json b/agents-common/src/test/resources/policyengine/test_policyengine_tag_hive.json index 04b9afea39..11f31e3177 100644 --- a/agents-common/src/test/resources/policyengine/test_policyengine_tag_hive.json +++ b/agents-common/src/test/resources/policyengine/test_policyengine_tag_hive.json @@ -143,11 +143,11 @@ "policyConditions": [ { "itemId":1, - "name":"ScriptConditionEvaluator", - "evaluator": "org.apache.ranger.plugin.conditionevaluator.RangerScriptConditionEvaluator", - "evaluatorOptions" : {"engineName":"JavaScript"}, - "label":"Script", - "description": "Script to execute" + "name":"expression", + "evaluator": "org.apache.ranger.plugin.conditionevaluator.RangerScriptConditionEvaluator", + "evaluatorOptions" : {"engineName":"JavaScript", "ui.isMultiline":"true"}, + "label":"Enter boolean expression", + "description": "Boolean expression" }, { "itemId":2, @@ -166,7 +166,7 @@ { "accesses":[{"type":"hive:select","isAllowed":true}],"users":["hive", "user1"],"groups":[],"delegateAdmin":false, "conditions":[{ - "type":"ScriptConditionEvaluator", + "type":"expression", "values":["if ( ctx.isAccessedBefore('expiry') ) ctx.result = true;"] }] } @@ -197,7 +197,7 @@ "denyExceptions":[ {"accesses":[{"type":"hive:select","isAllowed":true}],"users":["hive", "user1"],"groups":[],"delegateAdmin":false, "conditions":[{ - "type":"ScriptConditionEvaluator", + "type":"expression", "values":["if ( ctx.isAccessedBefore('expiry') ) ctx.result = true;"] }] } diff --git a/agents-common/src/test/resources/policyengine/test_policyengine_tag_hive_filebased.json b/agents-common/src/test/resources/policyengine/test_policyengine_tag_hive_filebased.json index c2cb0b3de4..6b2863ad85 100644 --- a/agents-common/src/test/resources/policyengine/test_policyengine_tag_hive_filebased.json +++ b/agents-common/src/test/resources/policyengine/test_policyengine_tag_hive_filebased.json @@ -149,11 +149,11 @@ "policyConditions": [ { "itemId":1, - "name":"ScriptConditionEvaluator", - "evaluator": "org.apache.ranger.plugin.conditionevaluator.RangerScriptConditionEvaluator", - "evaluatorOptions" : {"engineName":"JavaScript"}, - "label":"Script", - "description": "Script to execute" + "name":"expression", + "evaluator": "org.apache.ranger.plugin.conditionevaluator.RangerScriptConditionEvaluator", + "evaluatorOptions" : {"engineName":"JavaScript", "ui.isMultiline":"true"}, + "label":"Enter boolean expression", + "description": "Boolean expression" }, { "itemId":2, @@ -172,8 +172,8 @@ { "accesses":[{"type":"hive:select","isAllowed":true}],"users":["hive", "user1"],"groups":[],"delegateAdmin":false, "conditions":[{ - "type":"ScriptConditionEvaluator", - "values":["if ( ctx.isAccessedBefore('activation_date') ) ctx.result = true;"] + "type":"expression", + "values":["if ( tagAttr.get('score') < 2 ) ctx.result = true;"] }] } ] @@ -203,7 +203,7 @@ "denyExceptions":[ {"accesses":[{"type":"hive:select","isAllowed":true}],"users":["hive", "user1"],"groups":[],"delegateAdmin":false, "conditions":[{ - "type":"ScriptConditionEvaluator", + "type":"expression", "values":["if ( ctx.isAccessedBefore('activation_date') ) ctx.result = true;"] }] } From 109f2218df687a2ce6085ec6e94d3c7d7664ff2d Mon Sep 17 00:00:00 2001 From: Sailaja Polavarapu Date: Wed, 13 Sep 2017 16:32:49 -0700 Subject: [PATCH 036/151] RANGER-1647: Missed upgrade patch file from previous checkin --- .../PatchForTagServiceDefUpdate_J10008.java | 202 ++++++++++++++++++ 1 file changed, 202 insertions(+) create mode 100644 security-admin/src/main/java/org/apache/ranger/patch/PatchForTagServiceDefUpdate_J10008.java diff --git a/security-admin/src/main/java/org/apache/ranger/patch/PatchForTagServiceDefUpdate_J10008.java b/security-admin/src/main/java/org/apache/ranger/patch/PatchForTagServiceDefUpdate_J10008.java new file mode 100644 index 0000000000..918fe1ecfc --- /dev/null +++ b/security-admin/src/main/java/org/apache/ranger/patch/PatchForTagServiceDefUpdate_J10008.java @@ -0,0 +1,202 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ranger.patch; + +import org.apache.commons.lang.StringUtils; +import org.apache.log4j.Logger; +import org.apache.ranger.biz.RangerBizUtil; +import org.apache.ranger.biz.ServiceDBStore; +import org.apache.ranger.common.JSONUtil; +import org.apache.ranger.common.RangerValidatorFactory; +import org.apache.ranger.common.StringUtil; +import org.apache.ranger.db.RangerDaoManager; +import org.apache.ranger.plugin.model.RangerServiceDef; +import org.apache.ranger.plugin.model.validation.RangerServiceDefValidator; +import org.apache.ranger.plugin.model.validation.RangerValidator.Action; +import org.apache.ranger.plugin.store.EmbeddedServiceDefsUtil; +import org.apache.ranger.service.RangerPolicyService; +import org.apache.ranger.service.XPermMapService; +import org.apache.ranger.service.XPolicyService; +import org.apache.ranger.util.CLIUtil; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +import org.apache.ranger.entity.XXServiceDef; +import java.util.List; +import java.util.Map; + +@Component +public class PatchForTagServiceDefUpdate_J10008 extends BaseLoader { + private static final Logger logger = Logger.getLogger(PatchForTagServiceDefUpdate_J10008.class); + public static final String SERVICEDBSTORE_SERVICEDEFBYNAME_TAG_NAME = "tag"; + public static final String SCRIPT_POLICY_CONDITION_NAME = "expression"; + + @Autowired + RangerDaoManager daoMgr; + + @Autowired + ServiceDBStore svcDBStore; + + @Autowired + JSONUtil jsonUtil; + + @Autowired + RangerPolicyService policyService; + + @Autowired + StringUtil stringUtil; + + @Autowired + XPolicyService xPolService; + + @Autowired + XPermMapService xPermMapService; + + @Autowired + RangerBizUtil bizUtil; + + @Autowired + RangerValidatorFactory validatorFactory; + + @Autowired + ServiceDBStore svcStore; + + public static void main(String[] args) { + logger.info("main()"); + try { + PatchForTagServiceDefUpdate_J10008 loader = (PatchForTagServiceDefUpdate_J10008) CLIUtil.getBean(PatchForTagServiceDefUpdate_J10008.class); + loader.init(); + while (loader.isMoreToProcess()) { + loader.load(); + } + logger.info("Load complete. Exiting!!!"); + System.exit(0); + } catch (Exception e) { + logger.error("Error loading", e); + System.exit(1); + } + } + + @Override + public void init() throws Exception { + // Do Nothing + } + + @Override + public void execLoad() { + logger.info("==> PatchForTagServiceDefUpdate.execLoad()"); + try { + updateTagServiceDef(); + } catch (Exception e) { + logger.error("Error whille updateTagServiceDef()data.", e); + } + logger.info("<== PatchForTagServiceDefUpdate.execLoad()"); + } + + @Override + public void printStats() { + logger.info("PatchForTagServiceDefUpdate data "); + } + + private void updateTagServiceDef(){ + RangerServiceDef embeddedTagServiceDef = null; + RangerServiceDef dbTagServiceDef = null; + List embeddedTagPolicyConditionDefs = null; + XXServiceDef xXServiceDefObj = null; + try{ + embeddedTagServiceDef=EmbeddedServiceDefsUtil.instance().getEmbeddedServiceDef(SERVICEDBSTORE_SERVICEDEFBYNAME_TAG_NAME); + if(embeddedTagServiceDef!=null){ + embeddedTagPolicyConditionDefs = embeddedTagServiceDef.getPolicyConditions(); + if (embeddedTagPolicyConditionDefs == null) { + logger.error("Policy Conditions are empyt in tag service def json"); + return; + } + + if (checkScriptPolicyCondPresent(embeddedTagPolicyConditionDefs) == false) { + logger.error(SCRIPT_POLICY_CONDITION_NAME + "policy condition not found!!"); + return; + } + + xXServiceDefObj = daoMgr.getXXServiceDef().findByName(SERVICEDBSTORE_SERVICEDEFBYNAME_TAG_NAME); + if (xXServiceDefObj == null) { + logger.error("Service def for " + SERVICEDBSTORE_SERVICEDEFBYNAME_TAG_NAME + " is not found!!"); + return; + } + + Map serviceDefOptionsPreUpdate=null; + String jsonStrPreUpdate=null; + jsonStrPreUpdate=xXServiceDefObj.getDefOptions(); + if (!StringUtils.isEmpty(jsonStrPreUpdate)) { + serviceDefOptionsPreUpdate=jsonUtil.jsonToMap(jsonStrPreUpdate); + } + xXServiceDefObj=null; + dbTagServiceDef=svcDBStore.getServiceDefByName(SERVICEDBSTORE_SERVICEDEFBYNAME_TAG_NAME); + + if(dbTagServiceDef!=null){ + dbTagServiceDef.setPolicyConditions(embeddedTagPolicyConditionDefs); + RangerServiceDefValidator validator = validatorFactory.getServiceDefValidator(svcStore); + validator.validate(dbTagServiceDef, Action.UPDATE); + + svcStore.updateServiceDef(dbTagServiceDef); + + xXServiceDefObj = daoMgr.getXXServiceDef().findByName(SERVICEDBSTORE_SERVICEDEFBYNAME_TAG_NAME); + if(xXServiceDefObj!=null) { + String jsonStrPostUpdate=xXServiceDefObj.getDefOptions(); + Map serviceDefOptionsPostUpdate = null; + if (!StringUtils.isEmpty(jsonStrPostUpdate)) { + serviceDefOptionsPostUpdate =jsonUtil.jsonToMap(jsonStrPostUpdate); + } + if (serviceDefOptionsPostUpdate != null && serviceDefOptionsPostUpdate.containsKey(RangerServiceDef.OPTION_ENABLE_DENY_AND_EXCEPTIONS_IN_POLICIES)) { + if(serviceDefOptionsPreUpdate == null || !serviceDefOptionsPreUpdate.containsKey(RangerServiceDef.OPTION_ENABLE_DENY_AND_EXCEPTIONS_IN_POLICIES)) { + String preUpdateValue = serviceDefOptionsPreUpdate == null ? null : serviceDefOptionsPreUpdate.get(RangerServiceDef.OPTION_ENABLE_DENY_AND_EXCEPTIONS_IN_POLICIES); + if (preUpdateValue == null) { + serviceDefOptionsPostUpdate.remove(RangerServiceDef.OPTION_ENABLE_DENY_AND_EXCEPTIONS_IN_POLICIES); + } else { + serviceDefOptionsPostUpdate.put(RangerServiceDef.OPTION_ENABLE_DENY_AND_EXCEPTIONS_IN_POLICIES, preUpdateValue); + } + xXServiceDefObj.setDefOptions(mapToJsonString(serviceDefOptionsPostUpdate)); + daoMgr.getXXServiceDef().update(xXServiceDefObj); + } + } + } + } + } + }catch(Exception e) + { + logger.error("Error while updating "+SERVICEDBSTORE_SERVICEDEFBYNAME_TAG_NAME+"service-def", e); + } + } + + private boolean checkScriptPolicyCondPresent(List policyCondDefs) { + boolean ret = false; + for(RangerServiceDef.RangerPolicyConditionDef policyCondDef : policyCondDefs) { + if ( SCRIPT_POLICY_CONDITION_NAME.equals(policyCondDef.getName()) ) { + ret = true ; + break; + } + } + return ret; + } + + private String mapToJsonString(Map map) throws Exception{ + String ret = null; + if(map != null) { + ret = jsonUtil.readMapToString(map); + } + return ret; + } +} From 3bf2d33b3dab94273c2a52195c2d44da3b96efab Mon Sep 17 00:00:00 2001 From: Mehul Parikh Date: Fri, 15 Sep 2017 18:10:14 +0530 Subject: [PATCH 037/151] RANGER-1651 : Improve Ranger and Ranger KMS REST Api documentation --- enunciate.xml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/enunciate.xml b/enunciate.xml index c91d1969bf..2bd313d257 100644 --- a/enunciate.xml +++ b/enunciate.xml @@ -34,4 +34,7 @@ + + + From 106a1f45f0de7b3bc2c4dfa6ad0520e79f287208 Mon Sep 17 00:00:00 2001 From: Nitin Galave Date: Thu, 14 Sep 2017 17:45:59 +0530 Subject: [PATCH 038/151] RANGER-1682: Clicking on export service after session timeout gets stuck indefinitely. --- security-admin/src/main/webapp/login.jsp | 5 +++++ .../main/webapp/scripts/models/RangerServiceDef.js | 8 ++++++++ .../src/main/webapp/scripts/models/XABaseModel.js | 4 ++-- .../src/main/webapp/scripts/utils/XAUtils.js | 6 ++++-- .../webapp/scripts/views/DownloadServicePolicy.js | 4 ++++ .../main/webapp/scripts/views/UploadServicePolicy.js | 12 ++++++++---- .../main/webapp/scripts/views/kms/KMSTableLayout.js | 9 +++++++++ .../views/permissions/ModulePermissionForm.js | 3 ++- .../webapp/scripts/views/policies/PermissionList.js | 8 ++++++++ .../webapp/scripts/views/policies/RangerPolicyRO.js | 9 ++++++++- .../scripts/views/policymanager/ServiceLayout.js | 2 +- .../main/webapp/scripts/views/reports/AuditLayout.js | 9 ++++++++- .../webapp/scripts/views/service/ServiceCreate.js | 3 +++ .../main/webapp/scripts/views/user/UserProfile.js | 8 +++++++- 14 files changed, 77 insertions(+), 13 deletions(-) diff --git a/security-admin/src/main/webapp/login.jsp b/security-admin/src/main/webapp/login.jsp index 39fb3274fa..b8f4cb982b 100644 --- a/security-admin/src/main/webapp/login.jsp +++ b/security-admin/src/main/webapp/login.jsp @@ -44,6 +44,11 @@ }); }; $(window).resize(updateBoxPosition); + var queryParams = JSON.parse('{"' + decodeURI((location.href.split('?')[1] || 'g=0').replace(/=/g, "\":\"")) + '"}'); + if(queryParams.sessionTimeout){ + window.alert('Session Timeout'); + location.replace("login.jsp"); + } setTimeout(updateBoxPosition, 50); }); diff --git a/security-admin/src/main/webapp/scripts/models/RangerServiceDef.js b/security-admin/src/main/webapp/scripts/models/RangerServiceDef.js index 704f37818d..73e19a1700 100644 --- a/security-admin/src/main/webapp/scripts/models/RangerServiceDef.js +++ b/security-admin/src/main/webapp/scripts/models/RangerServiceDef.js @@ -107,6 +107,14 @@ define(function(require){ return {results : results}; } return {results : results}; + }, + transport : function (options) { + $.ajax(options).error(function(respones) { + XAUtils.defaultErrorHandler('error',respones); + this.success({ + resultSize : 0 + }); + }); } }, formatResult : function(result){ diff --git a/security-admin/src/main/webapp/scripts/models/XABaseModel.js b/security-admin/src/main/webapp/scripts/models/XABaseModel.js index 608321083f..cb0691c308 100644 --- a/security-admin/src/main/webapp/scripts/models/XABaseModel.js +++ b/security-admin/src/main/webapp/scripts/models/XABaseModel.js @@ -41,9 +41,9 @@ define(function(require){ }, bindErrorEvents :function(){ //Moved require inside fuctn expression due to ie issue - this.bind("error", function(e){ + this.bind("error", function(e, error){ var XAUtils = require('utils/XAUtils'); - XAUtils.defaultErrorHandler(undefined, e); + XAUtils.defaultErrorHandler(undefined, error, e); }); }, /** diff --git a/security-admin/src/main/webapp/scripts/utils/XAUtils.js b/security-admin/src/main/webapp/scripts/utils/XAUtils.js index 1a86177a4b..a7a6285a6d 100644 --- a/security-admin/src/main/webapp/scripts/utils/XAUtils.js +++ b/security-admin/src/main/webapp/scripts/utils/XAUtils.js @@ -609,7 +609,9 @@ define(function(require) { XAUtils.defaultErrorHandler = function(model, error) { var App = require('App'); var vError = require('views/common/ErrorView'); - if(!_.isUndefined(model) && !_.isUndefined(model.modelName) && model.modelName == XAEnums.ClassTypes.CLASS_TYPE_XA_ACCESS_AUDIT.modelName){ + if(!_.isUndefined(model) && !_.isUndefined(model.modelName) + && model.modelName == XAEnums.ClassTypes.CLASS_TYPE_XA_ACCESS_AUDIT.modelName + && error.status !== 419){ return; } if (error.status == 404) { @@ -621,7 +623,7 @@ define(function(require) { status : error.status })); } else if (error.status == 419) { - window.location = 'login.jsp' + window.location = 'login.jsp?sessionTimeout=true'; } }; XAUtils.select2Focus = function(event) { diff --git a/security-admin/src/main/webapp/scripts/views/DownloadServicePolicy.js b/security-admin/src/main/webapp/scripts/views/DownloadServicePolicy.js index fb10562c79..1b1a4aa83f 100644 --- a/security-admin/src/main/webapp/scripts/views/DownloadServicePolicy.js +++ b/security-admin/src/main/webapp/scripts/views/DownloadServicePolicy.js @@ -80,6 +80,10 @@ define(function(require){ } }, + error : function(data,status,response){ + XAUtil.blockUI('unblock'); + XAUtil.defaultErrorHandler(status,data); + }, }); }, onRender: function() { diff --git a/security-admin/src/main/webapp/scripts/views/UploadServicePolicy.js b/security-admin/src/main/webapp/scripts/views/UploadServicePolicy.js index 338fffb964..62a1fcff24 100644 --- a/security-admin/src/main/webapp/scripts/views/UploadServicePolicy.js +++ b/security-admin/src/main/webapp/scripts/views/UploadServicePolicy.js @@ -207,11 +207,15 @@ define(function(require){ error : function(response,model){ XAUtil.blockUI('unblock'); if ( response && response.responseJSON && response.responseJSON.msgDesc){ - XAUtil.notifyError('Error', response.responseJSON.msgDesc); - } else { + if(response.status == '419'){ + XAUtil.defaultErrorHandler(model,response); + }else{ + XAUtil.notifyError('Error', response.responseJSON.msgDesc); + } + } else { XAUtil.notifyError('Error', 'File import failed.'); - } - } + } + } }); }, onAddClick : function(){ diff --git a/security-admin/src/main/webapp/scripts/views/kms/KMSTableLayout.js b/security-admin/src/main/webapp/scripts/views/kms/KMSTableLayout.js index fbbee46d66..2749cea52e 100755 --- a/security-admin/src/main/webapp/scripts/views/kms/KMSTableLayout.js +++ b/security-admin/src/main/webapp/scripts/views/kms/KMSTableLayout.js @@ -266,6 +266,7 @@ define(function(require){ this.ui.selectServiceName.select2({ maximumSelectionSize : 1, closeOnSelect : true, + allowClear: true, width :'220px', placeholder : 'Please select KMS service', initSelection : function (element, callback) { @@ -284,6 +285,14 @@ define(function(require){ return { results : results }; } return { results : results }; + }, + transport: function (options) { + $.ajax(options).error(function(respones) { + XAUtil.defaultErrorHandler('error',respones); + this.success({ + resultSize : 0 + }); + }); } }, formatResult : function(result){ diff --git a/security-admin/src/main/webapp/scripts/views/permissions/ModulePermissionForm.js b/security-admin/src/main/webapp/scripts/views/permissions/ModulePermissionForm.js index aa4c332326..d9c522c47e 100644 --- a/security-admin/src/main/webapp/scripts/views/permissions/ModulePermissionForm.js +++ b/security-admin/src/main/webapp/scripts/views/permissions/ModulePermissionForm.js @@ -178,8 +178,9 @@ define(function(require) { return { results : results}; }, transport: function (options) { - $.ajax(options).error(function() { + $.ajax(options).error(function(respones) { console.log("ajax failed"); + XAUtil.defaultErrorHandler('error',respones); this.success({ resultSize : 0 }); diff --git a/security-admin/src/main/webapp/scripts/views/policies/PermissionList.js b/security-admin/src/main/webapp/scripts/views/policies/PermissionList.js index bb64984054..0803945e76 100644 --- a/security-admin/src/main/webapp/scripts/views/policies/PermissionList.js +++ b/security-admin/src/main/webapp/scripts/views/policies/PermissionList.js @@ -223,6 +223,14 @@ define(function(require) { return {results : results}; } return {results : results}; + }, + transport: function (options) { + $.ajax(options).error(function(respones) { + XAUtil.defaultErrorHandler('error',respones); + this.success({ + resultSize : 0 + }); + }); } }, formatResult : function(result){ diff --git a/security-admin/src/main/webapp/scripts/views/policies/RangerPolicyRO.js b/security-admin/src/main/webapp/scripts/views/policies/RangerPolicyRO.js index b08447311e..d0a86836e5 100644 --- a/security-admin/src/main/webapp/scripts/views/policies/RangerPolicyRO.js +++ b/security-admin/src/main/webapp/scripts/views/policies/RangerPolicyRO.js @@ -83,7 +83,14 @@ define(function(require) { this.policy.fetchByEventTime({ async: false, cache: false, - data : data + data : data, + error : function(error , response){ + if (response && response.status === 419 ) { + XAUtils.defaultErrorHandler(error , response); + } else { + XAUtils.showErrorMsg(response.responseJSON.msgDesc); + } + } }); }, diff --git a/security-admin/src/main/webapp/scripts/views/policymanager/ServiceLayout.js b/security-admin/src/main/webapp/scripts/views/policymanager/ServiceLayout.js index c551887b6d..ab42b5ca28 100644 --- a/security-admin/src/main/webapp/scripts/views/policymanager/ServiceLayout.js +++ b/security-admin/src/main/webapp/scripts/views/policymanager/ServiceLayout.js @@ -231,7 +231,7 @@ define(function(require){ }, error :function(model, response) { XAUtil.blockUI('unblock'); - if(!_.isUndefined(response) && !_.isUndefined(response.responseJSON) && !_.isUndefined(response.responseJSON.msgDesc)){ + if(!_.isUndefined(response) && !_.isUndefined(response.responseJSON) && !_.isUndefined(response.responseJSON.msgDesc && response.status !='419')){ XAUtil.notifyError('Error', response.responseJSON.msgDesc); } } diff --git a/security-admin/src/main/webapp/scripts/views/reports/AuditLayout.js b/security-admin/src/main/webapp/scripts/views/reports/AuditLayout.js index f2c8448f4d..e9d3675fbc 100644 --- a/security-admin/src/main/webapp/scripts/views/reports/AuditLayout.js +++ b/security-admin/src/main/webapp/scripts/views/reports/AuditLayout.js @@ -619,7 +619,14 @@ define(function(require) { var fullTrxLogListForTrxId = new VXTrxLogList(); fullTrxLogListForTrxId.getFullTrxLogListForTrxId(this.model.get('transactionId'),{ - cache : false + cache : false, + error : function(response , error){ + if (response && response.status === 419 ) { + XAUtils.defaultErrorHandler(error , response); + } else { + XAUtils.showErrorMsg(response.responseJSON.msgDesc); + } + } }).done(function(coll,mm){ XAUtils.blockUI('unblock'); fullTrxLogListForTrxId = new VXTrxLogList(coll.vXTrxLogs); diff --git a/security-admin/src/main/webapp/scripts/views/service/ServiceCreate.js b/security-admin/src/main/webapp/scripts/views/service/ServiceCreate.js index c2b78ab72b..54444fb1ad 100644 --- a/security-admin/src/main/webapp/scripts/views/service/ServiceCreate.js +++ b/security-admin/src/main/webapp/scripts/views/service/ServiceCreate.js @@ -238,6 +238,9 @@ define(function(require){ } }, error: function (msResponse, options) { + if(msResponse.status === 419){ + XAUtil.defaultErrorHandler(options , msResponse); + } bootbox.alert("Connection Failed."); } }); diff --git a/security-admin/src/main/webapp/scripts/views/user/UserProfile.js b/security-admin/src/main/webapp/scripts/views/user/UserProfile.js index cac80a81ce..8e60b53b92 100644 --- a/security-admin/src/main/webapp/scripts/views/user/UserProfile.js +++ b/security-admin/src/main/webapp/scripts/views/user/UserProfile.js @@ -140,7 +140,11 @@ define(function(require){ that.form.fields.name.setError(response.responseJSON.msgDesc); XAUtil.notifyError('Error', response.responseJSON.msgDesc); }else { - XAUtil.notifyError('Error', 'Error occurred while updating user profile!!'); + if(model.status == 419){ + XAUtil.defaultErrorHandler(response , model); + }else{ + XAUtil.notifyError('Error', 'Error occurred while updating user profile!!'); + } } } }); @@ -168,6 +172,8 @@ define(function(require){ that.form.fields.reEnterPassword.setError(localization.tt('validationMessages.newPasswordError')); }else if((msResponse.responseJSON.msgDesc) == "serverMsg.userMgrOldPassword"){ that.form.fields.oldPassword.setError(localization.tt('validationMessages.oldPasswordRepeatError')); + }else if(msResponse.status == 419){ + XAUtil.defaultErrorHandler(options , msResponse); } else { that.form.fields.oldPassword.setError(localization.tt('validationMessages.oldPasswordError')); } From 5a77c8cf2303a840aad31440cd126d5ab876f204 Mon Sep 17 00:00:00 2001 From: fatimaawez Date: Fri, 15 Sep 2017 15:27:39 +0530 Subject: [PATCH 039/151] RANGER-1730 : Utility script that will list the users with a given role --- .../scripts/rolebasedusersearchutil.py | 159 ++++++++++ .../cliutil/RoleBasedUserSearchUtil.java | 271 ++++++++++++++++++ .../cliutil/TestRoleBasedUserSearchUtil.java | 139 +++++++++ src/main/assembly/admin-web.xml | 3 +- 4 files changed, 571 insertions(+), 1 deletion(-) create mode 100644 security-admin/scripts/rolebasedusersearchutil.py create mode 100644 security-admin/src/main/java/org/apache/ranger/patch/cliutil/RoleBasedUserSearchUtil.java create mode 100644 security-admin/src/test/java/org/apache/ranger/patch/cliutil/TestRoleBasedUserSearchUtil.java diff --git a/security-admin/scripts/rolebasedusersearchutil.py b/security-admin/scripts/rolebasedusersearchutil.py new file mode 100644 index 0000000000..d651461f66 --- /dev/null +++ b/security-admin/scripts/rolebasedusersearchutil.py @@ -0,0 +1,159 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. See accompanying LICENSE file. +# + +import os +import re +import sys +import errno +import shlex +import logging +import subprocess +import platform +import fileinput +import getpass +import shutil +from xml.etree import ElementTree as ET +from os.path import basename +from subprocess import Popen,PIPE +from datetime import date +from datetime import datetime +from operator import contains + + +os_name = platform.system() +os_name = os_name.upper() + +msgPrompt = "Enter the below options" +msgCommand = "Usage : python rolebasedusersearchutil.py -u -p -r " +msgRoleList = " can be ROLE_USER/ROLE_SYS_ADMIN/ROLE_KEY_ADMIN" + + + +if os_name == "LINUX": + RANGER_ADMIN_HOME = os.getenv("RANGER_ADMIN_HOME") + if RANGER_ADMIN_HOME is None: + RANGER_ADMIN_HOME = os.getcwd() +elif os_name == "WINDOWS": + RANGER_ADMIN_HOME = os.getenv("RANGER_ADMIN_HOME") + +def log(msg,type): + if type == 'info': + logging.info(" %s",msg) + if type == 'debug': + logging.debug(" %s",msg) + if type == 'warning': + logging.warning(" %s",msg) + if type == 'exception': + logging.exception(" %s",msg) + if type == 'error': + logging.error(" %s",msg) + +def main(argv): + FORMAT = '%(asctime)-15s %(message)s' + logging.basicConfig(format=FORMAT, level=logging.DEBUG) + ews_lib = os.path.join(RANGER_ADMIN_HOME,"ews","lib") + app_home = os.path.join(RANGER_ADMIN_HOME,"ews","webapp") + ranger_log = os.path.join(RANGER_ADMIN_HOME,"ews","logs") + + if os.environ['JAVA_HOME'] == "": + log("[E] ---------- JAVA_HOME environment property not defined, aborting installation. ----------", "error") + sys.exit(1) + JAVA_BIN=os.path.join(os.environ['JAVA_HOME'],'bin','java') + if os_name == "WINDOWS" : + JAVA_BIN = JAVA_BIN+'.exe' + if os.path.isfile(JAVA_BIN): + pass + else: + while os.path.isfile(JAVA_BIN) == False: + log("Enter java executable path: :","info") + JAVA_BIN=raw_input() + log("[I] Using Java:" + str(JAVA_BIN),"info") + userName = "" + password = "" + userRole = "" + userNameMsgFlag = False + passwordMsgFlag = False + userRoleMsgFlag = False + userroleFlag = False + + if len(argv) == 1: + print msgPrompt + " or \n" + msgCommand + "\n " +msgRoleList + userName = raw_input('Enter a user name: ') + password = getpass.getpass('Enter a user password:') + userRole = raw_input('Enter a role: ') + elif len(argv) > 1 and len(argv) < 8 : + for i in range(1, len(sys.argv)) : + if sys.argv[i] == "-u" : + if len(argv)-1 > i+1 or len(argv)-1 == i+1: + userName = sys.argv[i+1] + continue + if sys.argv[i] == "-p" : + if len(argv)-1 > i+1 or len(argv)-1 == i+1: + password = sys.argv[i+1] + continue + if sys.argv[i] == "-r" : + if len(argv)-1 > i+1 or len(argv)-1 == i+1: + userRole = sys.argv[i+1] + userroleFlag = True + continue + else: + log("[E] Invalid argument list.", "error") + log("[I] " + msgCommand + "\n " + msgRoleList, "info") + sys.exit(1) + + if userName == "" : + userNameMsgFlag = True + elif userName != "" : + if userName.lower() == "-p" or userName.lower() == "-r" or userName.lower() == "-u" : + userNameMsgFlag = True + if password == "" : + passwordMsgFlag = True + elif password.lower() == "-p" or password.lower() == "-r" or password.lower() == "-u" : + passwordMsgFlag = True + if userroleFlag == True : + if userRole == "": + userRoleMsgFlag = True + elif userRole != "": + if userRole.lower() == "-p" or userRole.lower() == "-r" or userRole.lower() == "-u": + userRoleMsgFlag = True + if userNameMsgFlag == True or passwordMsgFlag == True or userRoleMsgFlag == True : + print msgPrompt + " or \n" + msgCommand + "\n " +msgRoleList + if userNameMsgFlag == True : + userName = raw_input('Enter a user name: ') + if passwordMsgFlag == True : + password = getpass.getpass("Enter user password:") + if userRoleMsgFlag == True : + userRole = raw_input('Enter a role: ') + if userName != "" and password != "" : + if os_name == "LINUX": + path = os.path.join("%s","WEB-INF","classes","conf:%s","WEB-INF","classes","lib","*:%s","WEB-INF",":%s","META-INF",":%s","WEB-INF","lib","*:%s","WEB-INF","classes",":%s","WEB-INF","classes","META-INF:%s/*")%(app_home ,app_home ,app_home, app_home, app_home, app_home ,app_home,ews_lib) + elif os_name == "WINDOWS": + path = os.path.join("%s","WEB-INF","classes","conf;%s","WEB-INF","classes","lib","*;%s","WEB-INF",";%s","META-INF",";%s","WEB-INF","lib","*;%s","WEB-INF","classes",";%s","WEB-INF","classes","META-INF" )%(app_home ,app_home ,app_home, app_home, app_home, app_home ,app_home) + if userRole != "" : + get_java_cmd = "%s -Dlogdir=%s -Dlog4j.configuration=db_patch.log4j.xml -cp %s org.apache.ranger.patch.cliutil.%s %s %s %s"%(JAVA_BIN,ranger_log,path,'RoleBasedUserSearchUtil',userName,password,userRole) + if userRole == "" : + get_java_cmd = "%s -Dlogdir=%s -Dlog4j.configuration=db_patch.log4j.xml -cp %s org.apache.ranger.patch.cliutil.%s %s %s "%(JAVA_BIN,ranger_log,path,'RoleBasedUserSearchUtil',userName,password) + if os_name == "LINUX": + ret = subprocess.call(shlex.split(get_java_cmd)) + elif os_name == "WINDOWS": + ret = subprocess.call(get_java_cmd) + if ret == 0: + log("[I] List fetched successfully","info") + else: + log("[E] Unable to fetch user list of given role ","error") + sys.exit(1) + else: + log("[E] Input Error","error") + +main(sys.argv) diff --git a/security-admin/src/main/java/org/apache/ranger/patch/cliutil/RoleBasedUserSearchUtil.java b/security-admin/src/main/java/org/apache/ranger/patch/cliutil/RoleBasedUserSearchUtil.java new file mode 100644 index 0000000000..0459be68e2 --- /dev/null +++ b/security-admin/src/main/java/org/apache/ranger/patch/cliutil/RoleBasedUserSearchUtil.java @@ -0,0 +1,271 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.patch.cliutil; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.MapUtils; +import org.apache.log4j.Logger; +import org.apache.ranger.biz.UserMgr; +import org.apache.ranger.biz.XUserMgr; +import org.apache.ranger.common.RangerConstants; +import org.apache.ranger.db.RangerDaoManager; +import org.apache.ranger.entity.XXPortalUser; +import org.apache.commons.lang.StringUtils; +import org.apache.ranger.patch.BaseLoader; +import org.apache.ranger.service.XUserService; +import org.apache.ranger.util.CLIUtil; +import org.apache.ranger.view.VXUser; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +@Component +public class RoleBasedUserSearchUtil extends BaseLoader { + + private static final Logger logger = Logger + .getLogger(RoleBasedUserSearchUtil.class); + @Autowired + XUserService xUserService; + + @Autowired + RangerDaoManager daoMgr; + + @Autowired + UserMgr userMgr; + + @Autowired + XUserMgr xUserMgr; + + public static Boolean checkRole = true; + public static String userLoginId = ""; + public static String currentPassword = ""; + public static String userRole = ""; + + public static void main(String[] args) { + logger.info("RoleBaseUserSearchUtil : main()"); + try { + RoleBasedUserSearchUtil loader = (RoleBasedUserSearchUtil) CLIUtil.getBean(RoleBasedUserSearchUtil.class); + loader.init(); + if (args.length == 3 || args.length == 2) { + userLoginId = args[0]; + currentPassword = args[1]; + if (args.length == 3) { + userRole = args[2]; + List roles = new ArrayList(); + roles.add(RangerConstants.ROLE_USER); + roles.add(RangerConstants.ROLE_SYS_ADMIN); + roles.add(RangerConstants.ROLE_KEY_ADMIN); + if (!StringUtils.isBlank(userRole)) { + userRole = userRole.toUpperCase(); + if (!roles.contains(userRole)) { + System.out.println("Invalid UserRole. Exiting!!!"); + logger.info("Invalid UserRole. Exiting!!!"); + System.exit(1); + } else { + checkRole = false; + } + } + } + if (StringUtils.isBlank(userLoginId)) { + System.out.println("Invalid login ID. Exiting!!!"); + logger.info("Invalid login ID. Exiting!!!"); + System.exit(1); + } + if (StringUtils.isBlank(currentPassword)) { + System.out.println("Invalid current password. Exiting!!!"); + logger.info("Invalid current password. Exiting!!!"); + System.exit(1); + } + while (loader.isMoreToProcess()) { + loader.load(); + } + logger.info("Load complete. Exiting!!!"); + System.exit(0); + } else { + System.out.println("RoleBaseUserSearchUtil: Incorrect Arguments \n Usage: \n "); + logger.error("RoleBaseUserSearchUtil: Incorrect Arguments \n Usage: \n "); + System.exit(1); + } + } catch (Exception e) { + logger.error("Error loading", e); + System.exit(1); + } + } + + @Override + public void init() throws Exception { + logger.info("==> RoleBaseUserSearchUtil.init()"); + } + + @Override + public void printStats() { + } + + @Override + public void execLoad() { + logger.info("==> RoleBaseUserSearchUtil.execLoad()"); + validateUserAndFetchUserList(); + logger.info("<== RoleBaseUserSearchUtil.execLoad()"); + } + + public void getUsersBasedOnRole(List userRoleList) { + try { + if (!CollectionUtils.isEmpty(userRoleList) && userRoleList != null) { + Map roleSysAdminMap = new HashMap(); + Map roleKeyAdminMap = new HashMap(); + Map roleUserMap = new HashMap(); + for (String userRole : userRoleList) { + List listXXPortalUser = daoMgr.getXXPortalUser().findByRole(userRole); + if (listXXPortalUser != null && !CollectionUtils.isEmpty(listXXPortalUser)) { + if (userRole.equalsIgnoreCase(RangerConstants.ROLE_SYS_ADMIN)) { + for (XXPortalUser xXPortalUser : listXXPortalUser) { + roleSysAdminMap.put(xXPortalUser.getLoginId(),userRole); + } + } else if (userRole.equalsIgnoreCase(RangerConstants.ROLE_KEY_ADMIN)) { + for (XXPortalUser xXPortalUser : listXXPortalUser) { + roleKeyAdminMap.put(xXPortalUser.getLoginId(),userRole); + } + } else if (userRole.equalsIgnoreCase(RangerConstants.ROLE_USER)) { + for (XXPortalUser xXPortalUser : listXXPortalUser) { + roleUserMap.put(xXPortalUser.getLoginId(),userRole); + } + } + } + } + if (MapUtils.isEmpty( roleSysAdminMap) && MapUtils.isEmpty(roleKeyAdminMap) && MapUtils.isEmpty(roleUserMap)) { + System.out.println("users with given user role are not there"); + logger.error("users with given user role are not there"); + System.exit(1); + } else { + if (!MapUtils.isEmpty(roleSysAdminMap)) { + for (String key : roleSysAdminMap.keySet()) { + System.out.println(roleSysAdminMap.get(key) + " : " + key); + } + } + if (!MapUtils.isEmpty(roleKeyAdminMap)) { + for (String key : roleKeyAdminMap.keySet()) { + System.out.println(roleKeyAdminMap.get(key) + " : " + key); + } + } + if (!MapUtils.isEmpty(roleUserMap)) { + for (String key : roleUserMap.keySet()) { + System.out.println(roleUserMap.get(key) + " : " + key); + } + } + if (userRoleList.contains(RangerConstants.ROLE_SYS_ADMIN)) { + System.out.println("ROLE_SYS_ADMIN Total Count : " + roleSysAdminMap.size()); + } + if (userRoleList.contains(RangerConstants.ROLE_KEY_ADMIN)) { + System.out.println("ROLE_KEY_ADMIN Total Count : " + roleKeyAdminMap.size()); + } + if (userRoleList.contains(RangerConstants.ROLE_USER)) { + System.out.println("ROLE_USER Total Count : " + roleUserMap.size()); + } + int total = roleSysAdminMap.size() + roleKeyAdminMap.size() + roleUserMap.size(); + System.out.println("Total Count : " + total); + } + } + + } catch (Exception e) { + logger.error("Error getting User's List with the mentioned role: "+ e.getMessage()); + } + } + + public void validateUserAndFetchUserList() { + userLoginId = userLoginId.toLowerCase(); + XXPortalUser xxPortalUser = daoMgr.getXXPortalUser().findByLoginId( + userLoginId); + Boolean isUserAuthorized = false; + if (xxPortalUser != null) { + String dbPassword = xxPortalUser.getPassword(); + String currentEncryptedPassword = null; + try { + currentEncryptedPassword = userMgr.encrypt(userLoginId,currentPassword); + if (currentEncryptedPassword != null && currentEncryptedPassword.equals(dbPassword)) { + VXUser vxUser = xUserService.getXUserByUserName(xxPortalUser.getLoginId()); + if (vxUser != null) { + List existingRole = (List) vxUser.getUserRoleList(); + List permissionList = daoMgr.getXXModuleDef().findAccessibleModulesByUserId(xxPortalUser.getId(), vxUser.getId()); + if (permissionList != null && permissionList.contains(RangerConstants.MODULE_USER_GROUPS) && !CollectionUtils.isEmpty(existingRole) && !StringUtils.isBlank(existingRole.get(0))) { + List userRoleList = new ArrayList(); + if (existingRole.get(0).equalsIgnoreCase(RangerConstants.ROLE_USER)) { + userRoleList.add(RangerConstants.ROLE_USER); + if (checkRole) { + getUsersBasedOnRole(userRoleList); + } else if (existingRole.get(0).equalsIgnoreCase(userRole) || userRole.equalsIgnoreCase(RangerConstants.ROLE_USER)) { + getUsersBasedOnRole(userRoleList); + } else { + isUserAuthorized = true; + } + } else if (existingRole.get(0).equalsIgnoreCase(RangerConstants.ROLE_SYS_ADMIN)) { + if (checkRole) { + userRoleList.add(RangerConstants.ROLE_SYS_ADMIN); + userRoleList.add(RangerConstants.ROLE_USER); + getUsersBasedOnRole(userRoleList); + } else if (existingRole.get(0).equalsIgnoreCase(userRole) || userRole.equalsIgnoreCase(RangerConstants.ROLE_USER)) { + userRoleList.add(userRole); + getUsersBasedOnRole(userRoleList); + } else { + isUserAuthorized = true; + } + } else if (existingRole.get(0).equalsIgnoreCase(RangerConstants.ROLE_KEY_ADMIN) || userRole.equalsIgnoreCase(RangerConstants.ROLE_USER)) { + if (checkRole) { + userRoleList.add(RangerConstants.ROLE_KEY_ADMIN); + userRoleList.add(RangerConstants.ROLE_USER); + getUsersBasedOnRole(userRoleList); + } else if (existingRole.get(0).equalsIgnoreCase(userRole) || userRole.equalsIgnoreCase(RangerConstants.ROLE_USER)) { + userRoleList.add(userRole); + getUsersBasedOnRole(userRoleList); + } else { + isUserAuthorized = true; + } + + } + if (isUserAuthorized == true) { + System.out.println("user is not authorized to fetch this list"); + logger.error("user is not authorized to fetch this list"); + System.exit(1); + } + } else { + System.out.println("user permission denied"); + logger.error("user permission denied"); + System.exit(1); + } + } + } else { + System.out.println("Invalid user password"); + logger.error("Invalid user password"); + System.exit(1); + } + } catch (Exception e) { + logger.error("Getting User's List with the mentioned role failure. Detail: \n",e); + System.exit(1); + } + } else { + System.out.println("User does not exist in DB!!"); + logger.error("User does not exist in DB"); + System.exit(1); + } + } +} \ No newline at end of file diff --git a/security-admin/src/test/java/org/apache/ranger/patch/cliutil/TestRoleBasedUserSearchUtil.java b/security-admin/src/test/java/org/apache/ranger/patch/cliutil/TestRoleBasedUserSearchUtil.java new file mode 100644 index 0000000000..83eab7aedd --- /dev/null +++ b/security-admin/src/test/java/org/apache/ranger/patch/cliutil/TestRoleBasedUserSearchUtil.java @@ -0,0 +1,139 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ranger.patch.cliutil; + +import static org.junit.Assert.fail; + +import java.util.ArrayList; +import java.util.List; +import org.apache.ranger.biz.UserMgr; +import org.apache.ranger.biz.XUserMgr; +import org.apache.ranger.common.RangerConstants; + +import org.apache.ranger.db.RangerDaoManager; +import org.apache.ranger.db.XXModuleDefDao; +import org.apache.ranger.db.XXPortalUserDao; +import org.apache.ranger.entity.XXPortalUser; +import org.apache.ranger.service.XUserService; +import org.apache.ranger.view.VXUser; +import org.apache.ranger.view.VXUserList; +import org.junit.FixMethodOrder; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.MethodSorters; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.runners.MockitoJUnitRunner; + +@RunWith(MockitoJUnitRunner.class) +@FixMethodOrder(MethodSorters.NAME_ASCENDING) +public class TestRoleBasedUserSearchUtil { + @Mock + XUserService xUserService; + + @Mock + RangerDaoManager daoMgr; + + @Mock + UserMgr userMgr; + + @Mock + XUserMgr xUserMgr; + @Mock + XXPortalUserDao xXPortalUserDao; + + + @InjectMocks + RoleBasedUserSearchUtil roleBasedUserSearchUtil = new RoleBasedUserSearchUtil(); + + + public TestRoleBasedUserSearchUtil() { + + } + @Test + public void TestGetUsersBasedOnRole() { + try { + XXPortalUser xXPortalUser = new XXPortalUser(); + xXPortalUser.setLoginId("testUser"); + xXPortalUser.setId(1L); + xXPortalUser.setFirstName("testUser"); + xXPortalUser.setPublicScreenName("testUser"); + xXPortalUser.setPassword("testUserPassword"); + List listXXPortalUser = new ArrayList(); + listXXPortalUser.add(xXPortalUser); + List userRoleList = new ArrayList(); + userRoleList.add("ROLE_SYS_ADMIN"); + + Mockito.when(daoMgr.getXXPortalUser()).thenReturn(xXPortalUserDao); + Mockito.when(xXPortalUserDao.findByRole(RangerConstants.ROLE_SYS_ADMIN)).thenReturn(listXXPortalUser); + + roleBasedUserSearchUtil.getUsersBasedOnRole(userRoleList); + + Mockito.verify(daoMgr).getXXPortalUser(); + Mockito.verify(xXPortalUserDao).findByRole(RangerConstants.ROLE_SYS_ADMIN); + + } catch(Exception e) { + fail("test failed due to: " + e.getMessage()); + } + } + @Test + public void TestValidateUserAndFetchUserList() { + List permissionList = new ArrayList(); + permissionList.add(RangerConstants.MODULE_USER_GROUPS); + String currentEncryptedPassword = "testpassword"; + XXPortalUser xxPortalUser = new XXPortalUser(); + xxPortalUser.setId(1L); + xxPortalUser.setLoginId("testUser"); + xxPortalUser.setPassword("testpassword"); + xxPortalUser.setFirstName("testUser"); + VXUser vxUser = new VXUser(); + vxUser.setId(1L); + VXUserList vXUserList = new VXUserList(); + List vXUsers = new ArrayList(); + vXUsers.add(vxUser); + vXUserList.setVXUsers(vXUsers ); + + List userRoleList = new ArrayList(); + userRoleList.add("ROLE_SYS_ADMIN"); + List listXXPortalUser = new ArrayList(); + listXXPortalUser.add(xxPortalUser); + vxUser.setUserRoleList(userRoleList); + XXModuleDefDao xXModuleDefDao = Mockito.mock(XXModuleDefDao.class); + + Mockito.when(daoMgr.getXXPortalUser()).thenReturn(xXPortalUserDao); + Mockito.when(xXPortalUserDao.findByLoginId(Mockito.anyString())).thenReturn(xxPortalUser); + Mockito.when(xUserService.getXUserByUserName(xxPortalUser.getLoginId())).thenReturn(vxUser); + Mockito.when(daoMgr.getXXModuleDef()).thenReturn(xXModuleDefDao); + Mockito.when(xXModuleDefDao.findAccessibleModulesByUserId(Mockito.anyLong(), Mockito.anyLong())).thenReturn(permissionList); + Mockito.when(userMgr.encrypt(Mockito.anyString(),Mockito.anyString())).thenReturn(currentEncryptedPassword); + Mockito.when(xXPortalUserDao.findByRole(Mockito.anyString())).thenReturn(listXXPortalUser); + + roleBasedUserSearchUtil.validateUserAndFetchUserList(); + Mockito.verify(daoMgr, Mockito.atLeast(2)).getXXPortalUser(); + Mockito.verify(xXPortalUserDao).findByLoginId(Mockito.anyString()); + Mockito.verify(xUserService).getXUserByUserName(xxPortalUser.getLoginId()); + Mockito.verify(daoMgr).getXXModuleDef(); + Mockito.verify(xXModuleDefDao).findAccessibleModulesByUserId(Mockito.anyLong(), Mockito.anyLong()); + Mockito.verify(userMgr).encrypt(Mockito.anyString(),Mockito.anyString()); + Mockito.verify(xXPortalUserDao, Mockito.atLeast(2)).findByRole(Mockito.anyString()); + + } + +} diff --git a/src/main/assembly/admin-web.xml b/src/main/assembly/admin-web.xml index cb1aad2e60..0e97818b4d 100644 --- a/src/main/assembly/admin-web.xml +++ b/src/main/assembly/admin-web.xml @@ -378,7 +378,8 @@ ranger_credential_helper.py deleteUserGroupUtil.py changepasswordutil.py - changeusernameutil.py + changeusernameutil.py + rolebasedusersearchutil.py 544 From 29f8918ffd6fc9aa8e0d5eb4256349ac27b23690 Mon Sep 17 00:00:00 2001 From: Nitin Galave Date: Thu, 14 Sep 2017 18:15:50 +0530 Subject: [PATCH 040/151] RANGER-1786: Need warning on external user role change. --- .../src/main/webapp/scripts/views/users/UserForm.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/security-admin/src/main/webapp/scripts/views/users/UserForm.js b/security-admin/src/main/webapp/scripts/views/users/UserForm.js index 492b14deb4..27e86a9c5c 100644 --- a/security-admin/src/main/webapp/scripts/views/users/UserForm.js +++ b/security-admin/src/main/webapp/scripts/views/users/UserForm.js @@ -54,6 +54,10 @@ define(function(require){ bindEvents : function(){ this.on('userRoleList:change', function(form, fieldEditor){ //this.userRoleListChange(form, fieldEditor); + if(this.model.get('userSource') === XAEnums.UserTypes.USER_EXTERNAL.value){ + var externalUserRoleProperty = " Warning !! : Please make sure that "+ this.model.get('name') + " user's role change performed here is consistent with ranger.usersync.group.based.role.assignment.rules property in ranger usersync configuration."; + XAUtils.alertPopup({msg : externalUserRoleProperty}); + } }); }, From fd6047c450acf499ae2bf33609679c54e3be6ebb Mon Sep 17 00:00:00 2001 From: Nitin Galave Date: Mon, 18 Sep 2017 11:20:07 +0530 Subject: [PATCH 041/151] RANGER-1787: User has to fill up all the allow and deny conditions items to create a knox policy. --- .../webapp/scripts/views/policies/PermissionList.js | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/security-admin/src/main/webapp/scripts/views/policies/PermissionList.js b/security-admin/src/main/webapp/scripts/views/policies/PermissionList.js index 0803945e76..92b8334bcc 100644 --- a/security-admin/src/main/webapp/scripts/views/policies/PermissionList.js +++ b/security-admin/src/main/webapp/scripts/views/policies/PermissionList.js @@ -247,12 +247,8 @@ define(function(require) { renderPerms :function(){ var that = this; this.perms = _.map(this.accessTypes,function(m){return {text:m.label, value:m.name};}); - this.perms.push({'value' : -1, 'text' : 'Select/Deselect All'}); - //set default access type 'select' for add new masking & row filter policies - if(this.perms.length == 2){ - if(!_.isUndefined(this.perms[0].value) && _.isEmpty(this.permsIds)){ - this.permsIds.push(this.perms[0].value); - } + if(this.perms.length > 1){ + this.perms.push({'value' : -1, 'text' : 'Select/Deselect All'}); } //create x-editable for permissions this.ui.addPerms.editable({ @@ -318,10 +314,6 @@ define(function(require) { this.ui.addPerms.attr('title','Components Permissions') this.ui.delegatedAdmin.parent('td').hide(); this.perms = _.map(this.accessTypes,function(m){return {text:m.label, value:m.name};}); - //select defatult access type if single component exists - if(this.perms.length == 1 && this.permsIds.length >= 0){ - this.permsIds.push(this.perms[0].value) - } var select2optn = { width :'600px' }; if(XAUtil.isMaskingPolicy(this.rangerPolicyType)){ select2optn = {width :'600px' , maximumSelectionSize : 1 }; From 9381159d671a5804fcb3a4a116e0d73566771a49 Mon Sep 17 00:00:00 2001 From: "peng.jianhua" Date: Mon, 18 Sep 2017 10:26:10 +0800 Subject: [PATCH 042/151] RANGER-1788:Install Ranger admin failure. Signed-off-by: Colm O hEigeartaigh --- .../patches/028-add-unique-constraint-on-table-x_group.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/security-admin/db/mysql/patches/028-add-unique-constraint-on-table-x_group.sql b/security-admin/db/mysql/patches/028-add-unique-constraint-on-table-x_group.sql index 2c5be737ea..078fb9900f 100644 --- a/security-admin/db/mysql/patches/028-add-unique-constraint-on-table-x_group.sql +++ b/security-admin/db/mysql/patches/028-add-unique-constraint-on-table-x_group.sql @@ -23,7 +23,7 @@ DECLARE loginID bigint(20); /* check unique constraint exist on group_name column or not */ if not exists (select * from information_schema.columns where table_schema=database() and table_name = 'x_group' and column_name='group_name' and column_key='UNI') then if not exists (select * from information_schema.table_constraints where table_schema=database() and table_name = 'x_group' and constraint_name='x_group_UK_group_name') then - ALTER IGNORE TABLE x_group ADD UNIQUE INDEX x_group_UK_group_name(group_name(767)); + ALTER TABLE x_group ADD UNIQUE INDEX x_group_UK_group_name(group_name(767)); -- ALTER TABLE x_group MODIFY COLUMN group_name varchar(767) NOT NULL, ADD CONSTRAINT x_group_UK_group_name UNIQUE(group_name(767)); end if; end if; @@ -32,7 +32,7 @@ DECLARE loginID bigint(20); if exists (select * from information_schema.columns where table_schema=database() and table_name = 'x_group_users' and column_name='user_id') then /* check unique constraint exist on group_name column or not */ if not exists (select * from information_schema.table_constraints where table_schema=database() and table_name = 'x_group_users' and constraint_name='x_group_users_UK_uid_gname') then - ALTER IGNORE TABLE x_group_users ADD UNIQUE INDEX x_group_users_UK_uid_gname(user_id,group_name(740)); + ALTER TABLE x_group_users ADD UNIQUE INDEX x_group_users_UK_uid_gname(user_id,group_name(740)); -- ALTER TABLE x_group_users MODIFY COLUMN group_name varchar(767), ADD CONSTRAINT x_group_users_UK_uid_gname UNIQUE(user_id,group_name(767)); end if; end if; From 0abed0efa16a94a46ca053b5e48f98d423c9f972 Mon Sep 17 00:00:00 2001 From: Sailaja Polavarapu Date: Tue, 26 Sep 2017 14:16:07 -0700 Subject: [PATCH 043/151] RANGER-1800: Usersync fails to update users and groups during incremental sync with nested groups and group first search enabled --- .../ldapusersync/process/LdapDeltaUserGroupBuilder.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/ugsync/src/main/java/org/apache/ranger/ldapusersync/process/LdapDeltaUserGroupBuilder.java b/ugsync/src/main/java/org/apache/ranger/ldapusersync/process/LdapDeltaUserGroupBuilder.java index 394bde291e..17682ba0c7 100644 --- a/ugsync/src/main/java/org/apache/ranger/ldapusersync/process/LdapDeltaUserGroupBuilder.java +++ b/ugsync/src/main/java/org/apache/ranger/ldapusersync/process/LdapDeltaUserGroupBuilder.java @@ -349,6 +349,7 @@ public void updateSink(UserGroupSink sink) throws Throwable { } List userList = new ArrayList<>(userSet); String transformGroupName = groupNameTransform(groupName); + LOG.debug("addOrUpdateGroup(): group = " + groupName + " users = " + userList); try { sink.addOrUpdateGroup(transformGroupName, userList); } catch (Throwable t) { @@ -751,8 +752,10 @@ private void getGroups(UserGroupSink sink) throws Throwable { } if (groupHierarchyLevels > 0) { + LOG.debug("deltaSyncGroupTime = " + deltaSyncGroupTime); if (deltaSyncGroupTime > 0) { - goUpGroupHierarchyLdap(groupNameMap.keySet(), groupHierarchyLevels-1); + LOG.info("LdapDeltaUserGroupBuilder.getGroups(): Going through group hierarchy for nested group evaluation for deltasync"); + goUpGroupHierarchyLdap(groupNameMap.keySet(), groupHierarchyLevels-1); } } @@ -942,7 +945,7 @@ private void goUpGroupHierarchyLdap(Set groupDNs, int groupHierarchyLeve } else { groupUserTable.put(gName, originalUserFullName, originalUserFullName); } - + groupNameMap.put(groupEntry.getNameInNamespace().toLowerCase(), gName); } LOG.info("No. of members in the group " + gName + " = " + userCount); } From bd0e82555df24f067f690ade6ee20c5b9e8354fb Mon Sep 17 00:00:00 2001 From: Sailaja Polavarapu Date: Tue, 26 Sep 2017 14:17:50 -0700 Subject: [PATCH 044/151] RANGER-1801: group user mapping updates to ranger admin fail when the mapping is already existed in ranger DB --- .../java/org/apache/ranger/biz/XUserMgr.java | 3 +++ .../org/apache/ranger/db/XXGroupUserDao.java | 18 ++++++++++++++++++ .../ranger/service/XGroupUserService.java | 14 ++++++++++++-- .../resources/META-INF/jpa_named_queries.xml | 5 +++++ 4 files changed, 38 insertions(+), 2 deletions(-) diff --git a/security-admin/src/main/java/org/apache/ranger/biz/XUserMgr.java b/security-admin/src/main/java/org/apache/ranger/biz/XUserMgr.java index 745030253b..abc0e0cd2a 100644 --- a/security-admin/src/main/java/org/apache/ranger/biz/XUserMgr.java +++ b/security-admin/src/main/java/org/apache/ranger/biz/XUserMgr.java @@ -576,6 +576,7 @@ public VXGroupUserInfo createXGroupUserFromMap( if (xUser != null) { // Add or update group user mapping only if the user already // exists in x_user table. + logger.debug(String.format("createXGroupUserFromMap(): Create or update group %s ", vXGroup.getName())); vXGroup = xGroupService.createXGroupWithOutLogin(vXGroup); vxGUInfo.setXgroupInfo(vXGroup); vxu.add(vXUser); @@ -585,6 +586,8 @@ public VXGroupUserInfo createXGroupUserFromMap( if (xXPortalUser.getUserSource() == RangerCommonEnums.USER_EXTERNAL) { vXGroupUser = xGroupUserService .createXGroupUserWithOutLogin(vXGroupUser); + logger.debug(String.format("createXGroupUserFromMap(): Create or update group user mapping with groupname = " + vXGroup.getName() + + " username = %s userId = %d", xXPortalUser.getLoginId(), xUser.getId())); } Collection reqRoleList = vXUser.getUserRoleList(); diff --git a/security-admin/src/main/java/org/apache/ranger/db/XXGroupUserDao.java b/security-admin/src/main/java/org/apache/ranger/db/XXGroupUserDao.java index 1a76d27db4..c8c105de47 100644 --- a/security-admin/src/main/java/org/apache/ranger/db/XXGroupUserDao.java +++ b/security-admin/src/main/java/org/apache/ranger/db/XXGroupUserDao.java @@ -27,6 +27,7 @@ import javax.persistence.NoResultException; +import org.apache.commons.lang.StringUtils; import org.apache.log4j.Logger; import org.apache.ranger.common.db.BaseDao; import org.apache.ranger.entity.XXGroupUser; @@ -114,4 +115,21 @@ public List findByGroupId(Long groupId) { } } + public XXGroupUser findByGroupNameAndUserId(String groupName, Long userId) { + if (StringUtils.isNotBlank(groupName) && userId != null) { + try { + return getEntityManager() + .createNamedQuery("XXGroupUser.findByGroupNameAndUserId", XXGroupUser.class) + .setParameter("userId", userId) + .setParameter("groupName", groupName) + .getSingleResult(); + } catch (NoResultException e) { + logger.debug(e.getMessage()); + } + } else { + logger.debug("userId and/or groupId not provided."); + return new XXGroupUser(); + } + return null; + } } diff --git a/security-admin/src/main/java/org/apache/ranger/service/XGroupUserService.java b/security-admin/src/main/java/org/apache/ranger/service/XGroupUserService.java index d1901d9c2a..8df205a0d1 100644 --- a/security-admin/src/main/java/org/apache/ranger/service/XGroupUserService.java +++ b/security-admin/src/main/java/org/apache/ranger/service/XGroupUserService.java @@ -83,7 +83,13 @@ protected void validateForUpdate(VXGroupUser vObj, XXGroupUser mObj) { } public VXGroupUser createXGroupUserWithOutLogin(VXGroupUser vxGroupUser) { - XXGroupUser xxGroupUser = new XXGroupUser(); + boolean groupUserMappingExists = true; + XXGroupUser xxGroupUser = daoManager.getXXGroupUser().findByGroupNameAndUserId(vxGroupUser.getName(), vxGroupUser.getUserId()); + if (xxGroupUser == null) { + xxGroupUser = new XXGroupUser(); + groupUserMappingExists = false; + } + XXGroup xGroup = daoManager.getXXGroup().findByGroupName(vxGroupUser.getName()); vxGroupUser.setParentGroupId(xGroup.getId()); xxGroupUser = mapViewToEntityBean(vxGroupUser, xxGroupUser, 0); @@ -92,7 +98,11 @@ public VXGroupUser createXGroupUserWithOutLogin(VXGroupUser vxGroupUser) { xxGroupUser.setAddedByUserId(createdByUserId); xxGroupUser.setUpdatedByUserId(createdByUserId); } - xxGroupUser = getDao().create(xxGroupUser); + if (groupUserMappingExists) { + xxGroupUser = getDao().update(xxGroupUser); + } else { + xxGroupUser = getDao().create(xxGroupUser); + } vxGroupUser = postCreate(xxGroupUser); return vxGroupUser; } diff --git a/security-admin/src/main/resources/META-INF/jpa_named_queries.xml b/security-admin/src/main/resources/META-INF/jpa_named_queries.xml index 68548a5da7..a212e59e43 100644 --- a/security-admin/src/main/resources/META-INF/jpa_named_queries.xml +++ b/security-admin/src/main/resources/META-INF/jpa_named_queries.xml @@ -189,6 +189,11 @@ + + SELECT obj FROM XXGroupUser obj WHERE obj.name=:groupName AND obj.userId=:userId + + + SELECT obj FROM XXTrxLog obj WHERE obj.transactionId = :transactionId From 90ed7025d9abfdd98b72349bf54b19957065a5e2 Mon Sep 17 00:00:00 2001 From: Bhavik Patel Date: Mon, 25 Sep 2017 12:31:16 +0530 Subject: [PATCH 045/151] RANGER-1748 : User is unable to update existing policy while importing policy from file Signed-off-by: pradeep --- .../org/apache/ranger/rest/ServiceREST.java | 81 ++++++++++++++++++- 1 file changed, 77 insertions(+), 4 deletions(-) diff --git a/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java b/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java index 5fa114d642..3703d1fa68 100644 --- a/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java +++ b/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java @@ -2018,21 +2018,34 @@ public void importPoliciesFromFile( } } String updateIfExists = request.getParameter(PARAM_UPDATE_IF_EXISTS); + String polResource = request.getParameter(SearchFilter.POL_RESOURCE); if (updateIfExists == null || updateIfExists.isEmpty()) { updateIfExists = "false"; } else if (updateIfExists.equalsIgnoreCase("true")) { isOverride = false; } - if (isOverride && updateIfExists.equalsIgnoreCase("false")){ + if (isOverride && "false".equalsIgnoreCase(updateIfExists) && StringUtils.isEmpty(polResource)) { if (LOG.isDebugEnabled()) { LOG.debug("Deleting Policy from provided services in servicesMapJson file..."); } - if (CollectionUtils.isNotEmpty(sourceServices) && CollectionUtils.isNotEmpty(destinationServices)){ - deletePoliciesProvidedInServiceMap(sourceServices, - destinationServices, null); + if (CollectionUtils.isNotEmpty(sourceServices) + && CollectionUtils.isNotEmpty(destinationServices)) { + deletePoliciesProvidedInServiceMap(sourceServices, destinationServices, null); } } + if ("true".equalsIgnoreCase(updateIfExists) && StringUtils.isNotEmpty(polResource)) { + if (LOG.isDebugEnabled()) { + LOG.debug( + "Deleting Policy from provided services in servicesMapJson file for specific resource..."); + } + if (CollectionUtils.isNotEmpty(sourceServices) + && CollectionUtils.isNotEmpty(destinationServices)) { + deletePoliciesForResource(sourceServices, destinationServices, polResource, request, + policies); + } + } + if (policies != null && !CollectionUtils.sizeIsEmpty(policies)){ for (RangerPolicy policyInJson: policies){ if (policyInJson != null){ @@ -2264,6 +2277,66 @@ private void deletePoliciesProvidedInServiceMap( } } + private void deletePoliciesForResource(List sourceServices, List destinationServices, + String resource, HttpServletRequest request, List exportPolicies) { + int totalDeletedPilicies = 0; + if (CollectionUtils.isNotEmpty(sourceServices) && CollectionUtils.isNotEmpty(destinationServices)) { + Set exportedPolicyNames = new HashSet(); + if (CollectionUtils.isNotEmpty(exportPolicies)) { + for (RangerPolicy rangerPolicy : exportPolicies) { + if (rangerPolicy != null) { + exportedPolicyNames.add(rangerPolicy.getName()); + } + } + } + for (int i = 0; i < sourceServices.size(); i++) { + if (!destinationServices.get(i).isEmpty()) { + RangerPolicyList servicePolicies = null; + servicePolicies = getServicePoliciesByName(destinationServices.get(i), request); + if (servicePolicies != null) { + List rangerPolicyList = servicePolicies.getPolicies(); + if (CollectionUtils.isNotEmpty(rangerPolicyList)) { + for (RangerPolicy rangerPolicy : rangerPolicyList) { + if (rangerPolicy != null) { + Map rangerPolicyResourceMap = rangerPolicy + .getResources(); + if (rangerPolicyResourceMap != null) { + RangerPolicy.RangerPolicyResource rangerPolicyResource = null; + if (rangerPolicyResourceMap.containsKey("path")) { + rangerPolicyResource = rangerPolicyResourceMap.get("path"); + } else if (rangerPolicyResourceMap.containsKey("database")) { + rangerPolicyResource = rangerPolicyResourceMap.get("database"); + } + if (rangerPolicyResource != null) { + if (CollectionUtils.isNotEmpty(rangerPolicyResource.getValues()) + && rangerPolicyResource.getValues().size() > 1) { + continue; + } + } + } + if (rangerPolicy.getId() != null) { + if (!exportedPolicyNames.contains(rangerPolicy.getName())) { + deletePolicy(rangerPolicy.getId()); + if (LOG.isDebugEnabled()) { + LOG.debug( + "Policy " + rangerPolicy.getName() + " deleted successfully."); + } + totalDeletedPilicies = totalDeletedPilicies + 1; + } + } + } + } + } + } + } + } + } + if (LOG.isDebugEnabled()) { + LOG.debug("Total Deleted Policy : " + totalDeletedPilicies); + } + } + + public List getPolicies(SearchFilter filter) { if(LOG.isDebugEnabled()) { LOG.debug("==> ServiceREST.getPolicies(filter)"); From 78ffe3f534398e0a4127827ad73671c136581b71 Mon Sep 17 00:00:00 2001 From: fatimaawez Date: Tue, 26 Sep 2017 19:05:53 +0530 Subject: [PATCH 046/151] RANGER-1806 : Good coding practice in Ranger recommended by static code analysis Signed-off-by: Mehul Parikh --- .../cliutil/RoleBasedUserSearchUtil.java | 353 +++++++++--------- .../cliutil/TestRoleBasedUserSearchUtil.java | 6 +- 2 files changed, 179 insertions(+), 180 deletions(-) diff --git a/security-admin/src/main/java/org/apache/ranger/patch/cliutil/RoleBasedUserSearchUtil.java b/security-admin/src/main/java/org/apache/ranger/patch/cliutil/RoleBasedUserSearchUtil.java index 0459be68e2..d3a28f7660 100644 --- a/security-admin/src/main/java/org/apache/ranger/patch/cliutil/RoleBasedUserSearchUtil.java +++ b/security-admin/src/main/java/org/apache/ranger/patch/cliutil/RoleBasedUserSearchUtil.java @@ -23,6 +23,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.MapUtils; @@ -63,59 +64,59 @@ public class RoleBasedUserSearchUtil extends BaseLoader { public static String userRole = ""; public static void main(String[] args) { - logger.info("RoleBaseUserSearchUtil : main()"); - try { - RoleBasedUserSearchUtil loader = (RoleBasedUserSearchUtil) CLIUtil.getBean(RoleBasedUserSearchUtil.class); - loader.init(); - if (args.length == 3 || args.length == 2) { - userLoginId = args[0]; - currentPassword = args[1]; - if (args.length == 3) { - userRole = args[2]; - List roles = new ArrayList(); - roles.add(RangerConstants.ROLE_USER); - roles.add(RangerConstants.ROLE_SYS_ADMIN); - roles.add(RangerConstants.ROLE_KEY_ADMIN); - if (!StringUtils.isBlank(userRole)) { - userRole = userRole.toUpperCase(); - if (!roles.contains(userRole)) { - System.out.println("Invalid UserRole. Exiting!!!"); - logger.info("Invalid UserRole. Exiting!!!"); - System.exit(1); - } else { - checkRole = false; - } - } - } - if (StringUtils.isBlank(userLoginId)) { - System.out.println("Invalid login ID. Exiting!!!"); - logger.info("Invalid login ID. Exiting!!!"); - System.exit(1); - } - if (StringUtils.isBlank(currentPassword)) { - System.out.println("Invalid current password. Exiting!!!"); - logger.info("Invalid current password. Exiting!!!"); - System.exit(1); - } - while (loader.isMoreToProcess()) { - loader.load(); - } - logger.info("Load complete. Exiting!!!"); - System.exit(0); - } else { - System.out.println("RoleBaseUserSearchUtil: Incorrect Arguments \n Usage: \n "); - logger.error("RoleBaseUserSearchUtil: Incorrect Arguments \n Usage: \n "); - System.exit(1); - } - } catch (Exception e) { - logger.error("Error loading", e); + logger.info("RoleBaseUserSearchUtil : main()"); + try { + RoleBasedUserSearchUtil loader = (RoleBasedUserSearchUtil) CLIUtil.getBean(RoleBasedUserSearchUtil.class); + loader.init(); + if (args.length == 3 || args.length == 2) { + userLoginId = args[0]; + currentPassword = args[1]; + if (args.length == 3) { + userRole = args[2]; + List roles = new ArrayList(); + roles.add(RangerConstants.ROLE_USER); + roles.add(RangerConstants.ROLE_SYS_ADMIN); + roles.add(RangerConstants.ROLE_KEY_ADMIN); + if (!StringUtils.isBlank(userRole)) { + userRole = userRole.toUpperCase(); + if (!roles.contains(userRole)) { + System.out.println("Invalid UserRole. Exiting!!!"); + logger.info("Invalid UserRole. Exiting!!!"); + System.exit(1); + } else { + checkRole = false; + } + } + } + if (StringUtils.isBlank(userLoginId)) { + System.out.println("Invalid login ID. Exiting!!!"); + logger.info("Invalid login ID. Exiting!!!"); + System.exit(1); + } + if (StringUtils.isBlank(currentPassword)) { + System.out.println("Invalid current password. Exiting!!!"); + logger.info("Invalid current password. Exiting!!!"); System.exit(1); - } + } + while (loader.isMoreToProcess()) { + loader.load(); + } + logger.info("Load complete. Exiting!!!"); + System.exit(0); + } else { + System.out.println("RoleBaseUserSearchUtil: Incorrect Arguments \n Usage: \n "); + logger.error("RoleBaseUserSearchUtil: Incorrect Arguments \n Usage: \n "); + System.exit(1); + } + } catch (Exception e) { + logger.error("Error loading", e); + System.exit(1); + } } @Override public void init() throws Exception { - logger.info("==> RoleBaseUserSearchUtil.init()"); + logger.info("==> RoleBaseUserSearchUtil.init()"); } @Override @@ -124,148 +125,146 @@ public void printStats() { @Override public void execLoad() { - logger.info("==> RoleBaseUserSearchUtil.execLoad()"); - validateUserAndFetchUserList(); - logger.info("<== RoleBaseUserSearchUtil.execLoad()"); + logger.info("==> RoleBaseUserSearchUtil.execLoad()"); + validateUserAndFetchUserList(); + logger.info("<== RoleBaseUserSearchUtil.execLoad()"); } public void getUsersBasedOnRole(List userRoleList) { - try { - if (!CollectionUtils.isEmpty(userRoleList) && userRoleList != null) { - Map roleSysAdminMap = new HashMap(); - Map roleKeyAdminMap = new HashMap(); - Map roleUserMap = new HashMap(); - for (String userRole : userRoleList) { - List listXXPortalUser = daoMgr.getXXPortalUser().findByRole(userRole); - if (listXXPortalUser != null && !CollectionUtils.isEmpty(listXXPortalUser)) { - if (userRole.equalsIgnoreCase(RangerConstants.ROLE_SYS_ADMIN)) { - for (XXPortalUser xXPortalUser : listXXPortalUser) { - roleSysAdminMap.put(xXPortalUser.getLoginId(),userRole); - } - } else if (userRole.equalsIgnoreCase(RangerConstants.ROLE_KEY_ADMIN)) { - for (XXPortalUser xXPortalUser : listXXPortalUser) { - roleKeyAdminMap.put(xXPortalUser.getLoginId(),userRole); - } - } else if (userRole.equalsIgnoreCase(RangerConstants.ROLE_USER)) { - for (XXPortalUser xXPortalUser : listXXPortalUser) { - roleUserMap.put(xXPortalUser.getLoginId(),userRole); - } - } - } + try { + if (!CollectionUtils.isEmpty(userRoleList) && userRoleList != null) { + Map roleSysAdminMap = new HashMap(); + Map roleKeyAdminMap = new HashMap(); + Map roleUserMap = new HashMap(); + for (String userRole : userRoleList) { + List listXXPortalUser = daoMgr.getXXPortalUser().findByRole(userRole); + if (listXXPortalUser != null && !CollectionUtils.isEmpty(listXXPortalUser)) { + if (userRole.equalsIgnoreCase(RangerConstants.ROLE_SYS_ADMIN)) { + for (XXPortalUser xXPortalUser : listXXPortalUser) { + roleSysAdminMap.put(xXPortalUser.getLoginId(),userRole); + } + } else if (userRole.equalsIgnoreCase(RangerConstants.ROLE_KEY_ADMIN)) { + for (XXPortalUser xXPortalUser : listXXPortalUser) { + roleKeyAdminMap.put(xXPortalUser.getLoginId(),userRole); } - if (MapUtils.isEmpty( roleSysAdminMap) && MapUtils.isEmpty(roleKeyAdminMap) && MapUtils.isEmpty(roleUserMap)) { - System.out.println("users with given user role are not there"); - logger.error("users with given user role are not there"); - System.exit(1); - } else { - if (!MapUtils.isEmpty(roleSysAdminMap)) { - for (String key : roleSysAdminMap.keySet()) { - System.out.println(roleSysAdminMap.get(key) + " : " + key); - } - } - if (!MapUtils.isEmpty(roleKeyAdminMap)) { - for (String key : roleKeyAdminMap.keySet()) { - System.out.println(roleKeyAdminMap.get(key) + " : " + key); - } - } - if (!MapUtils.isEmpty(roleUserMap)) { - for (String key : roleUserMap.keySet()) { - System.out.println(roleUserMap.get(key) + " : " + key); - } - } - if (userRoleList.contains(RangerConstants.ROLE_SYS_ADMIN)) { - System.out.println("ROLE_SYS_ADMIN Total Count : " + roleSysAdminMap.size()); - } - if (userRoleList.contains(RangerConstants.ROLE_KEY_ADMIN)) { - System.out.println("ROLE_KEY_ADMIN Total Count : " + roleKeyAdminMap.size()); - } - if (userRoleList.contains(RangerConstants.ROLE_USER)) { - System.out.println("ROLE_USER Total Count : " + roleUserMap.size()); - } - int total = roleSysAdminMap.size() + roleKeyAdminMap.size() + roleUserMap.size(); - System.out.println("Total Count : " + total); + } else if (userRole.equalsIgnoreCase(RangerConstants.ROLE_USER)) { + for (XXPortalUser xXPortalUser : listXXPortalUser) { + roleUserMap.put(xXPortalUser.getLoginId(),userRole); } + } + } + } + if (MapUtils.isEmpty( roleSysAdminMap) && MapUtils.isEmpty(roleKeyAdminMap) && MapUtils.isEmpty(roleUserMap)) { + System.out.println("users with given user role are not there"); + logger.error("users with given user role are not there"); + System.exit(1); + } else { + if (!MapUtils.isEmpty(roleSysAdminMap)) { + for(Entry entry : roleSysAdminMap.entrySet()){ + System.out.println(entry.getValue() + " : " + entry.getKey()); + } + } + if (!MapUtils.isEmpty(roleKeyAdminMap)) { + for(Entry entry : roleKeyAdminMap.entrySet()){ + System.out.println(entry.getValue() + " : " + entry.getKey()); + } } - - } catch (Exception e) { - logger.error("Error getting User's List with the mentioned role: "+ e.getMessage()); - } + if (!MapUtils.isEmpty(roleUserMap)) { + for(Entry entry : roleUserMap.entrySet()){ + System.out.println(entry.getValue() + " : " + entry.getKey()); + } + } + if (userRoleList.contains(RangerConstants.ROLE_SYS_ADMIN)) { + System.out.println("ROLE_SYS_ADMIN Total Count : " + roleSysAdminMap.size()); + } + if (userRoleList.contains(RangerConstants.ROLE_KEY_ADMIN)) { + System.out.println("ROLE_KEY_ADMIN Total Count : " + roleKeyAdminMap.size()); + } + if (userRoleList.contains(RangerConstants.ROLE_USER)) { + System.out.println("ROLE_USER Total Count : " + roleUserMap.size()); + } + int total = roleSysAdminMap.size() + roleKeyAdminMap.size() + roleUserMap.size(); + System.out.println("Total Count : " + total); + } + } + + } catch (Exception e) { + logger.error("Error getting User's List with the mentioned role: "+ e.getMessage()); + } } public void validateUserAndFetchUserList() { - userLoginId = userLoginId.toLowerCase(); - XXPortalUser xxPortalUser = daoMgr.getXXPortalUser().findByLoginId( - userLoginId); - Boolean isUserAuthorized = false; - if (xxPortalUser != null) { - String dbPassword = xxPortalUser.getPassword(); - String currentEncryptedPassword = null; - try { - currentEncryptedPassword = userMgr.encrypt(userLoginId,currentPassword); - if (currentEncryptedPassword != null && currentEncryptedPassword.equals(dbPassword)) { - VXUser vxUser = xUserService.getXUserByUserName(xxPortalUser.getLoginId()); - if (vxUser != null) { - List existingRole = (List) vxUser.getUserRoleList(); - List permissionList = daoMgr.getXXModuleDef().findAccessibleModulesByUserId(xxPortalUser.getId(), vxUser.getId()); - if (permissionList != null && permissionList.contains(RangerConstants.MODULE_USER_GROUPS) && !CollectionUtils.isEmpty(existingRole) && !StringUtils.isBlank(existingRole.get(0))) { - List userRoleList = new ArrayList(); - if (existingRole.get(0).equalsIgnoreCase(RangerConstants.ROLE_USER)) { - userRoleList.add(RangerConstants.ROLE_USER); - if (checkRole) { - getUsersBasedOnRole(userRoleList); - } else if (existingRole.get(0).equalsIgnoreCase(userRole) || userRole.equalsIgnoreCase(RangerConstants.ROLE_USER)) { - getUsersBasedOnRole(userRoleList); - } else { - isUserAuthorized = true; - } - } else if (existingRole.get(0).equalsIgnoreCase(RangerConstants.ROLE_SYS_ADMIN)) { - if (checkRole) { - userRoleList.add(RangerConstants.ROLE_SYS_ADMIN); - userRoleList.add(RangerConstants.ROLE_USER); - getUsersBasedOnRole(userRoleList); - } else if (existingRole.get(0).equalsIgnoreCase(userRole) || userRole.equalsIgnoreCase(RangerConstants.ROLE_USER)) { - userRoleList.add(userRole); - getUsersBasedOnRole(userRoleList); - } else { - isUserAuthorized = true; - } - } else if (existingRole.get(0).equalsIgnoreCase(RangerConstants.ROLE_KEY_ADMIN) || userRole.equalsIgnoreCase(RangerConstants.ROLE_USER)) { - if (checkRole) { - userRoleList.add(RangerConstants.ROLE_KEY_ADMIN); - userRoleList.add(RangerConstants.ROLE_USER); - getUsersBasedOnRole(userRoleList); - } else if (existingRole.get(0).equalsIgnoreCase(userRole) || userRole.equalsIgnoreCase(RangerConstants.ROLE_USER)) { - userRoleList.add(userRole); - getUsersBasedOnRole(userRoleList); - } else { - isUserAuthorized = true; - } - - } - if (isUserAuthorized == true) { - System.out.println("user is not authorized to fetch this list"); - logger.error("user is not authorized to fetch this list"); - System.exit(1); - } - } else { - System.out.println("user permission denied"); - logger.error("user permission denied"); - System.exit(1); - } - } - } else { - System.out.println("Invalid user password"); - logger.error("Invalid user password"); - System.exit(1); + userLoginId = userLoginId.toLowerCase(); + XXPortalUser xxPortalUser = daoMgr.getXXPortalUser().findByLoginId(userLoginId); + Boolean isUserAuthorized = false; + if (xxPortalUser != null) { + String dbPassword = xxPortalUser.getPassword(); + String currentEncryptedPassword = null; + try { + currentEncryptedPassword = userMgr.encrypt(userLoginId,currentPassword); + if (currentEncryptedPassword != null && currentEncryptedPassword.equals(dbPassword)) { + VXUser vxUser = xUserService.getXUserByUserName(xxPortalUser.getLoginId()); + if (vxUser != null) { + List existingRole = (List) vxUser.getUserRoleList(); + List permissionList = daoMgr.getXXModuleDef().findAccessibleModulesByUserId(xxPortalUser.getId(), vxUser.getId()); + if (permissionList != null && permissionList.contains(RangerConstants.MODULE_USER_GROUPS) && !CollectionUtils.isEmpty(existingRole) && !StringUtils.isBlank(existingRole.get(0))) { + List userRoleList = new ArrayList(); + if (existingRole.get(0).equalsIgnoreCase(RangerConstants.ROLE_USER)) { + userRoleList.add(RangerConstants.ROLE_USER); + if (checkRole) { + getUsersBasedOnRole(userRoleList); + } else if (existingRole.get(0).equalsIgnoreCase(userRole) || userRole.equalsIgnoreCase(RangerConstants.ROLE_USER)) { + getUsersBasedOnRole(userRoleList); + } else { + isUserAuthorized = true; + } + } else if (existingRole.get(0).equalsIgnoreCase(RangerConstants.ROLE_SYS_ADMIN)) { + if (checkRole) { + userRoleList.add(RangerConstants.ROLE_SYS_ADMIN); + userRoleList.add(RangerConstants.ROLE_USER); + getUsersBasedOnRole(userRoleList); + } else if (existingRole.get(0).equalsIgnoreCase(userRole) || userRole.equalsIgnoreCase(RangerConstants.ROLE_USER)) { + userRoleList.add(userRole); + getUsersBasedOnRole(userRoleList); + } else { + isUserAuthorized = true; + } + } else if (existingRole.get(0).equalsIgnoreCase(RangerConstants.ROLE_KEY_ADMIN) || userRole.equalsIgnoreCase(RangerConstants.ROLE_USER)) { + if (checkRole) { + userRoleList.add(RangerConstants.ROLE_KEY_ADMIN); + userRoleList.add(RangerConstants.ROLE_USER); + getUsersBasedOnRole(userRoleList); + } else if (existingRole.get(0).equalsIgnoreCase(userRole) || userRole.equalsIgnoreCase(RangerConstants.ROLE_USER)) { + userRoleList.add(userRole); + getUsersBasedOnRole(userRoleList); + } else { + isUserAuthorized = true; + } } - } catch (Exception e) { - logger.error("Getting User's List with the mentioned role failure. Detail: \n",e); + if (isUserAuthorized == true) { + System.out.println("user is not authorized to fetch this list"); + logger.error("user is not authorized to fetch this list"); + System.exit(1); + } + } else { + System.out.println("user permission denied"); + logger.error("user permission denied"); System.exit(1); + } } - } else { - System.out.println("User does not exist in DB!!"); - logger.error("User does not exist in DB"); + } else { + System.out.println("Invalid user password"); + logger.error("Invalid user password"); System.exit(1); + } + } catch (Exception e) { + logger.error("Getting User's List with the mentioned role failure. Detail: \n",e); + System.exit(1); } + } else { + System.out.println("User does not exist in DB!!"); + logger.error("User does not exist in DB"); + System.exit(1); + } } } \ No newline at end of file diff --git a/security-admin/src/test/java/org/apache/ranger/patch/cliutil/TestRoleBasedUserSearchUtil.java b/security-admin/src/test/java/org/apache/ranger/patch/cliutil/TestRoleBasedUserSearchUtil.java index 83eab7aedd..891ac10630 100644 --- a/security-admin/src/test/java/org/apache/ranger/patch/cliutil/TestRoleBasedUserSearchUtil.java +++ b/security-admin/src/test/java/org/apache/ranger/patch/cliutil/TestRoleBasedUserSearchUtil.java @@ -86,7 +86,7 @@ public void TestGetUsersBasedOnRole() { roleBasedUserSearchUtil.getUsersBasedOnRole(userRoleList); - Mockito.verify(daoMgr).getXXPortalUser(); + Mockito.verify(xXPortalUserDao).findByRole(RangerConstants.ROLE_SYS_ADMIN); } catch(Exception e) { @@ -126,10 +126,10 @@ public void TestValidateUserAndFetchUserList() { Mockito.when(xXPortalUserDao.findByRole(Mockito.anyString())).thenReturn(listXXPortalUser); roleBasedUserSearchUtil.validateUserAndFetchUserList(); - Mockito.verify(daoMgr, Mockito.atLeast(2)).getXXPortalUser(); + Mockito.verify(xXPortalUserDao).findByLoginId(Mockito.anyString()); Mockito.verify(xUserService).getXUserByUserName(xxPortalUser.getLoginId()); - Mockito.verify(daoMgr).getXXModuleDef(); + Mockito.verify(xXModuleDefDao).findAccessibleModulesByUserId(Mockito.anyLong(), Mockito.anyLong()); Mockito.verify(userMgr).encrypt(Mockito.anyString(),Mockito.anyString()); Mockito.verify(xXPortalUserDao, Mockito.atLeast(2)).findByRole(Mockito.anyString()); From 6134db8c821daccacb6df0035ed26523b5fb1e5f Mon Sep 17 00:00:00 2001 From: fatimaawez Date: Tue, 26 Sep 2017 14:59:41 +0530 Subject: [PATCH 047/151] RANGER-1727 : Ranger allows user to change an external user's password with 'null' old password Signed-off-by: Velmurugan Periasamy --- .../java/org/apache/ranger/biz/UserMgr.java | 41 +++++++++++++++---- .../java/org/apache/ranger/biz/XUserMgr.java | 18 ++++++-- .../org/apache/ranger/biz/TestXUserMgr.java | 4 ++ 3 files changed, 52 insertions(+), 11 deletions(-) diff --git a/security-admin/src/main/java/org/apache/ranger/biz/UserMgr.java b/security-admin/src/main/java/org/apache/ranger/biz/UserMgr.java index 94a087cf9f..35d9b410e9 100644 --- a/security-admin/src/main/java/org/apache/ranger/biz/UserMgr.java +++ b/security-admin/src/main/java/org/apache/ranger/biz/UserMgr.java @@ -401,7 +401,13 @@ public VXResponse changePassword(VXPasswordChange pwdChange) { logger.warn("SECURITY:changePassword(). User not found. LoginId="+ pwdChange.getLoginId()); throw restErrorUtil.createRESTException("serverMsg.userMgrInvalidUser",MessageEnums.DATA_NOT_FOUND, null, null,pwdChange.getLoginId()); } - + if (gjUser.getUserSource() == RangerCommonEnums.USER_EXTERNAL) { + logger.info("SECURITY:changePassword().Ranger External Users cannot change password. LoginId=" + pwdChange.getLoginId()); + VXResponse vXResponse = new VXResponse(); + vXResponse.setStatusCode(HttpServletResponse.SC_FORBIDDEN); + vXResponse.setMsgDesc("SECURITY:changePassword().Ranger External Users cannot change password. LoginId=" + pwdChange.getLoginId()); + throw restErrorUtil.generateRESTException(vXResponse); + } //check current password and provided old password is same or not String encryptedOldPwd = encrypt(pwdChange.getLoginId(),pwdChange.getOldPassword()); if (!stringUtil.equals(encryptedOldPwd, gjUser.getPassword())) { @@ -484,9 +490,12 @@ public VXPortalUser changeEmailAddress(XXPortalUser gjUser, String saltEncodedpasswd = encrypt(gjUser.getLoginId(), changeEmail.getOldPassword()); - + if (gjUser.getUserSource() == RangerCommonEnums.USER_APP) { gjUser.setPassword(saltEncodedpasswd); - + } + else if (gjUser.getUserSource() == RangerCommonEnums.USER_EXTERNAL) { + gjUser.setPassword(gjUser.getPassword()); + } daoManager.getXXPortalUser().update(gjUser); return mapXXPortalUserVXPortalUser(gjUser); } @@ -1243,7 +1252,7 @@ public boolean isUserInRole(Long userId, String role) { public XXPortalUser updateUserWithPass(VXPortalUser userProfile) { String updatedPassword = userProfile.getPassword(); - XXPortalUser xXPortalUser = this.updateUser(userProfile); + XXPortalUser xXPortalUser = this.updateUser(userProfile); if (xXPortalUser == null) { return null; @@ -1264,8 +1273,13 @@ public XXPortalUser updateUserWithPass(VXPortalUser userProfile) { String encryptedNewPwd = encrypt(xXPortalUser.getLoginId(), updatedPassword); - xXPortalUser.setPassword(encryptedNewPwd); - xXPortalUser = daoManager.getXXPortalUser().update(xXPortalUser); + if (xXPortalUser.getUserSource() != RangerCommonEnums.USER_EXTERNAL) { + xXPortalUser.setPassword(encryptedNewPwd); + } + else if (xXPortalUser.getUserSource() != RangerCommonEnums.USER_EXTERNAL) { + xXPortalUser.setPassword(xXPortalUser.getPassword()); + } + xXPortalUser = daoManager.getXXPortalUser().update(xXPortalUser); } return xXPortalUser; } @@ -1283,7 +1297,13 @@ public XXPortalUser updatePasswordInSHA256(String userName,String userPassword,b } String dbOldPwd =xXPortalUser.getPassword(); String encryptedNewPwd = encrypt(xXPortalUser.getLoginId(),userPassword); - xXPortalUser.setPassword(encryptedNewPwd); + if (xXPortalUser.getUserSource() != RangerCommonEnums.USER_EXTERNAL) { + xXPortalUser.setPassword(encryptedNewPwd); + } + else if (xXPortalUser.getUserSource() != RangerCommonEnums.USER_EXTERNAL) { + xXPortalUser.setPassword(xXPortalUser.getPassword()); + } + xXPortalUser = daoManager.getXXPortalUser().update(xXPortalUser); if(xXPortalUser!=null && logAudits){ String dbNewPwd=xXPortalUser.getPassword(); @@ -1360,7 +1380,12 @@ public XXPortalUser updateOldUserName(String userLoginId,String newUserName, Str xXPortalUser.setLoginId(newUserName); // The old password needs to be encrypted by the new user name String updatedPwd = encrypt(newUserName,currentPassword); - xXPortalUser.setPassword(updatedPwd); + if (xXPortalUser.getUserSource() == RangerCommonEnums.USER_APP) { + xXPortalUser.setPassword(updatedPwd); + } + else if (xXPortalUser.getUserSource() == RangerCommonEnums.USER_EXTERNAL) { + xXPortalUser.setPassword(xXPortalUser.getPassword()); + } xXPortalUser = daoManager.getXXPortalUser().update(xXPortalUser); List trxLogList = new ArrayList(); XXTrxLog xTrxLog = new XXTrxLog(); diff --git a/security-admin/src/main/java/org/apache/ranger/biz/XUserMgr.java b/security-admin/src/main/java/org/apache/ranger/biz/XUserMgr.java index abc0e0cd2a..8d3b751f7f 100644 --- a/security-admin/src/main/java/org/apache/ranger/biz/XUserMgr.java +++ b/security-admin/src/main/java/org/apache/ranger/biz/XUserMgr.java @@ -386,10 +386,16 @@ public VXUser updateXUser(VXUser vXUser) { && password.equals(hiddenPasswordString)) { vXPortalUser.setPassword(oldUserProfile.getPassword()); } - else if(password != null){ - validatePassword(vXUser); - vXPortalUser.setPassword(password); + else if(password != null){ + validatePassword(vXUser); + if (oldUserProfile.getUserSource() == RangerCommonEnums.USER_EXTERNAL) { + vXPortalUser.setPassword(oldUserProfile.getPassword()); + } + else if(oldUserProfile.getUserSource() == RangerCommonEnums.USER_APP) + { + vXPortalUser.setPassword(password); } + } Collection groupIdList = vXUser.getGroupIdList(); XXPortalUser xXPortalUser = new XXPortalUser(); xXPortalUser = userMgr.updateUserWithPass(vXPortalUser); @@ -441,7 +447,13 @@ else if(password != null){ // There is nothing to log anything in XXUser so far. vXUser = xUserService.updateResource(vXUser); vXUser.setUserRoleList(roleList); + if (oldUserProfile.getUserSource() == RangerCommonEnums.USER_APP) { vXUser.setPassword(password); + } + else if (oldUserProfile.getUserSource() == RangerCommonEnums.USER_EXTERNAL) { + vXUser.setPassword(oldUserProfile.getPassword()); + } + List trxLogList = xUserService.getTransactionLog(vXUser, oldUserProfile, "update"); vXUser.setPassword(hiddenPasswordString); diff --git a/security-admin/src/test/java/org/apache/ranger/biz/TestXUserMgr.java b/security-admin/src/test/java/org/apache/ranger/biz/TestXUserMgr.java index 88984b5edc..b6ef5725c7 100644 --- a/security-admin/src/test/java/org/apache/ranger/biz/TestXUserMgr.java +++ b/security-admin/src/test/java/org/apache/ranger/biz/TestXUserMgr.java @@ -353,7 +353,11 @@ public void test11CreateXUser() { public void test12UpdateXUser() { setup(); VXUser vxUser = vxUser(); + vxUser.setUserSource(RangerCommonEnums.USER_APP); + vxUser.setName("name"); Mockito.when(xUserService.updateResource(vxUser)).thenReturn(vxUser); + VXPortalUser vXPortalUser = new VXPortalUser(); + Mockito.when(userMgr.getUserProfileByLoginId(vxUser.getName())).thenReturn(vXPortalUser); VXUser dbvxUser = xUserMgr.updateXUser(vxUser); Assert.assertNotNull(dbvxUser); From 997d7c3d0c87b842b77eead7ca32dd382fe24150 Mon Sep 17 00:00:00 2001 From: Nikhil P Date: Wed, 27 Sep 2017 15:27:26 +0530 Subject: [PATCH 048/151] RANGER-1779 : last resource gets duplicated during update policy if policy is created through public api rest call Signed-off-by: Mehul Parikh --- .../org/apache/ranger/biz/ServiceDBStore.java | 24 ++++++++++++------- ...hForNifiResourceUpdateExclude_J10011.java} | 12 ++++++---- 2 files changed, 23 insertions(+), 13 deletions(-) rename security-admin/src/main/java/org/apache/ranger/patch/{PatchForNifiResourceUpdateExclude_J10008.java => PatchForNifiResourceUpdateExclude_J10011.java} (93%) diff --git a/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java b/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java index 63fdf4f1bc..5f5f239a5e 100644 --- a/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java +++ b/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java @@ -27,6 +27,7 @@ import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashMap; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -2925,15 +2926,20 @@ private void createNewResourcesForPolicy(RangerPolicy policy, XXPolicy xPolicy, xPolRes = daoMgr.getXXPolicyResource().create(xPolRes); List values = policyRes.getValues(); - if(CollectionUtils.isNotEmpty(values)){ - for(int i = 0; i < values.size(); i++) { - if(values.get(i)!=null){ - XXPolicyResourceMap xPolResMap = new XXPolicyResourceMap(); - xPolResMap = (XXPolicyResourceMap) rangerAuditFields.populateAuditFields(xPolResMap, xPolRes); - xPolResMap.setResourceId(xPolRes.getId()); - xPolResMap.setValue(values.get(i)); - xPolResMap.setOrder(i); - xPolResMap = daoMgr.getXXPolicyResourceMap().create(xPolResMap); + if (CollectionUtils.isNotEmpty(values)) { + Set uniqueValues = new LinkedHashSet(values); + int i = 0; + if (CollectionUtils.isNotEmpty(uniqueValues)) { + for (String uniqValue : uniqueValues) { + if (!StringUtils.isEmpty(uniqValue)) { + XXPolicyResourceMap xPolResMap = new XXPolicyResourceMap(); + xPolResMap = (XXPolicyResourceMap) rangerAuditFields.populateAuditFields(xPolResMap,xPolRes); + xPolResMap.setResourceId(xPolRes.getId()); + xPolResMap.setValue(uniqValue); + xPolResMap.setOrder(i); + xPolResMap = daoMgr.getXXPolicyResourceMap().create(xPolResMap); + i++; + } } } } diff --git a/security-admin/src/main/java/org/apache/ranger/patch/PatchForNifiResourceUpdateExclude_J10008.java b/security-admin/src/main/java/org/apache/ranger/patch/PatchForNifiResourceUpdateExclude_J10011.java similarity index 93% rename from security-admin/src/main/java/org/apache/ranger/patch/PatchForNifiResourceUpdateExclude_J10008.java rename to security-admin/src/main/java/org/apache/ranger/patch/PatchForNifiResourceUpdateExclude_J10011.java index 634082c049..0025eb89ba 100644 --- a/security-admin/src/main/java/org/apache/ranger/patch/PatchForNifiResourceUpdateExclude_J10008.java +++ b/security-admin/src/main/java/org/apache/ranger/patch/PatchForNifiResourceUpdateExclude_J10011.java @@ -39,9 +39,14 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; +/** + * Disables the Nifi plugin's exclude toggle in Ranger UI. + * After running this patch user wont be able to add exclude resource policies in NIFI. + * + */ @Component -public class PatchForNifiResourceUpdateExclude_J10008 extends BaseLoader { - private static final Logger logger = Logger.getLogger(PatchForHiveServiceDefUpdate_J10006.class); +public class PatchForNifiResourceUpdateExclude_J10011 extends BaseLoader { + private static final Logger logger = Logger.getLogger(PatchForNifiResourceUpdateExclude_J10011.class); @Autowired RangerDaoManager daoMgr; @@ -66,7 +71,7 @@ public class PatchForNifiResourceUpdateExclude_J10008 extends BaseLoader { public static void main(String[] args) { logger.info("main()"); try { - PatchForNifiResourceUpdateExclude_J10008 loader = (PatchForNifiResourceUpdateExclude_J10008) CLIUtil.getBean(PatchForNifiResourceUpdateExclude_J10008.class); + PatchForNifiResourceUpdateExclude_J10011 loader = (PatchForNifiResourceUpdateExclude_J10011) CLIUtil.getBean(PatchForNifiResourceUpdateExclude_J10011.class); loader.init(); while (loader.isMoreToProcess()) { loader.load(); @@ -135,7 +140,6 @@ private void updateNifiServiceDef(){ } if (ret == null) { logger.error("Error while updating " + EmbeddedServiceDefsUtil.EMBEDDED_SERVICEDEF_NIFI_NAME+ "service-def"); - System.exit(1); } } catch (Exception e) { logger.error("Error while updating " + EmbeddedServiceDefsUtil.EMBEDDED_SERVICEDEF_NIFI_NAME + "service-def", e); From 186233c64a0d7ef30fb039e0a73a06357ff32146 Mon Sep 17 00:00:00 2001 From: ni3galave Date: Fri, 29 Sep 2017 12:40:39 +0530 Subject: [PATCH 049/151] RANGER-1756: Handle role related restrictions for users having User role. Signed-off-by: Mehul Parikh --- .../hadoop/security/SecureClientLogin.java | 3 +-- .../org/apache/ranger/rest/XUserREST.java | 25 +++++++++++++++++-- .../src/main/webapp/scripts/utils/XAUtils.js | 4 ++- 3 files changed, 27 insertions(+), 5 deletions(-) diff --git a/agents-common/src/main/java/org/apache/hadoop/security/SecureClientLogin.java b/agents-common/src/main/java/org/apache/hadoop/security/SecureClientLogin.java index e6b3387f97..140d87e656 100644 --- a/agents-common/src/main/java/org/apache/hadoop/security/SecureClientLogin.java +++ b/agents-common/src/main/java/org/apache/hadoop/security/SecureClientLogin.java @@ -71,7 +71,6 @@ public synchronized static Subject loginUserFromKeytab(String user, String path, } public synchronized static Subject loginUserWithPassword(String user, String password) throws IOException { - String tmpPass = password; try { Subject subject = new Subject(); SecureClientLoginConfiguration loginConf = new SecureClientLoginConfiguration(false, user, password); @@ -80,7 +79,7 @@ public synchronized static Subject loginUserWithPassword(String user, String pas login.login(); return login.getSubject(); } catch (LoginException le) { - throw new IOException("Login failure for " + user + " using password " + tmpPass.replaceAll(".","*"), le); + throw new IOException("Login failure for " + user + " using password ****", le); } } diff --git a/security-admin/src/main/java/org/apache/ranger/rest/XUserREST.java b/security-admin/src/main/java/org/apache/ranger/rest/XUserREST.java index 739ea05bc4..5a58346096 100644 --- a/security-admin/src/main/java/org/apache/ranger/rest/XUserREST.java +++ b/security-admin/src/main/java/org/apache/ranger/rest/XUserREST.java @@ -20,6 +20,8 @@ package org.apache.ranger.rest; import java.util.HashMap; +import java.util.List; +import java.util.Random; import javax.servlet.http.HttpServletRequest; import javax.ws.rs.DELETE; @@ -31,12 +33,14 @@ import javax.ws.rs.Produces; import javax.ws.rs.core.Context; +import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang.StringUtils; import org.apache.ranger.biz.RangerBizUtil; import org.apache.ranger.biz.SessionMgr; import org.apache.ranger.biz.XUserMgr; import org.apache.ranger.common.MessageEnums; import org.apache.ranger.common.RESTErrorUtil; +import org.apache.ranger.common.RangerConstants; import org.apache.ranger.common.SearchCriteria; import org.apache.ranger.common.SearchUtil; import org.apache.ranger.common.StringUtil; @@ -346,18 +350,35 @@ public void deleteXUser(@PathParam("id") Long id, @Produces({ "application/xml", "application/json" }) @PreAuthorize("@rangerPreAuthSecurityHandler.isAPIAccessible(\"" + RangerAPIList.SEARCH_X_USERS + "\")") public VXUserList searchXUsers(@Context HttpServletRequest request) { + String UserRoleParamName = RangerConstants.ROLE_USER; SearchCriteria searchCriteria = searchUtil.extractCommonCriterias( request, xUserService.sortFields); - + String userName = null; + if(request != null && request.getUserPrincipal() != null){ + userName = request.getUserPrincipal().getName(); + } searchUtil.extractString(request, searchCriteria, "name", "User name",null); searchUtil.extractString(request, searchCriteria, "emailAddress", "Email Address", null); searchUtil.extractInt(request, searchCriteria, "userSource", "User Source"); searchUtil.extractInt(request, searchCriteria, "isVisible", "User Visibility"); searchUtil.extractInt(request, searchCriteria, "status", "User Status"); - searchUtil.extractStringList(request, searchCriteria, "userRoleList", "User Role List", "userRoleList", null, + List userRolesList = searchUtil.extractStringList(request, searchCriteria, "userRoleList", "User Role List", "userRoleList", null, null); searchUtil.extractString(request, searchCriteria, "userRole", "UserRole", null); + if (CollectionUtils.isNotEmpty(userRolesList) && CollectionUtils.size(userRolesList) == 1 && userRolesList.get(0).equalsIgnoreCase(UserRoleParamName)) { + if (!(searchCriteria.getParamList().containsKey("name"))) { + searchCriteria.addParam("name", userName); + } + else if ((searchCriteria.getParamList().containsKey("name")) && userName.contains((String) searchCriteria.getParamList().get("name"))) { + searchCriteria.addParam("name", userName); + } + else { + String randomString = new Random().toString(); + searchCriteria.addParam("name", randomString); + } + } + return xUserMgr.searchXUsers(searchCriteria); } diff --git a/security-admin/src/main/webapp/scripts/utils/XAUtils.js b/security-admin/src/main/webapp/scripts/utils/XAUtils.js index a7a6285a6d..7f00911530 100644 --- a/security-admin/src/main/webapp/scripts/utils/XAUtils.js +++ b/security-admin/src/main/webapp/scripts/utils/XAUtils.js @@ -1209,7 +1209,9 @@ define(function(require) { _.each(XAEnums.UserRoles,function(val, key){ if(SessionMgr.isKeyAdmin() && XAEnums.UserRoles.ROLE_SYS_ADMIN.value != val.value){ userRoleList.push(key) - }else if(!SessionMgr.isKeyAdmin() && XAEnums.UserRoles.ROLE_KEY_ADMIN.value != val.value){ + }else if(SessionMgr.isSystemAdmin() && XAEnums.UserRoles.ROLE_KEY_ADMIN.value != val.value){ + userRoleList.push(key) + }else if(SessionMgr.isUser() && XAEnums.UserRoles.ROLE_USER.value == val.value){ userRoleList.push(key) } }) From 2d1d5c0357b60e3bcc601a8f35ea3e6e4101202f Mon Sep 17 00:00:00 2001 From: pradeep Date: Sat, 30 Sep 2017 22:34:36 +0530 Subject: [PATCH 050/151] RANGER-1820: Duplicate entries should be deleted before creation of unique index on x_group and x_group_users table Signed-off-by: Velmurugan Periasamy --- ...028-delete-xgroup-duplicate-references.sql | 96 +++++++++++++++++++ ...dd-unique-constraint-on-table-x_group.sql} | 0 ...028-delete-xgroup-duplicate-references.sql | 65 +++++++++++++ ...dd-unique-constraint-on-table-x_group.sql} | 0 ...028-delete-xgroup-duplicate-references.sql | 68 +++++++++++++ ...dd-unique-constraint-on-table-x_group.sql} | 0 ...028-delete-xgroup-duplicate-references.sql | 75 +++++++++++++++ ...dd-unique-constraint-on-table-x_group.sql} | 0 ...028-delete-xgroup-duplicate-references.sql | 81 ++++++++++++++++ ...dd-unique-constraint-on-table-x_group.sql} | 0 10 files changed, 385 insertions(+) create mode 100644 security-admin/db/mysql/patches/028-delete-xgroup-duplicate-references.sql rename security-admin/db/mysql/patches/{028-add-unique-constraint-on-table-x_group.sql => 029-add-unique-constraint-on-table-x_group.sql} (100%) create mode 100644 security-admin/db/oracle/patches/028-delete-xgroup-duplicate-references.sql rename security-admin/db/oracle/patches/{028-add-unique-constraint-on-table-x_group.sql => 029-add-unique-constraint-on-table-x_group.sql} (100%) create mode 100644 security-admin/db/postgres/patches/028-delete-xgroup-duplicate-references.sql rename security-admin/db/postgres/patches/{028-add-unique-constraint-on-table-x_group.sql => 029-add-unique-constraint-on-table-x_group.sql} (100%) create mode 100644 security-admin/db/sqlanywhere/patches/028-delete-xgroup-duplicate-references.sql rename security-admin/db/sqlanywhere/patches/{028-add-unique-constraint-on-table-x_group.sql => 029-add-unique-constraint-on-table-x_group.sql} (100%) create mode 100644 security-admin/db/sqlserver/patches/028-delete-xgroup-duplicate-references.sql rename security-admin/db/sqlserver/patches/{028-add-unique-constraint-on-table-x_group.sql => 029-add-unique-constraint-on-table-x_group.sql} (100%) diff --git a/security-admin/db/mysql/patches/028-delete-xgroup-duplicate-references.sql b/security-admin/db/mysql/patches/028-delete-xgroup-duplicate-references.sql new file mode 100644 index 0000000000..811c5bb2fb --- /dev/null +++ b/security-admin/db/mysql/patches/028-delete-xgroup-duplicate-references.sql @@ -0,0 +1,96 @@ +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +-- -------------------------------------------------------------------------------- +-- Procedure which shall remove duplicate entries from x_group table +-- Duplicate entries were previously created due to unavailablity of unique index +-- -------------------------------------------------------------------------------- +DELIMITER $$ + +DROP PROCEDURE if exists deleteXGroupDuplicateReferences $$ +CREATE PROCEDURE `deleteXGroupDuplicateReferences`() +BEGIN +Block1: BEGIN + +DECLARE donecursor1 INT; +DECLARE group_name1 varchar(1024); +DECLARE mingroupid1 bigint; +DECLARE id2 bigint; + +DECLARE cursor1 CURSOR FOR + SELECT group_name,min(id) FROM x_group GROUP BY group_name HAVING count(group_name)>1; + +DECLARE CONTINUE HANDLER FOR NOT FOUND SET donecursor1 = 1; + OPEN cursor1; + REPEAT + FETCH cursor1 into group_name1, mingroupid1; + Block2: BEGIN + DECLARE donecursor2 INT DEFAULT 0; + DECLARE cursor2 CURSOR FOR SELECT id FROM x_group WHERE group_name= group_name1 AND id > mingroupid1; + DECLARE CONTINUE HANDLER FOR NOT FOUND SET donecursor2 = 1; + OPEN cursor2; + REPEAT + FETCH cursor2 INTO id2; + UPDATE x_group_users SET p_group_id=mingroupid1 where p_group_id=id2; + UNTIL donecursor2 END REPEAT; + CLOSE cursor2; + END Block2; + UNTIL donecursor1 END REPEAT; + CLOSE cursor1; +END Block1; + +Block3: BEGIN + +DECLARE donecursor3 INT; +DECLARE group_name3 varchar(1024); +DECLARE user_id3 bigint; +DECLARE minrowid3 bigint; + +DECLARE cursor3 CURSOR FOR + SELECT group_name,user_id,min(id) FROM x_group_users GROUP BY group_name,user_id HAVING count(1)>1; + +DECLARE CONTINUE HANDLER FOR NOT FOUND SET donecursor3 = 1; + OPEN cursor3; + REPEAT + FETCH cursor3 into group_name3, user_id3, minrowid3; + DELETE FROM x_group_users WHERE group_name=group_name3 AND user_id=user_id3 AND id > minrowid3; + UNTIL donecursor3 END REPEAT; + CLOSE cursor3; +END Block3; + +Block4: BEGIN + +DECLARE donecursor4 INT; +DECLARE group_name4 varchar(1024); +DECLARE group_id4 bigint; +DECLARE minrowid4 bigint; + +DECLARE cursor4 CURSOR FOR + SELECT group_name,min(id) FROM x_group GROUP BY group_name HAVING count(1)>1; + +DECLARE CONTINUE HANDLER FOR NOT FOUND SET donecursor4 = 1; + OPEN cursor4; + REPEAT + FETCH cursor4 into group_name4, minrowid4; + DELETE FROM x_group WHERE group_name=group_name4 AND id > minrowid4; + UNTIL donecursor4 END REPEAT; + CLOSE cursor4; +END Block4; + +END $$ +DELIMITER ; +call deleteXGroupDuplicateReferences(); + +DROP PROCEDURE if exists deleteXGroupDuplicateReferences; diff --git a/security-admin/db/mysql/patches/028-add-unique-constraint-on-table-x_group.sql b/security-admin/db/mysql/patches/029-add-unique-constraint-on-table-x_group.sql similarity index 100% rename from security-admin/db/mysql/patches/028-add-unique-constraint-on-table-x_group.sql rename to security-admin/db/mysql/patches/029-add-unique-constraint-on-table-x_group.sql diff --git a/security-admin/db/oracle/patches/028-delete-xgroup-duplicate-references.sql b/security-admin/db/oracle/patches/028-delete-xgroup-duplicate-references.sql new file mode 100644 index 0000000000..7c017f9e7e --- /dev/null +++ b/security-admin/db/oracle/patches/028-delete-xgroup-duplicate-references.sql @@ -0,0 +1,65 @@ +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +DECLARE + donecursor1 number:=0; + group_name1 VARCHAR2(1024); + mingroupid1 number:=0; + id2 number:=0; + group_name3 VARCHAR2(1024); + user_id3 number:=0; + minrowid3 number:=0; + group_name4 VARCHAR2(1024); + group_id4 number:=0; + minrowid4 number:=0; + + CURSOR cursor1 IS + SELECT group_name, min(id) FROM x_group GROUP BY group_name HAVING count(group_name)>1; + CURSOR cursor2 IS + SELECT id FROM x_group WHERE group_name = group_name1 AND id > mingroupid1; + CURSOR cursor3 IS + SELECT group_name,user_id,min(id) FROM x_group_users GROUP BY group_name,user_id HAVING count(1)>1; + CURSOR cursor4 IS + SELECT group_name,min(id) FROM x_group GROUP BY group_name HAVING count(1)>1; + +BEGIN + OPEN cursor1; + LOOP + FETCH cursor1 into group_name1, mingroupid1; + EXIT WHEN cursor1%notfound; + OPEN cursor2; + LOOP + FETCH cursor2 INTO id2; + EXIT WHEN cursor2%notfound; + UPDATE x_group_users SET p_group_id=mingroupid1 where p_group_id=id2; + END LOOP; + CLOSE cursor2; + END LOOP; + CLOSE cursor1; + OPEN cursor3; + LOOP + FETCH cursor3 into group_name3, user_id3, minrowid3; + EXIT WHEN cursor3%notfound; + DELETE FROM x_group_users WHERE group_name=group_name3 AND user_id=user_id3 AND id > minrowid3; + END LOOP; + CLOSE cursor3; + OPEN cursor4; + LOOP + FETCH cursor4 into group_name4, minrowid4; + EXIT WHEN cursor4%notfound; + DELETE FROM x_group WHERE group_name=group_name4 AND id > minrowid4; + END LOOP; + CLOSE cursor4; +END;/ \ No newline at end of file diff --git a/security-admin/db/oracle/patches/028-add-unique-constraint-on-table-x_group.sql b/security-admin/db/oracle/patches/029-add-unique-constraint-on-table-x_group.sql similarity index 100% rename from security-admin/db/oracle/patches/028-add-unique-constraint-on-table-x_group.sql rename to security-admin/db/oracle/patches/029-add-unique-constraint-on-table-x_group.sql diff --git a/security-admin/db/postgres/patches/028-delete-xgroup-duplicate-references.sql b/security-admin/db/postgres/patches/028-delete-xgroup-duplicate-references.sql new file mode 100644 index 0000000000..6fbb49ea3c --- /dev/null +++ b/security-admin/db/postgres/patches/028-delete-xgroup-duplicate-references.sql @@ -0,0 +1,68 @@ +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +select 'delimiter start'; +CREATE OR REPLACE FUNCTION delete_xgroup_duplicate_references() +RETURNS void AS $$ +DECLARE + donecursor1 BIGINT:=0; + group_name1 VARCHAR(1024); + mingroupid1 BIGINT:=0; + id2 BIGINT:=0; + group_name3 VARCHAR(1024); + user_id3 BIGINT:=0; + minrowid3 BIGINT:=0; + group_name4 VARCHAR(1024); + group_id4 BIGINT:=0; + minrowid4 BIGINT:=0; + + cursor1 cursor for SELECT group_name, min(id) FROM x_group GROUP BY group_name HAVING count(group_name)>1; + cursor2 cursor for SELECT id FROM x_group WHERE group_name = group_name1 AND id > mingroupid1; + cursor3 cursor for SELECT group_name,user_id,min(id) FROM x_group_users GROUP BY group_name,user_id HAVING count(1)>1; + cursor4 cursor for SELECT group_name,min(id) FROM x_group GROUP BY group_name HAVING count(1)>1; + +BEGIN + OPEN cursor1; + LOOP + FETCH cursor1 into group_name1, mingroupid1; + EXIT WHEN NOT FOUND; + OPEN cursor2; + LOOP + FETCH cursor2 INTO id2; + EXIT WHEN NOT FOUND; + UPDATE x_group_users SET p_group_id=mingroupid1 where p_group_id=id2; + END LOOP; + CLOSE cursor2; + END LOOP; + CLOSE cursor1; + + OPEN cursor3; + LOOP + FETCH cursor3 into group_name3, user_id3, minrowid3; + EXIT WHEN NOT FOUND; + DELETE FROM x_group_users WHERE group_name=group_name3 AND user_id=user_id3 AND id > minrowid3; + END LOOP; + CLOSE cursor3; + + OPEN cursor4; + LOOP + FETCH cursor4 into group_name4, minrowid4; + EXIT WHEN NOT FOUND; + DELETE FROM x_group WHERE group_name=group_name4 AND id > minrowid4; + END LOOP; + CLOSE cursor4; +END; +$$ LANGUAGE plpgsql; +select delete_xgroup_duplicate_references(); +select 'delimiter end'; \ No newline at end of file diff --git a/security-admin/db/postgres/patches/028-add-unique-constraint-on-table-x_group.sql b/security-admin/db/postgres/patches/029-add-unique-constraint-on-table-x_group.sql similarity index 100% rename from security-admin/db/postgres/patches/028-add-unique-constraint-on-table-x_group.sql rename to security-admin/db/postgres/patches/029-add-unique-constraint-on-table-x_group.sql diff --git a/security-admin/db/sqlanywhere/patches/028-delete-xgroup-duplicate-references.sql b/security-admin/db/sqlanywhere/patches/028-delete-xgroup-duplicate-references.sql new file mode 100644 index 0000000000..46ef9100f0 --- /dev/null +++ b/security-admin/db/sqlanywhere/patches/028-delete-xgroup-duplicate-references.sql @@ -0,0 +1,75 @@ +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +CREATE OR REPLACE PROCEDURE delete_xgroup_duplicate_references() +BEGIN + DECLARE donecursor1 bigint; + DECLARE group_name1 varchar(1024); + DECLARE mingroupid1 bigint; + DECLARE id2 bigint; + DECLARE group_name3 varchar(1024); + DECLARE user_id3 bigint; + DECLARE minrowid3 bigint; + DECLARE group_name4 varchar(1024); + DECLARE group_id4 bigint; + DECLARE minrowid4 bigint; + + DECLARE cursor1 CURSOR FOR SELECT group_name, min(id) FROM x_group GROUP BY group_name HAVING count(group_name)>1; + DECLARE cursor2 CURSOR FOR SELECT id FROM x_group WHERE group_name = group_name1 AND id > mingroupid1; + DECLARE cursor3 CURSOR FOR SELECT group_name,user_id,min(id) FROM x_group_users GROUP BY group_name,user_id HAVING count(1)>1; + DECLARE cursor4 CURSOR FOR SELECT group_name,min(id) FROM x_group GROUP BY group_name HAVING count(1)>1; + + SET donecursor1=0; + SET mingroupid1=0; + SET id2=0; + SET user_id3=0; + SET minrowid3=0; + SET group_id4=0; + SET minrowid4=0; + + OPEN cursor1; + loopc1: LOOP + FETCH cursor1 INTO group_name1, mingroupid1; + IF SQLCODE <> 0 THEN LEAVE loopc1 END IF; + OPEN cursor2; + loopc2: LOOP + FETCH cursor2 INTO id2; + IF SQLCODE <> 0 THEN LEAVE loopc2 END IF; + UPDATE x_group_users SET p_group_id=mingroupid1 where p_group_id=id2; + END LOOP; + CLOSE cursor2; + END LOOP; + CLOSE cursor1; + + OPEN cursor3; + loopc3: LOOP + FETCH cursor3 INTO group_name3, user_id3, minrowid3; + IF SQLCODE <> 0 THEN LEAVE loopc3 END IF; + DELETE FROM x_group_users WHERE group_name=group_name3 AND user_id=user_id3 AND id > minrowid3; + END LOOP; + CLOSE cursor3; + + OPEN cursor4; + loopc4: LOOP + FETCH cursor4 INTO group_name4, minrowid4; + IF SQLCODE <> 0 THEN LEAVE loopc4 END IF; + DELETE FROM x_group WHERE group_name=group_name4 AND id > minrowid4; + END LOOP; + CLOSE cursor4; +END; +GO +EXEC delete_xgroup_duplicate_references; +GO +exit \ No newline at end of file diff --git a/security-admin/db/sqlanywhere/patches/028-add-unique-constraint-on-table-x_group.sql b/security-admin/db/sqlanywhere/patches/029-add-unique-constraint-on-table-x_group.sql similarity index 100% rename from security-admin/db/sqlanywhere/patches/028-add-unique-constraint-on-table-x_group.sql rename to security-admin/db/sqlanywhere/patches/029-add-unique-constraint-on-table-x_group.sql diff --git a/security-admin/db/sqlserver/patches/028-delete-xgroup-duplicate-references.sql b/security-admin/db/sqlserver/patches/028-delete-xgroup-duplicate-references.sql new file mode 100644 index 0000000000..e8c56b0f90 --- /dev/null +++ b/security-admin/db/sqlserver/patches/028-delete-xgroup-duplicate-references.sql @@ -0,0 +1,81 @@ +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +IF (OBJECT_ID('delete_xgroup_duplicate_references') IS NOT NULL) +BEGIN + DROP PROCEDURE [dbo].[delete_xgroup_duplicate_references] +END +GO +CREATE PROCEDURE delete_xgroup_duplicate_references +AS BEGIN + DECLARE @donecursor1 bigint + DECLARE @group_name1 varchar(1024) + DECLARE @mingroupid1 bigint + DECLARE @id2 bigint + DECLARE @group_name3 varchar(1024) + DECLARE @user_id3 bigint + DECLARE @minrowid3 bigint + DECLARE @group_name4 varchar(1024) + DECLARE @group_id4 bigint + DECLARE @minrowid4 bigint + DECLARE cursor1 CURSOR FOR SELECT group_name, min(id) FROM x_group GROUP BY group_name HAVING count(group_name)>1 + OPEN cursor1 + FETCH NEXT FROM cursor1 INTO @group_name1, @mingroupid1 + WHILE (@@FETCH_STATUS = 0) + BEGIN + DECLARE cursor2 CURSOR FOR SELECT id FROM x_group WHERE group_name = @group_name1 AND id > @mingroupid1 + OPEN cursor2 + FETCH NEXT FROM cursor2 INTO @id2 + WHILE (@@FETCH_STATUS = 0) + BEGIN + UPDATE x_group_users SET p_group_id=@mingroupid1 where p_group_id=@id2 + FETCH NEXT FROM cursor2 INTO @id2 + END + CLOSE cursor2 + DEALLOCATE cursor2 + FETCH NEXT FROM cursor1 INTO @group_name1, @mingroupid1 + END + CLOSE cursor1 + DEALLOCATE cursor1 + + DECLARE cursor3 CURSOR FOR SELECT group_name,user_id,min(id) FROM x_group_users GROUP BY group_name,user_id HAVING count(1)>1 + OPEN cursor3 + FETCH NEXT FROM cursor3 INTO @group_name3, @user_id3, @minrowid3 + WHILE (@@FETCH_STATUS = 0) + BEGIN + DELETE FROM x_group_users WHERE group_name=@group_name3 AND user_id=@user_id3 AND id > @minrowid3 + FETCH NEXT FROM cursor3 INTO @group_name3, @user_id3, @minrowid3 + END + CLOSE cursor3 + DEALLOCATE cursor3 + + DECLARE cursor4 CURSOR FOR SELECT group_name,min(id) FROM x_group GROUP BY group_name HAVING count(1)>1 + OPEN cursor4 + FETCH NEXT FROM cursor4 INTO @group_name4, @minrowid4 + WHILE (@@FETCH_STATUS = 0) + BEGIN + DELETE FROM x_group WHERE group_name=@group_name4 AND id > @minrowid4 + FETCH NEXT FROM cursor4 INTO @group_name4, @minrowid4 + END + CLOSE cursor4 + DEALLOCATE cursor4 +END +GO +IF (OBJECT_ID('delete_xgroup_duplicate_references') IS NOT NULL) +BEGIN + EXEC delete_xgroup_duplicate_references +END +GO +exit \ No newline at end of file diff --git a/security-admin/db/sqlserver/patches/028-add-unique-constraint-on-table-x_group.sql b/security-admin/db/sqlserver/patches/029-add-unique-constraint-on-table-x_group.sql similarity index 100% rename from security-admin/db/sqlserver/patches/028-add-unique-constraint-on-table-x_group.sql rename to security-admin/db/sqlserver/patches/029-add-unique-constraint-on-table-x_group.sql From dd03e8cec2e3f45d406369e6cd23b8792ba14159 Mon Sep 17 00:00:00 2001 From: fatimaawez Date: Sat, 30 Sep 2017 15:38:51 +0530 Subject: [PATCH 051/151] RANGER-1818 : Good coding practice in Ranger recommended by static code analysis Signed-off-by: Velmurugan Periasamy --- .../authentication/unix/jaas/PamLoginModule.java | 14 +++++++++----- .../ranger/authentication/PasswordValidator.java | 4 ++-- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/unixauthclient/src/main/java/org/apache/ranger/authentication/unix/jaas/PamLoginModule.java b/unixauthclient/src/main/java/org/apache/ranger/authentication/unix/jaas/PamLoginModule.java index 0b3d2e6faf..803e3e81f1 100644 --- a/unixauthclient/src/main/java/org/apache/ranger/authentication/unix/jaas/PamLoginModule.java +++ b/unixauthclient/src/main/java/org/apache/ranger/authentication/unix/jaas/PamLoginModule.java @@ -19,6 +19,7 @@ package org.apache.ranger.authentication.unix.jaas; +import org.apache.commons.lang.StringUtils; import org.jvnet.libpam.PAM; import org.jvnet.libpam.PAMException; import org.jvnet.libpam.UnixUser; @@ -147,11 +148,14 @@ private boolean performLogin() throws LoginException { try { - UnixUser user = _pam.authenticate(_username, _password); - _principal = new PamPrincipal(user); - _authSucceeded = true; - - return true; + if (StringUtils.isNotEmpty(_password)) { + UnixUser user = _pam.authenticate(_username, _password); + _principal = new PamPrincipal(user); + _authSucceeded = true; + return true; + } else { + throw new PAMException("Password is Null or Empty!!!"); + } } catch (PAMException ex) { diff --git a/unixauthservice/src/main/java/org/apache/ranger/authentication/PasswordValidator.java b/unixauthservice/src/main/java/org/apache/ranger/authentication/PasswordValidator.java index 20ced8944c..d3e25fe499 100644 --- a/unixauthservice/src/main/java/org/apache/ranger/authentication/PasswordValidator.java +++ b/unixauthservice/src/main/java/org/apache/ranger/authentication/PasswordValidator.java @@ -112,9 +112,9 @@ public void run() { } catch(Throwable t) { if (userName != null && writer != null ) { - String res = "FAILED: unable to validate due to error " + t; + String res = "FAILED: unable to validate due to error " + t.getMessage(); writer.println(res); - LOG.error("Response [" + res + "] for user: " + userName, t); + LOG.error("Response [" + res + "] for user: " + userName+","+ t.getMessage()); } } finally { From f0a90a88773580cd2c703c935c6f065c09d19b97 Mon Sep 17 00:00:00 2001 From: fatimaawez Date: Sat, 30 Sep 2017 11:49:05 +0530 Subject: [PATCH 052/151] RANGER-1817 : Audit to Solr fails to log when the number of columns are in large number Signed-off-by: Mehul Parikh --- security-admin/contrib/solr_for_audit_setup/conf/managed-schema | 1 + 1 file changed, 1 insertion(+) diff --git a/security-admin/contrib/solr_for_audit_setup/conf/managed-schema b/security-admin/contrib/solr_for_audit_setup/conf/managed-schema index ee1d894059..4d032f5534 100644 --- a/security-admin/contrib/solr_for_audit_setup/conf/managed-schema +++ b/security-admin/contrib/solr_for_audit_setup/conf/managed-schema @@ -29,6 +29,7 @@ + From 2ce742cf5e82d0c76023d0cac8cae0db34c24d33 Mon Sep 17 00:00:00 2001 From: ni3galave Date: Wed, 4 Oct 2017 15:18:41 +0530 Subject: [PATCH 053/151] RANGER-1819: Not able to delete group that is having special character(ampersand) from ranger admin Signed-off-by: Mehul Parikh --- .../org/apache/ranger/rest/XUserREST.java | 36 +++++++++++-- .../webapp/scripts/model_bases/VXGroupBase.js | 10 ++-- .../webapp/scripts/model_bases/VXUserBase.js | 4 +- .../scripts/views/users/UserTableLayout.js | 51 ++++++++++--------- .../authentication/PasswordValidator.java | 2 +- 5 files changed, 66 insertions(+), 37 deletions(-) diff --git a/security-admin/src/main/java/org/apache/ranger/rest/XUserREST.java b/security-admin/src/main/java/org/apache/ranger/rest/XUserREST.java index 5a58346096..a07c243af2 100644 --- a/security-admin/src/main/java/org/apache/ranger/rest/XUserREST.java +++ b/security-admin/src/main/java/org/apache/ranger/rest/XUserREST.java @@ -354,7 +354,7 @@ public VXUserList searchXUsers(@Context HttpServletRequest request) { SearchCriteria searchCriteria = searchUtil.extractCommonCriterias( request, xUserService.sortFields); String userName = null; - if(request != null && request.getUserPrincipal() != null){ + if (request.getUserPrincipal() != null){ userName = request.getUserPrincipal().getName(); } searchUtil.extractString(request, searchCriteria, "name", "User name",null); @@ -370,7 +370,7 @@ public VXUserList searchXUsers(@Context HttpServletRequest request) { if (!(searchCriteria.getParamList().containsKey("name"))) { searchCriteria.addParam("name", userName); } - else if ((searchCriteria.getParamList().containsKey("name")) && userName.contains((String) searchCriteria.getParamList().get("name"))) { + else if ((searchCriteria.getParamList().containsKey("name")) && userName!= null && userName.contains((String) searchCriteria.getParamList().get("name"))) { searchCriteria.addParam("name", userName); } else { @@ -1122,4 +1122,34 @@ public void deleteSingleGroupByGroupName(@Context HttpServletRequest request, @P xUserMgr.deleteXGroup(vxGroup.getId(), forceDelete); } } -} + + @DELETE + @Path("/secure/users/id/{userId}") + @Produces({ "application/xml", "application/json" }) + @PreAuthorize("hasRole('ROLE_SYS_ADMIN')") + public void deleteSingleUserByUserId(@Context HttpServletRequest request, @PathParam("userId") Long userId) { + String forceDeleteStr = request.getParameter("forceDelete"); + boolean forceDelete = false; + if (StringUtils.isNotEmpty(forceDeleteStr) && "true".equalsIgnoreCase(forceDeleteStr)) { + forceDelete = true; + } + if (userId != null) { + xUserMgr.deleteXUser(userId, forceDelete); + } + } + + @DELETE + @Path("/secure/groups/id/{groupId}") + @Produces({ "application/xml", "application/json" }) + @PreAuthorize("hasRole('ROLE_SYS_ADMIN')") + public void deleteSingleGroupByGroupId(@Context HttpServletRequest request, @PathParam("groupId") Long groupId) { + String forceDeleteStr = request.getParameter("forceDelete"); + boolean forceDelete = false; + if (StringUtils.isNotEmpty(forceDeleteStr) && "true".equalsIgnoreCase(forceDeleteStr)) { + forceDelete = true; + } + if (groupId != null) { + xUserMgr.deleteXGroup(groupId, forceDelete); + } + } +} \ No newline at end of file diff --git a/security-admin/src/main/webapp/scripts/model_bases/VXGroupBase.js b/security-admin/src/main/webapp/scripts/model_bases/VXGroupBase.js index b86120b457..fce4cf6f3a 100644 --- a/security-admin/src/main/webapp/scripts/model_bases/VXGroupBase.js +++ b/security-admin/src/main/webapp/scripts/model_bases/VXGroupBase.js @@ -88,14 +88,12 @@ define(function(require){ initialize: function() { this.modelName = 'VXGroupBase'; }, - deleteGroups : function(groupNameValues, options){ - var url = this.urlRoot+'/'+ groupNameValues + '?forceDelete=true'; - - options = _.extend({ + deleteGroups : function(groupId, options){ + var url = this.urlRoot + '/id/' + groupId + '?forceDelete=true'; + options = _.extend({ contentType : 'application/json', dataType : 'json', - - }, options); + }, options); return this.constructor.nonCrudOperation.call(this, url, 'DELETE', options); }, diff --git a/security-admin/src/main/webapp/scripts/model_bases/VXUserBase.js b/security-admin/src/main/webapp/scripts/model_bases/VXUserBase.js index c4a0d6cdea..5f598746d5 100644 --- a/security-admin/src/main/webapp/scripts/model_bases/VXUserBase.js +++ b/security-admin/src/main/webapp/scripts/model_bases/VXUserBase.js @@ -42,8 +42,8 @@ define(function(require){ this.modelName = 'VXUserBase'; }, - deleteUsers : function(userNameValues,options){ - var url = this.urlRoot +'/'+ userNameValues +'?forceDelete=true'; + deleteUsers : function(userId,options){ + var url = this.urlRoot + '/id/' + userId +'?forceDelete=true'; options = _.extend({ contentType : 'application/json', dataType : 'json', diff --git a/security-admin/src/main/webapp/scripts/views/users/UserTableLayout.js b/security-admin/src/main/webapp/scripts/views/users/UserTableLayout.js index 65a1d18e6b..9febd99310 100644 --- a/security-admin/src/main/webapp/scripts/views/users/UserTableLayout.js +++ b/security-admin/src/main/webapp/scripts/views/users/UserTableLayout.js @@ -485,15 +485,16 @@ define(function(require){ var message = ''; collection.each(function(obj){ if(obj.selected){ - selArr.push(obj.get('name')); + selArr.push({"value" : obj.get('name') , "id" : obj.get('id')}); } }); var vXStrings = []; var jsonUsers = {}; for(var i in selArr) { - var item = selArr[i]; + var itemName = selArr[i].value , itemId = selArr[i].id; vXStrings.push({ - "value" : item, + "value" : itemName, + "id" : itemId }); } jsonUsers.vXStrings = vXStrings; @@ -522,33 +523,33 @@ define(function(require){ var model = new VXUser(); var count = 0 , notDeletedUserName = ""; _.map(jsonUsers.vXStrings , function(m){ - model.deleteUsers(m.value,{ - success: function(response,options){ - count += 1; - that.userCollection(jsonUsers.vXStrings.length, count, notDeletedUserName) - }, - error:function(response,options){ - count += 1; - notDeletedUserName += m.value + ", "; - that.userCollection(jsonUsers.vXStrings.length, count, notDeletedUserName) - } - }); + model.deleteUsers(m.id,{ + success: function(response,options){ + count += 1; + that.userCollection(jsonUsers.vXStrings.length, count, notDeletedUserName) + }, + error:function(response,options){ + count += 1; + notDeletedUserName += m.value + ", "; + that.userCollection(jsonUsers.vXStrings.length, count, notDeletedUserName) + } + }); }); }else { var model = new VXGroup(); var count = 0, notDeletedGroupName =""; _.map(jsonUsers.vXStrings, function(m){ - model.deleteGroups(m.value,{ - success: function(response){ - count += 1; - that.groupCollection(jsonUsers.vXStrings.length,count,notDeletedGroupName) - }, - error:function(response,options){ - count += 1; - notDeletedGroupName += m.value + ", "; - that.groupCollection(jsonUsers.vXStrings.length,count, notDeletedGroupName) - } - }) + model.deleteGroups(m.id,{ + success: function(response){ + count += 1; + that.groupCollection(jsonUsers.vXStrings.length,count,notDeletedGroupName) + }, + error:function(response,options){ + count += 1; + notDeletedGroupName += m.value + ", "; + that.groupCollection(jsonUsers.vXStrings.length,count, notDeletedGroupName) + } + }) }); } } diff --git a/unixauthservice/src/main/java/org/apache/ranger/authentication/PasswordValidator.java b/unixauthservice/src/main/java/org/apache/ranger/authentication/PasswordValidator.java index d3e25fe499..59cc9144a0 100644 --- a/unixauthservice/src/main/java/org/apache/ranger/authentication/PasswordValidator.java +++ b/unixauthservice/src/main/java/org/apache/ranger/authentication/PasswordValidator.java @@ -114,7 +114,7 @@ public void run() { if (userName != null && writer != null ) { String res = "FAILED: unable to validate due to error " + t.getMessage(); writer.println(res); - LOG.error("Response [" + res + "] for user: " + userName+","+ t.getMessage()); + LOG.error("Response [" + res + "] for user: " + userName+", "+ t.getMessage()); } } finally { From e656b9a0beb62e412eb70c6c26e90dc0039e8f3d Mon Sep 17 00:00:00 2001 From: pradeep Date: Mon, 9 Oct 2017 17:26:21 +0530 Subject: [PATCH 054/151] RANGER-1826: Import of bulk policies is causing OOM and Apparent Deadlock --- .../org/apache/ranger/biz/ServiceDBStore.java | 63 ++++++++++++++++++- .../common/RangerServicePoliciesCache.java | 2 +- .../org/apache/ranger/common/db/BaseDao.java | 19 ++++++ .../org/apache/ranger/rest/ServiceREST.java | 12 +++- .../apache/ranger/biz/TestServiceDBStore.java | 28 ++++----- 5 files changed, 105 insertions(+), 19 deletions(-) diff --git a/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java b/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java index 5f5f239a5e..9de40d955a 100644 --- a/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java +++ b/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java @@ -1657,9 +1657,11 @@ public void deleteService(Long id) throws Exception { } List policies = daoMgr.getXXPolicy().findByServiceId(service.getId()); + RangerPolicy rangerPolicy =null; for(XXPolicy policy : policies) { LOG.info("Deleting Policy, policyName: " + policy.getName()); - deletePolicy(policy.getId()); + rangerPolicy = getPolicy(policy.getId()); + deletePolicy(rangerPolicy); } XXServiceConfigMapDao configDao = daoMgr.getXXServiceConfigMap(); @@ -2008,6 +2010,34 @@ public void deletePolicy(Long policyId) throws Exception { LOG.info("Policy Deleted Successfully. PolicyName : " + policyName); } + public void deletePolicy(RangerPolicy policy) throws Exception { + if(policy == null) { + return; + } + if(LOG.isDebugEnabled()) { + LOG.debug("==> ServiceDBStore.deletePolicy(" + policy.getId() + ")"); + } + RangerService service = getServiceByName(policy.getService()); + if(service == null) { + throw new Exception("service does not exist - name='" + policy.getService()); + } + Long version = policy.getVersion(); + if(version == null) { + version = Long.valueOf(1); + LOG.info("Found Version Value: `null`, so setting value of version to 1, While updating object, version should not be null."); + } else { + version = Long.valueOf(version.longValue() + 1); + } + policy.setVersion(version); + List trxLogList = policyService.getTransactionLog(policy, RangerPolicyService.OPERATION_DELETE_CONTEXT); + deleteExistingPolicyItemsNative(policy); + deleteExistingPolicyResourcesNative(policy); + daoMgr.getXXPolicy().deletePolicyIDReference("id",policy.getId()); + handlePolicyUpdate(service, true); + dataHistService.createObjectDataHistory(policy, RangerDataHistService.ACTION_DELETE); + bizUtil.createTrxLog(trxLogList); + } + @Override public RangerPolicy getPolicy(Long id) throws Exception { return policyService.read(id); @@ -3017,6 +3047,37 @@ private Boolean deleteExistingPolicyResources(RangerPolicy policy) { return true; } + private Boolean deleteExistingPolicyItemsNative(RangerPolicy policy) { + if(policy == null) { + return false; + } + XXPolicyItemDao policyItemDao = daoMgr.getXXPolicyItem(); + List policyItems = policyItemDao.findByPolicyId(policy.getId()); + for(XXPolicyItem policyItem : policyItems) { + Long polItemId = policyItem.getId(); + daoMgr.getXXPolicyItemRowFilterInfo().deletePolicyIDReference("policy_item_id", polItemId); + daoMgr.getXXPolicyItemDataMaskInfo().deletePolicyIDReference("policy_item_id", polItemId); + daoMgr.getXXPolicyItemGroupPerm().deletePolicyIDReference("policy_item_id", polItemId); + daoMgr.getXXPolicyItemUserPerm().deletePolicyIDReference("policy_item_id", polItemId); + daoMgr.getXXPolicyItemCondition().deletePolicyIDReference("policy_item_id", polItemId); + daoMgr.getXXPolicyItemAccess().deletePolicyIDReference("policy_item_id", polItemId); + } + daoMgr.getXXPolicyItem().deletePolicyIDReference("policy_id", policy.getId()); + return true; + } + + private Boolean deleteExistingPolicyResourcesNative(RangerPolicy policy) { + if(policy == null) { + return false; + } + List resources = daoMgr.getXXPolicyResource().findByPolicyId(policy.getId()); + for(XXPolicyResource resource : resources) { + daoMgr.getXXPolicyResourceMap().deletePolicyIDReference("resource_id", resource.getId()); + daoMgr.getXXPolicyResource().deletePolicyIDReference("id", resource.getId()); + } + return true; + } + @Override public Boolean getPopulateExistingBaseFields() { return populateExistingBaseFields; diff --git a/security-admin/src/main/java/org/apache/ranger/common/RangerServicePoliciesCache.java b/security-admin/src/main/java/org/apache/ranger/common/RangerServicePoliciesCache.java index 7d1f28c4a5..eb20f693e1 100644 --- a/security-admin/src/main/java/org/apache/ranger/common/RangerServicePoliciesCache.java +++ b/security-admin/src/main/java/org/apache/ranger/common/RangerServicePoliciesCache.java @@ -270,7 +270,7 @@ private void pruneUnusedPolicyAttributes(List policies) { policy.setCreateTime(null); policy.setUpdatedBy(null); policy.setUpdateTime(null); - policy.setGuid(null); + // policy.setGuid(null); /* this is used by import policy */ // policy.setName(null); /* this is used by GUI in policy list page */ // policy.setDescription(null); /* this is used by export policy */ policy.setResourceSignature(null); diff --git a/security-admin/src/main/java/org/apache/ranger/common/db/BaseDao.java b/security-admin/src/main/java/org/apache/ranger/common/db/BaseDao.java index c2832ea931..51c0de56c2 100644 --- a/security-admin/src/main/java/org/apache/ranger/common/db/BaseDao.java +++ b/security-admin/src/main/java/org/apache/ranger/common/db/BaseDao.java @@ -250,6 +250,25 @@ public void updateUserIDReference(String paramName,long oldID) { } } + public boolean deletePolicyIDReference(String paramName,long oldID) { + Table table = tClass.getAnnotation(Table.class); + if(table != null) { + String tableName = table.name(); + String query = "delete from " + tableName + " where " +paramName+"=" + oldID; + if (logger.isDebugEnabled()) { + logger.debug("Delete Query:" + query); + } + int count=getEntityManager().createNativeQuery(query).executeUpdate(); + getEntityManager().flush(); + if(count>0){ + return true; + } + }else{ + logger.warn("Required annotation `Table` not found"); + } + return false; + } + public String getDBVersion(){ String dbVersion="Not Available"; String query ="SELECT 1"; diff --git a/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java b/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java index 3703d1fa68..9330edd969 100644 --- a/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java +++ b/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java @@ -2249,6 +2249,7 @@ private void deletePoliciesProvidedInServiceMap( int totalDeletedPilicies = 0; if (CollectionUtils.isNotEmpty(sourceServices) && CollectionUtils.isNotEmpty(destinationServices)) { + RangerPolicyValidator validator = validatorFactory.getPolicyValidator(svcStore); for (int i = 0; i < sourceServices.size(); i++) { if (!destinationServices.get(i).isEmpty()) { RangerPolicyList servicePolicies = null; @@ -2258,12 +2259,17 @@ private void deletePoliciesProvidedInServiceMap( if (CollectionUtils.isNotEmpty(rangerPolicyList)) { for (RangerPolicy rangerPolicy : rangerPolicyList) { if (rangerPolicy != null) { - if (rangerPolicy.getId() != null){ - deletePolicy(rangerPolicy.getId()); + try { + validator.validate(rangerPolicy.getId(), Action.DELETE); + ensureAdminAccess(rangerPolicy.getService(), rangerPolicy.getResources()); + svcStore.deletePolicy(rangerPolicy); + totalDeletedPilicies = totalDeletedPilicies + 1; if (LOG.isDebugEnabled()) { LOG.debug("Policy " + rangerPolicy.getName() + " deleted successfully." ); + LOG.debug("TotalDeletedPilicies: " +totalDeletedPilicies); } - totalDeletedPilicies = totalDeletedPilicies + 1; + } catch(Throwable excp) { + LOG.error("deletePolicy(" + rangerPolicy.getId() + ") failed", excp); } } } diff --git a/security-admin/src/test/java/org/apache/ranger/biz/TestServiceDBStore.java b/security-admin/src/test/java/org/apache/ranger/biz/TestServiceDBStore.java index cf3485ee88..cd7a605b5b 100644 --- a/security-admin/src/test/java/org/apache/ranger/biz/TestServiceDBStore.java +++ b/security-admin/src/test/java/org/apache/ranger/biz/TestServiceDBStore.java @@ -1389,8 +1389,8 @@ public void test21deleteService() throws Exception { policyItem.setUpdateTime(new Date()); policyItemList.add(policyItem); - List policyItemDataMaskInfoList = new ArrayList(); - List policyItemRowFilterInfoList = new ArrayList(); + //List policyItemDataMaskInfoList = new ArrayList(); + //List policyItemRowFilterInfoList = new ArrayList(); List policyItemConditionList = new ArrayList(); XXPolicyItemCondition policyItemCondition = new XXPolicyItemCondition(); @@ -1504,34 +1504,34 @@ public void test21deleteService() throws Exception { .thenReturn(policyItemList); Mockito.when(daoManager.getXXPolicyItemDataMaskInfo()).thenReturn(xxPolicyItemDataMaskInfoDao); - Mockito.when(xxPolicyItemDataMaskInfoDao.findByPolicyItemId(policyItem.getId())).thenReturn(policyItemDataMaskInfoList); + //Mockito.when(xxPolicyItemDataMaskInfoDao.findByPolicyItemId(policyItem.getId())).thenReturn(policyItemDataMaskInfoList); Mockito.when(daoManager.getXXPolicyItemRowFilterInfo()).thenReturn(xxPolicyItemRowFilterInfoDao); - Mockito.when(xxPolicyItemRowFilterInfoDao.findByPolicyItemId(policyItem.getId())).thenReturn(policyItemRowFilterInfoList); + //Mockito.when(xxPolicyItemRowFilterInfoDao.findByPolicyItemId(policyItem.getId())).thenReturn(policyItemRowFilterInfoList); Mockito.when(daoManager.getXXPolicyItemCondition()).thenReturn( xPolicyItemConditionDao); - Mockito.when( + /*Mockito.when( xPolicyItemConditionDao.findByPolicyItemId(policyItemCondition .getId())).thenReturn(policyItemConditionList); - + */ Mockito.when(daoManager.getXXPolicyItemGroupPerm()).thenReturn( xPolicyItemGroupPermDao); - Mockito.when( + /*Mockito.when( xPolicyItemGroupPermDao.findByPolicyItemId(policyItem.getId())) .thenReturn(policyItemGroupPermList); - + */ Mockito.when(daoManager.getXXPolicyItemUserPerm()).thenReturn( xPolicyItemUserPermDao); - Mockito.when(xPolicyItemUserPermDao.findByPolicyItemId(Id)).thenReturn( - policyItemUserPermList); + /*Mockito.when(xPolicyItemUserPermDao.findByPolicyItemId(Id)).thenReturn( + policyItemUserPermList);*/ Mockito.when(daoManager.getXXPolicyItemAccess()).thenReturn( xPolicyItemAccessDao); - Mockito.when( + /*Mockito.when( xPolicyItemAccessDao.findByPolicyItemId(policyItemAccess .getId())).thenReturn(policyItemAccessList); - + */ Mockito.when(daoManager.getXXPolicyResource()).thenReturn( xPolicyResourceDao); Mockito.when(xPolicyResourceDao.findByPolicyId(policyResource.getId())) @@ -1539,10 +1539,10 @@ public void test21deleteService() throws Exception { Mockito.when(daoManager.getXXPolicyResourceMap()).thenReturn( xPolicyResourceMapDao); - Mockito.when( + /*Mockito.when( xPolicyResourceMapDao.findByPolicyResId(policyResourceMap .getId())).thenReturn(policyResourceMapList); - + */ Mockito.when(daoManager.getXXService()).thenReturn(xServiceDao); Mockito.when(xServiceDao.getById(Id)).thenReturn(xService); From 9ac77670278895b62400af61e8c351b13550e8ed Mon Sep 17 00:00:00 2001 From: ni3galave Date: Tue, 10 Oct 2017 18:57:34 +0530 Subject: [PATCH 055/151] RANGER-1176: Ranger admin does not allow to create / update a policy with only delegate admin permission. Signed-off-by: Mehul Parikh --- .../java/org/apache/ranger/biz/XUserMgr.java | 12 ++--- .../scripts/modules/globalize/message/en.js | 3 +- .../views/policies/RangerPolicyCreate.js | 13 ++++- .../views/policies/RangerPolicyForm.js | 16 ++++--- .../webapp/templates/helpers/XAHelpers.js | 3 ++ .../PlugableServicePolicyUpdateDiff_tmpl.html | 48 +++++++++---------- 6 files changed, 55 insertions(+), 40 deletions(-) diff --git a/security-admin/src/main/java/org/apache/ranger/biz/XUserMgr.java b/security-admin/src/main/java/org/apache/ranger/biz/XUserMgr.java index 8d3b751f7f..f9e6991339 100644 --- a/security-admin/src/main/java/org/apache/ranger/biz/XUserMgr.java +++ b/security-admin/src/main/java/org/apache/ranger/biz/XUserMgr.java @@ -386,15 +386,13 @@ public VXUser updateXUser(VXUser vXUser) { && password.equals(hiddenPasswordString)) { vXPortalUser.setPassword(oldUserProfile.getPassword()); } + else if(oldUserProfile != null && oldUserProfile.getUserSource() == RangerCommonEnums.USER_EXTERNAL && password != null){ + vXPortalUser.setPassword(oldUserProfile.getPassword()); + logger.debug("User is trrying to change external user password which we are not allowing it to change"); + } else if(password != null){ validatePassword(vXUser); - if (oldUserProfile.getUserSource() == RangerCommonEnums.USER_EXTERNAL) { - vXPortalUser.setPassword(oldUserProfile.getPassword()); - } - else if(oldUserProfile.getUserSource() == RangerCommonEnums.USER_APP) - { - vXPortalUser.setPassword(password); - } + vXPortalUser.setPassword(password); } Collection groupIdList = vXUser.getGroupIdList(); XXPortalUser xXPortalUser = new XXPortalUser(); diff --git a/security-admin/src/main/webapp/scripts/modules/globalize/message/en.js b/security-admin/src/main/webapp/scripts/modules/globalize/message/en.js index 3338633139..8c9f284188 100644 --- a/security-admin/src/main/webapp/scripts/modules/globalize/message/en.js +++ b/security-admin/src/main/webapp/scripts/modules/globalize/message/en.js @@ -395,7 +395,8 @@ define(function(require) { plsSelectUserToSetVisibility :' Please select user to set visibility or selected user is already visible/hidden.', plsSelectGroupToSetVisibility:' Please select group to set visibility or selected group is already visible/hidden.', activationTimeDelayMsg :'Policy activation time delayed by more than 1hr from last update time.', - pleaseSelectAccessTypeForTagMasking : 'Please select access type first to enable add masking options.' + pleaseSelectAccessTypeForTagMasking : 'Please select access type first to enable add masking options.', + addUserOrGroupForDelegateAdmin : 'Please select user/group for the selected permission(s)', }, plcHldr : { diff --git a/security-admin/src/main/webapp/scripts/views/policies/RangerPolicyCreate.js b/security-admin/src/main/webapp/scripts/views/policies/RangerPolicyCreate.js index df13b7c243..1475dd97c1 100644 --- a/security-admin/src/main/webapp/scripts/views/policies/RangerPolicyCreate.js +++ b/security-admin/src/main/webapp/scripts/views/policies/RangerPolicyCreate.js @@ -163,8 +163,10 @@ define(function(require){ var userPerm = (validateObj1.userPerm || validateObj2.userPerm || validateObj3.userPerm || validateObj4.userPerm); var groupPerm = (validateObj1.groupPermSet || validateObj2.groupPermSet - || validateObj3.groupPermSet || validateObj4.groupPermSet) - if((!validateObj1.auditLoggin) && !(groupPerm || userPerm)){ + || validateObj3.groupPermSet || validateObj4.groupPermSet); + var delegatePerm = (validateObj1.delegateAdmin || validateObj2.delegateAdmin + || validateObj3.delegateAdmin || validateObj4.delegateAdmin); + if((!validateObj1.auditLoggin) && !(groupPerm || userPerm || delegatePerm )){ XAUtil.alertPopup({ msg :localization.tt('msg.yourAuditLogginIsOff') }); return; } @@ -177,6 +179,13 @@ define(function(require){ }, validatePolicyItem : function(validateObj){ var that = this, valid = false; + //DelegateAdmin checks + if((validateObj.groupSet || validateObj.userSet) && validateObj.delegateAdmin){ + return true; + }else if(validateObj.delegateAdmin && !(validateObj.groupSet || validateObj.userSet)) { + this.popupCallBack(localization.tt('msg.addUserOrGroupForDelegateAdmin'),validateObj); + return false; + } valid = (validateObj.groupSet && validateObj.permSet) || (validateObj.userSet && validateObj.userPerm); if(!valid){ if((!validateObj.groupSet && !validateObj.userSet) && (validateObj.condSet)) { diff --git a/security-admin/src/main/webapp/scripts/views/policies/RangerPolicyForm.js b/security-admin/src/main/webapp/scripts/views/policies/RangerPolicyForm.js index 6f27d5db06..e235cdacfa 100644 --- a/security-admin/src/main/webapp/scripts/views/policies/RangerPolicyForm.js +++ b/security-admin/src/main/webapp/scripts/views/policies/RangerPolicyForm.js @@ -427,7 +427,6 @@ define(function(require){ var RangerPolicyItemAccessList = Backbone.Collection.extend(); var rangerPlcItemAccessList = new RangerPolicyItemAccessList(m.get('accesses')); policyItem.set('accesses', rangerPlcItemAccessList) - policyItemList.add(policyItem) } if(!_.isUndefined(m.get('dataMaskInfo'))){ policyItem.set("dataMaskInfo",m.get("dataMaskInfo")); @@ -435,6 +434,7 @@ define(function(require){ if(!_.isUndefined(m.get('rowFilterInfo'))){ policyItem.set("rowFilterInfo",m.get("rowFilterInfo")); } + policyItemList.add(policyItem); } @@ -655,16 +655,17 @@ define(function(require){ return JSON.stringify(context); }, formValidation : function(coll){ - var groupSet = false,permSet = false,groupPermSet = false, + var groupSet = false , permSet = false , groupPermSet = false , delegateAdmin = false , userSet=false, userPerm = false, userPermSet =false,breakFlag =false, condSet = false,customMaskSet = true; console.log('validation called..'); coll.each(function(m){ if(_.isEmpty(m.attributes)) return; - if(m.has('groupName') || m.has('userName') || m.has('accesses') ){ + if(m.has('groupName') || m.has('userName') || m.has('accesses') || m.has('delegateAdmin') ){ if(! breakFlag){ groupSet = m.has('groupName') ? true : false; userSet = m.has('userName') ? true : false; - permSet = m.has('accesses') ? true : false; + permSet = m.has('accesses') ? true : false; + delegateAdmin = m.has('delegateAdmin') ? m.get('delegateAdmin') : false; if(groupSet && permSet){ groupPermSet = true; userPermSet = false; @@ -672,7 +673,9 @@ define(function(require){ userPermSet = true; groupPermSet = false; }else{ - breakFlag=true; + if(!((userSet || groupSet) && delegateAdmin)){ + breakFlag=true; + } } } } @@ -692,7 +695,8 @@ define(function(require){ userSet : userSet, isUsers:userPermSet, auditLoggin : auditStatus, condSet : condSet, - customMaskSet : customMaskSet + customMaskSet : customMaskSet, + delegateAdmin : delegateAdmin, }; if(groupSet || userSet){ obj['permSet'] = groupSet ? permSet : false; diff --git a/security-admin/src/main/webapp/templates/helpers/XAHelpers.js b/security-admin/src/main/webapp/templates/helpers/XAHelpers.js index 17668805cd..9363c6b36c 100644 --- a/security-admin/src/main/webapp/templates/helpers/XAHelpers.js +++ b/security-admin/src/main/webapp/templates/helpers/XAHelpers.js @@ -310,6 +310,9 @@ case '>=': return (v1 >= v2) ? options.fn(this) : options.inverse(this); break; + case '||': + return (v1 || v2) ? options.fn(this) : options.inverse(this); + break; default: return options.inverse(this); break; diff --git a/security-admin/src/main/webapp/templates/reports/PlugableServicePolicyUpdateDiff_tmpl.html b/security-admin/src/main/webapp/templates/reports/PlugableServicePolicyUpdateDiff_tmpl.html index 524b18feb8..cac71d979c 100644 --- a/security-admin/src/main/webapp/templates/reports/PlugableServicePolicyUpdateDiff_tmpl.html +++ b/security-admin/src/main/webapp/templates/reports/PlugableServicePolicyUpdateDiff_tmpl.html @@ -76,7 +76,7 @@
Allow PolicyItems :

Old Value

{{#each oldPolicyItems}}
    - {{#if this.permissions}} + {{#ifCond this.users '||' this.groups }}
  1. Groups: {{#if_eq this.groups compare=0}} <empty> @@ -111,7 +111,7 @@

    Old Value

  2. Delegate Admin: {{this.delegateAdmin}}
    3. {{else}}
  3. <empty>
    4. - {{/if}} + {{/ifCond}}

{{/each}} @@ -119,7 +119,7 @@

Old Value

New Value

{{#each newPolicyItems}}
    - {{#if this.permissions}} + {{#ifCond this.users '||' this.groups}}
  1. Groups: {{#if_eq this.groups compare=0}} <empty> @@ -154,7 +154,7 @@

    New Value

  2. Delegate Admin: {{this.delegateAdmin}}
    3. {{else}}
  3. <empty>
    4. - {{/if}} + {{/ifCond}}

{{/each}} @@ -168,7 +168,7 @@
Allow Exception PolicyItems :

Old Value

{{#each oldAllowExceptionPolicyItems}}
    - {{#if this.permissions}} + {{#ifCond this.users '||' this.groups}}
  1. Groups: {{#if_eq this.groups compare=0}} <empty> @@ -203,7 +203,7 @@

    Old Value

  2. Delegate Admin: {{this.delegateAdmin}}
    3. {{else}}
  3. <empty>
    4. - {{/if}} + {{/ifCond}}

{{/each}} @@ -211,7 +211,7 @@

Old Value

New Value

{{#each newAllowExceptionPolicyItems}}
    - {{#if this.permissions}} + {{#ifCond this.users '||' this.groups}}
  1. Groups: {{#if_eq this.groups compare=0}} <empty> @@ -246,7 +246,7 @@

    New Value

  2. Delegate Admin: {{this.delegateAdmin}}
    3. {{else}}
  3. <empty>
    4. - {{/if}} + {{/ifCond}}

{{/each}} @@ -261,7 +261,7 @@
Deny PolicyItems :

Old Value

{{#each oldDenyPolicyItems}}
    - {{#if this.permissions}} + {{#ifCond this.users '||' this.groups}}
  1. Groups: {{#if_eq this.groups compare=0}} <empty> @@ -296,7 +296,7 @@

    Old Value

  2. Delegate Admin: {{this.delegateAdmin}}
    3. {{else}}
  3. <empty>
    4. - {{/if}} + {{/ifCond}}

{{/each}} @@ -304,7 +304,7 @@

Old Value

New Value

{{#each newDenyPolicyItems}}
    - {{#if this.permissions}} + {{#ifCond this.users '||' this.groups}}
  1. Groups: {{#if_eq this.groups compare=0}} <empty> @@ -339,7 +339,7 @@

    New Value

  2. Delegate Admin: {{this.delegateAdmin}}
    3. {{else}}
  3. <empty>
    4. - {{/if}} + {{/ifCond}}

{{/each}} @@ -355,7 +355,7 @@
Deny Exception PolicyItems :

Old Value

{{#each oldDenyExceptionPolicyItems}}
    - {{#if this.permissions}} + {{#ifCond this.users '||' this.groups}}
  1. Groups: {{#if_eq this.groups compare=0}} <empty> @@ -390,7 +390,7 @@

    Old Value

  2. Delegate Admin: {{this.delegateAdmin}}
    3. {{else}}
  3. <empty>
    4. - {{/if}} + {{/ifCond}}

{{/each}} @@ -398,7 +398,7 @@

Old Value

New Value

{{#each newDenyExceptionPolicyItems}}
    - {{#if this.permissions}} + {{#ifCond this.users '||' this.groups}}
  1. Groups: {{#if_eq this.groups compare=0}} <empty> @@ -433,7 +433,7 @@

    New Value

  2. Delegate Admin: {{this.delegateAdmin}}
    3. {{else}}
  3. <empty>
    4. - {{/if}} + {{/ifCond}}

{{/each}} @@ -448,7 +448,7 @@
Masking Policy Items :

Old Value

{{#each oldMaskPolicyItems}}
    - {{#if this.accesses}} + {{#ifCond this.users '||' this.groups}}
  1. Groups: {{#if_eq this.groups compare=0}} <empty> @@ -480,7 +480,7 @@

    Old Value

    {{/if}} {{else}}
  2. <empty>
    3. - {{/if}} + {{/ifCond}}

{{/each}} @@ -488,7 +488,7 @@

Old Value

New Value

{{#each newMaskPolicyItems}}
    - {{#if this.accesses}} + {{#ifCond this.users '||' this.groups}}
  1. Groups: {{#if_eq this.groups compare=0}} <empty> @@ -521,7 +521,7 @@

    New Value

    {{else}}
  2. <empty>
    3. - {{/if}} + {{/ifCond}}

{{/each}} @@ -536,7 +536,7 @@
Row Level Filter Policy Items :

Old Value

{{#each oldRowFilterPolicyItems}}
    - {{#if this.accesses}} + {{#ifCond this.users '||' this.groups}}
  1. Groups: {{#if_eq this.groups compare=0}} <empty> @@ -568,7 +568,7 @@

    Old Value

    {{/if}} {{else}}
  2. <empty>
    3. - {{/if}} + {{/ifCond}}

{{/each}} @@ -576,7 +576,7 @@

Old Value

New Value

{{#each newRowFilterPolicyItems}}
    - {{#if this.accesses}} + {{#ifCond this.users '||' this.groups}}
  1. Groups: {{#if_eq this.groups compare=0}} <empty> @@ -609,7 +609,7 @@

    New Value

    {{else}}
  2. <empty>
    3. - {{/if}} + {{/ifCond}}

{{/each}} From c2e1ec9c79e47491c465cb4967aa5b08fbcda360 Mon Sep 17 00:00:00 2001 From: pradeep Date: Fri, 13 Oct 2017 13:45:20 +0530 Subject: [PATCH 056/151] RANGER-1832: Export REST API should return exact matching results if polResource param is provided --- .../org/apache/ranger/common/ServiceUtil.java | 54 +++++++++++++++++++ .../org/apache/ranger/rest/ServiceREST.java | 1 + 2 files changed, 55 insertions(+) diff --git a/security-admin/src/main/java/org/apache/ranger/common/ServiceUtil.java b/security-admin/src/main/java/org/apache/ranger/common/ServiceUtil.java index 6864c5a54a..c3c39dc82f 100644 --- a/security-admin/src/main/java/org/apache/ranger/common/ServiceUtil.java +++ b/security-admin/src/main/java/org/apache/ranger/common/ServiceUtil.java @@ -1563,5 +1563,59 @@ private Integer getAssetType(RangerService service, String serviceName) { return assetType; } + + public List getMatchingPoliciesForResource(HttpServletRequest request, + List policyLists) { + List policies = new ArrayList(); + if (request != null) { + String resource = request.getParameter(SearchFilter.POL_RESOURCE); + String serviceType = request.getParameter(SearchFilter.SERVICE_TYPE); + if (!StringUtil.isEmpty(resource) && !StringUtil.isEmpty(serviceType)) { + List resourceList = null; + Map rangerPolicyResourceMap = null; + RangerPolicy.RangerPolicyResource rangerPolicyResource = null; + for (RangerPolicy rangerPolicy : policyLists) { + if (rangerPolicy != null) { + rangerPolicyResourceMap = rangerPolicy.getResources(); + if (rangerPolicyResourceMap != null) { + if (rangerPolicyResourceMap.containsKey("path")) { + rangerPolicyResource = rangerPolicyResourceMap.get("path"); + if (rangerPolicyResource != null) { + resourceList = rangerPolicyResource.getValues(); + if (CollectionUtils.isNotEmpty(resourceList) && resourceList.size() == 1) { + String resourcePath = resourceList.get(0); + if (!StringUtil.isEmpty(resourcePath)) { + if (resourcePath.equals(resource) + || resourcePath.startsWith(resource + "/")) { + policies.add(rangerPolicy); + } + } + } + } + } else if (rangerPolicyResourceMap.containsKey("database")) { + rangerPolicyResource = rangerPolicyResourceMap.get("database"); + if (rangerPolicyResource != null) { + resourceList = rangerPolicyResource.getValues(); + if (CollectionUtils.isNotEmpty(resourceList) && resourceList.size() == 1) { + String resourcePath = resourceList.get(0); + if (!StringUtil.isEmpty(resourcePath)) { + if (resourcePath.equals(resource)) { + policies.add(rangerPolicy); + } + } + } + } + } + } + } + } + policyLists.clear(); + if (CollectionUtils.isNotEmpty(policies)) { + policyLists.addAll(policies); + } + } + } + return policyLists; + } } diff --git a/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java b/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java index 9330edd969..1fc002ff65 100644 --- a/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java +++ b/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java @@ -2227,6 +2227,7 @@ private List getAllFilteredPolicyList(SearchFilter filter, } } } + policyLists=serviceUtil.getMatchingPoliciesForResource(request, policyLists); Map orderedPolicies = new TreeMap(); if (!CollectionUtils.isEmpty(policyLists)) { From 448182cd3789a69b23fcb9bf09a65935e23740dc Mon Sep 17 00:00:00 2001 From: Abhay Kulkarni Date: Wed, 11 Oct 2017 17:06:22 -0700 Subject: [PATCH 057/151] RANGER-1834: row filter policies are not being returned by policy search Signed-off-by: Madhan Neethiraj --- .../RangerDefaultPolicyResourceMatcher.java | 21 ++++++-- .../RangerPolicyResourceMatcher.java | 2 + .../org/apache/ranger/biz/ServiceDBStore.java | 53 +++++++++---------- 3 files changed, 44 insertions(+), 32 deletions(-) diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyresourcematcher/RangerDefaultPolicyResourceMatcher.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyresourcematcher/RangerDefaultPolicyResourceMatcher.java index be10b95466..8f1e102556 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyresourcematcher/RangerDefaultPolicyResourceMatcher.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyresourcematcher/RangerDefaultPolicyResourceMatcher.java @@ -46,7 +46,7 @@ public class RangerDefaultPolicyResourceMatcher implements RangerPolicyResourceM private static final Log LOG = LogFactory.getLog(RangerDefaultPolicyResourceMatcher.class); protected RangerServiceDef serviceDef = null; - protected RangerPolicy policy = null; + protected int policyType; protected Map policyResources = null; private Map matchers = null; @@ -70,14 +70,23 @@ public void setServiceDef(RangerServiceDef serviceDef) { @Override public void setPolicy(RangerPolicy policy) { - this.policy = policy; - setPolicyResources(policy == null ? null : policy.getResources()); + if (policy == null) { + setPolicyResources(null, RangerPolicy.POLICY_TYPE_ACCESS); + } else { + setPolicyResources(policy.getResources(), policy.getPolicyType() == null ? RangerPolicy.POLICY_TYPE_ACCESS : policy.getPolicyType()); + } } @Override public void setPolicyResources(Map policyResources) { + setPolicyResources(policyResources, RangerPolicy.POLICY_TYPE_ACCESS); + } + + @Override + public void setPolicyResources(Map policyResources, int policyType) { this.policyResources = policyResources; + this.policyType = policyType; } @Override @@ -98,7 +107,6 @@ public void init() { Set policyResourceKeySet = policyResources.keySet(); RangerServiceDefHelper serviceDefHelper = new RangerServiceDefHelper(serviceDef, false); - int policyType = policy != null && policy.getPolicyType() != null ? policy.getPolicyType() : RangerPolicy.POLICY_TYPE_ACCESS; Set> validResourceHierarchies = serviceDefHelper.getResourceHierarchies(policyType); for (List validResourceHierarchy : validResourceHierarchies) { @@ -371,6 +379,10 @@ public boolean isMatch(RangerPolicy policy, MatchScope scope, Map resources = policy.getResources(); if (MapUtils.isNotEmpty(resources)) { @@ -539,7 +551,6 @@ private boolean isValid(RangerAccessResource resource) { aValidHierarchy = firstValidResourceDefHierarchy; } else { RangerServiceDefHelper serviceDefHelper = new RangerServiceDefHelper(serviceDef, false); - int policyType = policy != null && policy.getPolicyType() != null ? policy.getPolicyType() : RangerPolicy.POLICY_TYPE_ACCESS; Set> validResourceHierarchies = serviceDefHelper.getResourceHierarchies(policyType); for (List resourceHierarchy : validResourceHierarchies) { diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyresourcematcher/RangerPolicyResourceMatcher.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyresourcematcher/RangerPolicyResourceMatcher.java index b4dc2c509a..9cc4bd6213 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyresourcematcher/RangerPolicyResourceMatcher.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyresourcematcher/RangerPolicyResourceMatcher.java @@ -37,6 +37,8 @@ enum MatchType { NONE, SELF, DESCENDANT, ANCESTOR }; void setPolicyResources(Map policyResources); + void setPolicyResources(Map policyResources, int policyType); + void init(); RangerServiceDef getServiceDef(); diff --git a/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java b/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java index 9de40d955a..d951090082 100644 --- a/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java +++ b/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java @@ -2320,48 +2320,47 @@ List getMatchers(RangerServiceDef serviceDef, Map policyTypes = new ArrayList<>(); if (StringUtils.isNotBlank(policyTypeStr)) { - policyType = Integer.parseInt(policyTypeStr); - } - - Set> validResourceHierarchies = serviceDefHelper.getResourceHierarchies(policyType, filterResources.keySet()); - - if (LOG.isDebugEnabled()) { - LOG.debug("Found " + validResourceHierarchies.size() + " valid resource hierarchies for key-set " + filterResources.keySet()); + policyTypes.add(Integer.parseInt(policyTypeStr)); + } else { + policyTypes.add(RangerPolicy.POLICY_TYPE_ACCESS); + policyTypes.add(RangerPolicy.POLICY_TYPE_DATAMASK); + policyTypes.add(RangerPolicy.POLICY_TYPE_ROWFILTER); } - List> resourceHierarchies = new ArrayList>(validResourceHierarchies); - - for (List validResourceHierarchy : resourceHierarchies) { + for (Integer policyType : policyTypes) { + Set> validResourceHierarchies = serviceDefHelper.getResourceHierarchies(policyType, filterResources.keySet()); if (LOG.isDebugEnabled()) { - LOG.debug("validResourceHierarchy:[" + validResourceHierarchy + "]"); + LOG.debug("Found " + validResourceHierarchies.size() + " valid resource hierarchies for key-set " + filterResources.keySet()); } - Map policyResources = new HashMap(); - - for (RangerResourceDef resourceDef : validResourceHierarchy) { + List> resourceHierarchies = new ArrayList>(validResourceHierarchies); - String resourceValue = filterResources.get(resourceDef.getName()); + for (List validResourceHierarchy : resourceHierarchies) { - if (StringUtils.isBlank(resourceValue)) { - resourceValue = RangerAbstractResourceMatcher.WILDCARD_ASTERISK; + if (LOG.isDebugEnabled()) { + LOG.debug("validResourceHierarchy:[" + validResourceHierarchy + "]"); } - policyResources.put(resourceDef.getName(), new RangerPolicyResource(resourceValue, false, resourceDef.getRecursiveSupported())); - } + Map policyResources = new HashMap(); - RangerDefaultPolicyResourceMatcher matcher = new RangerDefaultPolicyResourceMatcher(); - matcher.setServiceDef(serviceDef); - matcher.setPolicyResources(policyResources); - matcher.init(); + for (RangerResourceDef resourceDef : validResourceHierarchy) { + policyResources.put(resourceDef.getName(), new RangerPolicyResource(filterResources.get(resourceDef.getName()), false, resourceDef.getRecursiveSupported())); + } - ret.add(matcher); + RangerDefaultPolicyResourceMatcher matcher = new RangerDefaultPolicyResourceMatcher(); + matcher.setServiceDef(serviceDef); + matcher.setPolicyResources(policyResources, policyType); + matcher.init(); - if (LOG.isDebugEnabled()) { - LOG.debug("Added matcher:[" + matcher + "]"); + ret.add(matcher); + + if (LOG.isDebugEnabled()) { + LOG.debug("Added matcher:[" + matcher + "]"); + } } } From 8fe42a8d4d8969f759a9c8655dc5c188663436ca Mon Sep 17 00:00:00 2001 From: pradeep Date: Mon, 16 Oct 2017 16:32:34 +0530 Subject: [PATCH 058/151] RANGER-1832: Export REST API should return exact matching results if polResource param is provided --- .../src/main/java/org/apache/ranger/rest/ServiceREST.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java b/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java index 1fc002ff65..8ea8313340 100644 --- a/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java +++ b/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java @@ -2227,7 +2227,10 @@ private List getAllFilteredPolicyList(SearchFilter filter, } } } - policyLists=serviceUtil.getMatchingPoliciesForResource(request, policyLists); + if (StringUtils.isNotEmpty(request.getParameter("resourceMatch")) + && "full".equalsIgnoreCase(request.getParameter("resourceMatch"))) { + policyLists = serviceUtil.getMatchingPoliciesForResource(request, policyLists); + } Map orderedPolicies = new TreeMap(); if (!CollectionUtils.isEmpty(policyLists)) { From 49f874fcf430eff8ec57e1864202f6c1d06eeef2 Mon Sep 17 00:00:00 2001 From: pradeep Date: Mon, 16 Oct 2017 09:28:55 +0530 Subject: [PATCH 059/151] RANGER-1838: Refactor Jisql dependencies --- jisql/pom.xml | 5 ---- .../util/outputformatter/CSVFormatter.java | 29 ++----------------- .../main/java/org/apache/util/sql/Jisql.java | 4 +-- src/main/assembly/admin-web.xml | 1 - src/main/assembly/kms.xml | 1 - 5 files changed, 3 insertions(+), 37 deletions(-) diff --git a/jisql/pom.xml b/jisql/pom.xml index 87113ce3f3..8deac5c465 100644 --- a/jisql/pom.xml +++ b/jisql/pom.xml @@ -28,11 +28,6 @@ .. - - net.sourceforge.javacsv - javacsv - 2.0 - net.sf.jopt-simple jopt-simple diff --git a/jisql/src/main/java/org/apache/util/outputformatter/CSVFormatter.java b/jisql/src/main/java/org/apache/util/outputformatter/CSVFormatter.java index 158e25ce06..d55ecbca17 100644 --- a/jisql/src/main/java/org/apache/util/outputformatter/CSVFormatter.java +++ b/jisql/src/main/java/org/apache/util/outputformatter/CSVFormatter.java @@ -22,15 +22,9 @@ import java.io.PrintStream; import java.sql.ResultSet; import java.sql.ResultSetMetaData; - -import java.nio.charset.Charset; - import joptsimple.OptionParser; import joptsimple.OptionSet; -import com.csvreader.CsvWriter; - - /** * This is the default formatter for Jisql. It outputs data in a "normal" * format that is similar to most other database command line formatters. @@ -117,8 +111,7 @@ public void formatHeader( PrintStream out, ResultSetMetaData metaData ) throws E /** - * Called to output the data. This class uses a third party library to output - * the CSV data. The library escapes the data as needed. + * Called to output the data. * * @param out the PrintStream to output data to. * @param resultSet the ResultSet for the row. @@ -126,25 +119,7 @@ public void formatHeader( PrintStream out, ResultSetMetaData metaData ) throws E * * */ - public void formatData( PrintStream out, ResultSet resultSet, ResultSetMetaData metaData ) throws Exception { - - CsvWriter csvWriter = new CsvWriter( out, delimiter, Charset.forName( "us-ascii" ) ); - - while( resultSet.next() ) { - int numColumns = metaData.getColumnCount(); - - for (int i = 1; i <= numColumns; i++) { - String result = resultSet.getString(i); - if( !resultSet.wasNull() ) - csvWriter.write( result ); - else - csvWriter.write( "" ); - } - - csvWriter.endRecord(); - } - - csvWriter.flush(); + public void formatData( PrintStream out, ResultSet resultSet, ResultSetMetaData metaData ) throws Exception{ } diff --git a/jisql/src/main/java/org/apache/util/sql/Jisql.java b/jisql/src/main/java/org/apache/util/sql/Jisql.java index cf5f2c44f8..53a6ca4fe9 100644 --- a/jisql/src/main/java/org/apache/util/sql/Jisql.java +++ b/jisql/src/main/java/org/apache/util/sql/Jisql.java @@ -62,9 +62,7 @@ * default is the term "go" on a single line like Sybase's isql or MS/SQL's * isql/osql. Note that there is a dependency on JOpt Simple in for the base - * configuration. Additionally, if you are using the CSVFormatter then it is - * dependent on Java CSV. - *

+ * configuration. * * * Options: diff --git a/src/main/assembly/admin-web.xml b/src/main/assembly/admin-web.xml index 0e97818b4d..4dc52fd41b 100644 --- a/src/main/assembly/admin-web.xml +++ b/src/main/assembly/admin-web.xml @@ -266,7 +266,6 @@ /jisql/lib false - net.sourceforge.javacsv:javacsv net.sf.jopt-simple:jopt-simple diff --git a/src/main/assembly/kms.xml b/src/main/assembly/kms.xml index e26dd33366..1709859acf 100755 --- a/src/main/assembly/kms.xml +++ b/src/main/assembly/kms.xml @@ -165,7 +165,6 @@ /jisql/lib false - net.sourceforge.javacsv:javacsv net.sf.jopt-simple:jopt-simple From eb8129534c1b76ba0dc2f4661a45b1c0c522c51c Mon Sep 17 00:00:00 2001 From: Abhay Kulkarni Date: Mon, 16 Oct 2017 14:09:21 -0700 Subject: [PATCH 060/151] RANGER-1795: Service should not be renamed if tagged service resources exist for it unless 'forceRename=true' option is specified --- .../ranger/plugin/store/ServiceStore.java | 6 +++- .../org/apache/ranger/biz/ServiceDBStore.java | 31 +++++++++++++------ .../ranger/db/XXServiceResourceDao.java | 12 +++++++ .../org/apache/ranger/rest/AssetREST.java | 2 +- .../org/apache/ranger/rest/PublicAPIs.java | 2 +- .../org/apache/ranger/rest/PublicAPIsv2.java | 10 +++--- .../org/apache/ranger/rest/ServiceREST.java | 20 ++++++++++-- .../resources/META-INF/jpa_named_queries.xml | 6 ++++ .../apache/ranger/biz/TestServiceDBStore.java | 13 ++++---- .../org/apache/ranger/rest/TestAssetREST.java | 5 +-- .../apache/ranger/rest/TestPublicAPIs.java | 5 +-- .../apache/ranger/rest/TestPublicAPIsv2.java | 14 +++++---- .../apache/ranger/rest/TestServiceREST.java | 10 ++++-- 13 files changed, 98 insertions(+), 38 deletions(-) diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/store/ServiceStore.java b/agents-common/src/main/java/org/apache/ranger/plugin/store/ServiceStore.java index 89c33269bc..2c57a6fd8d 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/store/ServiceStore.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/store/ServiceStore.java @@ -20,6 +20,7 @@ package org.apache.ranger.plugin.store; import java.util.List; +import java.util.Map; import org.apache.ranger.plugin.model.RangerPolicy; import org.apache.ranger.plugin.model.RangerService; @@ -28,6 +29,9 @@ import org.apache.ranger.plugin.util.ServicePolicies; public interface ServiceStore { + + String OPTION_FORCE_RENAME = "forceRename"; + void init() throws Exception; RangerServiceDef createServiceDef(RangerServiceDef serviceDef) throws Exception; @@ -48,7 +52,7 @@ public interface ServiceStore { RangerService createService(RangerService service) throws Exception; - RangerService updateService(RangerService service) throws Exception; + RangerService updateService(RangerService service, Map options) throws Exception; void deleteService(Long id) throws Exception; diff --git a/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java b/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java index d951090082..6a1ef09771 100644 --- a/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java +++ b/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java @@ -61,6 +61,7 @@ import org.apache.ranger.plugin.policyresourcematcher.RangerPolicyResourceMatcher; import org.apache.ranger.plugin.resourcematcher.RangerAbstractResourceMatcher; import org.apache.ranger.plugin.service.RangerBaseService; +import org.apache.ranger.plugin.store.ServiceStore; import org.apache.ranger.plugin.util.PasswordUtils; import org.apache.ranger.common.JSONUtil; import org.apache.ranger.common.PropertiesUtil; @@ -1463,7 +1464,7 @@ public RangerService createService(RangerService service) throws Exception { } @Override - public RangerService updateService(RangerService service) throws Exception { + public RangerService updateService(RangerService service, Map options) throws Exception { if(LOG.isDebugEnabled()) { LOG.debug("==> ServiceDBStore.updateService()"); } @@ -1481,13 +1482,26 @@ public RangerService updateService(RangerService service) throws Exception { boolean renamed = !StringUtils.equalsIgnoreCase(service.getName(), existingName); if(renamed) { - XXService newNameService = daoMgr.getXXService().findByName(service.getName()); + XXService newNameService = daoMgr.getXXService().findByName(service.getName()); - if(newNameService != null) { - throw restErrorUtil.createRESTException("another service already exists with name '" - + service.getName() + "'. ID=" + newNameService.getId(), MessageEnums.DATA_NOT_UPDATABLE); - } - } + if (newNameService != null) { + throw restErrorUtil.createRESTException("another service already exists with name '" + + service.getName() + "'. ID=" + newNameService.getId(), MessageEnums.DATA_NOT_UPDATABLE); + } + + long countOfTaggedResources = daoMgr.getXXServiceResource().countTaggedResourcesInServiceId(existing.getId()); + + Boolean isForceRename = options != null && options.get(ServiceStore.OPTION_FORCE_RENAME) != null ? (Boolean) options.get(ServiceStore.OPTION_FORCE_RENAME) : Boolean.FALSE; + + if (countOfTaggedResources != 0L) { + if (isForceRename) { + LOG.warn("Forcing the renaming of service from " + existingName + " to " + service.getName() + " although it is associated with " + countOfTaggedResources + + " service-resources!"); + } else { + throw restErrorUtil.createRESTException("Service " + existingName + " cannot be renamed, as it has associated service-resources", MessageEnums.DATA_NOT_UPDATABLE); + } + } + } Map configs = service.getConfigs(); Map validConfigs = validateRequiredConfigParams(service, configs); @@ -3922,7 +3936,7 @@ private void updateServiceWithCustomProperty() { chkServiceUpdate = true; } if(chkServiceUpdate){ - updateService(rangerService); + updateService(rangerService, null); if(LOG.isDebugEnabled()){ LOG.debug("Updated service "+rangerService.getName()+" with custom properties in secure environment"); } @@ -3931,7 +3945,6 @@ private void updateServiceWithCustomProperty() { } } catch (Throwable e) { LOG.fatal("updateServiceWithCustomProperty failed with exception : "+e.getMessage()); - return; } } diff --git a/security-admin/src/main/java/org/apache/ranger/db/XXServiceResourceDao.java b/security-admin/src/main/java/org/apache/ranger/db/XXServiceResourceDao.java index 98599921f0..f87c0ae596 100644 --- a/security-admin/src/main/java/org/apache/ranger/db/XXServiceResourceDao.java +++ b/security-admin/src/main/java/org/apache/ranger/db/XXServiceResourceDao.java @@ -84,6 +84,18 @@ public List findTaggedResourcesInServiceId(Long serviceId) { } } + public long countTaggedResourcesInServiceId(Long serviceId) { + if (serviceId == null) { + return -1; + } + try { + return getEntityManager().createNamedQuery("XXServiceResource.countTaggedResourcesInServiceId", Long.class) + .setParameter("serviceId", serviceId).getSingleResult(); + } catch (NoResultException e) { + return -1; + } + } + public List findForServicePlugin(Long serviceId) { if (serviceId == null) { return new ArrayList(); diff --git a/security-admin/src/main/java/org/apache/ranger/rest/AssetREST.java b/security-admin/src/main/java/org/apache/ranger/rest/AssetREST.java index 54226d97c4..3c274e3faa 100644 --- a/security-admin/src/main/java/org/apache/ranger/rest/AssetREST.java +++ b/security-admin/src/main/java/org/apache/ranger/rest/AssetREST.java @@ -190,7 +190,7 @@ public VXAsset updateXAsset(VXAsset vXAsset) { RangerService service = serviceUtil.toRangerService(vXAsset); - RangerService updatedService = serviceREST.updateService(service); + RangerService updatedService = serviceREST.updateService(service, null); VXAsset ret = serviceUtil.toVXAsset(updatedService); diff --git a/security-admin/src/main/java/org/apache/ranger/rest/PublicAPIs.java b/security-admin/src/main/java/org/apache/ranger/rest/PublicAPIs.java index d3c22d7226..7818eb591e 100644 --- a/security-admin/src/main/java/org/apache/ranger/rest/PublicAPIs.java +++ b/security-admin/src/main/java/org/apache/ranger/rest/PublicAPIs.java @@ -144,7 +144,7 @@ public VXRepository updateRepository(VXRepository vXRepository, RangerService service = serviceUtil.toRangerService(vXAsset); service.setVersion(existing.getVersion()); - RangerService updatedService = serviceREST.updateService(service); + RangerService updatedService = serviceREST.updateService(service, null); VXAsset retvXAsset = serviceUtil.toVXAsset(updatedService); diff --git a/security-admin/src/main/java/org/apache/ranger/rest/PublicAPIsv2.java b/security-admin/src/main/java/org/apache/ranger/rest/PublicAPIsv2.java index fa3c68ed69..0281c94a5e 100644 --- a/security-admin/src/main/java/org/apache/ranger/rest/PublicAPIsv2.java +++ b/security-admin/src/main/java/org/apache/ranger/rest/PublicAPIsv2.java @@ -209,7 +209,8 @@ public RangerService createService(RangerService service) { @Path("/api/service/{id}") @PreAuthorize("@rangerPreAuthSecurityHandler.isAPISpnegoAccessible()") @Produces({ "application/json", "application/xml" }) - public RangerService updateService(RangerService service, @PathParam("id") Long id) { + public RangerService updateService(RangerService service, @PathParam("id") Long id, + @Context HttpServletRequest request) { // if service.id is specified, it should be same as the param 'id' if(service.getId() == null) { service.setId(id); @@ -217,7 +218,7 @@ public RangerService updateService(RangerService service, @PathParam("id") Long throw restErrorUtil.createRESTException(HttpServletResponse.SC_BAD_REQUEST , "service id mismatch", true); } - return serviceREST.updateService(service); + return serviceREST.updateService(service, request); } @@ -226,7 +227,8 @@ public RangerService updateService(RangerService service, @PathParam("id") Long @PreAuthorize("@rangerPreAuthSecurityHandler.isAPISpnegoAccessible()") @Produces({ "application/json", "application/xml" }) public RangerService updateServiceByName(RangerService service, - @PathParam("name") String name) { + @PathParam("name") String name, + @Context HttpServletRequest request) { // ignore service.id - if specified. Retrieve using the given name and use id from the retrieved object RangerService existingService = getServiceByName(name); service.setId(existingService.getId()); @@ -237,7 +239,7 @@ public RangerService updateServiceByName(RangerService service, service.setName(existingService.getName()); } - return serviceREST.updateService(service); + return serviceREST.updateService(service, request); } /* diff --git a/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java b/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java index 8ea8313340..6e6d241a14 100644 --- a/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java +++ b/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java @@ -101,6 +101,7 @@ import org.apache.ranger.plugin.service.ResourceLookupContext; import org.apache.ranger.plugin.store.PList; import org.apache.ranger.plugin.store.EmbeddedServiceDefsUtil; +import org.apache.ranger.plugin.store.ServiceStore; import org.apache.ranger.plugin.util.GrantRevokeRequest; import org.apache.ranger.plugin.util.RangerAccessRequestUtil; import org.apache.ranger.plugin.util.RangerPerfTracer; @@ -670,7 +671,8 @@ public RangerService createService(RangerService service) { @Path("/services/{id}") @Produces({ "application/json", "application/xml" }) @PreAuthorize("@rangerPreAuthSecurityHandler.isAPIAccessible(\"" + RangerAPIList.UPDATE_SERVICE + "\")") - public RangerService updateService(RangerService service) { + public RangerService updateService(RangerService service, + @Context HttpServletRequest request) { if(LOG.isDebugEnabled()) { LOG.debug("==> ServiceREST.updateService(): " + service); } @@ -693,7 +695,9 @@ public RangerService updateService(RangerService service) { XXServiceDef xxServiceDef = daoManager.getXXServiceDef().findByName(service.getType()); bizUtil.hasKMSPermissions("Service", xxServiceDef.getImplclassname()); - ret = svcStore.updateService(service); + Map options = getOptions(request); + + ret = svcStore.updateService(service, options); } catch(WebApplicationException excp) { throw excp; } catch(Throwable excp) { @@ -3304,4 +3308,16 @@ private void validateGrantRevokeRequest(GrantRevokeRequest request){ } } } + + private Map getOptions(HttpServletRequest request) { + Map ret = null; + if (request != null) { + String isForceRenameOption = request.getParameter(ServiceStore.OPTION_FORCE_RENAME); + if (StringUtils.isNotBlank(isForceRenameOption)) { + ret = new HashMap(); + ret.put(ServiceStore.OPTION_FORCE_RENAME, Boolean.valueOf(isForceRenameOption)); + } + } + return ret; + } } diff --git a/security-admin/src/main/resources/META-INF/jpa_named_queries.xml b/security-admin/src/main/resources/META-INF/jpa_named_queries.xml index a212e59e43..786b4bfdd7 100644 --- a/security-admin/src/main/resources/META-INF/jpa_named_queries.xml +++ b/security-admin/src/main/resources/META-INF/jpa_named_queries.xml @@ -1085,6 +1085,12 @@ + + select count(obj.id) from XXServiceResource obj where obj.serviceId = :serviceId and obj.id in + (select tagResMap.resourceId from XXTagResourceMap tagResMap) + + + select obj from XXServiceResource obj, XXService service where service.id = :serviceId and service.tagService is not null and obj.serviceId = service.id and obj.id in diff --git a/security-admin/src/test/java/org/apache/ranger/biz/TestServiceDBStore.java b/security-admin/src/test/java/org/apache/ranger/biz/TestServiceDBStore.java index cd7a605b5b..44523fb22d 100644 --- a/security-admin/src/test/java/org/apache/ranger/biz/TestServiceDBStore.java +++ b/security-admin/src/test/java/org/apache/ranger/biz/TestServiceDBStore.java @@ -1215,10 +1215,12 @@ public void test20updateService() throws Exception { XXServiceConfigDefDao xServiceConfigDefDao = Mockito .mock(XXServiceConfigDefDao.class); XXUserDao xUserDao = Mockito.mock(XXUserDao.class); - XXUser xUser = Mockito.mock(XXUser.class); + XXUser xUser = Mockito.mock(XXUser.class); + XXServiceResourceDao xServiceResourceDao = Mockito.mock(XXServiceResourceDao.class); VXUser vXUser = null; RangerService rangerService = rangerService(); + Map options = null; String name = "fdfdfds"; List trxLogList = new ArrayList(); @@ -1247,12 +1249,9 @@ public void test20updateService() throws Exception { xServiceConfigDefList.add(serviceConfigDefObj); Mockito.when(daoManager.getXXServiceConfigDef()).thenReturn( xServiceConfigDefDao); - Mockito.when(xServiceConfigDefDao.findByServiceDefName(name)) - .thenReturn(xServiceConfigDefList); - - Mockito.when(svcService.getTransactionLog(rangerService, xService, 0)) - .thenReturn(trxLogList); + Mockito.when(daoManager.getXXServiceResource()).thenReturn(xServiceResourceDao); + Mockito.when(xServiceResourceDao.countTaggedResourcesInServiceId(xService.getId())).thenReturn(0L); Mockito.when(svcService.update(rangerService)) .thenReturn(rangerService); Mockito.when(daoManager.getXXService()).thenReturn(xServiceDao); @@ -1303,7 +1302,7 @@ public void test20updateService() throws Exception { Mockito.when(xServiceVersionInfoDao.update(xServiceVersionInfo)).thenReturn(xServiceVersionInfo); RangerService dbRangerService = serviceDBStore - .updateService(rangerService); + .updateService(rangerService, options); Assert.assertNotNull(dbRangerService); Assert.assertEquals(dbRangerService, rangerService); Assert.assertEquals(dbRangerService.getId(), rangerService.getId()); diff --git a/security-admin/src/test/java/org/apache/ranger/rest/TestAssetREST.java b/security-admin/src/test/java/org/apache/ranger/rest/TestAssetREST.java index af07e60ab5..5e4c68e8a2 100644 --- a/security-admin/src/test/java/org/apache/ranger/rest/TestAssetREST.java +++ b/security-admin/src/test/java/org/apache/ranger/rest/TestAssetREST.java @@ -325,14 +325,15 @@ public void testCreateXAsset() { @Test public void testUpdateXAsset() { RangerService rangerService = rangerService(Id); + HttpServletRequest request = null; VXAsset vXAsset = vXAsset(Id); Mockito.when(serviceUtil.toRangerService(vXAsset)).thenReturn(rangerService); - Mockito.when(serviceREST.updateService(rangerService)).thenReturn(rangerService); + Mockito.when(serviceREST.updateService(rangerService, request)).thenReturn(rangerService); Mockito.when(serviceUtil.toVXAsset(rangerService)).thenReturn(vXAsset); VXAsset asset = assetREST.updateXAsset(vXAsset); Assert.assertNotNull(asset); Assert.assertEquals(vXAsset, asset); - Mockito.verify(serviceREST).updateService(rangerService); + Mockito.verify(serviceREST).updateService(rangerService, request); Mockito.verify(serviceUtil).toRangerService(vXAsset); Mockito.verify(serviceUtil).toVXAsset(rangerService); } diff --git a/security-admin/src/test/java/org/apache/ranger/rest/TestPublicAPIs.java b/security-admin/src/test/java/org/apache/ranger/rest/TestPublicAPIs.java index 9faae937c0..11c709c3b5 100644 --- a/security-admin/src/test/java/org/apache/ranger/rest/TestPublicAPIs.java +++ b/security-admin/src/test/java/org/apache/ranger/rest/TestPublicAPIs.java @@ -293,6 +293,7 @@ public void test2createRepository() throws Exception { public void test3updateRepository() throws Exception { VXAsset vXAsset = new VXAsset(); RangerService rangerService = rangerService(); + HttpServletRequest request = null; VXRepository vXRepository = vXRepository(rangerService); XXService xService = xService(); XXServiceDao xServiceDao = Mockito.mock(XXServiceDao.class); @@ -300,7 +301,7 @@ public void test3updateRepository() throws Exception { Mockito.when(xServiceDao.getById(Id)).thenReturn(xService); Mockito.when(serviceUtil.publicObjecttoVXAsset(vXRepository)).thenReturn(vXAsset); Mockito.when(serviceUtil.toRangerService(vXAsset)).thenReturn(rangerService); - Mockito.when(serviceREST.updateService(rangerService)).thenReturn(rangerService); + Mockito.when(serviceREST.updateService(rangerService, request)).thenReturn(rangerService); Mockito.when(serviceUtil.toVXAsset(rangerService)).thenReturn(vXAsset); Mockito.when(serviceUtil.vXAssetToPublicObject(vXAsset)).thenReturn(vXRepository); VXRepository dbVXRepository = publicAPIs.updateRepository(vXRepository, Id); @@ -311,7 +312,7 @@ public void test3updateRepository() throws Exception { vXRepository.getId()); Assert.assertEquals(dbVXRepository.getName(), vXRepository.getName()); - Mockito.verify(serviceREST).updateService(rangerService); + Mockito.verify(serviceREST).updateService(rangerService, request); Mockito.verify(serviceUtil).publicObjecttoVXAsset(vXRepository); Mockito.verify(serviceUtil).toRangerService(vXAsset); Mockito.verify(serviceUtil).toVXAsset(rangerService); diff --git a/security-admin/src/test/java/org/apache/ranger/rest/TestPublicAPIsv2.java b/security-admin/src/test/java/org/apache/ranger/rest/TestPublicAPIsv2.java index 761ed03ea3..527b9a2807 100644 --- a/security-admin/src/test/java/org/apache/ranger/rest/TestPublicAPIsv2.java +++ b/security-admin/src/test/java/org/apache/ranger/rest/TestPublicAPIsv2.java @@ -403,31 +403,33 @@ public void test12createService() throws Exception { @Test public void test13updateService() throws Exception { RangerService rangerService = rangerService(); - Mockito.when(serviceREST.updateService(rangerService)).thenReturn(rangerService); - RangerService dbRangerService = publicAPIsv2.updateService(rangerService, Id); + HttpServletRequest request = null; + Mockito.when(serviceREST.updateService(rangerService, request)).thenReturn(rangerService); + RangerService dbRangerService = publicAPIsv2.updateService(rangerService, Id, request); Assert.assertNotNull(dbRangerService); Assert.assertEquals(dbRangerService, rangerService); Assert.assertEquals(dbRangerService.getId(), rangerService.getId()); Assert.assertEquals(dbRangerService.getName(), rangerService.getName()); - Mockito.verify(serviceREST).updateService(rangerService); + Mockito.verify(serviceREST).updateService(rangerService, request); } @Test public void test14updateServiceByName() throws Exception { RangerService rangerService = rangerService(); + HttpServletRequest request = null; String name = rangerService.getName(); Mockito.when(serviceREST.getServiceByName(name)).thenReturn(rangerService); - Mockito.when(serviceREST.updateService(rangerService)).thenReturn(rangerService); - RangerService dbRangerService = publicAPIsv2.updateServiceByName(rangerService, name); + Mockito.when(serviceREST.updateService(rangerService, request)).thenReturn(rangerService); + RangerService dbRangerService = publicAPIsv2.updateServiceByName(rangerService, name, request); Assert.assertNotNull(dbRangerService); Assert.assertEquals(dbRangerService, rangerService); Assert.assertEquals(dbRangerService.getId(), rangerService.getId()); Assert.assertEquals(dbRangerService.getName(), rangerService.getName()); - Mockito.verify(serviceREST).updateService(rangerService); + Mockito.verify(serviceREST).updateService(rangerService, request); Mockito.verify(serviceREST).getServiceByName(name); } diff --git a/security-admin/src/test/java/org/apache/ranger/rest/TestServiceREST.java b/security-admin/src/test/java/org/apache/ranger/rest/TestServiceREST.java index 805694838f..68908b2057 100644 --- a/security-admin/src/test/java/org/apache/ranger/rest/TestServiceREST.java +++ b/security-admin/src/test/java/org/apache/ranger/rest/TestServiceREST.java @@ -543,6 +543,9 @@ public void test8updateServiceDef() throws Exception { RangerService rangerService = rangerService(); XXServiceDef xServiceDef = serviceDef(); + HttpServletRequest request = null; + Map options = null; + XXServiceDefDao xServiceDefDao = Mockito.mock(XXServiceDefDao.class); Mockito.when(validatorFactory.getServiceValidator(svcStore)) .thenReturn(serviceValidator); @@ -552,11 +555,12 @@ public void test8updateServiceDef() throws Exception { .thenReturn(xServiceDef); Mockito.when( - svcStore.updateService((RangerService) Mockito.anyObject())) + + svcStore.updateService((RangerService) Mockito.any(), (Map) Mockito.any())) .thenReturn(rangerService); RangerService dbRangerService = serviceREST - .updateService(rangerService); + .updateService(rangerService, request); Assert.assertNotNull(dbRangerService); Assert.assertNotNull(dbRangerService); Assert.assertEquals(rangerService, dbRangerService); @@ -580,7 +584,7 @@ public void test8updateServiceDef() throws Exception { dbRangerService.getUpdatedBy()); Mockito.verify(validatorFactory).getServiceValidator(svcStore); Mockito.verify(daoManager).getXXServiceDef(); - Mockito.verify(svcStore).updateService(rangerService); + Mockito.verify(svcStore).updateService(rangerService, options); } @Test From 616a646d1f8275a1c61c7b0ff4e1135dbcbcb0b7 Mon Sep 17 00:00:00 2001 From: Abhay Kulkarni Date: Mon, 16 Oct 2017 16:33:52 -0700 Subject: [PATCH 061/151] RANGER-1841: Audit log record for 'use dbName' hive command contains large number of tags --- .../hive/authorizer/RangerHiveAuditHandler.java | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/hive-agent/src/main/java/org/apache/ranger/authorization/hive/authorizer/RangerHiveAuditHandler.java b/hive-agent/src/main/java/org/apache/ranger/authorization/hive/authorizer/RangerHiveAuditHandler.java index 9dea37a138..89bc0d8761 100644 --- a/hive-agent/src/main/java/org/apache/ranger/authorization/hive/authorizer/RangerHiveAuditHandler.java +++ b/hive-agent/src/main/java/org/apache/ranger/authorization/hive/authorizer/RangerHiveAuditHandler.java @@ -54,6 +54,16 @@ AuthzAuditEvent createAuditEvent(RangerAccessResult result, String accessType, S auditEvent.setResourcePath(resourcePath); auditEvent.setResourceType("@" + resourceType); // to be consistent with earlier release + if (request instanceof RangerHiveAccessRequest && resource instanceof RangerHiveResource) { + RangerHiveAccessRequest hiveAccessRequest = (RangerHiveAccessRequest) request; + RangerHiveResource hiveResource = (RangerHiveResource) resource; + + if (hiveAccessRequest.getHiveAccessType() == HiveAccessType.USE && hiveResource.getObjectType() == HiveObjectType.DATABASE) { + // this should happen only for SHOWDATABASES and USE commands + auditEvent.setTags(null); + } + } + return auditEvent; } From d3527546f10c514cc0b4069d7d830bf46e8726f8 Mon Sep 17 00:00:00 2001 From: pradeep Date: Wed, 18 Oct 2017 22:13:04 +0530 Subject: [PATCH 062/151] RANGER-1820: Update consolidated db schema script for SQLServer DB --- .../sqlserver/optimized/current/ranger_core_db_sqlserver.sql | 3 +++ 1 file changed, 3 insertions(+) diff --git a/security-admin/db/sqlserver/optimized/current/ranger_core_db_sqlserver.sql b/security-admin/db/sqlserver/optimized/current/ranger_core_db_sqlserver.sql index 27257e3fa8..a79c954491 100644 --- a/security-admin/db/sqlserver/optimized/current/ranger_core_db_sqlserver.sql +++ b/security-admin/db/sqlserver/optimized/current/ranger_core_db_sqlserver.sql @@ -2989,6 +2989,7 @@ INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active) VALUES ('026',CURRENT_TIMESTAMP,'Ranger 0.7.0',CURRENT_TIMESTAMP,'localhost','Y'); INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active) VALUES ('027',CURRENT_TIMESTAMP,'Ranger 0.7.0',CURRENT_TIMESTAMP,'localhost','Y'); INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active) VALUES ('028',CURRENT_TIMESTAMP,'Ranger 0.7.0',CURRENT_TIMESTAMP,'localhost','Y'); +INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active) VALUES ('029',CURRENT_TIMESTAMP,'Ranger 0.7.0',CURRENT_TIMESTAMP,'localhost','Y'); INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active) VALUES ('DB_PATCHES',CURRENT_TIMESTAMP,'Ranger 0.7.0',CURRENT_TIMESTAMP,'localhost','Y'); INSERT INTO x_user_module_perm (user_id,module_id,create_time,update_time,added_by_id,upd_by_id,is_allowed) VALUES (1,3,CURRENT_TIMESTAMP,CURRENT_TIMESTAMP,1,1,1); INSERT INTO x_user_module_perm (user_id,module_id,create_time,update_time,added_by_id,upd_by_id,is_allowed) VALUES (1,1,CURRENT_TIMESTAMP,CURRENT_TIMESTAMP,1,1,1); @@ -3015,6 +3016,8 @@ INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active) VALUES ('J10005',CURRENT_TIMESTAMP,'Ranger 0.7.0',CURRENT_TIMESTAMP,'localhost','Y'); INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active) VALUES ('J10006',CURRENT_TIMESTAMP,'Ranger 0.7.0',CURRENT_TIMESTAMP,'localhost','Y'); INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active) VALUES ('J10007',CURRENT_TIMESTAMP,'Ranger 0.7.0',CURRENT_TIMESTAMP,'localhost','Y'); +INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active) VALUES ('J10008',CURRENT_TIMESTAMP,'Ranger 0.7.0',CURRENT_TIMESTAMP,'localhost','Y'); +INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active) VALUES ('J10011',CURRENT_TIMESTAMP,'Ranger 0.7.0',CURRENT_TIMESTAMP,'localhost','Y'); INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active) VALUES ('JAVA_PATCHES',CURRENT_TIMESTAMP,'Ranger 0.7.0',CURRENT_TIMESTAMP,'localhost','Y'); GO CREATE VIEW [dbo].[vx_trx_log] AS From 3968c7f79589bd6d2bdef185126a20c5707e7ecb Mon Sep 17 00:00:00 2001 From: Abhay Kulkarni Date: Wed, 18 Oct 2017 12:28:06 -0700 Subject: [PATCH 063/151] RANGER-1843: Tag enricher performance improvement in identifying tags for resource being accessed --- .../contextenricher/RangerTagEnricher.java | 165 +++++++++++++----- .../validation/RangerServiceDefHelper.java | 24 ++- 2 files changed, 144 insertions(+), 45 deletions(-) diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java index 5f0a422dc7..4a3a95062b 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java @@ -27,9 +27,11 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.ranger.authorization.hadoop.config.RangerConfiguration; +import org.apache.ranger.plugin.model.RangerPolicy; import org.apache.ranger.plugin.model.RangerServiceDef; import org.apache.ranger.plugin.model.RangerServiceResource; import org.apache.ranger.plugin.model.RangerTag; +import org.apache.ranger.plugin.model.validation.RangerServiceDefHelper; import org.apache.ranger.plugin.policyengine.RangerAccessRequest; import org.apache.ranger.plugin.policyengine.RangerAccessResource; import org.apache.ranger.plugin.policyresourcematcher.RangerDefaultPolicyResourceMatcher; @@ -46,6 +48,7 @@ import java.io.Reader; import java.io.Writer; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -61,6 +64,7 @@ public class RangerTagEnricher extends RangerAbstractContextEnricher { public static final String TAG_REFRESHER_POLLINGINTERVAL_OPTION = "tagRefresherPollingInterval"; public static final String TAG_RETRIEVER_CLASSNAME_OPTION = "tagRetrieverClassName"; public static final String TAG_DISABLE_TRIE_PREFILTER_OPTION = "disableTrieLookupPrefilter"; + public static final int[] allPolicyTypes = new int[] {RangerPolicy.POLICY_TYPE_ACCESS, RangerPolicy.POLICY_TYPE_DATAMASK, RangerPolicy.POLICY_TYPE_ROWFILTER}; private RangerTagRefresher tagRefresher = null; private RangerTagRetriever tagRetriever = null; @@ -148,6 +152,52 @@ public void enrich(RangerAccessRequest request) { } } + /* + * This class implements a cache of result of look-up of keyset of policy-resources for each of the collections of hierarchies + * for policy types: access, datamask and rowfilter. If a keyset is examined for validity in a hierarchy of a policy-type, + * then that record is maintained in this cache for later look-up. + * + * The basic idea is that with a large number of tagged service-resources, this cache will speed up performance as well as put + * a cap on the upper bound because it is expected that the cardinality of set of all possible keysets for all resource-def + * combinations in a service-def will be much smaller than the number of service-resources. + */ + + static private class ResourceHierarchies { + private final Map, Boolean> accessHierarchies = new HashMap<>(); + private final Map, Boolean> dataMaskHierarchies = new HashMap<>(); + private final Map, Boolean> rowFilterHierarchies = new HashMap<>(); + + public Boolean isValidHierarchy(int policyType, Collection resourceKeys) { + switch (policyType) { + case RangerPolicy.POLICY_TYPE_ACCESS: + return accessHierarchies.get(resourceKeys); + case RangerPolicy.POLICY_TYPE_DATAMASK: + return dataMaskHierarchies.get(resourceKeys); + case RangerPolicy.POLICY_TYPE_ROWFILTER: + return rowFilterHierarchies.get(resourceKeys); + default: + return null; + } + } + + public void addHierarchy(int policyType, Collection resourceKeys, Boolean isValid) { + switch (policyType) { + case RangerPolicy.POLICY_TYPE_ACCESS: + accessHierarchies.put(resourceKeys, isValid); + break; + case RangerPolicy.POLICY_TYPE_DATAMASK: + dataMaskHierarchies.put(resourceKeys, isValid); + break; + case RangerPolicy.POLICY_TYPE_ROWFILTER: + rowFilterHierarchies.put(resourceKeys, isValid); + break; + default: + LOG.error("unknown policy-type " + policyType); + break; + } + } + } + public void setServiceTags(final ServiceTags serviceTags) { if (serviceTags == null || CollectionUtils.isEmpty(serviceTags.getServiceResources())) { @@ -157,29 +207,52 @@ public void setServiceTags(final ServiceTags serviceTags) { List resourceMatchers = new ArrayList(); + RangerServiceDefHelper serviceDefHelper = new RangerServiceDefHelper(serviceDef, false); + List serviceResources = serviceTags.getServiceResources(); - if (CollectionUtils.isNotEmpty(serviceResources)) { + ResourceHierarchies hierarchies = new ResourceHierarchies(); + + for (RangerServiceResource serviceResource : serviceResources) { + final Collection resourceKeys = serviceResource.getResourceElements().keySet(); - for (RangerServiceResource serviceResource : serviceResources) { - RangerDefaultPolicyResourceMatcher matcher = new RangerDefaultPolicyResourceMatcher(); + for (int policyType : allPolicyTypes) { + Boolean isValidHierarchy = hierarchies.isValidHierarchy(policyType, resourceKeys); - matcher.setServiceDef(this.serviceDef); - matcher.setPolicyResources(serviceResource.getResourceElements()); + if (isValidHierarchy == null) { // hierarchy not yet validated + isValidHierarchy = Boolean.FALSE; - if (LOG.isDebugEnabled()) { - LOG.debug("RangerTagEnricher.setServiceTags() - Initializing matcher with (resource=" + serviceResource - + ", serviceDef=" + this.serviceDef.getName() + ")"); + for (List hierarchy : serviceDefHelper.getResourceHierarchies(policyType)) { + if (serviceDefHelper.hierarchyHasAllResources(hierarchy, resourceKeys)) { + isValidHierarchy = Boolean.TRUE; + + break; + } + } + hierarchies.addHierarchy(policyType, resourceKeys, isValidHierarchy); } - matcher.init(); - RangerServiceResourceMatcher serviceResourceMatcher = new RangerServiceResourceMatcher(serviceResource, matcher); - resourceMatchers.add(serviceResourceMatcher); - } + if (isValidHierarchy) { + RangerDefaultPolicyResourceMatcher matcher = new RangerDefaultPolicyResourceMatcher(); + + matcher.setServiceDef(this.serviceDef); + matcher.setPolicyResources(serviceResource.getResourceElements(), policyType); + + if (LOG.isDebugEnabled()) { + LOG.debug("RangerTagEnricher.setServiceTags() - Initializing matcher with (resource=" + serviceResource + + ", serviceDef=" + this.serviceDef.getName() + ")"); + } + matcher.init(); + + RangerServiceResourceMatcher serviceResourceMatcher = new RangerServiceResourceMatcher(serviceResource, matcher); + resourceMatchers.add(serviceResourceMatcher); + } + } } + Map> serviceResourceTrie = null; if (!disableTrieLookupPrefilter) { @@ -287,18 +360,17 @@ private List getEvaluators(RangerAccessResource re List ret = null; - final List serviceResourceMatchers = enrichedServiceTags.getServiceResourceMatchers(); final Map> serviceResourceTrie = enrichedServiceTags.getServiceResourceTrie(); - if (resource == null || resource.getKeys() == null || resource.getKeys().size() == 0 || serviceResourceTrie == null) { - ret = serviceResourceMatchers; + if (resource == null || resource.getKeys() == null || resource.getKeys().isEmpty() || serviceResourceTrie == null) { + ret = enrichedServiceTags.getServiceResourceMatchers(); } else { Set resourceKeys = resource.getKeys(); + List> serviceResourceMatchersList = null; + List smallestList = null; if (CollectionUtils.isNotEmpty(resourceKeys)) { - boolean isRetModifiable = false; - for (String resourceName : resourceKeys) { RangerResourceTrie trie = serviceResourceTrie.get(resourceName); @@ -306,38 +378,43 @@ private List getEvaluators(RangerAccessResource re continue; } - List resourceEvaluators = trie.getEvaluatorsForResource(resource.getValue(resourceName)); - - if (CollectionUtils.isEmpty(resourceEvaluators)) { // no policies for this resource, bail out - ret = null; - } else if (ret == null) { // initialize ret with policies found for this resource - ret = resourceEvaluators; - } else { // remove matchers from ret that are not in resourceEvaluators - if (isRetModifiable) { - ret.retainAll(resourceEvaluators); - } else { - final List shorterList; - final List longerList; - - if (ret.size() < resourceEvaluators.size()) { - shorterList = ret; - longerList = resourceEvaluators; - } else { - shorterList = resourceEvaluators; - longerList = ret; - } + List serviceResourceMatchers = trie.getEvaluatorsForResource(resource.getValue(resourceName)); - ret = new ArrayList<>(shorterList); - ret.retainAll(longerList); - isRetModifiable = true; - } + if (CollectionUtils.isEmpty(serviceResourceMatchers)) { // no policies for this resource, bail out + serviceResourceMatchersList = null; + smallestList = null; + break; } - if (CollectionUtils.isEmpty(ret)) { // if no matcher exists, bail out and return empty list - ret = null; - break; + if (smallestList == null) { + smallestList = serviceResourceMatchers; + } else { + if (serviceResourceMatchersList == null) { + serviceResourceMatchersList = new ArrayList<>(); + serviceResourceMatchersList.add(smallestList); + } + serviceResourceMatchersList.add(serviceResourceMatchers); + + if (smallestList.size() > serviceResourceMatchers.size()) { + smallestList = serviceResourceMatchers; + } } } + if (serviceResourceMatchersList != null) { + ret = new ArrayList<>(smallestList); + for (List serviceResourceMatchers : serviceResourceMatchersList) { + if (serviceResourceMatchers != smallestList) { + // remove policies from ret that are not in serviceResourceMatchers + ret.retainAll(serviceResourceMatchers); + if (CollectionUtils.isEmpty(ret)) { // if no policy exists, bail out and return empty list + ret = null; + break; + } + } + } + } else { + ret = smallestList; + } } } diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerServiceDefHelper.java b/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerServiceDefHelper.java index 210eb3d5e1..13a1669f73 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerServiceDefHelper.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerServiceDefHelper.java @@ -180,7 +180,7 @@ public Set> getResourceHierarchies(Integer policyType, C Set> ret = new HashSet>(); for (List hierarchy : getResourceHierarchies(policyType)) { - if (getAllResourceNames(hierarchy).containsAll(keys)) { + if (hierarchyHasAllResources(hierarchy, keys)) { ret.add(hierarchy); } } @@ -188,6 +188,28 @@ public Set> getResourceHierarchies(Integer policyType, C return ret; } + public boolean hierarchyHasAllResources(List hierarchy, Collection resourceNames) { + boolean foundAllResourceKeys = true; + + for (String resourceKey : resourceNames) { + boolean found = false; + + for (RangerResourceDef resourceDef : hierarchy) { + if (resourceDef.getName().equals(resourceKey)) { + found = true; + break; + } + } + + if (!found) { + foundAllResourceKeys = false; + break; + } + } + + return foundAllResourceKeys; + } + public Set getMandatoryResourceNames(List hierarchy) { Set result = new HashSet(hierarchy.size()); for (RangerResourceDef resourceDef : hierarchy) { From 8c92df64d8ccb03b59c5ba89d43632fc181c5c9c Mon Sep 17 00:00:00 2001 From: ni3galave Date: Mon, 23 Oct 2017 10:40:56 +0530 Subject: [PATCH 064/151] RANGER-1853: Masking functions based on custom masking of string types fails to unescape quotes properly Signed-off-by: Mehul Parikh --- .../src/main/webapp/scripts/views/policies/PermissionList.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/security-admin/src/main/webapp/scripts/views/policies/PermissionList.js b/security-admin/src/main/webapp/scripts/views/policies/PermissionList.js index 92b8334bcc..821a86b716 100644 --- a/security-admin/src/main/webapp/scripts/views/policies/PermissionList.js +++ b/security-admin/src/main/webapp/scripts/views/policies/PermissionList.js @@ -485,7 +485,7 @@ define(function(require) { }); this.$el.find('input[data-id="maskTypeCustom"]').on('change', function(e){ if(!_.isUndefined(that.model.get('dataMaskInfo'))){ - that.model.get('dataMaskInfo').valueExpr = _.escape(e.currentTarget.value); + that.model.get('dataMaskInfo').valueExpr = (e.currentTarget.value); } }).trigger('change'); if(!this.accessPermSetForTagMasking){ @@ -724,7 +724,7 @@ define(function(require) { }); this.$el.find('input[data-id="maskTypeCustom"]').on('change', function(e){ if(!_.isUndefined(that.model.get('dataMaskInfo'))){ - that.model.get('dataMaskInfo').valueExpr = _.escape(e.currentTarget.value); + that.model.get('dataMaskInfo').valueExpr = (e.currentTarget.value); } }).trigger('change'); }, From e5a272cf8e61ed0bf9700bdec8d48ab1b5e93016 Mon Sep 17 00:00:00 2001 From: Nikhil P Date: Tue, 17 Oct 2017 16:16:58 +0530 Subject: [PATCH 065/151] RANGER-1828 : Good coding practice-add additional headers in ranger Signed-off-by: Mehul Parikh --- .../web/filter/RangerSecurityContextFormationFilter.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/security-admin/src/main/java/org/apache/ranger/security/web/filter/RangerSecurityContextFormationFilter.java b/security-admin/src/main/java/org/apache/ranger/security/web/filter/RangerSecurityContextFormationFilter.java index 721dd44023..dc1e106906 100644 --- a/security-admin/src/main/java/org/apache/ranger/security/web/filter/RangerSecurityContextFormationFilter.java +++ b/security-admin/src/main/java/org/apache/ranger/security/web/filter/RangerSecurityContextFormationFilter.java @@ -139,6 +139,9 @@ public void doFilter(ServletRequest request, ServletResponse response, } HttpServletResponse res = (HttpServletResponse)response; res.setHeader("X-Frame-Options", "DENY" ); + res.setHeader("X-Content-Type-Options", "nosniff"); + res.setHeader("X-XSS-Protection", "1; mode=block"); + res.setHeader("Strict-Transport-Security", "max-age=31536000; includeSubDomains"); chain.doFilter(request, res); } finally { From 8320f0847725cd2ed4a23f0b2c99f23d129d0192 Mon Sep 17 00:00:00 2001 From: Nikhil P Date: Fri, 10 Nov 2017 12:32:38 +0530 Subject: [PATCH 066/151] RANGER-1868 : Good coding practice in Ranger recommended by static code analysis Signed-off-by: Mehul Parikh --- .../java/org/apache/hadoop/security/SecureClientLogin.java | 3 +++ .../org/apache/ranger/services/storm/client/StormClient.java | 1 + 2 files changed, 4 insertions(+) diff --git a/agents-common/src/main/java/org/apache/hadoop/security/SecureClientLogin.java b/agents-common/src/main/java/org/apache/hadoop/security/SecureClientLogin.java index 140d87e656..0f0da96c70 100644 --- a/agents-common/src/main/java/org/apache/hadoop/security/SecureClientLogin.java +++ b/agents-common/src/main/java/org/apache/hadoop/security/SecureClientLogin.java @@ -49,6 +49,7 @@ public synchronized static Subject loginUserFromKeytab(String user, String path) SecureClientLoginConfiguration loginConf = new SecureClientLoginConfiguration(true, user, path); LoginContext login = new LoginContext("hadoop-keytab-kerberos", subject, null, loginConf); subject.getPrincipals().add(new User(user, AuthenticationMethod.KERBEROS, login)); + login.logout(); login.login(); return login.getSubject(); } catch (LoginException le) { @@ -63,6 +64,7 @@ public synchronized static Subject loginUserFromKeytab(String user, String path, LoginContext login = new LoginContext("hadoop-keytab-kerberos", subject, null, loginConf); KerberosName.setRules(nameRules); subject.getPrincipals().add(new User(user, AuthenticationMethod.KERBEROS, login)); + login.logout(); login.login(); return login.getSubject(); } catch (LoginException le) { @@ -76,6 +78,7 @@ public synchronized static Subject loginUserWithPassword(String user, String pas SecureClientLoginConfiguration loginConf = new SecureClientLoginConfiguration(false, user, password); LoginContext login = new LoginContext("hadoop-keytab-kerberos", subject, null, loginConf); subject.getPrincipals().add(new User(user, AuthenticationMethod.KERBEROS, login)); + login.logout(); login.login(); return login.getSubject(); } catch (LoginException le) { diff --git a/storm-agent/src/main/java/org/apache/ranger/services/storm/client/StormClient.java b/storm-agent/src/main/java/org/apache/ranger/services/storm/client/StormClient.java index b72a9a238b..cd622f4103 100644 --- a/storm-agent/src/main/java/org/apache/ranger/services/storm/client/StormClient.java +++ b/storm-agent/src/main/java/org/apache/ranger/services/storm/client/StormClient.java @@ -308,6 +308,7 @@ public AppConfigurationEntry[] getAppConfigurationEntry( null, loginConf); LOG.debug("executeUnderKerberos():Logging in.."); + loginContext.logout(); loginContext.login(); LOG.info("Init Login: using username/password"); loginSubj = loginContext.getSubject(); From 02e2c7d431d7a63f3e7409c0215ade1d86d97078 Mon Sep 17 00:00:00 2001 From: Abhay Kulkarni Date: Fri, 10 Nov 2017 19:21:15 -0800 Subject: [PATCH 067/151] RANGER-1883: TagSync should reuse kerberos ticket in REST calls to Ranger Admin (cherry picked from commit 98cb80e3335e7c9588b9ad5b57667d3421fba4e6) --- .../sink/tagadmin/TagAdminRESTSink.java | 76 ++++++++++++++----- 1 file changed, 56 insertions(+), 20 deletions(-) diff --git a/tagsync/src/main/java/org/apache/ranger/tagsync/sink/tagadmin/TagAdminRESTSink.java b/tagsync/src/main/java/org/apache/ranger/tagsync/sink/tagadmin/TagAdminRESTSink.java index b1225c2f52..4f6761f8b7 100644 --- a/tagsync/src/main/java/org/apache/ranger/tagsync/sink/tagadmin/TagAdminRESTSink.java +++ b/tagsync/src/main/java/org/apache/ranger/tagsync/sink/tagadmin/TagAdminRESTSink.java @@ -27,6 +27,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.security.SecureClientLogin; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.ranger.admin.client.datatype.RESTResponse; import org.apache.ranger.tagsync.model.TagSink; import org.apache.ranger.plugin.util.RangerRESTClient; @@ -36,6 +37,7 @@ import javax.security.auth.Subject; import javax.servlet.http.HttpServletResponse; +import java.io.IOException; import java.security.PrivilegedAction; import java.util.Map; import java.util.Properties; @@ -95,16 +97,33 @@ public boolean initialize(Properties properties) { if (StringUtils.isNotBlank(restUrl)) { tagRESTClient = new RangerRESTClient(restUrl, sslConfigFile); - if(!(!StringUtils.isEmpty(authenticationType) && authenticationType.trim().equalsIgnoreCase(AUTH_TYPE_KERBEROS) && SecureClientLogin.isKerberosCredentialExists(principal, keytab))){ + if(isKerberosEnabled()) { + Subject subject = null; + try { + subject = SecureClientLogin.loginUserFromKeytab(principal, keytab, nameRules); + } catch(IOException exception) { + LOG.error("Could not get Subject from principal:[" + principal + "], keytab:[" + keytab + "], nameRules:[" + nameRules + "]", exception); + } + if (subject != null) { + try { + UserGroupInformation.loginUserFromSubject(subject); + ret = true; + } catch (IOException exception) { + LOG.error("Failed to get UGI from Subject:[" + subject + "]"); + } + } + } else { tagRESTClient.setBasicAuthInfo(userName, password); + ret = true; } - uploadWorkItems = new LinkedBlockingQueue(); - - ret = true; } else { LOG.error("No value specified for property 'ranger.tagsync.tagadmin.rest.url'!"); } + if (ret) { + uploadWorkItems = new LinkedBlockingQueue(); + } + if(LOG.isDebugEnabled()) { LOG.debug("<== TagAdminRESTSink.initialize(), result=" + ret); } @@ -133,26 +152,43 @@ public ServiceTags upload(ServiceTags toUpload) throws Exception { return ret; } + private boolean isKerberosEnabled() { + return !StringUtils.isEmpty(authenticationType) && authenticationType.trim().equalsIgnoreCase(AUTH_TYPE_KERBEROS) && SecureClientLogin.isKerberosCredentialExists(principal, keytab); + } + private ServiceTags doUpload(ServiceTags serviceTags) throws Exception { - if(!StringUtils.isEmpty(authenticationType) && authenticationType.trim().equalsIgnoreCase(AUTH_TYPE_KERBEROS) && SecureClientLogin.isKerberosCredentialExists(principal, keytab)){ + if(isKerberosEnabled()) { try{ - Subject sub = SecureClientLogin.loginUserFromKeytab(principal, keytab, nameRules); - if(LOG.isDebugEnabled()) { - LOG.debug("Using Principal = "+ principal + ", keytab = "+keytab); + UserGroupInformation userGroupInformation = UserGroupInformation.getLoginUser(); + if (userGroupInformation != null) { + try { + userGroupInformation.checkTGTAndReloginFromKeytab(); + } catch (IOException ioe) { + LOG.error("Error renewing TGT and relogin", ioe); + userGroupInformation = null; + } } - final ServiceTags serviceTag = serviceTags; - ServiceTags ret = Subject.doAs(sub, new PrivilegedAction() { - @Override - public ServiceTags run() { - try{ - return uploadServiceTags(serviceTag); - }catch (Exception e) { - LOG.error("Upload of service-tags failed with message ", e); - } - return null; + if (userGroupInformation != null) { + if (LOG.isDebugEnabled()) { + LOG.debug("Using Principal = " + principal + ", keytab = " + keytab); } - }); - return ret; + final ServiceTags serviceTag = serviceTags; + ServiceTags ret = userGroupInformation.doAs(new PrivilegedAction() { + @Override + public ServiceTags run() { + try { + return uploadServiceTags(serviceTag); + } catch (Exception e) { + LOG.error("Upload of service-tags failed with message ", e); + } + return null; + } + }); + return ret; + } else { + LOG.error("Failed to get UserGroupInformation.getLoginUser()"); + return null; // This will cause retries !!! + } }catch(Exception e){ LOG.error("Upload of service-tags failed with message ", e); } From 8ebad64ef72831ac0d667affbf110abd7296b624 Mon Sep 17 00:00:00 2001 From: Abhay Kulkarni Date: Mon, 13 Nov 2017 18:56:03 -0800 Subject: [PATCH 068/151] RANGER-1883: Part 2 - All tag sources and sinks should use the same kerberos identity and reuse it Signed-off-by: Madhan Neethiraj (cherry picked from commit 10051777edf21c5ab7914c0670e23040d105dc4c) --- .../ranger/tagsync/process/TagSyncConfig.java | 6 ++ .../tagsync/process/TagSynchronizer.java | 94 +++++++++++++++++-- .../sink/tagadmin/TagAdminRESTSink.java | 52 +++------- .../source/atlasrest/AtlasRESTTagSource.java | 30 +----- .../source/atlasrest/AtlasRESTUtil.java | 39 ++++---- 5 files changed, 132 insertions(+), 89 deletions(-) diff --git a/tagsync/src/main/java/org/apache/ranger/tagsync/process/TagSyncConfig.java b/tagsync/src/main/java/org/apache/ranger/tagsync/process/TagSyncConfig.java index 3f3509753b..697c7cc641 100644 --- a/tagsync/src/main/java/org/apache/ranger/tagsync/process/TagSyncConfig.java +++ b/tagsync/src/main/java/org/apache/ranger/tagsync/process/TagSyncConfig.java @@ -98,6 +98,8 @@ public class TagSyncConfig extends Configuration { private static final String TAGSYNC_KERBEROS_PRICIPAL = "ranger.tagsync.kerberos.principal"; private static final String TAGSYNC_KERBEROS_KEYTAB = "ranger.tagsync.kerberos.keytab"; + public static final String TAGSYNC_KERBEROS_IDENTITY = "tagsync.kerberos.identity"; + private static String LOCAL_HOSTNAME = "unknown"; private Properties props; @@ -399,6 +401,10 @@ static public long getTagSourceRetryInitializationInterval(Properties prop) { return ret; } + static public String getTagsyncKerberosIdentity(Properties prop) { + return prop.getProperty(TAGSYNC_KERBEROS_IDENTITY); + } + private TagSyncConfig() { super(false); init(); diff --git a/tagsync/src/main/java/org/apache/ranger/tagsync/process/TagSynchronizer.java b/tagsync/src/main/java/org/apache/ranger/tagsync/process/TagSynchronizer.java index d36ecaeacc..b07cd34ba5 100644 --- a/tagsync/src/main/java/org/apache/ranger/tagsync/process/TagSynchronizer.java +++ b/tagsync/src/main/java/org/apache/ranger/tagsync/process/TagSynchronizer.java @@ -22,10 +22,14 @@ import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.MapUtils; import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.security.SecureClientLogin; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.log4j.Logger; import org.apache.ranger.tagsync.model.TagSink; import org.apache.ranger.tagsync.model.TagSource; +import javax.security.auth.Subject; +import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -36,6 +40,8 @@ public class TagSynchronizer { private static final Logger LOG = Logger.getLogger(TagSynchronizer.class); + private static final String AUTH_TYPE_KERBEROS = "kerberos"; + private static final String TAGSYNC_SOURCE_BASE = "ranger.tagsync.source."; private static final String PROP_CLASS_NAME = "class"; @@ -97,15 +103,19 @@ public boolean initialize() { printConfigurationProperties(properties); - boolean ret = false; + boolean ret = initializeKerberosIdentity(properties); - LOG.info("Initializing TAG source and sink"); + if (ret) { + LOG.info("Initializing TAG source and sink"); - tagSink = initializeTagSink(properties); + tagSink = initializeTagSink(properties); - if (tagSink != null) { - initializeTagSources(); - ret = true; + if (tagSink != null) { + initializeTagSources(); + ret = true; + } + } else { + LOG.error("Error initializing kerberos identity"); } if (LOG.isDebugEnabled()) { @@ -344,6 +354,78 @@ static private TagSource getTagSourceFromConfig(Properties props, return tagSource; } + private static boolean initializeKerberosIdentity(Properties props) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> TagSynchronizer.initializeKerberosIdentity()"); + } + + boolean ret = false; + + String authenticationType = TagSyncConfig.getAuthenticationType(props); + String principal = TagSyncConfig.getKerberosPrincipal(props); + String keytab = TagSyncConfig.getKerberosKeytab(props); + String nameRules = TagSyncConfig.getNameRules(props); + + if (LOG.isDebugEnabled()) { + if (LOG.isDebugEnabled()) { + LOG.debug("authenticationType=" + authenticationType); + LOG.debug("principal=" + principal); + LOG.debug("keytab" + keytab); + LOG.debug("nameRules=" + nameRules); + } + } + final boolean isKerberized = !StringUtils.isEmpty(authenticationType) && authenticationType.trim().equalsIgnoreCase(AUTH_TYPE_KERBEROS) && SecureClientLogin.isKerberosCredentialExists(principal, keytab); + + if (isKerberized) { + if (LOG.isDebugEnabled()) { + LOG.debug("Trying to get kerberos identitiy"); + } + Subject subject = null; + try { + subject = SecureClientLogin.loginUserFromKeytab(principal, keytab, nameRules); + } catch(IOException exception) { + LOG.error("Could not get Subject from principal:[" + principal + "], keytab:[" + keytab + "], nameRules:[" + nameRules + "]", exception); + } + + UserGroupInformation kerberosIdentity; + + if (subject != null) { + try { + UserGroupInformation.loginUserFromSubject(subject); + kerberosIdentity = UserGroupInformation.getLoginUser(); + if (kerberosIdentity != null) { + props.put(TagSyncConfig.TAGSYNC_KERBEROS_IDENTITY, kerberosIdentity.getUserName()); + if (LOG.isDebugEnabled()) { + LOG.debug("Got UGI, user:[" + kerberosIdentity.getUserName() + "]"); + } + ret = true; + } else { + LOG.error("KerberosIdentity is null!"); + } + } catch (IOException exception) { + LOG.error("Failed to get UGI from Subject:[" + subject + "]", exception); + } + } + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("Not configured for Kerberos Authentication"); + } + props.remove(TagSyncConfig.TAGSYNC_KERBEROS_IDENTITY); + + ret = true; + } + + if (!ret) { + props.remove(TagSyncConfig.TAGSYNC_KERBEROS_IDENTITY); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== TagSynchronizer.initializeKerberosIdentity() : " + ret); + } + + return ret; + } + private static String getStringProperty(Properties props, String propName) { String ret = null; diff --git a/tagsync/src/main/java/org/apache/ranger/tagsync/sink/tagadmin/TagAdminRESTSink.java b/tagsync/src/main/java/org/apache/ranger/tagsync/sink/tagadmin/TagAdminRESTSink.java index 4f6761f8b7..c34b6eadd8 100644 --- a/tagsync/src/main/java/org/apache/ranger/tagsync/sink/tagadmin/TagAdminRESTSink.java +++ b/tagsync/src/main/java/org/apache/ranger/tagsync/sink/tagadmin/TagAdminRESTSink.java @@ -26,7 +26,6 @@ import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.security.SecureClientLogin; import org.apache.hadoop.security.UserGroupInformation; import org.apache.ranger.admin.client.datatype.RESTResponse; import org.apache.ranger.tagsync.model.TagSink; @@ -34,7 +33,6 @@ import org.apache.ranger.plugin.util.SearchFilter; import org.apache.ranger.plugin.util.ServiceTags; import org.apache.ranger.tagsync.process.TagSyncConfig; -import javax.security.auth.Subject; import javax.servlet.http.HttpServletResponse; import java.io.IOException; @@ -55,18 +53,13 @@ public class TagAdminRESTSink implements TagSink, Runnable { private static final String REST_URL_IMPORT_SERVICETAGS_RESOURCE = REST_PREFIX + MODULE_PREFIX + "/importservicetags/"; - private static final String AUTH_TYPE_KERBEROS = "kerberos"; - private long rangerAdminConnectionCheckInterval; private RangerRESTClient tagRESTClient = null; + private boolean isKerberized; + private BlockingQueue uploadWorkItems; - - private String authenticationType; - private String principal; - private String keytab; - private String nameRules; private Thread myThread = null; @@ -83,47 +76,28 @@ public boolean initialize(Properties properties) { String userName = TagSyncConfig.getTagAdminUserName(properties); String password = TagSyncConfig.getTagAdminPassword(properties); rangerAdminConnectionCheckInterval = TagSyncConfig.getTagAdminConnectionCheckInterval(properties); - authenticationType = TagSyncConfig.getAuthenticationType(properties); - nameRules = TagSyncConfig.getNameRules(properties); - principal = TagSyncConfig.getKerberosPrincipal(properties); - keytab = TagSyncConfig.getKerberosKeytab(properties); + isKerberized = TagSyncConfig.getTagsyncKerberosIdentity(properties) != null; + if (LOG.isDebugEnabled()) { LOG.debug("restUrl=" + restUrl); LOG.debug("sslConfigFile=" + sslConfigFile); LOG.debug("userName=" + userName); - LOG.debug("rangerAdminConnectionCheckInterval" + rangerAdminConnectionCheckInterval); + LOG.debug("rangerAdminConnectionCheckInterval=" + rangerAdminConnectionCheckInterval); + LOG.debug("isKerberized=" + isKerberized); } if (StringUtils.isNotBlank(restUrl)) { tagRESTClient = new RangerRESTClient(restUrl, sslConfigFile); - if(isKerberosEnabled()) { - Subject subject = null; - try { - subject = SecureClientLogin.loginUserFromKeytab(principal, keytab, nameRules); - } catch(IOException exception) { - LOG.error("Could not get Subject from principal:[" + principal + "], keytab:[" + keytab + "], nameRules:[" + nameRules + "]", exception); - } - if (subject != null) { - try { - UserGroupInformation.loginUserFromSubject(subject); - ret = true; - } catch (IOException exception) { - LOG.error("Failed to get UGI from Subject:[" + subject + "]"); - } - } - } else { + if(!isKerberized) { tagRESTClient.setBasicAuthInfo(userName, password); - ret = true; } + uploadWorkItems = new LinkedBlockingQueue(); + ret = true; } else { LOG.error("No value specified for property 'ranger.tagsync.tagadmin.rest.url'!"); } - if (ret) { - uploadWorkItems = new LinkedBlockingQueue(); - } - if(LOG.isDebugEnabled()) { LOG.debug("<== TagAdminRESTSink.initialize(), result=" + ret); } @@ -152,12 +126,8 @@ public ServiceTags upload(ServiceTags toUpload) throws Exception { return ret; } - private boolean isKerberosEnabled() { - return !StringUtils.isEmpty(authenticationType) && authenticationType.trim().equalsIgnoreCase(AUTH_TYPE_KERBEROS) && SecureClientLogin.isKerberosCredentialExists(principal, keytab); - } - private ServiceTags doUpload(ServiceTags serviceTags) throws Exception { - if(isKerberosEnabled()) { + if(isKerberized) { try{ UserGroupInformation userGroupInformation = UserGroupInformation.getLoginUser(); if (userGroupInformation != null) { @@ -170,7 +140,7 @@ private ServiceTags doUpload(ServiceTags serviceTags) throws Exception { } if (userGroupInformation != null) { if (LOG.isDebugEnabled()) { - LOG.debug("Using Principal = " + principal + ", keytab = " + keytab); + LOG.debug("Using Principal = " + userGroupInformation.getUserName()); } final ServiceTags serviceTag = serviceTags; ServiceTags ret = userGroupInformation.doAs(new PrivilegedAction() { diff --git a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlasrest/AtlasRESTTagSource.java b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlasrest/AtlasRESTTagSource.java index 7da4ed4824..4e0ae90881 100644 --- a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlasrest/AtlasRESTTagSource.java +++ b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlasrest/AtlasRESTTagSource.java @@ -29,7 +29,6 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.security.SecureClientLogin; import org.apache.ranger.plugin.util.RangerRESTClient; import org.apache.ranger.tagsync.model.AbstractTagSource; import org.apache.ranger.plugin.util.ServiceTags; @@ -47,16 +46,9 @@ public class AtlasRESTTagSource extends AbstractTagSource implements Runnable { private static final Log LOG = LogFactory.getLog(AtlasRESTTagSource.class); - static final String AUTH_TYPE_KERBEROS = "kerberos"; - private long sleepTimeBetweenCycleInMillis; - AtlasRESTUtil atlasRESTUtil = null; - - private String authenticationType; - private String principal; - private String keytab; - private String nameRules; + private AtlasRESTUtil atlasRESTUtil = null; private Thread myThread = null; @@ -103,30 +95,18 @@ public boolean initialize(Properties properties) { boolean ret = AtlasResourceMapperUtil.initializeAtlasResourceMappers(properties); sleepTimeBetweenCycleInMillis = TagSyncConfig.getTagSourceAtlasDownloadIntervalInMillis(properties); + final boolean isKerberized = TagSyncConfig.getTagsyncKerberosIdentity(properties) != null; String restUrl = TagSyncConfig.getAtlasRESTEndpoint(properties); String sslConfigFile = TagSyncConfig.getAtlasRESTSslConfigFile(properties); String userName = TagSyncConfig.getAtlasRESTUserName(properties); String password = TagSyncConfig.getAtlasRESTPassword(properties); - authenticationType = TagSyncConfig.getAuthenticationType(properties); - nameRules = TagSyncConfig.getNameRules(properties); - principal = TagSyncConfig.getKerberosPrincipal(properties); - keytab = TagSyncConfig.getKerberosKeytab(properties); - - final boolean kerberized = StringUtils.isNotEmpty(authenticationType) - && authenticationType.trim().equalsIgnoreCase(AtlasRESTTagSource.AUTH_TYPE_KERBEROS) - && SecureClientLogin.isKerberosCredentialExists(principal, keytab); - if (LOG.isDebugEnabled()) { LOG.debug("restUrl=" + restUrl); LOG.debug("sslConfigFile=" + sslConfigFile); LOG.debug("userName=" + userName); - LOG.debug("authenticationType=" + authenticationType); - LOG.debug("principal=" + principal); - LOG.debug("keytab=" + keytab); - LOG.debug("nameRules=" + nameRules); - LOG.debug("kerberized=" + kerberized); + LOG.debug("kerberized=" + isKerberized); } if (StringUtils.isNotEmpty(restUrl)) { @@ -135,10 +115,10 @@ public boolean initialize(Properties properties) { } RangerRESTClient atlasRESTClient = new RangerRESTClient(restUrl, sslConfigFile); - if (!kerberized) { + if (!isKerberized) { atlasRESTClient.setBasicAuthInfo(userName, password); } - atlasRESTUtil = new AtlasRESTUtil(atlasRESTClient, kerberized, authenticationType, principal, keytab, nameRules); + atlasRESTUtil = new AtlasRESTUtil(atlasRESTClient, isKerberized); } else { LOG.info("AtlasEndpoint not specified, Initial download of Atlas-entities cannot be done."); ret = false; diff --git a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlasrest/AtlasRESTUtil.java b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlasrest/AtlasRESTUtil.java index 167fe68596..00a101e727 100644 --- a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlasrest/AtlasRESTUtil.java +++ b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlasrest/AtlasRESTUtil.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -28,14 +28,14 @@ import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.MapUtils; import org.apache.commons.lang.StringUtils; -import org.apache.hadoop.security.SecureClientLogin; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.log4j.Logger; import org.apache.ranger.admin.client.datatype.RESTResponse; import org.apache.ranger.plugin.util.RangerRESTClient; import org.apache.ranger.tagsync.source.atlas.AtlasEntityWithTraits; import org.apache.ranger.tagsync.source.atlas.AtlasResourceMapperUtil; -import javax.security.auth.Subject; +import java.io.IOException; import java.security.PrivilegedAction; import java.util.ArrayList; import java.util.HashMap; @@ -66,22 +66,17 @@ public class AtlasRESTUtil { private final Gson gson = new Gson(); private final RangerRESTClient atlasRESTClient; - private final String principal; - private final String keytab; - private final String nameRules; - private final boolean kerberized; - public AtlasRESTUtil(RangerRESTClient atlasRESTClient, boolean kerberized, String authenticationType, String principal, String keytab, String nameRules) { + private final boolean isKerberized; + + public AtlasRESTUtil(RangerRESTClient atlasRESTClient, boolean isKerberized) { if (LOG.isDebugEnabled()) { LOG.debug("==> AtlasRESTUtil()"); } - this.kerberized = kerberized; - this.atlasRESTClient = atlasRESTClient; - this.principal = principal; - this.keytab = keytab; - this.nameRules = nameRules; + + this.isKerberized = isKerberized; if (LOG.isDebugEnabled()) { LOG.debug("<== AtlasRESTUtil()"); @@ -249,13 +244,23 @@ private Map atlasAPI(final String endpoint) { Map ret = new HashMap(); try { - if (kerberized) { + UserGroupInformation userGroupInformation = null; + if (isKerberized) { + userGroupInformation = UserGroupInformation.getLoginUser(); + + try { + userGroupInformation.checkTGTAndReloginFromKeytab(); + } catch (IOException ioe) { + LOG.error("Error renewing TGT and relogin", ioe); + userGroupInformation = null; + } + } + if (userGroupInformation != null) { LOG.debug("Using kerberos authentication"); - Subject sub = SecureClientLogin.loginUserFromKeytab(principal, keytab, nameRules); if(LOG.isDebugEnabled()) { - LOG.debug("Using Principal = "+ principal + ", keytab = "+keytab); + LOG.debug("Using Principal = "+ userGroupInformation.getUserName()); } - ret = Subject.doAs(sub, new PrivilegedAction>() { + ret = userGroupInformation.doAs(new PrivilegedAction>() { @Override public Map run() { try{ From 30b1188fe54788bcca3216dbeeb2f956e5cb9c9d Mon Sep 17 00:00:00 2001 From: Madhan Neethiraj Date: Tue, 21 Nov 2017 11:03:53 -0800 Subject: [PATCH 069/151] RANGER-1897: tagsync update to replace Atlas V1 API usage with Atlas V2 API for tag-download using REST --- pom.xml | 9 +- src/main/assembly/tagsync.xml | 12 +- tagsync/pom.xml | 42 ++- .../atlas/AtlasHbaseResourceMapper.java | 19 +- .../source/atlas/AtlasHdfsResourceMapper.java | 17 +- .../source/atlas/AtlasHiveResourceMapper.java | 19 +- .../atlas/AtlasKafkaResourceMapper.java | 15 +- .../source/atlas/AtlasNotificationMapper.java | 317 ++++++++++++++--- .../source/atlas/AtlasResourceMapper.java | 7 + .../source/atlas/AtlasResourceMapperUtil.java | 25 ++ .../tagsync/source/atlas/AtlasTagSource.java | 48 +-- .../source/atlasrest/AtlasRESTTagSource.java | 129 ++++--- .../source/atlasrest/AtlasRESTUtil.java | 325 ------------------ 13 files changed, 512 insertions(+), 472 deletions(-) delete mode 100644 tagsync/src/main/java/org/apache/ranger/tagsync/source/atlasrest/AtlasRESTUtil.java diff --git a/pom.xml b/pom.xml index 80de97e8bf..cc09475f94 100644 --- a/pom.xml +++ b/pom.xml @@ -125,7 +125,7 @@ 3.2 3.1 1.8.2 - 0.7-incubating + 0.8.2-SNAPSHOT 14.0 2.5 1.3.7 @@ -357,7 +357,12 @@ com.webcohesion.enunciate enunciate-core-annotations 2.8.0 - + + + com.sun.jersey.contribs + jersey-multipart + ${jersey-bundle.version} + diff --git a/src/main/assembly/tagsync.xml b/src/main/assembly/tagsync.xml index 0b17151b95..26b42cae30 100644 --- a/src/main/assembly/tagsync.xml +++ b/src/main/assembly/tagsync.xml @@ -40,12 +40,16 @@ com.google.inject:guice:jar:${guice.version} com.google.inject.extensions:guice-multibindings:jar:${guice.version} com.sun.jersey:jersey-bundle:jar:${jersey-bundle.version} + com.sun.jersey.contribs:jersey-multipart:jar:${sun-jersey-bundle.version} com.thoughtworks.paranamer:paranamer:jar:${paranamer.version} com.yammer.metrics:metrics-core org.apache.atlas:atlas-notification:jar:${atlas.version} org.apache.atlas:atlas-typesystem:jar:${atlas.version} org.apache.atlas:atlas-client:jar:${atlas.version} + org.apache.atlas:atlas-client-common:jar:${atlas.version} + org.apache.atlas:atlas-client-v2:jar:${atlas.version} org.apache.atlas:atlas-common:jar:${atlas.version} + org.apache.atlas:atlas-intg:jar:${atlas.version} org.apache.hadoop:hadoop-auth org.apache.hadoop:hadoop-common org.apache.kafka:kafka_${scala.binary.version}:jar:${kafka.version} @@ -55,10 +59,10 @@ org.apache.ranger:ranger-plugins-common org.apache.ranger:ranger-util org.apache.zookeeper:zookeeper:jar:${zookeeper.version} - org.codehaus.jackson:jackson-core-asl - org.codehaus.jackson:jackson-jaxrs - org.codehaus.jackson:jackson-mapper-asl - org.codehaus.jackson:jackson-xc + org.codehaus.jackson:jackson-core-asl:jar:${codehaus.jackson.version} + org.codehaus.jackson:jackson-jaxrs:jar:${codehaus.jackson.version} + org.codehaus.jackson:jackson-mapper-asl:jar:${codehaus.jackson.version} + org.codehaus.jackson:jackson-xc:jar:${codehaus.jackson.version} org.codehaus.jettison:jettison:jar:${jettison.version} org.json4s:json4s-native_${scala.binary.version}:jar:${json4s.version} org.json4s:json4s-core_${scala.binary.version}:jar:${json4s.version} diff --git a/tagsync/pom.xml b/tagsync/pom.xml index 42e9d2ff95..417a12f9ef 100644 --- a/tagsync/pom.xml +++ b/tagsync/pom.xml @@ -54,6 +54,11 @@ jersey-bundle ${jersey-bundle.version} + + com.sun.jersey.contribs + jersey-multipart + ${sun-jersey-bundle.version} + commons-cli commons-cli @@ -109,6 +114,26 @@ jettison ${jettison.version} + + org.codehaus.jackson + jackson-core-asl + ${codehaus.jackson.version} + + + org.codehaus.jackson + jackson-jaxrs + ${codehaus.jackson.version} + + + org.codehaus.jackson + jackson-mapper-asl + ${codehaus.jackson.version} + + + org.codehaus.jackson + jackson-xc + ${codehaus.jackson.version} + org.apache.atlas atlas-notification @@ -121,7 +146,17 @@ org.apache.atlas - atlas-client + atlas-client-v1 + ${atlas.version} + + + org.apache.atlas + atlas-client-common + ${atlas.version} + + + org.apache.atlas + atlas-client-v2 ${atlas.version} @@ -143,6 +178,11 @@ + + org.apache.atlas + atlas-intg + ${atlas.version} + org.apache.kafka kafka_${scala.binary.version} diff --git a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasHbaseResourceMapper.java b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasHbaseResourceMapper.java index 8b36a31879..00615e4eb5 100644 --- a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasHbaseResourceMapper.java +++ b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasHbaseResourceMapper.java @@ -22,6 +22,7 @@ import java.util.Map; import java.util.HashMap; +import org.apache.atlas.model.instance.AtlasEntityHeader; import org.apache.atlas.typesystem.IReferenceableInstance; import org.apache.commons.lang.StringUtils; import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyResource; @@ -48,7 +49,23 @@ public AtlasHbaseResourceMapper() { @Override public RangerServiceResource buildResource(final IReferenceableInstance entity) throws Exception { + String entityGuid = entity.getId() != null ? entity.getId()._getId() : null; + String entityType = entity.getTypeName(); String qualifiedName = getEntityAttribute(entity, ENTITY_ATTRIBUTE_QUALIFIED_NAME, String.class); + + return getServiceResource(entityGuid, entityType, qualifiedName); + } + + @Override + public RangerServiceResource buildResource(final AtlasEntityHeader entity) throws Exception { + String entityGuid = entity.getGuid(); + String entityType = entity.getTypeName(); + String qualifiedName = getEntityAttribute(entity, ENTITY_ATTRIBUTE_QUALIFIED_NAME, String.class); + + return getServiceResource(entityGuid, entityType, qualifiedName); + } + + private RangerServiceResource getServiceResource(String entityGuid, String entityType, String qualifiedName) throws Exception { if (StringUtils.isEmpty(qualifiedName)) { throw new Exception("attribute '" + ENTITY_ATTRIBUTE_QUALIFIED_NAME + "' not found in entity"); } @@ -63,8 +80,6 @@ public RangerServiceResource buildResource(final IReferenceableInstance entity) throwExceptionWithMessage("cluster-name not found in attribute '" + ENTITY_ATTRIBUTE_QUALIFIED_NAME + "': " + qualifiedName); } - String entityType = entity.getTypeName(); - String entityGuid = entity.getId() != null ? entity.getId()._getId() : null; String serviceName = getRangerServiceName(clusterName); Map elements = new HashMap(); diff --git a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasHdfsResourceMapper.java b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasHdfsResourceMapper.java index 06bff9067a..d9708594fc 100644 --- a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasHdfsResourceMapper.java +++ b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasHdfsResourceMapper.java @@ -22,6 +22,7 @@ import java.util.HashMap; import java.util.Map; +import org.apache.atlas.model.instance.AtlasEntityHeader; import org.apache.atlas.typesystem.IReferenceableInstance; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.fs.Path; @@ -57,10 +58,25 @@ public String getRangerServiceName(String clusterName) { @Override public RangerServiceResource buildResource(final IReferenceableInstance entity) throws Exception { + String entityGuid = entity.getId() != null ? entity.getId()._getId() : null; String path = getEntityAttribute(entity, ENTITY_ATTRIBUTE_PATH, String.class); String clusterName = getEntityAttribute(entity, ENTITY_ATTRIBUTE_CLUSTER_NAME, String.class); String qualifiedName = getEntityAttribute(entity, ENTITY_ATTRIBUTE_QUALIFIED_NAME, String.class); + return getServiceResource(entityGuid, path, clusterName, qualifiedName); + } + + @Override + public RangerServiceResource buildResource(final AtlasEntityHeader entity) throws Exception { + String entityGuid = entity.getGuid(); + String path = getEntityAttribute(entity, ENTITY_ATTRIBUTE_PATH, String.class); + String clusterName = getEntityAttribute(entity, ENTITY_ATTRIBUTE_CLUSTER_NAME, String.class); + String qualifiedName = getEntityAttribute(entity, ENTITY_ATTRIBUTE_QUALIFIED_NAME, String.class); + + return getServiceResource(entityGuid, path, clusterName, qualifiedName); + } + + private RangerServiceResource getServiceResource(String entityGuid, String path, String clusterName, String qualifiedName) throws Exception { if(StringUtils.isEmpty(path)) { path = getResourceNameFromQualifiedName(qualifiedName); @@ -81,7 +97,6 @@ public RangerServiceResource buildResource(final IReferenceableInstance entity) } } - String entityGuid = entity.getId() != null ? entity.getId()._getId() : null; String serviceName = getRangerServiceName(clusterName); Boolean isExcludes = Boolean.FALSE; Boolean isRecursive = Boolean.TRUE; diff --git a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasHiveResourceMapper.java b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasHiveResourceMapper.java index a35962239c..84d1226d16 100644 --- a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasHiveResourceMapper.java +++ b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasHiveResourceMapper.java @@ -22,6 +22,7 @@ import java.util.Map; import java.util.HashMap; +import org.apache.atlas.model.instance.AtlasEntityHeader; import org.apache.atlas.typesystem.IReferenceableInstance; import org.apache.commons.lang.StringUtils; import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyResource; @@ -47,7 +48,23 @@ public AtlasHiveResourceMapper() { @Override public RangerServiceResource buildResource(final IReferenceableInstance entity) throws Exception { + String entityGuid = entity.getId() != null ? entity.getId()._getId() : null; + String entityType = entity.getTypeName(); String qualifiedName = getEntityAttribute(entity, ENTITY_ATTRIBUTE_QUALIFIED_NAME, String.class); + + return getServiceResource(entityGuid, entityType, qualifiedName); + } + + @Override + public RangerServiceResource buildResource(final AtlasEntityHeader entity) throws Exception { + String entityGuid = entity.getGuid(); + String entityType = entity.getTypeName(); + String qualifiedName = getEntityAttribute(entity, ENTITY_ATTRIBUTE_QUALIFIED_NAME, String.class); + + return getServiceResource(entityGuid, entityType, qualifiedName); + } + + private RangerServiceResource getServiceResource(String entityGuid, String entityType, String qualifiedName) throws Exception { if (StringUtils.isEmpty(qualifiedName)) { throw new Exception("attribute '" + ENTITY_ATTRIBUTE_QUALIFIED_NAME + "' not found in entity"); } @@ -62,8 +79,6 @@ public RangerServiceResource buildResource(final IReferenceableInstance entity) throwExceptionWithMessage("cluster-name not found in attribute '" + ENTITY_ATTRIBUTE_QUALIFIED_NAME + "': " + qualifiedName); } - String entityType = entity.getTypeName(); - String entityGuid = entity.getId() != null ? entity.getId()._getId() : null; String serviceName = getRangerServiceName(clusterName); String[] resources = resourceStr.split(QUALIFIED_NAME_DELIMITER); String dbName = resources.length > 0 ? resources[0] : null; diff --git a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasKafkaResourceMapper.java b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasKafkaResourceMapper.java index 9f1fc2db47..0c0247f4b9 100644 --- a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasKafkaResourceMapper.java +++ b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasKafkaResourceMapper.java @@ -22,6 +22,7 @@ import java.util.HashMap; import java.util.Map; +import org.apache.atlas.model.instance.AtlasEntityHeader; import org.apache.atlas.typesystem.IReferenceableInstance; import org.apache.commons.lang.StringUtils; import org.apache.ranger.plugin.model.RangerPolicy; @@ -42,8 +43,21 @@ public AtlasKafkaResourceMapper() { @Override public RangerServiceResource buildResource(final IReferenceableInstance entity) throws Exception { + String entityGuid = entity.getId() != null ? entity.getId()._getId() : null; String qualifiedName = getEntityAttribute(entity, ENTITY_ATTRIBUTE_QUALIFIED_NAME, String.class); + return getServiceResource(entityGuid, qualifiedName); + } + + @Override + public RangerServiceResource buildResource(final AtlasEntityHeader entity) throws Exception { + String entityGuid = entity.getGuid(); + String qualifiedName = getEntityAttribute(entity, ENTITY_ATTRIBUTE_QUALIFIED_NAME, String.class); + + return getServiceResource(entityGuid, qualifiedName); + } + + private RangerServiceResource getServiceResource(String entityGuid, String qualifiedName) throws Exception { String topic = getResourceNameFromQualifiedName(qualifiedName); if(StringUtils.isEmpty(topic)) { @@ -67,7 +81,6 @@ public RangerServiceResource buildResource(final IReferenceableInstance entity) elements.put(RANGER_TYPE_KAFKA_TOPIC, new RangerPolicyResource(topic, isExcludes, isRecursive)); - String entityGuid = entity.getId() != null ? entity.getId()._getId() : null; String serviceName = getRangerServiceName(clusterName); return new RangerServiceResource(entityGuid, serviceName, elements); diff --git a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasNotificationMapper.java b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasNotificationMapper.java index 922317e8ce..f42c90864f 100644 --- a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasNotificationMapper.java +++ b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasNotificationMapper.java @@ -20,7 +20,16 @@ package org.apache.ranger.tagsync.source.atlas; import org.apache.atlas.AtlasException; +import org.apache.atlas.model.discovery.AtlasSearchResult; +import org.apache.atlas.model.instance.AtlasClassification; +import org.apache.atlas.model.instance.AtlasEntity; +import org.apache.atlas.model.instance.AtlasEntityHeader; +import org.apache.atlas.model.typedef.AtlasBaseTypeDef; import org.apache.atlas.notification.entity.EntityNotification; +import org.apache.atlas.type.AtlasBuiltInTypes; +import org.apache.atlas.type.AtlasClassificationType; +import org.apache.atlas.type.AtlasStructType.AtlasAttribute; +import org.apache.atlas.type.AtlasTypeRegistry; import org.apache.atlas.typesystem.IReferenceableInstance; import org.apache.atlas.typesystem.IStruct; import org.apache.atlas.typesystem.persistence.Id; @@ -35,10 +44,9 @@ import org.apache.ranger.plugin.model.RangerTagDef.RangerTagAttributeDef; import org.apache.ranger.plugin.util.ServiceTags; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.text.DateFormat; +import java.text.SimpleDateFormat; +import java.util.*; public class AtlasNotificationMapper { private static final Log LOG = LogFactory.getLog(AtlasNotificationMapper.class); @@ -46,6 +54,17 @@ public class AtlasNotificationMapper { private static Map unhandledEventTypes = new HashMap(); + private static final ThreadLocal DATE_FORMATTER = new ThreadLocal() { + @Override + protected DateFormat initialValue() { + SimpleDateFormat dateFormat = new SimpleDateFormat(AtlasBaseTypeDef.SERIALIZED_DATE_FORMAT_STR); + + dateFormat.setTimeZone(TimeZone.getTimeZone("UTC")); + + return dateFormat; + } + }; + private static void logUnhandledEntityNotification(EntityNotification entityNotification) { final int REPORTING_INTERVAL_FOR_UNHANDLED_ENTITYTYPE_IN_MILLIS = 5 * 60 * 1000; // 5 minutes @@ -134,6 +153,7 @@ static private boolean isNotificationHandled(EntityNotification entityNotificati case ENTITY_UPDATE: case ENTITY_DELETE: case TRAIT_ADD: + case TRAIT_UPDATE: case TRAIT_DELETE: { ret = true; break; @@ -175,7 +195,6 @@ static private ServiceTags buildServiceTagsForEntityDeleteNotification(AtlasEnti } static private Map buildServiceTags(List entitiesWithTraits) throws Exception { - Map ret = new HashMap(); for (AtlasEntityWithTraits element : entitiesWithTraits) { @@ -189,11 +208,163 @@ static private Map buildServiceTags(List serviceTagsMap) throws Exception { + ServiceTags ret = null; + IReferenceableInstance entity = entityWithTraits.getEntity(); + RangerServiceResource serviceResource = AtlasResourceMapperUtil.getRangerServiceResource(entity); + + if (serviceResource != null) { + List tags = getTags(entityWithTraits); + List tagDefs = getTagDefs(entityWithTraits); + String serviceName = serviceResource.getServiceName(); + + ret = createOrGetServiceTags(serviceTagsMap, serviceName); + + if (serviceTagsMap == null || CollectionUtils.isNotEmpty(tags)) { + serviceResource.setId((long) ret.getServiceResources().size()); + ret.getServiceResources().add(serviceResource); + + List tagIds = new ArrayList<>(); + + if (CollectionUtils.isNotEmpty(tags)) { + for (RangerTag tag : tags) { + tag.setId((long) ret.getTags().size()); + ret.getTags().put(tag.getId(), tag); + + tagIds.add(tag.getId()); + } + } + ret.getResourceToTagIds().put(serviceResource.getId(), tagIds); + + if (CollectionUtils.isNotEmpty(tagDefs)) { + for (RangerTagDef tagDef : tagDefs) { + tagDef.setId((long) ret.getTagDefinitions().size()); + ret.getTagDefinitions().put(tagDef.getId(), tagDef); + } + } + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("Entity " + entityWithTraits + " does not have any tags associated with it when full-sync is being done."); + LOG.debug("Will not add this entity to serviceTags, so that this entity, if exists, will be removed from ranger"); + } + } + } else { + LOG.error("Failed to build serviceResource for entity:" + entity.getId()._getId()); + } + + return ret; + } + + static private ServiceTags createOrGetServiceTags(Map serviceTagsMap, String serviceName) { + ServiceTags ret = serviceTagsMap == null ? null : serviceTagsMap.get(serviceName); + + if (ret == null) { + ret = new ServiceTags(); + + if (serviceTagsMap != null) { + serviceTagsMap.put(serviceName, ret); + } + + ret.setOp(ServiceTags.OP_ADD_OR_UPDATE); + ret.setServiceName(serviceName); + } + + return ret; + } + + static private List getTags(AtlasEntityWithTraits entityWithTraits) { + List ret = new ArrayList(); + IReferenceableInstance entity = entityWithTraits != null ? entityWithTraits.getEntity() : null; + + if(entity != null && CollectionUtils.isNotEmpty(entity.getTraits())) { + for (String traitName : entity.getTraits()) { + IStruct trait = entity.getTrait(traitName); + Map tagAttrs = new HashMap(); + + try { + Map attrs = trait.getValuesMap(); + + if(MapUtils.isNotEmpty(attrs)) { + for (Map.Entry attrEntry : attrs.entrySet()) { + String attrName = attrEntry.getKey(); + Object attrValue = attrEntry.getValue(); + + tagAttrs.put(attrName, attrValue != null ? attrValue.toString() : null); + } + } + + } catch (AtlasException exception) { + LOG.error("Could not get values for trait:" + trait.getTypeName(), exception); + } + + ret.add(new RangerTag(null, trait.getTypeName(), tagAttrs, RangerTag.OWNER_SERVICERESOURCE)); + } + } + + return ret; + } + + static private List getTagDefs(AtlasEntityWithTraits entityWithTraits) { + List ret = new ArrayList(); + IReferenceableInstance entity = entityWithTraits != null ? entityWithTraits.getEntity() : null; + + if(entity != null && CollectionUtils.isNotEmpty(entity.getTraits())) { + for (String traitName : entity.getTraits()) { + IStruct trait = entity.getTrait(traitName); + RangerTagDef tagDef = new RangerTagDef(trait.getTypeName(), "Atlas"); + + try { + Map attrs = trait.getValuesMap(); + + if(MapUtils.isNotEmpty(attrs)) { + for (String attrName : attrs.keySet()) { + tagDef.getAttributeDefs().add(new RangerTagAttributeDef(attrName, "string")); + } + } + } catch (AtlasException exception) { + LOG.error("Could not get values for trait:" + trait.getTypeName(), exception); + } + + ret.add(tagDef); + } + } + + return ret; + } + + public static Map processSearchResult(AtlasSearchResult result, AtlasTypeRegistry typeRegistry) { + Map ret = null; + + try { + ret = buildServiceTags(result, typeRegistry); + } catch (Exception exception) { + LOG.error("Failed to build serviceTags", exception); + } + + return ret; + } + + static private Map buildServiceTags(AtlasSearchResult result, AtlasTypeRegistry typeRegistry) throws Exception { + Map ret = new HashMap<>(); + + for (AtlasEntityHeader entity : result.getEntities()) { + if (entity != null && entity.getStatus() == AtlasEntity.Status.ACTIVE) { + buildServiceTags(entity, typeRegistry, ret); + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("Ignoring entity because its State is not ACTIVE: " + entity); + } + } + } + // Remove duplicate tag definitions if(CollectionUtils.isNotEmpty(ret.values())) { for (ServiceTags serviceTag : ret.values()) { if(MapUtils.isNotEmpty(serviceTag.getTagDefinitions())) { - Map uniqueTagDefs = new HashMap(); + Map uniqueTagDefs = new HashMap<>(); for (RangerTagDef tagDef : serviceTag.getTagDefinitions().values()) { RangerTagDef existingTagDef = uniqueTagDefs.get(tagDef.getName()); @@ -239,25 +410,22 @@ static private Map buildServiceTags(List serviceTagsMap) throws Exception { - ServiceTags ret = null; - IReferenceableInstance entity = entityWithTraits.getEntity(); - RangerServiceResource serviceResource = AtlasResourceMapperUtil.getRangerServiceResource(entity); + static private ServiceTags buildServiceTags(AtlasEntityHeader entity, AtlasTypeRegistry typeRegistry, Map serviceTagsMap) throws Exception { + ServiceTags ret = null; + RangerServiceResource serviceResource = AtlasResourceMapperUtil.getRangerServiceResource(entity); if (serviceResource != null) { - - List tags = getTags(entityWithTraits); - List tagDefs = getTagDefs(entityWithTraits); + List tags = getTags(entity, typeRegistry); + List tagDefs = getTagDefs(entity); String serviceName = serviceResource.getServiceName(); ret = createOrGetServiceTags(serviceTagsMap, serviceName); if (serviceTagsMap == null || CollectionUtils.isNotEmpty(tags)) { - serviceResource.setId((long) ret.getServiceResources().size()); ret.getServiceResources().add(serviceResource); - List tagIds = new ArrayList(); + List tagIds = new ArrayList<>(); if (CollectionUtils.isNotEmpty(tags)) { for (RangerTag tag : tags) { @@ -277,90 +445,125 @@ static private ServiceTags buildServiceTags(AtlasEntityWithTraits entityWithTrai } } else { if (LOG.isDebugEnabled()) { - LOG.debug("Entity " + entityWithTraits + " does not have any tags associated with it when full-sync is being done."); + LOG.debug("Entity " + entity + " does not have any tags associated with it when full-sync is being done."); LOG.debug("Will not add this entity to serviceTags, so that this entity, if exists, will be removed from ranger"); } } } else { - LOG.error("Failed to build serviceResource for entity:" + entity.getId()._getId()); + LOG.error("Failed to build serviceResource for entity:" + entity.getGuid()); } return ret; } - static private ServiceTags createOrGetServiceTags(Map serviceTagsMap, String serviceName) { - ServiceTags ret = serviceTagsMap == null ? null : serviceTagsMap.get(serviceName); + static private List getTags(AtlasEntityHeader entity, AtlasTypeRegistry typeRegistry) { + List ret = new ArrayList<>(); - if (ret == null) { - ret = new ServiceTags(); + if(entity != null && CollectionUtils.isNotEmpty(entity.getClassificationNames())) { + List classifications = entity.getClassifications(); - if (serviceTagsMap != null) { - serviceTagsMap.put(serviceName, ret); + for (AtlasClassification classification : classifications) { + ret.add(getRangerTag(classification, typeRegistry)); + + List superClassifications = getSuperClassifications(classification, typeRegistry); + + if (CollectionUtils.isNotEmpty(superClassifications)) { + for (AtlasClassification superClassification : superClassifications) { + ret.add(getRangerTag(superClassification, typeRegistry)); + } + } } + } - ret.setOp(ServiceTags.OP_ADD_OR_UPDATE); - ret.setServiceName(serviceName); + return ret; + } + + static private List getTagDefs(AtlasEntityHeader entity) { + List ret = new ArrayList<>(); + + if(entity != null && CollectionUtils.isNotEmpty(entity.getClassificationNames())) { + List traits = entity.getClassifications(); + + for (AtlasClassification trait : traits) { + RangerTagDef tagDef = new RangerTagDef(trait.getTypeName(), "Atlas"); + + if(MapUtils.isNotEmpty(trait.getAttributes())) { + for (String attrName : trait.getAttributes().keySet()) { + tagDef.getAttributeDefs().add(new RangerTagAttributeDef(attrName, "string")); + } + } + + ret.add(tagDef); + } } return ret; } - static private List getTags(AtlasEntityWithTraits entityWithTraits) { - List ret = new ArrayList(); + static private List getSuperClassifications(AtlasClassification classification, AtlasTypeRegistry typeRegistry) { + List ret = null; + AtlasClassificationType classificationType = typeRegistry.getClassificationTypeByName(classification.getTypeName()); - if(entityWithTraits != null && CollectionUtils.isNotEmpty(entityWithTraits.getAllTraits())) { - List traits = entityWithTraits.getAllTraits(); + if (classificationType != null && CollectionUtils.isNotEmpty(classificationType.getAllSuperTypes())) { + ret = new ArrayList<>(classificationType.getAllSuperTypes().size()); - for (IStruct trait : traits) { - Map tagAttrs = new HashMap(); + for (String superTypeName : classificationType.getAllSuperTypes()) { + AtlasClassification superClassification = new AtlasClassification(superTypeName); - try { - Map attrs = trait.getValuesMap(); + if (MapUtils.isNotEmpty(classification.getAttributes())) { + AtlasClassificationType superClassificationType = typeRegistry.getClassificationTypeByName(superTypeName); - if(MapUtils.isNotEmpty(attrs)) { - for (Map.Entry attrEntry : attrs.entrySet()) { - String attrName = attrEntry.getKey(); - Object attrValue = attrEntry.getValue(); + if (superClassificationType != null && MapUtils.isNotEmpty(superClassificationType.getAllAttributes())) { + Map superClassificationAttributes = new HashMap<>(); - tagAttrs.put(attrName, attrValue != null ? attrValue.toString() : null); + for (Map.Entry entry : classification.getAttributes().entrySet()) { + String attrName = entry.getKey(); + + if (superClassificationType.getAllAttributes().containsKey(attrName)) { + superClassificationAttributes.put(attrName, entry.getValue()); + } } + + superClassification.setAttributes(superClassificationAttributes); } - } catch (AtlasException exception) { - LOG.error("Could not get values for trait:" + trait.getTypeName(), exception); } - ret.add(new RangerTag(null, trait.getTypeName(), tagAttrs, RangerTag.OWNER_SERVICERESOURCE)); + ret.add(superClassification); } } return ret; } - static private List getTagDefs(AtlasEntityWithTraits entityWithTraits) { - List ret = new ArrayList(); + static private RangerTag getRangerTag(AtlasClassification classification, AtlasTypeRegistry typeRegistry) { + final Map tagAttrs; - if(entityWithTraits != null && CollectionUtils.isNotEmpty(entityWithTraits.getAllTraits())) { - List traits = entityWithTraits.getAllTraits(); + if(MapUtils.isNotEmpty(classification.getAttributes())) { + tagAttrs = new HashMap<>(); - for (IStruct trait : traits) { - RangerTagDef tagDef = new RangerTagDef(trait.getTypeName(), "Atlas"); + for (Map.Entry attrEntry : classification.getAttributes().entrySet()) { + String attrName = attrEntry.getKey(); + Object attrValue = attrEntry.getValue(); - try { - Map attrs = trait.getValuesMap(); + // V2 Atlas APIs have date attributes as number; convert the value to earlier version format, so that + // Ranger conditions can recognize the value correctly + if (attrValue instanceof Number) { + AtlasClassificationType classificationType = typeRegistry.getClassificationTypeByName(classification.getTypeName()); + AtlasAttribute attribute = (classificationType != null) ? classificationType.getAttribute(attrName) : null; - if(MapUtils.isNotEmpty(attrs)) { - for (String attrName : attrs.keySet()) { - tagDef.getAttributeDefs().add(new RangerTagAttributeDef(attrName, "string")); - } + if (attribute != null && attribute.getAttributeType() instanceof AtlasBuiltInTypes.AtlasDateType) { + Date dateValue = new Date(((Number)attrValue).longValue()); + + attrValue = DATE_FORMATTER.get().format(dateValue); } - } catch (AtlasException exception) { - LOG.error("Could not get values for trait:" + trait.getTypeName(), exception); } - ret.add(tagDef); + tagAttrs.put(attrName, attrValue != null ? attrValue.toString() : null); } + } else { + tagAttrs = Collections.emptyMap(); } - return ret; + return new RangerTag(null, classification.getTypeName(), tagAttrs, RangerTag.OWNER_SERVICERESOURCE); } } diff --git a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasResourceMapper.java b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasResourceMapper.java index 8ececdf70f..a2ad796a68 100644 --- a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasResourceMapper.java +++ b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasResourceMapper.java @@ -23,6 +23,7 @@ import java.util.Map; import org.apache.atlas.AtlasException; +import org.apache.atlas.model.instance.AtlasEntityHeader; import org.apache.atlas.typesystem.IReferenceableInstance; import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; @@ -75,6 +76,8 @@ public void initialize(Properties properties) { abstract public RangerServiceResource buildResource(final IReferenceableInstance entity) throws Exception; + abstract public RangerServiceResource buildResource(final AtlasEntityHeader entity) throws Exception; + protected String getCustomRangerServiceName(String atlasInstanceName) { if(properties != null) { String propName = TAGSYNC_SERVICENAME_MAPPER_PROP_PREFIX + componentName @@ -132,6 +135,10 @@ static protected T getEntityAttribute(IReferenceableInstance entity, String return ret; } + static protected T getEntityAttribute(AtlasEntityHeader entity, String name, Class type) { + return getAttribute(entity.getAttributes(), name, type); + } + static protected T getAttribute(Map map, String name, Class type) { return type.cast(map.get(name)); } diff --git a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasResourceMapperUtil.java b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasResourceMapperUtil.java index f9f0eafdf4..d004bff89f 100644 --- a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasResourceMapperUtil.java +++ b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasResourceMapperUtil.java @@ -19,6 +19,7 @@ package org.apache.ranger.tagsync.source.atlas; +import org.apache.atlas.model.instance.AtlasEntityHeader; import org.apache.atlas.typesystem.IReferenceableInstance; import org.apache.commons.lang.StringUtils; import org.apache.ranger.plugin.model.RangerServiceResource; @@ -74,6 +75,30 @@ public static RangerServiceResource getRangerServiceResource(IReferenceableInsta return resource; } + public static RangerServiceResource getRangerServiceResource(AtlasEntityHeader atlasEntity) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> getRangerServiceResource(" + atlasEntity.getGuid() +")"); + } + + RangerServiceResource resource = null; + + AtlasResourceMapper mapper = atlasResourceMappers.get(atlasEntity.getTypeName()); + + if (mapper != null) { + try { + resource = mapper.buildResource(atlasEntity); + } catch (Exception exception) { + LOG.error("Could not get serviceResource for atlas entity:" + atlasEntity.getGuid() + ": ", exception); + } + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== getRangerServiceResource(" + atlasEntity.getGuid() +"): resource=" + resource); + } + + return resource; + } + static public boolean initializeAtlasResourceMappers(Properties properties) { final String MAPPER_NAME_DELIMITER = ","; diff --git a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasTagSource.java b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasTagSource.java index 12b02d9435..95ff8ec002 100644 --- a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasTagSource.java +++ b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasTagSource.java @@ -20,26 +20,22 @@ package org.apache.ranger.tagsync.source.atlas; -import com.google.inject.Guice; -import com.google.inject.Injector; -import com.google.inject.Provider; - -import org.apache.commons.lang.StringUtils; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; - +import org.apache.atlas.kafka.NotificationProvider; import org.apache.atlas.notification.NotificationConsumer; import org.apache.atlas.notification.NotificationInterface; -import org.apache.atlas.notification.NotificationModule; import org.apache.atlas.notification.entity.EntityNotification; - -import org.apache.ranger.tagsync.model.AbstractTagSource; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.ranger.plugin.util.ServiceTags; +import org.apache.ranger.tagsync.model.AbstractTagSource; +import org.apache.atlas.kafka.AtlasKafkaMessage; +import org.apache.kafka.common.TopicPartition; import java.io.IOException; import java.io.InputStream; -import java.util.Properties; import java.util.List; +import java.util.Properties; public class AtlasTagSource extends AbstractTagSource { private static final Log LOG = LogFactory.getLog(AtlasTagSource.class); @@ -102,12 +98,7 @@ public boolean initialize(Properties properties) { } if (ret) { - NotificationModule notificationModule = new NotificationModule(); - - Injector injector = Guice.createInjector(notificationModule); - - Provider consumerProvider = injector.getProvider(NotificationInterface.class); - NotificationInterface notification = consumerProvider.get(); + NotificationInterface notification = NotificationProvider.get(); List> iterators = notification.createConsumers(NotificationInterface.NotificationType.ENTITIES, 1); consumerTask = new ConsumerRunnable(iterators.get(0)); @@ -163,15 +154,6 @@ private ConsumerRunnable(NotificationConsumer consumer) { this.consumer = consumer; } - private boolean hasNext() { - boolean ret = false; - try { - ret = consumer.hasNext(); - } catch (Exception exception) { - LOG.error("EntityNotification consumer threw exception, IGNORING...:", exception); - } - return ret; - } @Override public void run() { @@ -180,8 +162,11 @@ public void run() { } while (true) { try { - if (hasNext()) { - EntityNotification notification = consumer.peek(); + List> messages = consumer.receive(1000L); + + for (AtlasKafkaMessage message : messages) { + EntityNotification notification = message != null ? message.getMessage() : null; + if (notification != null) { if (LOG.isDebugEnabled()) { LOG.debug("Notification=" + getPrintableEntityNotification(notification)); @@ -191,11 +176,12 @@ public void run() { if (serviceTags != null) { updateSink(serviceTags); } + + TopicPartition partition = new TopicPartition("ATLAS_ENTITIES", message.getPartition()); + consumer.commit(partition, message.getOffset()); } else { LOG.error("Null entityNotification received from Kafka!! Ignoring.."); } - // Move iterator forward - consumer.next(); } } catch (Exception exception) { LOG.error("Caught exception..: ", exception); diff --git a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlasrest/AtlasRESTTagSource.java b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlasrest/AtlasRESTTagSource.java index 4e0ae90881..239f143a5c 100644 --- a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlasrest/AtlasRESTTagSource.java +++ b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlasrest/AtlasRESTTagSource.java @@ -23,48 +23,52 @@ import com.google.gson.GsonBuilder; -import org.apache.commons.collections.CollectionUtils; +import org.apache.atlas.AtlasClientV2; +import org.apache.atlas.AtlasServiceException; +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.model.SearchFilter; +import org.apache.atlas.model.discovery.AtlasSearchResult; +import org.apache.atlas.model.discovery.SearchParameters; +import org.apache.atlas.model.typedef.AtlasTypesDef; +import org.apache.atlas.type.AtlasType; +import org.apache.atlas.type.AtlasTypeRegistry; +import org.apache.atlas.type.AtlasTypeRegistry.AtlasTransientTypeRegistry; import org.apache.commons.collections.MapUtils; import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.ranger.plugin.util.RangerRESTClient; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.ranger.tagsync.model.AbstractTagSource; import org.apache.ranger.plugin.util.ServiceTags; import org.apache.ranger.tagsync.model.TagSink; import org.apache.ranger.tagsync.process.TagSyncConfig; import org.apache.ranger.tagsync.process.TagSynchronizer; -import org.apache.ranger.tagsync.source.atlas.AtlasEntityWithTraits; import org.apache.ranger.tagsync.source.atlas.AtlasNotificationMapper; import org.apache.ranger.tagsync.source.atlas.AtlasResourceMapperUtil; -import java.util.List; +import java.io.IOException; import java.util.Map; import java.util.Properties; public class AtlasRESTTagSource extends AbstractTagSource implements Runnable { private static final Log LOG = LogFactory.getLog(AtlasRESTTagSource.class); - private long sleepTimeBetweenCycleInMillis; - - private AtlasRESTUtil atlasRESTUtil = null; - - private Thread myThread = null; + private long sleepTimeBetweenCycleInMillis; + private String[] restUrls = null; + private boolean isKerberized = false; + private String[] userNamePassword = null; + private Thread myThread = null; public static void main(String[] args) { - - AtlasRESTTagSource atlasRESTTagSource = new AtlasRESTTagSource(); - - TagSyncConfig config = TagSyncConfig.getInstance(); - - Properties props = config.getProperties(); + TagSyncConfig config = TagSyncConfig.getInstance(); + Properties props = config.getProperties(); + TagSink tagSink = TagSynchronizer.initializeTagSink(props); TagSynchronizer.printConfigurationProperties(props); - TagSink tagSink = TagSynchronizer.initializeTagSink(props); - if (tagSink != null) { + AtlasRESTTagSource atlasRESTTagSource = new AtlasRESTTagSource(); if (atlasRESTTagSource.initialize(props)) { try { @@ -79,46 +83,45 @@ public static void main(String[] args) { LOG.error("AtlasRESTTagSource initialized failed, exiting."); System.exit(1); } - } else { LOG.error("TagSink initialialization failed, exiting."); System.exit(1); } - } + @Override public boolean initialize(Properties properties) { if (LOG.isDebugEnabled()) { LOG.debug("==> AtlasRESTTagSource.initialize()"); } + sleepTimeBetweenCycleInMillis = TagSyncConfig.getTagSourceAtlasDownloadIntervalInMillis(properties); + boolean ret = AtlasResourceMapperUtil.initializeAtlasResourceMappers(properties); - sleepTimeBetweenCycleInMillis = TagSyncConfig.getTagSourceAtlasDownloadIntervalInMillis(properties); - final boolean isKerberized = TagSyncConfig.getTagsyncKerberosIdentity(properties) != null; + String sslConfigFile = TagSyncConfig.getAtlasRESTSslConfigFile(properties); + + this.isKerberized = TagSyncConfig.getTagsyncKerberosIdentity(properties) != null; + this.userNamePassword = new String[] { TagSyncConfig.getAtlasRESTUserName(properties), TagSyncConfig.getAtlasRESTPassword(properties) }; - String restUrl = TagSyncConfig.getAtlasRESTEndpoint(properties); - String sslConfigFile = TagSyncConfig.getAtlasRESTSslConfigFile(properties); - String userName = TagSyncConfig.getAtlasRESTUserName(properties); - String password = TagSyncConfig.getAtlasRESTPassword(properties); + String restEndpoint = TagSyncConfig.getAtlasRESTEndpoint(properties); if (LOG.isDebugEnabled()) { - LOG.debug("restUrl=" + restUrl); + LOG.debug("restEndpoint=" + restEndpoint); LOG.debug("sslConfigFile=" + sslConfigFile); - LOG.debug("userName=" + userName); + LOG.debug("userName=" + userNamePassword[0]); LOG.debug("kerberized=" + isKerberized); } - if (StringUtils.isNotEmpty(restUrl)) { - if (!restUrl.endsWith("/")) { - restUrl += "/"; - } - RangerRESTClient atlasRESTClient = new RangerRESTClient(restUrl, sslConfigFile); + if (StringUtils.isNotEmpty(restEndpoint)) { + this.restUrls = restEndpoint.split(","); - if (!isKerberized) { - atlasRESTClient.setBasicAuthInfo(userName, password); + for (int i = 0; i < restUrls.length; i++) { + if (!restUrls[i].endsWith("/")) { + restUrls[i] += "/"; + } } - atlasRESTUtil = new AtlasRESTUtil(atlasRESTClient, isKerberized); + } else { LOG.info("AtlasEndpoint not specified, Initial download of Atlas-entities cannot be done."); ret = false; @@ -133,7 +136,6 @@ public boolean initialize(Properties properties) { @Override public boolean start() { - myThread = new Thread(this); myThread.setDaemon(true); myThread.start(); @@ -150,21 +152,17 @@ public void stop() { @Override public void run() { - if (LOG.isDebugEnabled()) { LOG.debug("==> AtlasRESTTagSource.run()"); } while (true) { - synchUp(); LOG.debug("Sleeping for [" + sleepTimeBetweenCycleInMillis + "] milliSeconds"); try { - Thread.sleep(sleepTimeBetweenCycleInMillis); - } catch (InterruptedException exception) { LOG.error("Interrupted..: ", exception); return; @@ -173,17 +171,40 @@ public void run() { } public void synchUp() { + SearchParameters searchParams = new SearchParameters(); + AtlasTypeRegistry typeRegistry = new AtlasTypeRegistry(); + AtlasTransientTypeRegistry tty = null; + AtlasSearchResult searchResult = null; + + searchParams.setClassification("*"); + searchParams.setIncludeClassificationAttributes(true); + searchParams.setOffset(0); + searchParams.setLimit(Integer.MAX_VALUE); + + try { + AtlasClientV2 atlasClient = getAtlasClient(); + + searchResult = atlasClient.facetedSearch(searchParams); - List atlasEntities = atlasRESTUtil.getAtlasEntities(); + AtlasTypesDef typesDef = atlasClient.getAllTypeDefs(new SearchFilter()); - if (CollectionUtils.isNotEmpty(atlasEntities)) { + tty = typeRegistry.lockTypeRegistryForUpdate(); + + tty.addTypes(typesDef); + } catch (AtlasServiceException | AtlasBaseException | IOException excp) { + LOG.error("failed to download tags from Atlas", excp); + } finally { + if (tty != null) { + typeRegistry.releaseTypeRegistryForUpdate(tty, true); + } + } + + if (searchResult != null) { if (LOG.isDebugEnabled()) { - for (AtlasEntityWithTraits element : atlasEntities) { - LOG.debug(element); - } + LOG.debug(AtlasType.toJson(searchResult)); } - Map serviceTagsMap = AtlasNotificationMapper.processAtlasEntities(atlasEntities); + Map serviceTagsMap = AtlasNotificationMapper.processSearchResult(searchResult, typeRegistry); if (MapUtils.isNotEmpty(serviceTagsMap)) { for (Map.Entry entry : serviceTagsMap.entrySet()) { @@ -195,6 +216,7 @@ public void synchUp() { LOG.debug("serviceTags=" + serviceTagsString); } + updateSink(entry.getValue()); } } @@ -202,5 +224,20 @@ public void synchUp() { } + private AtlasClientV2 getAtlasClient() throws IOException { + final AtlasClientV2 ret; + + if (isKerberized) { + UserGroupInformation ugi = UserGroupInformation.getLoginUser(); + + ugi.checkTGTAndReloginFromKeytab(); + + ret = new AtlasClientV2(ugi, ugi.getShortUserName(), restUrls); + } else { + ret = new AtlasClientV2(restUrls, userNamePassword); + } + + return ret; + } } diff --git a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlasrest/AtlasRESTUtil.java b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlasrest/AtlasRESTUtil.java deleted file mode 100644 index 00a101e727..0000000000 --- a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlasrest/AtlasRESTUtil.java +++ /dev/null @@ -1,325 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ranger.tagsync.source.atlasrest; - -import com.google.gson.Gson; -import com.sun.jersey.api.client.ClientResponse; -import com.sun.jersey.api.client.WebResource; -import org.apache.atlas.typesystem.IReferenceableInstance; -import org.apache.atlas.typesystem.IStruct; -import org.apache.atlas.typesystem.Struct; -import org.apache.atlas.typesystem.json.InstanceSerialization; -import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.collections.MapUtils; -import org.apache.commons.lang.StringUtils; -import org.apache.hadoop.security.UserGroupInformation; -import org.apache.log4j.Logger; -import org.apache.ranger.admin.client.datatype.RESTResponse; -import org.apache.ranger.plugin.util.RangerRESTClient; -import org.apache.ranger.tagsync.source.atlas.AtlasEntityWithTraits; -import org.apache.ranger.tagsync.source.atlas.AtlasResourceMapperUtil; - -import java.io.IOException; -import java.security.PrivilegedAction; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; - -@SuppressWarnings("unchecked") -public class AtlasRESTUtil { - private static final Logger LOG = Logger.getLogger(AtlasRESTUtil.class); - - private static final String REST_MIME_TYPE_JSON = "application/json"; - private static final String API_ATLAS_TYPES = "api/atlas/types"; - private static final String API_ATLAS_ENTITIES = "api/atlas/entities?type="; - private static final String API_ATLAS_ENTITY = "api/atlas/entities/"; - private static final String API_ATLAS_TYPE = "api/atlas/types/"; - - private static final String RESULTS_ATTRIBUTE = "results"; - private static final String DEFINITION_ATTRIBUTE = "definition"; - private static final String VALUES_ATTRIBUTE = "values"; - private static final String TRAITS_ATTRIBUTE = "traits"; - private static final String TYPE_NAME_ATTRIBUTE = "typeName"; - private static final String TRAIT_TYPES_ATTRIBUTE = "traitTypes"; - private static final String SUPER_TYPES_ATTRIBUTE = "superTypes"; - private static final String ATTRIBUTE_DEFINITIONS_ATTRIBUTE = "attributeDefinitions"; - private static final String NAME_ATTRIBUTE = "name"; - - private final Gson gson = new Gson(); - - private final RangerRESTClient atlasRESTClient; - - private final boolean isKerberized; - - public AtlasRESTUtil(RangerRESTClient atlasRESTClient, boolean isKerberized) { - if (LOG.isDebugEnabled()) { - LOG.debug("==> AtlasRESTUtil()"); - } - - this.atlasRESTClient = atlasRESTClient; - - this.isKerberized = isKerberized; - - if (LOG.isDebugEnabled()) { - LOG.debug("<== AtlasRESTUtil()"); - } - } - - public List getAtlasEntities() { - if (LOG.isDebugEnabled()) { - LOG.debug("==> getAtlasEntities()"); - } - - List ret = new ArrayList(); - - Map typesResponse = atlasAPI(API_ATLAS_TYPES); - - List types = getAttribute(typesResponse, RESULTS_ATTRIBUTE, List.class); - - if (CollectionUtils.isNotEmpty(types)) { - - for (String type : types) { - - if (!AtlasResourceMapperUtil.isEntityTypeHandled(type)) { - if (LOG.isDebugEnabled()) { - LOG.debug("Not fetching Atlas entities of type: " + type); - } - continue; - } - - Map entitiesResponse = atlasAPI(API_ATLAS_ENTITIES + type); - - List guids = getAttribute(entitiesResponse, RESULTS_ATTRIBUTE, List.class); - - if (CollectionUtils.isEmpty(guids)) { - if (LOG.isDebugEnabled()) { - LOG.debug("No Atlas entities for type: " + type); - } - continue; - } - - for (String guid : guids) { - - Map entityResponse = atlasAPI(API_ATLAS_ENTITY + guid); - - Map definition = getAttribute(entityResponse, DEFINITION_ATTRIBUTE, Map.class); - - Map traitsAttribute = getAttribute(definition, TRAITS_ATTRIBUTE, Map.class); - - List allTraits = new LinkedList<>(); - - if (MapUtils.isNotEmpty(traitsAttribute)) { - - for (Map.Entry entry : traitsAttribute.entrySet()) { - - Map trait = (Map) entry.getValue(); - - Map traitValues = getAttribute(trait, VALUES_ATTRIBUTE, Map.class); - String traitTypeName = getAttribute(trait, TYPE_NAME_ATTRIBUTE, String.class); - - if (StringUtils.isEmpty(traitTypeName)) { - continue; - } - - List superTypes = getTraitSuperTypes(getTraitType(traitTypeName), traitValues); - - Struct trait1 = new Struct(traitTypeName, traitValues); - - allTraits.add(trait1); - allTraits.addAll(superTypes); - } - } - - IReferenceableInstance entity = InstanceSerialization.fromJsonReferenceable(gson.toJson(definition), true); - - if (entity != null) { - AtlasEntityWithTraits atlasEntity = new AtlasEntityWithTraits(entity, allTraits); - ret.add(atlasEntity); - } else { - if (LOG.isInfoEnabled()) { - LOG.info("Could not create Atlas entity from its definition, type=" + type + ", guid=" + guid); - } - } - - } - - } - if (LOG.isDebugEnabled()) { - LOG.debug("<== getAtlasEntities()"); - } - } - - return ret; - } - - private Map getTraitType(String traitName) { - if (LOG.isDebugEnabled()) { - LOG.debug("==> getTraitType(" + traitName + ")"); - } - Map ret = null; - - Map typeResponse = atlasAPI(API_ATLAS_TYPE + traitName); - - Map definition = getAttribute(typeResponse, DEFINITION_ATTRIBUTE, Map.class); - - List traitTypes = getAttribute(definition, TRAIT_TYPES_ATTRIBUTE, List.class); - - if (CollectionUtils.isNotEmpty(traitTypes)) { - ret = (Map) traitTypes.get(0); - } - - if (LOG.isDebugEnabled()) { - LOG.debug("<== getTraitType(" + traitName + ")"); - } - return ret; - } - - private List getTraitSuperTypes(Map traitType, Map values) { - - if (LOG.isDebugEnabled()) { - LOG.debug("==> getTraitSuperTypes()"); - } - List ret = new LinkedList<>(); - - if (traitType != null) { - - List superTypeNames = getAttribute(traitType, SUPER_TYPES_ATTRIBUTE, List.class); - - if (CollectionUtils.isNotEmpty(superTypeNames)) { - for (String superTypeName : superTypeNames) { - - Map superTraitType = getTraitType(superTypeName); - - if (superTraitType != null) { - List> attributeDefinitions = (List) superTraitType.get(ATTRIBUTE_DEFINITIONS_ATTRIBUTE); - - Map superTypeValues = new HashMap<>(); - for (Map attributeDefinition : attributeDefinitions) { - - String attributeName = attributeDefinition.get(NAME_ATTRIBUTE).toString(); - if (values.containsKey(attributeName)) { - superTypeValues.put(attributeName, values.get(attributeName)); - } - } - - List superTraits = getTraitSuperTypes(getTraitType(superTypeName), values); - - Struct superTrait = new Struct(superTypeName, superTypeValues); - - ret.add(superTrait); - ret.addAll(superTraits); - } - } - } - } - if (LOG.isDebugEnabled()) { - LOG.debug("<== getTraitSuperTypes()"); - } - return ret; - } - - private Map atlasAPI(final String endpoint) { - - if (LOG.isDebugEnabled()) { - LOG.debug("==> atlasAPI(" + endpoint + ")"); - } - Map ret = new HashMap(); - - try { - UserGroupInformation userGroupInformation = null; - if (isKerberized) { - userGroupInformation = UserGroupInformation.getLoginUser(); - - try { - userGroupInformation.checkTGTAndReloginFromKeytab(); - } catch (IOException ioe) { - LOG.error("Error renewing TGT and relogin", ioe); - userGroupInformation = null; - } - } - if (userGroupInformation != null) { - LOG.debug("Using kerberos authentication"); - if(LOG.isDebugEnabled()) { - LOG.debug("Using Principal = "+ userGroupInformation.getUserName()); - } - ret = userGroupInformation.doAs(new PrivilegedAction>() { - @Override - public Map run() { - try{ - return executeAtlasAPI(endpoint); - }catch (Exception e) { - LOG.error("Atlas API failed with message : ", e); - } - return null; - } - }); - } else { - LOG.debug("Using basic authentication"); - ret = executeAtlasAPI(endpoint); - } - } catch (Exception exception) { - LOG.error("Exception when fetching Atlas objects.", exception); - ret = null; - } - - if (LOG.isDebugEnabled()) { - LOG.debug("<== atlasAPI(" + endpoint + ")"); - } - return ret; - } - - private Map executeAtlasAPI(final String endpoint) { - - if (LOG.isDebugEnabled()) { - LOG.debug("==> executeAtlasAPI(" + endpoint + ")"); - } - - Map ret = new HashMap(); - - try { - final WebResource webResource = atlasRESTClient.getResource(endpoint); - - ClientResponse response = webResource.accept(REST_MIME_TYPE_JSON).type(REST_MIME_TYPE_JSON).get(ClientResponse.class); - - if (response != null && response.getStatus() == 200) { - ret = response.getEntity(ret.getClass()); - } else { - RESTResponse resp = RESTResponse.fromClientResponse(response); - LOG.error("Error getting atlas data request=" + webResource.toString() - + ", response=" + resp.toString()); - } - } catch (Exception exception) { - LOG.error("Exception when fetching Atlas objects.", exception); - ret = null; - } - - if (LOG.isDebugEnabled()) { - LOG.debug("<== executeAtlasAPI(" + endpoint + ")"); - } - - return ret; - } - - private T getAttribute(Map map, String name, Class type) { - return MapUtils.isNotEmpty(map) ? type.cast(map.get(name)) : null; - } - -} From 29cd16cbb690d6972e7ce24558350b5dee3468ba Mon Sep 17 00:00:00 2001 From: Abhay Kulkarni Date: Wed, 20 Dec 2017 19:41:25 -0800 Subject: [PATCH 070/151] RANGER-1937: Ranger tagsync should process ENTITY_CREATE notification, to support Atlas import feature --- .../source/atlas/AtlasNotificationMapper.java | 37 +++++++++---------- 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasNotificationMapper.java b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasNotificationMapper.java index f42c90864f..8641d609dd 100644 --- a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasNotificationMapper.java +++ b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasNotificationMapper.java @@ -103,26 +103,18 @@ public static ServiceTags processEntityNotification(EntityNotification entityNot try { IReferenceableInstance entity = entityNotification.getEntity(); - if (entity != null && AtlasResourceMapperUtil.isEntityTypeHandled(entity.getTypeName())) { - AtlasEntityWithTraits entityWithTraits = new AtlasEntityWithTraits(entity, entityNotification.getAllTraits()); - if (entityNotification.getOperationType() == EntityNotification.OperationType.ENTITY_DELETE) { - ret = buildServiceTagsForEntityDeleteNotification(entityWithTraits); - } else { - if (entity.getId().getState() == Id.EntityState.ACTIVE) { - ret = buildServiceTags(entityWithTraits, null); - } else { - if (LOG.isDebugEnabled()) { - LOG.debug("Ignoring entityNotification for entity that is not ACTIVE: " + entityWithTraits); - } - } - } + AtlasEntityWithTraits entityWithTraits = new AtlasEntityWithTraits(entity, entityNotification.getAllTraits()); + + if (entityNotification.getOperationType() == EntityNotification.OperationType.ENTITY_DELETE) { + ret = buildServiceTagsForEntityDeleteNotification(entityWithTraits); } else { - logUnhandledEntityNotification(entityNotification); + ret = buildServiceTags(entityWithTraits, null); } - } catch (Exception exception) { LOG.error("createServiceTags() failed!! ", exception); } + } else { + logUnhandledEntityNotification(entityNotification); } return ret; } @@ -144,12 +136,11 @@ static private boolean isNotificationHandled(EntityNotification entityNotificati EntityNotification.OperationType opType = entityNotification.getOperationType(); - if(opType != null) { + if (opType != null) { switch (opType) { - case ENTITY_CREATE: { - LOG.debug("ENTITY_CREATE notification is not handled, as Ranger will get necessary information from any subsequent TRAIT_ADDED notification"); + case ENTITY_CREATE: + ret = CollectionUtils.isNotEmpty(entityNotification.getAllTraits()); break; - } case ENTITY_UPDATE: case ENTITY_DELETE: case TRAIT_ADD: @@ -160,6 +151,14 @@ static private boolean isNotificationHandled(EntityNotification entityNotificati } default: LOG.error(opType + ": unknown notification received - not handled"); + break; + } + if (ret) { + final IReferenceableInstance entity = entityNotification.getEntity(); + + ret = entity != null + && entity.getId().getState() == Id.EntityState.ACTIVE + && AtlasResourceMapperUtil.isEntityTypeHandled(entity.getTypeName()); } } From 7acbe78921b609944ed53f63bcfaae31cdb17205 Mon Sep 17 00:00:00 2001 From: rmani Date: Thu, 21 Dec 2017 14:15:55 -0800 Subject: [PATCH 071/151] RANGER-1943:Ranger Solr authorization is skipped when collection is empty or null Signed-off-by: rmani --- .../solr/authorizer/RangerSolrAuthorizer.java | 57 +++++++++++-------- 1 file changed, 34 insertions(+), 23 deletions(-) diff --git a/plugin-solr/src/main/java/org/apache/ranger/authorization/solr/authorizer/RangerSolrAuthorizer.java b/plugin-solr/src/main/java/org/apache/ranger/authorization/solr/authorizer/RangerSolrAuthorizer.java index 0c32eb18aa..78b4ac4f3c 100644 --- a/plugin-solr/src/main/java/org/apache/ranger/authorization/solr/authorizer/RangerSolrAuthorizer.java +++ b/plugin-solr/src/main/java/org/apache/ranger/authorization/solr/authorizer/RangerSolrAuthorizer.java @@ -29,6 +29,8 @@ import java.util.Set; import javax.security.auth.login.Configuration; + +import org.apache.commons.collections.CollectionUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.security.UserGroupInformation; @@ -188,23 +190,38 @@ public AuthorizationResponse authorize(AuthorizationContext context) { ip = context.getHttpHeader("REMOTE_ADDR"); } - // Create the list of requests for access check. Each field is - // broken - // into a request List rangerRequests = new ArrayList(); - for (CollectionRequest collectionRequest : context - .getCollectionRequests()) { + List collectionRequests = context.getCollectionRequests(); + if (CollectionUtils.isEmpty(collectionRequests)) { + // if Collection is empty we set the collection to *. This happens when LIST is done. RangerAccessRequestImpl requestForCollection = createRequest( userName, userGroups, ip, eventTime, context, - collectionRequest); + null); if (requestForCollection != null) { rangerRequests.add(requestForCollection); } + } else { + // Create the list of requests for access check. Each field is + // broken + // into a request + for (CollectionRequest collectionRequest : context + .getCollectionRequests()) { + + RangerAccessRequestImpl requestForCollection = createRequest( + userName, userGroups, ip, eventTime, context, + collectionRequest); + if (requestForCollection != null) { + rangerRequests.add(requestForCollection); + } + } + } + if (logger.isDebugEnabled()) { logger.debug("rangerRequests.size()=" + rangerRequests.size()); } + try { // Let's check the access for each request/resource for (RangerAccessRequestImpl rangerRequest : rangerRequests) { @@ -313,25 +330,19 @@ private RangerAccessRequestImpl createRequest(String userName, String accessType = mapToRangerAccessType(context); String action = accessType; - - if (collectionRequest.collectionName != null) { - RangerAccessRequestImpl rangerRequest = createBaseRequest(userName, - userGroups, ip, eventTime); - RangerAccessResourceImpl rangerResource = new RangerAccessResourceImpl(); - rangerResource.setValue(KEY_COLLECTION, - collectionRequest.collectionName); - rangerRequest.setResource(rangerResource); - rangerRequest.setAccessType(accessType); - rangerRequest.setAction(action); - - return rangerRequest; + RangerAccessRequestImpl rangerRequest = createBaseRequest(userName, + userGroups, ip, eventTime); + RangerAccessResourceImpl rangerResource = new RangerAccessResourceImpl(); + if (collectionRequest == null) { + rangerResource.setValue(KEY_COLLECTION, "*"); + } else { + rangerResource.setValue(KEY_COLLECTION, collectionRequest.collectionName); } - - logger.fatal("Can't create RangerRequest oject. userName=" - + userName + ", accessType=" + accessType + ", ip=" + ip - + ", collectionRequest=" + collectionRequest); + rangerRequest.setResource(rangerResource); + rangerRequest.setAccessType(accessType); + rangerRequest.setAction(action); - return null; + return rangerRequest; } private RangerAccessRequestImpl createBaseRequest(String userName, From 07333fd66412dacc3895826ecaa72d26c7cac8f6 Mon Sep 17 00:00:00 2001 From: Kevin Risden Date: Tue, 19 Dec 2017 19:16:52 -0600 Subject: [PATCH 072/151] RANGER-1938: Enable DocValues for more fields in Solr Signed-off-by: Velmurugan Periasamy --- .../solr_for_audit_setup/conf/managed-schema | 36 +++++++++---------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/security-admin/contrib/solr_for_audit_setup/conf/managed-schema b/security-admin/contrib/solr_for_audit_setup/conf/managed-schema index 4d032f5534..6c87af7cf9 100644 --- a/security-admin/contrib/solr_for_audit_setup/conf/managed-schema +++ b/security-admin/contrib/solr_for_audit_setup/conf/managed-schema @@ -15,16 +15,16 @@ See the License for the specific language governing permissions and limitations under the License. --> - + id - - - + + + - + @@ -32,13 +32,13 @@ - + - - - - + + + + @@ -50,15 +50,15 @@ - - - - - - + + + + + + - + @@ -90,4 +90,4 @@ - + \ No newline at end of file From bcc1412ec419ecedbe7a9f0d2f04c38328d0ba6b Mon Sep 17 00:00:00 2001 From: ni3galave Date: Tue, 2 Jan 2018 10:59:48 +0530 Subject: [PATCH 073/151] RANGER-1944: Action filter for Admin Audit is not working Signed-off-by: Mehul Parikh --- .../src/main/webapp/scripts/utils/XAGlobals.js | 11 +++++++++++ .../webapp/scripts/views/reports/AuditLayout.js | 15 ++++++++++----- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/security-admin/src/main/webapp/scripts/utils/XAGlobals.js b/security-admin/src/main/webapp/scripts/utils/XAGlobals.js index d16e5deadf..d55ab4a1d2 100644 --- a/security-admin/src/main/webapp/scripts/utils/XAGlobals.js +++ b/security-admin/src/main/webapp/scripts/utils/XAGlobals.js @@ -73,5 +73,16 @@ define(function(require){ 'Permissions' : ['modulePermissionsAction','modulePermissionEditAction'] }, }; + XAGlobals.ActionType = { + Create : { value: 'create', label: 'Create' }, + Update : { value: 'update', label: 'Update' }, + Delete : { value: 'delete', label: 'Delete' }, + Password_Change : { value: 'password change', label: 'Password Change' }, + Export_Json : { value: 'EXPORT JSON', label: 'Export Json' }, + Export_Csv : { value: 'EXPORT CSV', label: 'Export Csv' }, + Export_Excel : { value: 'EXPORT EXCEL', label: 'Export Excel' }, + Import_End : { value: 'IMPORT END', label: 'Import End' }, + Import_Start : { value: 'IMPORT START', label: 'Import Start'} + }; return XAGlobals; }); diff --git a/security-admin/src/main/webapp/scripts/views/reports/AuditLayout.js b/security-admin/src/main/webapp/scripts/views/reports/AuditLayout.js index e9d3675fbc..c5d7d0e21b 100644 --- a/security-admin/src/main/webapp/scripts/views/reports/AuditLayout.js +++ b/security-admin/src/main/webapp/scripts/views/reports/AuditLayout.js @@ -370,11 +370,11 @@ define(function(require) { var that = this; var searchOpt = ["Audit Type", "User", "Actions", "Session Id", "Start Date", "End Date"]; var serverAttrName = [{text : "Audit Type", label :"objectClassType",'multiple' : true, 'optionsArr' : XAUtils.enumToSelectLabelValuePairs(XAEnums.ClassTypes)}, - {text : "User", label :"owner"}, - {text : "Actions", label :"action"},{text : "Session Id", label :"sessionId"}, - {text : 'Start Date',label :'startDate'},{text : 'End Date',label :'endDate'} ]; + {text : "User", label :"owner"}, {text : "Session Id", label :"sessionId"}, + {text : 'Start Date',label :'startDate'},{text : 'End Date',label :'endDate'}, + {text : "Actions", label :"action",'multiple' : true, 'optionsArr' : XAUtils.enumToSelectLabelValuePairs(XAGlobals.ActionType)},]; - var auditList = [],query = ''; + var auditList = [],query = '', actionTypeList = []; _.each(XAEnums.ClassTypes, function(obj){ if((obj.value == XAEnums.ClassTypes.CLASS_TYPE_XA_ASSET.value) || (obj.value == XAEnums.ClassTypes.CLASS_TYPE_XA_RESOURCE.value) @@ -384,6 +384,11 @@ define(function(require) { || (obj.value == XAEnums.ClassTypes.CLASS_TYPE_XA_GROUP.value)) auditList.push({label :obj.label, value :obj.label+''}); }); + _.each(XAGlobals.ActionType, function(obj){ + if(obj.label){ + actionTypeList.push({label :obj.label, value :obj.label}) + } + }) if(!_.isUndefined(App.sessionId)){ App.vsHistory.admin = [] ; query = '"Session Id": "'+App.sessionId+'"'; @@ -404,7 +409,7 @@ define(function(require) { callback(auditList); break; case 'Actions': - callback(["Create","Update","Delete","Password Change","Export Json","Export Csv","Export Excel","Import End","Import Start"]); + callback(actionTypeList); break; case 'Start Date' : var endDate, models = that.visualSearch.searchQuery.where({category:"End Date"}); From bfac532ed654a0230d5f69e1e0532207026d8d67 Mon Sep 17 00:00:00 2001 From: pradeep Date: Fri, 12 Jan 2018 11:25:21 +0530 Subject: [PATCH 074/151] RANGER-1952: Allow user's email Address to be blank on emailchange API as per RANGER-978 --- .../main/java/org/apache/ranger/biz/UserMgr.java | 7 ++----- .../main/java/org/apache/ranger/biz/XUserMgr.java | 15 +++++++++++++++ 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/security-admin/src/main/java/org/apache/ranger/biz/UserMgr.java b/security-admin/src/main/java/org/apache/ranger/biz/UserMgr.java index 35d9b410e9..f17c6d1cbe 100644 --- a/security-admin/src/main/java/org/apache/ranger/biz/UserMgr.java +++ b/security-admin/src/main/java/org/apache/ranger/biz/UserMgr.java @@ -456,16 +456,13 @@ public VXPortalUser changeEmailAddress(XXPortalUser gjUser, VXPasswordChange changeEmail) { checkAccessForUpdate(gjUser); if (StringUtils.isEmpty(changeEmail.getEmailAddress())) { - throw restErrorUtil.createRESTException( - "serverMsg.userMgrInvalidEmail", - MessageEnums.INVALID_INPUT_DATA, changeEmail.getId(), - "emailAddress", changeEmail.toString()); + changeEmail.setEmailAddress(null); } String encryptedOldPwd = encrypt(gjUser.getLoginId(), changeEmail.getOldPassword()); - if (!stringUtil.validateEmail(changeEmail.getEmailAddress())) { + if (!StringUtils.isEmpty(changeEmail.getEmailAddress()) && !stringUtil.validateEmail(changeEmail.getEmailAddress())) { logger.info("Invalid email address." + changeEmail); throw restErrorUtil.createRESTException( "serverMsg.userMgrInvalidEmail", diff --git a/security-admin/src/main/java/org/apache/ranger/biz/XUserMgr.java b/security-admin/src/main/java/org/apache/ranger/biz/XUserMgr.java index f9e6991339..e6e82d1d2e 100644 --- a/security-admin/src/main/java/org/apache/ranger/biz/XUserMgr.java +++ b/security-admin/src/main/java/org/apache/ranger/biz/XUserMgr.java @@ -1367,6 +1367,7 @@ public VXStringList setUserRolesByExternalID(Long userId, List vString if(vXUser!=null && roleListNewProfile.size()>0){ VXPortalUser oldUserProfile = userMgr.getUserProfileByLoginId(vXUser.getName()); if(oldUserProfile!=null){ + denySelfRoleChange(oldUserProfile.getLoginId()); updateUserRolesPermissions(oldUserProfile,roleListNewProfile); portalUserRoleList = daoManager.getXXPortalUserRole().findByUserId(oldUserProfile.getId()); return getStringListFromUserRoleList(portalUserRoleList); @@ -1389,6 +1390,7 @@ public VXStringList setUserRolesByName(String userName, List vStringRo if(userName!=null && roleListNewProfile.size()>0){ VXPortalUser oldUserProfile = userMgr.getUserProfileByLoginId(userName); if(oldUserProfile!=null){ + denySelfRoleChange(oldUserProfile.getLoginId()); updateUserRolesPermissions(oldUserProfile,roleListNewProfile); List portalUserRoleList = daoManager.getXXPortalUserRole().findByUserId(oldUserProfile.getId()); return getStringListFromUserRoleList(portalUserRoleList); @@ -2216,4 +2218,17 @@ private void validatePassword(VXUser vXUser) { throw restErrorUtil.createRESTException("serverMsg.xuserMgrValidatePassword", MessageEnums.INVALID_PASSWORD, null, "Password cannot be blank/null", null); } } + + public void denySelfRoleChange(String userName) { + UserSessionBase session = ContextUtil.getCurrentUserSession(); + if (session != null && session.getXXPortalUser()!=null) { + if (userName.equals(session.getXXPortalUser().getLoginId())) { + throw restErrorUtil.create403RESTException("Permission" + + " denied. LoggedInUser=" + + (session != null ? session.getXXPortalUser().getId() + : "Not Logged In") + + " ,isn't permitted to change its own role."); + } + } + } } From 3c1f7e2386dc38ad5f081a7c196fc1d88507aa53 Mon Sep 17 00:00:00 2001 From: Sailaja Polavarapu Date: Fri, 19 Jan 2018 15:17:37 -0800 Subject: [PATCH 075/151] RANGER-1957: Fixed code to sync all the users during periodic sync when there are updates to groups -- ranger-0.7 branch --- .../process/LdapDeltaUserGroupBuilder.java | 53 +++++++++++-------- 1 file changed, 30 insertions(+), 23 deletions(-) diff --git a/ugsync/src/main/java/org/apache/ranger/ldapusersync/process/LdapDeltaUserGroupBuilder.java b/ugsync/src/main/java/org/apache/ranger/ldapusersync/process/LdapDeltaUserGroupBuilder.java index 17682ba0c7..2852b32071 100644 --- a/ugsync/src/main/java/org/apache/ranger/ldapusersync/process/LdapDeltaUserGroupBuilder.java +++ b/ugsync/src/main/java/org/apache/ranger/ldapusersync/process/LdapDeltaUserGroupBuilder.java @@ -372,6 +372,12 @@ private void getUsers(UserGroupSink sink) throws Throwable { new PagedResultsControl(pagedResultsSize, Control.NONCRITICAL) }); } DateFormat dateFormat = new SimpleDateFormat("yyyyMMddhhmmss"); + if (groupSearchFirstEnabled && groupUserTable.rowKeySet().size() != 0) { + // Fix RANGER-1957: Perform full sync when group search is enabled and when there are updates to the groups + deltaSyncUserTime = 0; + deltaSyncUserTimeStamp = dateFormat.format(new Date(0)); + } + extendedUserSearchFilter = "(objectclass=" + userObjectClass + ")(|(uSNChanged>=" + deltaSyncUserTime + ")(modifyTimestamp>=" + deltaSyncUserTimeStamp + "Z))"; if (userSearchFilter != null && !userSearchFilter.trim().isEmpty()) { @@ -498,32 +504,13 @@ private void getUsers(UserGroupSink sink) throws Throwable { LOG.error("sink.addOrUpdateUserGroups failed with exception: " + t.getMessage() + ", for user: " + transformUserName + " and groups: " + groupList); } - counter++; - if (counter <= 2000) { - if (LOG.isInfoEnabled()) { - LOG.info("Updating user count: " + counter - + ", userName: " + userName + ", groupList: " - + groupList); - } - if ( counter == 2000 ) { - LOG.info("===> 2000 user records have been synchronized so far. From now on, only a summary progress log will be written for every 100 users. To continue to see detailed log for every user, please enable Trace level logging. <==="); - } - } else { - if (LOG.isTraceEnabled()) { - LOG.trace("Updating user count: " + counter - + ", userName: " + userName + ", groupList: " - + groupList); - } else { - if ( counter % 100 == 0) { - LOG.info("Synced " + counter + " users till now"); - } - } - } + counter++; } else { // If the user from the search result is present in the group user table, // then addorupdate user to ranger admin. LOG.debug("Chekcing if the user " + userFullName + " is part of the retrieved groups"); - if (groupUserTable.containsColumn(userFullName) || groupUserTable.containsColumn(userName)) { + if ((groupUserTable.containsColumn(userFullName) || groupUserTable.containsColumn(userName)) + && !userNameMap.containsKey(userFullName)) { String transformUserName = userNameTransform(userName); try { sink.addOrUpdateUser(transformUserName); @@ -538,9 +525,29 @@ private void getUsers(UserGroupSink sink) throws Throwable { LOG.debug("Updating groupUserTable " + entry.getValue() + " with: " + transformUserName + " for " + entry.getKey()); groupUserTable.put(entry.getKey(), userFullName, transformUserName); } + counter++; } } + if (counter <= 2000) { + if (LOG.isInfoEnabled()) { + LOG.info("Updating user count: " + counter + + ", userName: " + userName); + } + if ( counter == 2000 ) { + LOG.info("===> 2000 user records have been synchronized so far. From now on, only a summary progress log will be written for every 100 users. To continue to see detailed log for every user, please enable Trace level logging. <==="); + } + } else { + if (LOG.isTraceEnabled()) { + LOG.trace("Updating user count: " + counter + + ", userName: " + userName); + } else { + if ( counter % 100 == 0) { + LOG.info("Synced " + counter + " users till now"); + } + } + } + } // Examine the paged results control response @@ -579,7 +586,7 @@ private void getUsers(UserGroupSink sink) throws Throwable { } if (deltaSyncUserTime < highestdeltaSyncUserTime) { // Incrementing highestdeltaSyncUserTime (for AD) in order to avoid search record repetition for next sync cycle. - deltaSyncUserTime = highestdeltaSyncUserTime+1; + deltaSyncUserTime = highestdeltaSyncUserTime + 1; // Incrementing the highest timestamp value (for Openldap) with 1sec in order to avoid search record repetition for next sync cycle. deltaSyncUserTimeStamp = dateFormat.format(new Date(highestdeltaSyncUserTime + 60l)); } From 4f2737b9a6ca1741404ac95c5909b4ec7e1affff Mon Sep 17 00:00:00 2001 From: pradeep Date: Mon, 22 Jan 2018 14:04:25 +0530 Subject: [PATCH 076/151] RANGER-1956: Update CredentialBuilder module --- .../java/org/apache/ranger/credentialapi/buildks.java | 8 -------- 1 file changed, 8 deletions(-) diff --git a/credentialbuilder/src/main/java/org/apache/ranger/credentialapi/buildks.java b/credentialbuilder/src/main/java/org/apache/ranger/credentialapi/buildks.java index 01868ab58c..043f44cfc2 100644 --- a/credentialbuilder/src/main/java/org/apache/ranger/credentialapi/buildks.java +++ b/credentialbuilder/src/main/java/org/apache/ranger/credentialapi/buildks.java @@ -40,14 +40,6 @@ public static void main(String[] args) { buildksOBJ.createCredential(args); }else if(command.equalsIgnoreCase("list")){ buildksOBJ.listCredential(args); - }else if(command.equalsIgnoreCase("get")){ - String credential=buildksOBJ.getCredential(args); - if(credential!=null){ - System.out.println(credential); - System.exit(0); - }else{ - System.exit(1); - } }else{ System.out.println(command +" is not supported in current version of CredentialBuilder API."); System.exit(1); From 0fa92661380f79a9d89e19128dabcc374f3c3b99 Mon Sep 17 00:00:00 2001 From: tsokorai Date: Mon, 22 Jan 2018 09:04:57 -0300 Subject: [PATCH 077/151] RANGER-1960: Take snapshot's table name into consideration for deletion Signed-off-by: rmani --- .../authorization/hbase/RangerAuthorizationCoprocessor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hbase-agent/src/main/java/org/apache/ranger/authorization/hbase/RangerAuthorizationCoprocessor.java b/hbase-agent/src/main/java/org/apache/ranger/authorization/hbase/RangerAuthorizationCoprocessor.java index fc1db4693b..038bf08eee 100644 --- a/hbase-agent/src/main/java/org/apache/ranger/authorization/hbase/RangerAuthorizationCoprocessor.java +++ b/hbase-agent/src/main/java/org/apache/ranger/authorization/hbase/RangerAuthorizationCoprocessor.java @@ -812,7 +812,7 @@ public void preDeleteColumn(ObserverContext c, Tab } @Override public void preDeleteSnapshot(ObserverContext ctx, SnapshotDescription snapshot) throws IOException { - requirePermission("deleteSnapshot", Permission.Action.ADMIN); + requirePermission("deleteSnapshot", snapshot.getTableBytes().toByteArray(), Permission.Action.ADMIN); } @Override public void preDeleteTable(ObserverContext c, TableName tableName) throws IOException { From 27829af8f9af8c48e975311f38bd1e0fc8237e95 Mon Sep 17 00:00:00 2001 From: Abhay Kulkarni Date: Tue, 30 Jan 2018 10:18:57 -0800 Subject: [PATCH 078/151] RANGER-1966: Policy engine initialization does not create context enrichers in some cases --- .../policyengine/RangerPolicyRepository.java | 32 +- .../plugin/policyengine/TestPolicyEngine.java | 7 + .../test_policyengine_tag_hive_mask.json | 434 ++++++++++++++++++ 3 files changed, 471 insertions(+), 2 deletions(-) create mode 100644 agents-common/src/test/resources/policyengine/test_policyengine_tag_hive_mask.json diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java index 1580766996..a8fa2922f8 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java @@ -584,7 +584,8 @@ private void init(RangerPolicyEngineOptions options) { this.rowFilterPolicyEvaluators = Collections.unmodifiableList(rowFilterPolicyEvaluators); List contextEnrichers = new ArrayList(); - if (CollectionUtils.isNotEmpty(this.policyEvaluators)) { + if (CollectionUtils.isNotEmpty(this.policyEvaluators) || CollectionUtils.isNotEmpty(this.dataMaskPolicyEvaluators) + || CollectionUtils.isNotEmpty(this.rowFilterPolicyEvaluators)) { if (CollectionUtils.isNotEmpty(serviceDef.getContextEnrichers())) { for (RangerServiceDef.RangerContextEnricherDef enricherDef : serviceDef.getContextEnrichers()) { if (enricherDef == null) { @@ -627,7 +628,7 @@ private void init(RangerPolicyEngineOptions options) { LOG.debug("dataMask policy evaluation order: #" + (++order) + " - policy id=" + policy.getId() + "; name=" + policy.getName() + "; evalOrder=" + policyEvaluator.getEvalOrder()); } - LOG.debug("rowFilter policy evaluation order: " + this.dataMaskPolicyEvaluators.size() + " policies"); + LOG.debug("rowFilter policy evaluation order: " + this.rowFilterPolicyEvaluators.size() + " policies"); order = 0; for(RangerPolicyEvaluator policyEvaluator : this.rowFilterPolicyEvaluators) { RangerPolicy policy = policyEvaluator.getPolicy(); @@ -873,6 +874,32 @@ public StringBuilder toString(StringBuilder sb) { } } } + sb.append("} "); + + sb.append("dataMaskPolicyEvaluators={"); + + if (this.dataMaskPolicyEvaluators != null) { + for (RangerPolicyEvaluator policyEvaluator : dataMaskPolicyEvaluators) { + if (policyEvaluator != null) { + sb.append(policyEvaluator).append(" "); + } + } + } + sb.append("} "); + + sb.append("rowFilterPolicyEvaluators={"); + + if (this.rowFilterPolicyEvaluators != null) { + for (RangerPolicyEvaluator policyEvaluator : rowFilterPolicyEvaluators) { + if (policyEvaluator != null) { + sb.append(policyEvaluator).append(" "); + } + } + } + sb.append("} "); + + sb.append("contextEnrichers={"); + if (contextEnrichers != null) { for (RangerContextEnricher contextEnricher : contextEnrichers) { if (contextEnricher != null) { @@ -880,6 +907,7 @@ public StringBuilder toString(StringBuilder sb) { } } } + sb.append("} "); sb.append("} "); diff --git a/agents-common/src/test/java/org/apache/ranger/plugin/policyengine/TestPolicyEngine.java b/agents-common/src/test/java/org/apache/ranger/plugin/policyengine/TestPolicyEngine.java index d4c16c1164..a82df2875a 100644 --- a/agents-common/src/test/java/org/apache/ranger/plugin/policyengine/TestPolicyEngine.java +++ b/agents-common/src/test/java/org/apache/ranger/plugin/policyengine/TestPolicyEngine.java @@ -289,6 +289,13 @@ public void testPolicyEngine_hiveMasking() { runTestsFromResourceFiles(resourceFiles); } + @Test + public void testPolicyEngine_hiveTagMasking() { + String[] resourceFiles = {"/policyengine/test_policyengine_tag_hive_mask.json"}; + + runTestsFromResourceFiles(resourceFiles); + } + @Test public void testPolicyEngine_owner() { String[] resourceFiles = {"/policyengine/test_policyengine_owner.json"}; diff --git a/agents-common/src/test/resources/policyengine/test_policyengine_tag_hive_mask.json b/agents-common/src/test/resources/policyengine/test_policyengine_tag_hive_mask.json new file mode 100644 index 0000000000..3945dce060 --- /dev/null +++ b/agents-common/src/test/resources/policyengine/test_policyengine_tag_hive_mask.json @@ -0,0 +1,434 @@ +{ + "serviceName": "hivedev", + "serviceDef": { + "name": "hive", + "id": 3, + "resources": [ + { + "name": "database", + "level": 1, + "mandatory": true, + "lookupSupported": true, + "matcher": "org.apache.ranger.plugin.resourcematcher.RangerDefaultResourceMatcher", + "matcherOptions": { + "wildCard": true, + "ignoreCase": true + }, + "label": "Hive Database", + "description": "Hive Database" + }, + { + "name": "table", + "level": 2, + "parent": "database", + "mandatory": true, + "lookupSupported": true, + "matcher": "org.apache.ranger.plugin.resourcematcher.RangerDefaultResourceMatcher", + "matcherOptions": { + "wildCard": true, + "ignoreCase": true + }, + "label": "Hive Table", + "description": "Hive Table" + }, + { + "name": "udf", + "level": 2, + "parent": "database", + "mandatory": true, + "lookupSupported": true, + "matcher": "org.apache.ranger.plugin.resourcematcher.RangerDefaultResourceMatcher", + "matcherOptions": { + "wildCard": true, + "ignoreCase": true + }, + "label": "Hive UDF", + "description": "Hive UDF" + }, + { + "name": "column", + "level": 3, + "parent": "table", + "mandatory": true, + "lookupSupported": true, + "matcher": "org.apache.ranger.plugin.resourcematcher.RangerDefaultResourceMatcher", + "matcherOptions": { + "wildCard": true, + "ignoreCase": true + }, + "label": "Hive Column", + "description": "Hive Column" + } + ], + "accessTypes": [ + { + "name": "select", + "label": "Select" + }, + { + "name": "update", + "label": "Update" + }, + { + "name": "create", + "label": "Create" + }, + { + "name": "grant", + "label": "Grant" + }, + { + "name": "drop", + "label": "Drop" + }, + { + "name": "alter", + "label": "Alter" + }, + { + "name": "index", + "label": "Index" + }, + { + "name": "lock", + "label": "Lock" + }, + { + "name": "all", + "label": "All", + "impliedGrants": [ + "select", + "update", + "create", + "grant", + "drop", + "alter", + "index", + "lock" + ] + } + ] + }, + "policies": [ + { + "id": 101, + "name": "db=*: audit-all-access", + "isEnabled": true, + "isAuditEnabled": true, + "resources": { + "database": { + "values": [ + "*" + ] + }, + "table": { + "values": [ + "*" + ] + }, + "column": { + "values": [ + "*" + ] + } + }, + "policyItems": [ + { + "accesses": [ + { + "type": "all", + "isAllowed": true + } + ], + "users": [ + "hive", + "user1", + "user2" + ], + "groups": [ + "public" + ], + "delegateAdmin": false + } + ] + }, + { + "id": 102, + "name": "db=*, udf=*: audit-all-access", + "isEnabled": true, + "isAuditEnabled": true, + "resources": { + "database": { + "values": [ + "*" + ] + }, + "udf": { + "values": [ + "*" + ] + } + }, + "policyItems": [ + { + "accesses": [ + { + "type": "all", + "isAllowed": true + } + ], + "users": [ + "hive", + "user1", + "user2" + ], + "groups": [ + "public" + ], + "delegateAdmin": false + } + ] + } + ], + "tagPolicyInfo": { + "serviceName": "tagdev", + "serviceDef": { + "name": "tag", + "id": 100, + "resources": [ + { + "itemId": 1, + "name": "tag", + "type": "string", + "level": 1, + "parent": "", + "mandatory": true, + "lookupSupported": true, + "recursiveSupported": false, + "excludesSupported": false, + "matcher": "org.apache.ranger.plugin.resourcematcher.RangerDefaultResourceMatcher", + "matcherOptions": { + "wildCard": true, + "ignoreCase": false + }, + "validationRegEx": "", + "validationMessage": "", + "uiHint": "", + "label": "TAG", + "description": "TAG" + } + ], + "accessTypes": [ + { + "itemId": 1, + "name": "hive:select", + "label": "hive:select" + }, + { + "itemId": 2, + "name": "hive:update", + "label": "hive:update" + }, + { + "itemId": 3, + "name": "hive:create", + "label": "hive:create" + }, + { + "itemId": 4, + "name": "hive:grant", + "label": "hive:grant" + }, + { + "itemId": 5, + "name": "hive:drop", + "label": "hive:drop" + }, + { + "itemId": 6, + "name": "hive:alter", + "label": "hive:alter" + }, + { + "itemId": 7, + "name": "hive:index", + "label": "hive:index" + }, + { + "itemId": 8, + "name": "hive:lock", + "label": "hive:lock" + }, + { + "itemId": 9, + "name": "hive:all", + "label": "hive:all", + "impliedGrants": [ + "hive:select", + "hive:update", + "hive:create", + "hive:grant", + "hive:drop", + "hive:alter", + "hive:index", + "hive:lock" + ] + } + ], + "contextEnrichers": [ + ], + "policyConditions": [ + { + "itemId": 1, + "name": "expression", + "evaluator": "org.apache.ranger.plugin.conditionevaluator.RangerScriptConditionEvaluator", + "evaluatorOptions": { + "engineName": "JavaScript", + "ui.isMultiline": "true" + }, + "label": "Enter boolean expression", + "description": "Boolean expression" + }, + { + "itemId": 2, + "name": "enforce-expiry", + "evaluator": "org.apache.ranger.plugin.conditionevaluator.RangerScriptTemplateConditionEvaluator", + "evaluatorOptions": { + "scriptTemplate": "ctx.isAccessedAfter('expiry_date');" + }, + "label": "Deny access after expiry_date?", + "description": "Deny access after expiry_date? (yes/no)" + } + ] + }, + "tagPolicies": [ + { + "id": 1, + "name": "RESTRICTED_TAG_POLICY", + "isEnabled": true, + "isAuditEnabled": true, + "policyType": 1, + "resources": { + "tag": { + "values": [ + "RESTRICTED" + ], + "isRecursive": false + } + }, + "dataMaskPolicyItems": [ + { + "accesses": [ + { + "type": "select", + "isAllowed": true + } + ], + "users": [ + "user1" + ], + "groups": [], + "delegateAdmin": false, + "dataMaskInfo": { + "dataMaskType": "MASK" + } + }, + { + "accesses": [ + { + "type": "select", + "isAllowed": true + } + ], + "users": [ + "user2" + ], + "groups": [], + "delegateAdmin": false, + "dataMaskInfo": { + "dataMaskType": "SHUFFLE" + } + } + ] + } + ] + }, + "tests": [ + { + "name": "'select ssn from employee.personal;' for user1 - maskType=MASK", + "request": { + "resource": { + "elements": { + "database": "employee", + "table": "personal", + "column": "ssn" + } + }, + "accessType": "select", + "user": "user1", + "userGroups": [], + "requestData": "select ssn from employee.personal;' for user1", + "context": { + "TAGS": "[{\"type\":\"RESTRICTED\"}]" + } + }, + "dataMaskResult": { + "maskType": "MASK", + "maskCondition": null, + "maskValue": null, + "policyId": 1 + } + }, + { + "name": "'select ssn from employee.personal;' for user2 - maskType=SHUFFLE", + "request": { + "resource": { + "elements": { + "database": "employee", + "table": "personal", + "column": "ssn" + } + }, + "accessType": "select", + "user": "user2", + "userGroups": [], + "requestData": "select ssn from employee.personal;' for user2", + "context": { + "TAGS": "[{\"type\":\"RESTRICTED\"}]" + } + }, + "dataMaskResult": { + "maskType": "SHUFFLE", + "maskCondition": null, + "maskValue": null, + "policyId": 1 + } + }, + { + "name": "'select ssn from employee.personal;' for hive - maskType=NONE", + "request": { + "resource": { + "elements": { + "database": "employee", + "table": "personal", + "column": "ssn" + } + }, + "accessType": "select", + "user": "hive", + "userGroups": [], + "requestData": "select ssn from employee.personal;' for hive", + "context": { + "TAGS": "[{\"type\":\"RESTRICTED\"}]" + } + }, + "dataMaskResult": { + "maskType": null, + "maskCondition": null, + "maskValue": null, + "policyId": -1 + } + } + ] +} + From 07fd69343f19c970d7cc0d13ec2b28f3f3fe5599 Mon Sep 17 00:00:00 2001 From: ni3galave Date: Thu, 25 Jan 2018 11:43:15 +0530 Subject: [PATCH 079/151] RANGER-1805: Code improvement to follow best practices in js Signed-off-by: Mehul Parikh --- .../webapp/scripts/modules/XAOverrides.js | 7 +++--- .../scripts/modules/globalize/message/en.js | 2 +- .../src/main/webapp/scripts/utils/XAEnums.js | 2 +- .../webapp/scripts/utils/XATemplateHelpers.js | 2 +- .../scripts/views/DownloadServicePolicy.js | 7 +++--- .../webapp/scripts/views/common/AddGroup.js | 2 -- .../scripts/views/policies/PermissionList.js | 10 -------- .../views/policymanager/ServiceLayout.js | 4 ++-- .../scripts/views/reports/AuditLayout.js | 6 ++--- .../scripts/views/reports/UserAccessLayout.js | 8 +++---- .../scripts/views/user/UserProfileForm.js | 2 +- .../webapp/templates/common/Footer_tmpl.html | 2 +- .../common/downloadservicepolicy_tmpl.html | 2 +- .../webapp/templates/helpers/XAHelpers.js | 23 +++++++++---------- .../templates/kms/KmsKeyCreate_tmpl.html | 2 +- .../policies/RangerPolicyCreate_tmpl.html | 2 +- .../templates/service/ServiceCreate_tmpl.html | 2 +- .../templates/users/GroupCreate_tmpl.html | 2 +- .../templates/users/UserCreate_tmpl.html | 2 +- 19 files changed, 36 insertions(+), 53 deletions(-) diff --git a/security-admin/src/main/webapp/scripts/modules/XAOverrides.js b/security-admin/src/main/webapp/scripts/modules/XAOverrides.js index 5810d5d147..12f351d7de 100644 --- a/security-admin/src/main/webapp/scripts/modules/XAOverrides.js +++ b/security-admin/src/main/webapp/scripts/modules/XAOverrides.js @@ -86,7 +86,7 @@ render: function () { this.$el.empty(); if(this.model.get(this.column.get("name")) != undefined){ - rawValue = (this.model.get(this.column.get("name"))); + var rawValue = (this.model.get(this.column.get("name"))); this.switchStatus = this.formatter.fromRaw(rawValue, this.model); } @@ -681,9 +681,8 @@ } }, getTemplate : function() { - var that = this , resourcesType ; - var optionsHtml="", selectTemplate = '',excludeSupportToggleDiv='', recursiveSupportToggleDiv=''; - this.preserveResourceValues = {},klass = ''; + var that = this , resourcesType , optionsHtml="" , selectTemplate = '', excludeSupportToggleDiv='', recursiveSupportToggleDiv='', klass = ''; + this.preserveResourceValues = {} ; if(this.resourcesAtSameLevel){ _.each(this.sameLevelOpts, function(option){ return optionsHtml += ""; },this); selectTemplate = ' + Required diff --git a/security-admin/src/main/webapp/templates/helpers/XAHelpers.js b/security-admin/src/main/webapp/templates/helpers/XAHelpers.js index 9363c6b36c..f8479e47ff 100644 --- a/security-admin/src/main/webapp/templates/helpers/XAHelpers.js +++ b/security-admin/src/main/webapp/templates/helpers/XAHelpers.js @@ -139,7 +139,7 @@ return moment(context).format(f); }else{ return context; // moment plugin not available. return data as is. - }; + } }); /* @@ -154,7 +154,6 @@ Handlebars.registerHelper('tt', function(str) { return localization.tt(str); - return str; }); Handlebars.registerHelper('getCopyrightDate', function() { @@ -187,7 +186,7 @@ Handlebars.registerHelper('customPermString', function(permsString,kclass) { if(permsString == "--") return permsString; - permArr = permsString.split(','); + var permArr = permsString.split(','); var cl = _.isObject(kclass) ? 'label label-info' : kclass; var tempArr = []; _.each(permArr, function(val){ @@ -424,7 +423,7 @@ html = ''+val+''; } else { if($.inArray(val, arr) < 0) - return html = ''+val+''; + return ''+val+''; } return html; }); @@ -442,9 +441,9 @@ isRemoved = false; }); if(isRemoved) - return html = ''+perm[type]+''; + return ''+perm[type]+''; } else { - return html = ''+perm[type]+''; + return ''+perm[type]+''; } } } else { @@ -456,9 +455,9 @@ isNewAdd = false; }); if(isNewAdd) - return html = ''+perm[type]+''; + return ''+perm[type]+''; } else { - return html = ''+perm[type]+''; + return ''+perm[type]+''; } } } @@ -479,9 +478,9 @@ isRemoved = false; }); if(isRemoved) - return html = ''+perm[type]+''; + return ''+perm[type]+''; } else { - return html = ''+perm[type]+''; + return ''+perm[type]+''; } } } else { @@ -493,9 +492,9 @@ isNewAdd = false; }); if(isNewAdd) - return html = ''+perm[type]+''; + return ''+perm[type]+''; } else { - return html = ''+perm[type]+''; + return ''+perm[type]+''; } } } diff --git a/security-admin/src/main/webapp/templates/kms/KmsKeyCreate_tmpl.html b/security-admin/src/main/webapp/templates/kms/KmsKeyCreate_tmpl.html index 2aaac43a82..b4b7f03e32 100644 --- a/security-admin/src/main/webapp/templates/kms/KmsKeyCreate_tmpl.html +++ b/security-admin/src/main/webapp/templates/kms/KmsKeyCreate_tmpl.html @@ -17,7 +17,7 @@

Key Detail

-
+
diff --git a/security-admin/src/main/webapp/templates/policies/RangerPolicyCreate_tmpl.html b/security-admin/src/main/webapp/templates/policies/RangerPolicyCreate_tmpl.html index 31252afd93..e5fcf70c22 100644 --- a/security-admin/src/main/webapp/templates/policies/RangerPolicyCreate_tmpl.html +++ b/security-admin/src/main/webapp/templates/policies/RangerPolicyCreate_tmpl.html @@ -29,7 +29,7 @@

{{tt 'h.createPolicy'}}

{{{infoMsg}}}
-
+
{{#if editPolicy}} diff --git a/security-admin/src/main/webapp/templates/users/GroupCreate_tmpl.html b/security-admin/src/main/webapp/templates/users/GroupCreate_tmpl.html index c387e68246..ffc29bf077 100644 --- a/security-admin/src/main/webapp/templates/users/GroupCreate_tmpl.html +++ b/security-admin/src/main/webapp/templates/users/GroupCreate_tmpl.html @@ -17,7 +17,7 @@

Group Detail

-
+
diff --git a/security-admin/src/main/webapp/templates/users/UserCreate_tmpl.html b/security-admin/src/main/webapp/templates/users/UserCreate_tmpl.html index 204e8327b4..13ce8cca45 100644 --- a/security-admin/src/main/webapp/templates/users/UserCreate_tmpl.html +++ b/security-admin/src/main/webapp/templates/users/UserCreate_tmpl.html @@ -26,7 +26,7 @@

User Detail

-
+
From 42c996a2d9ea4b4445a30153441f95f9ac54077d Mon Sep 17 00:00:00 2001 From: Colm O hEigeartaigh Date: Mon, 12 Feb 2018 16:25:12 +0000 Subject: [PATCH 080/151] Ignoring Kafka SASL/SSL test --- .../kafka/authorizer/KafkaRangerAuthorizerSASLSSLTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/plugin-kafka/src/test/java/org/apache/ranger/authorization/kafka/authorizer/KafkaRangerAuthorizerSASLSSLTest.java b/plugin-kafka/src/test/java/org/apache/ranger/authorization/kafka/authorizer/KafkaRangerAuthorizerSASLSSLTest.java index cef9f124ac..277474f565 100644 --- a/plugin-kafka/src/test/java/org/apache/ranger/authorization/kafka/authorizer/KafkaRangerAuthorizerSASLSSLTest.java +++ b/plugin-kafka/src/test/java/org/apache/ranger/authorization/kafka/authorizer/KafkaRangerAuthorizerSASLSSLTest.java @@ -64,6 +64,7 @@ * * Clients and services authenticate to Kafka using the SASL SSL protocol as part of this test. */ +@org.junit.Ignore("This is failing on some VMs") public class KafkaRangerAuthorizerSASLSSLTest { private static KafkaServerStartable kafkaServer; From 0c948119dbb109e65c8a811a49f537f478d6eda7 Mon Sep 17 00:00:00 2001 From: pradeep Date: Mon, 12 Feb 2018 20:15:32 +0530 Subject: [PATCH 081/151] RANGER-1980: Build failure for Ranger 0.7 branch --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index cc09475f94..b76e4e30a5 100644 --- a/pom.xml +++ b/pom.xml @@ -125,7 +125,7 @@ 3.2 3.1 1.8.2 - 0.8.2-SNAPSHOT + 0.8.2 14.0 2.5 1.3.7 From 21b88027195b97a64094b6a4bd8694f3228feedc Mon Sep 17 00:00:00 2001 From: Zsombor Gegesy Date: Fri, 23 Mar 2018 11:41:15 +0100 Subject: [PATCH 082/151] RANGER-2035: backport to 0.7 - fix the null implClass handling, which is possible on oracle db --- .../org/apache/ranger/biz/RangerBizUtil.java | 33 ++++++++----------- .../org/apache/ranger/rest/ServiceREST.java | 8 +++-- 2 files changed, 18 insertions(+), 23 deletions(-) diff --git a/security-admin/src/main/java/org/apache/ranger/biz/RangerBizUtil.java b/security-admin/src/main/java/org/apache/ranger/biz/RangerBizUtil.java index 4d6227d2e5..8b8ce36da4 100644 --- a/security-admin/src/main/java/org/apache/ranger/biz/RangerBizUtil.java +++ b/security-admin/src/main/java/org/apache/ranger/biz/RangerBizUtil.java @@ -1452,14 +1452,12 @@ public Boolean hasAccess(XXDBBase xxDbBase, RangerBaseModelObject baseModel) { if (xxDbBase != null && xxDbBase instanceof XXServiceDef) { XXServiceDef xServiceDef = (XXServiceDef) xxDbBase; String implClass = xServiceDef.getImplclassname(); - if (implClass == null) { - return false; - } - - if (isKeyAdmin && implClass.equals(EmbeddedServiceDefsUtil.KMS_IMPL_CLASS_NAME)) { - return true; - } else if ((isSysAdmin || isUser) && !implClass.equals(EmbeddedServiceDefsUtil.KMS_IMPL_CLASS_NAME)) { - return true; + if (EmbeddedServiceDefsUtil.KMS_IMPL_CLASS_NAME.equals(implClass)) { + // KMS case + return isKeyAdmin; + } else { + // Other cases - implClass can be null! + return isSysAdmin || isUser; } } @@ -1474,18 +1472,13 @@ public Boolean hasAccess(XXDBBase xxDbBase, RangerBaseModelObject baseModel) { XXService xService = (XXService) xxDbBase; XXServiceDef xServiceDef = daoManager.getXXServiceDef().getById(xService.getType()); String implClass = xServiceDef.getImplclassname(); - if (implClass == null) { - return false; - } - - if (isKeyAdmin && implClass.equals(EmbeddedServiceDefsUtil.KMS_IMPL_CLASS_NAME)) { - return true; - } else if (isUser && !implClass.equals(EmbeddedServiceDefsUtil.KMS_IMPL_CLASS_NAME)) { - return true; + if (EmbeddedServiceDefsUtil.KMS_IMPL_CLASS_NAME.equals(implClass)) { + // KMS case + return isKeyAdmin; + } else { + // Other cases - implClass can be null! + return isUser; } - // else if ((isSysAdmin || isUser) && !implClass.equals(EmbeddedServiceDefsUtil.KMS_IMPL_CLASS_NAME)) { - // return true; - // } } return false; } @@ -1517,7 +1510,7 @@ public void hasKMSPermissions(String objType, String implClassName) { // TODO: As of now we are allowing SYS_ADMIN to create/update/read/delete all the // services including KMS - if (objType.equalsIgnoreCase("Service-Def") && session.isUserAdmin() && implClassName.equals(EmbeddedServiceDefsUtil.KMS_IMPL_CLASS_NAME)) { + if (objType.equalsIgnoreCase("Service-Def") && session.isUserAdmin() && EmbeddedServiceDefsUtil.KMS_IMPL_CLASS_NAME.equals(implClassName)) { throw restErrorUtil.createRESTException("System Admin cannot create/update/delete KMS " + objType, MessageEnums.OPER_NO_PERMISSION); } diff --git a/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java b/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java index 6e6d241a14..e2a0c29ab9 100644 --- a/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java +++ b/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java @@ -319,7 +319,9 @@ public void deleteServiceDef(@PathParam("id") Long id, @Context HttpServletReque bizUtil.hasAdminPermissions("Service-Def"); XXServiceDef xServiceDef = daoManager.getXXServiceDef().getById(id); - bizUtil.hasKMSPermissions("Service-Def", xServiceDef.getImplclassname()); + if (xServiceDef != null) { + bizUtil.hasKMSPermissions("Service-Def", xServiceDef.getImplclassname()); + } String forceDeleteStr = request.getParameter("forceDelete"); boolean forceDelete = false; @@ -3053,13 +3055,13 @@ void ensureAdminAccess(String serviceName, Map res XXServiceDef xServiceDef = daoManager.getXXServiceDef().getById(xService.getType()); if (isAdmin) { - if (xServiceDef.getImplclassname().equals(EmbeddedServiceDefsUtil.KMS_IMPL_CLASS_NAME)) { + if (EmbeddedServiceDefsUtil.KMS_IMPL_CLASS_NAME.equals(xServiceDef.getImplclassname())) { throw restErrorUtil.createRESTException( "KMS Policies/Services/Service-Defs are not accessible for user '" + userName + "'.", MessageEnums.OPER_NO_PERMISSION); } } else if (isKeyAdmin) { - if (!xServiceDef.getImplclassname().equals(EmbeddedServiceDefsUtil.KMS_IMPL_CLASS_NAME)) { + if (!EmbeddedServiceDefsUtil.KMS_IMPL_CLASS_NAME.equals(xServiceDef.getImplclassname())) { throw restErrorUtil.createRESTException( "Only KMS Policies/Services/Service-Defs are accessible for user '" + userName + "'.", MessageEnums.OPER_NO_PERMISSION); From 9e7760c6fedf8e39dcfe37fce54084578f2a8864 Mon Sep 17 00:00:00 2001 From: Abhay Kulkarni Date: Thu, 12 Apr 2018 22:04:20 -0700 Subject: [PATCH 083/151] RANGER-2066: Hbase column family access is authorized by a tagged column in the column family --- .../contextenricher/RangerTagEnricher.java | 3 +- .../policyengine/RangerTagAccessRequest.java | 1 + .../RangerDefaultPolicyEvaluator.java | 43 ++++++++++--------- .../test_policyengine_tag_hdfs.json | 4 +- .../test_policyengine_tag_hive.json | 21 +++++++-- 5 files changed, 46 insertions(+), 26 deletions(-) diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java index 4a3a95062b..858a7a4cde 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java @@ -324,10 +324,11 @@ private Set findMatchingTags(final RangerAccessRequest request if (request.isAccessTypeAny()) { isMatched = matchType != RangerPolicyResourceMatcher.MatchType.NONE; } else if (request.getResourceMatchingScope() == RangerAccessRequest.ResourceMatchingScope.SELF_OR_DESCENDANTS) { - isMatched = matchType == RangerPolicyResourceMatcher.MatchType.SELF || matchType == RangerPolicyResourceMatcher.MatchType.DESCENDANT; + isMatched = matchType != RangerPolicyResourceMatcher.MatchType.NONE; } else { isMatched = matchType == RangerPolicyResourceMatcher.MatchType.SELF || matchType == RangerPolicyResourceMatcher.MatchType.ANCESTOR; } + if (isMatched) { if (ret == null) { ret = new HashSet(); diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerTagAccessRequest.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerTagAccessRequest.java index dbdcacd11f..cf590f9aaa 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerTagAccessRequest.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerTagAccessRequest.java @@ -52,6 +52,7 @@ public RangerTagAccessRequest(RangerTagForEval resourceTag, RangerServiceDef tag super.setRemoteIPAddress(request.getRemoteIPAddress()); super.setForwardedAddresses(request.getForwardedAddresses()); super.setSessionId(request.getSessionId()); + super.setResourceMatchingScope(request.getResourceMatchingScope()); } public RangerPolicyResourceMatcher.MatchType getMatchType() { return matchType; diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerDefaultPolicyEvaluator.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerDefaultPolicyEvaluator.java index 71c076d03f..b1d63376ef 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerDefaultPolicyEvaluator.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerDefaultPolicyEvaluator.java @@ -169,21 +169,25 @@ public void evaluate(RangerAccessRequest request, RangerAccessResult result) { if (request != null && result != null) { if (!result.getIsAccessDetermined() || !result.getIsAuditedDetermined()) { - RangerPolicyResourceMatcher.MatchType matchType = resourceMatcher != null ? resourceMatcher.getMatchType(request.getResource(), request.getContext()) : RangerPolicyResourceMatcher.MatchType.NONE; + RangerPolicyResourceMatcher.MatchType matchType; + + if (RangerTagAccessRequest.class.isInstance(request)) { + matchType = ((RangerTagAccessRequest) request).getMatchType(); + } else { + matchType = resourceMatcher != null ? resourceMatcher.getMatchType(request.getResource(), request.getContext()) : RangerPolicyResourceMatcher.MatchType.NONE; + } final boolean isMatched; + if (request.isAccessTypeAny()) { isMatched = matchType != RangerPolicyResourceMatcher.MatchType.NONE; } else if (request.getResourceMatchingScope() == RangerAccessRequest.ResourceMatchingScope.SELF_OR_DESCENDANTS) { - isMatched = matchType == RangerPolicyResourceMatcher.MatchType.SELF || matchType == RangerPolicyResourceMatcher.MatchType.DESCENDANT; + isMatched = matchType != RangerPolicyResourceMatcher.MatchType.NONE; } else { isMatched = matchType == RangerPolicyResourceMatcher.MatchType.SELF || matchType == RangerPolicyResourceMatcher.MatchType.ANCESTOR; } if (isMatched) { - if (RangerTagAccessRequest.class.isInstance(request)) { - matchType = ((RangerTagAccessRequest) request).getMatchType(); - } if (!result.getIsAuditedDetermined()) { if (isAuditEnabled()) { result.setIsAudited(true); @@ -410,17 +414,15 @@ public void getResourceAccessInfo(RangerAccessRequest request, RangerResourceAcc if(LOG.isDebugEnabled()) { LOG.debug("==> RangerDefaultPolicyEvaluator.getResourceAccessInfo(" + request + ", " + result + ")"); } - RangerPolicyResourceMatcher.MatchType matchType = resourceMatcher != null ? resourceMatcher.getMatchType(request.getResource(), request.getContext()) : RangerPolicyResourceMatcher.MatchType.NONE; - - final boolean isMatched; - if (request.isAccessTypeAny()) { - isMatched = matchType != RangerPolicyResourceMatcher.MatchType.NONE; - } else if (request.getResourceMatchingScope() == RangerAccessRequest.ResourceMatchingScope.SELF_OR_DESCENDANTS) { - isMatched = matchType == RangerPolicyResourceMatcher.MatchType.SELF || matchType == RangerPolicyResourceMatcher.MatchType.DESCENDANT; - } else { - isMatched = matchType == RangerPolicyResourceMatcher.MatchType.SELF || matchType == RangerPolicyResourceMatcher.MatchType.ANCESTOR; + RangerPolicyResourceMatcher.MatchType matchType; + if (RangerTagAccessRequest.class.isInstance(request)) { + matchType = ((RangerTagAccessRequest) request).getMatchType(); + } else { + matchType = resourceMatcher != null ? resourceMatcher.getMatchType(request.getResource(), request.getContext()) : RangerPolicyResourceMatcher.MatchType.NONE; } + final boolean isMatched = matchType != RangerPolicyResourceMatcher.MatchType.NONE; + if (isMatched) { if (CollectionUtils.isNotEmpty(allowEvaluators)) { @@ -470,7 +472,6 @@ public void getResourceAccessInfo(RangerAccessRequest request, RangerResourceAcc } } - protected void evaluatePolicyItems(RangerAccessRequest request, RangerAccessResult result, boolean isResourceMatch) { if(LOG.isDebugEnabled()) { LOG.debug("==> RangerDefaultPolicyEvaluator.evaluatePolicyItems(" + request + ", " + result + ", " + isResourceMatch + ")"); @@ -486,16 +487,18 @@ protected void evaluatePolicyItems(RangerAccessRequest request, RangerAccessResu RangerPolicy policy = getPolicy(); if(matchedPolicyItem.getPolicyItemType() == RangerPolicyItemEvaluator.POLICY_ITEM_TYPE_DENY) { - if(isResourceMatch) { + if(isResourceMatch || !request.isAccessTypeAny()) { result.setIsAllowed(false); result.setPolicyId(policy.getId()); result.setReason(matchedPolicyItem.getComments()); } } else { - if(! result.getIsAllowed()) { // if access is not yet allowed by another policy - result.setIsAllowed(true); - result.setPolicyId(policy.getId()); - result.setReason(matchedPolicyItem.getComments()); + if(isResourceMatch || request.isAccessTypeAny()) { + if(! result.getIsAllowed()) { // if access is not yet allowed by another policy + result.setIsAllowed(true); + result.setPolicyId(policy.getId()); + result.setReason(matchedPolicyItem.getComments()); + } } } } diff --git a/agents-common/src/test/resources/policyengine/test_policyengine_tag_hdfs.json b/agents-common/src/test/resources/policyengine/test_policyengine_tag_hdfs.json index b4941cd198..eb2251c3c5 100644 --- a/agents-common/src/test/resources/policyengine/test_policyengine_tag_hdfs.json +++ b/agents-common/src/test/resources/policyengine/test_policyengine_tag_hdfs.json @@ -215,7 +215,7 @@ "userGroups": [ ], "requestData": "read /resource", "context": { - "TAGS": "[{\"type\":\"PII\", \"matchType\":1}]" + "TAGS": "[{\"type\":\"PII\", \"matchType\": \"SELF\"}]" } }, "result": { "isAudited": true, "isAllowed": false, "policyId": 101 } @@ -371,7 +371,7 @@ "userGroups": [ ], "requestData": "read /resource", "context": { - "TAGS": "[{\"type\":\"Unaudited-TAG\", \"matchType\":1}]" + "TAGS": "[{\"type\":\"Unaudited-TAG\", \"matchType\": \"SELF\"}]" } }, "result": { "isAudited": true, "isAllowed": false, "policyId": 1 } diff --git a/agents-common/src/test/resources/policyengine/test_policyengine_tag_hive.json b/agents-common/src/test/resources/policyengine/test_policyengine_tag_hive.json index 11f31e3177..d66f6e86ba 100644 --- a/agents-common/src/test/resources/policyengine/test_policyengine_tag_hive.json +++ b/agents-common/src/test/resources/policyengine/test_policyengine_tag_hive.json @@ -31,7 +31,10 @@ "lock" ] } - ] + ], + "options": { + "enableDenyAndExceptionsInPolicies":"true" + } }, "policies":[ @@ -39,7 +42,10 @@ "resources":{"database":{"values":["*"]},"table":{"values":["*"]},"column":{"values":["*"]}}, "policyItems":[ {"accesses":[{"type":"all","isAllowed":true}],"users":["hive", "user1", "user2"],"groups":["public"],"delegateAdmin":false} - ] + ], + "allowExceptions":[ + {"accesses":[{"type":"all","isAllowed":true}],"users":["testuser"],"groups":[],"delegateAdmin":false} + ] }, {"id":102,"name":"db=*, udf=*: audit-all-access","isEnabled":true,"isAuditEnabled":true, "resources":{"database":{"values":["*"]},"udf":{"values":["*"]}}, @@ -219,12 +225,21 @@ }, "tests":[ + {"name":"DENY 'select ssn from employee.personal;' for testuser using EXPIRES_ON tag with DESCENDANT match", + "request":{ + "resource":{"elements":{"database":"employee", "table":"personal", "column":"ssn"}}, + "accessType":"select","user":"testuser","userGroups":[],"requestData":"select ssn from employee.personal;' for testuser", + + "context": {"TAGS":"[{\"type\":\"EXPIRES_ON\", \"attributes\":{\"expiry_date\":\"2026-06-15T15:05:15.000Z\"}, \"matchType\":\"DESCENDANT\"}]"} + }, + "result":{"isAudited":true,"isAllowed":false,"policyId":-1} + }, {"name":"ALLOW 'select ssn from employee.personal;' for user1 using EXPIRES_ON tag", "request":{ "resource":{"elements":{"database":"employee", "table":"personal", "column":"ssn"}}, "accessType":"select","user":"user1","userGroups":[],"requestData":"select ssn from employee.personal;' for user1", - "context": {"TAGS":"[{\"type\":\"EXPIRES_ON\", \"attributes\":{\"expiry_date\":\"2026-06-15T15:05:15.000Z\"}, \"matchType\":1}]"} + "context": {"TAGS":"[{\"type\":\"EXPIRES_ON\", \"attributes\":{\"expiry_date\":\"2026-06-15T15:05:15.000Z\"}, \"matchType\":\"SELF\"}]"} }, "result":{"isAudited":true,"isAllowed":true,"policyId":101} }, From 7e3963bc76e452504c6551cc787c35b556d0fd7d Mon Sep 17 00:00:00 2001 From: Abhay Kulkarni Date: Wed, 18 Apr 2018 14:33:34 -0700 Subject: [PATCH 084/151] RANGER-2066: Fix regression --- .../policyevaluator/RangerDefaultPolicyEvaluator.java | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerDefaultPolicyEvaluator.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerDefaultPolicyEvaluator.java index b1d63376ef..6126f603b9 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerDefaultPolicyEvaluator.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerDefaultPolicyEvaluator.java @@ -493,12 +493,10 @@ protected void evaluatePolicyItems(RangerAccessRequest request, RangerAccessResu result.setReason(matchedPolicyItem.getComments()); } } else { - if(isResourceMatch || request.isAccessTypeAny()) { - if(! result.getIsAllowed()) { // if access is not yet allowed by another policy - result.setIsAllowed(true); - result.setPolicyId(policy.getId()); - result.setReason(matchedPolicyItem.getComments()); - } + if(! result.getIsAllowed()) { // if access is not yet allowed by another policy + result.setIsAllowed(true); + result.setPolicyId(policy.getId()); + result.setReason(matchedPolicyItem.getComments()); } } } From 126ff6ee04e580dcf8b924f76df0e3917221106e Mon Sep 17 00:00:00 2001 From: Abhay Kulkarni Date: Thu, 17 May 2018 08:51:40 -0700 Subject: [PATCH 085/151] RANGER-2104: Ranger tagsync should ignore ENTITY_UPDATE events if the updated entity does not have associated traits --- pom.xml | 1 + src/main/assembly/tagsync.xml | 1 + .../source/atlas/AtlasNotificationMapper.java | 53 ++++++++++++------ .../tagsync/source/atlas/AtlasTagSource.java | 54 ++++++++++++++++--- 4 files changed, 85 insertions(+), 24 deletions(-) diff --git a/pom.xml b/pom.xml index b76e4e30a5..2ec6768ff7 100644 --- a/pom.xml +++ b/pom.xml @@ -129,6 +129,7 @@ 14.0 2.5 1.3.7 + 1.4.1 1.1.3 1.7.1 1.55 diff --git a/src/main/assembly/tagsync.xml b/src/main/assembly/tagsync.xml index 26b42cae30..5139937ecd 100644 --- a/src/main/assembly/tagsync.xml +++ b/src/main/assembly/tagsync.xml @@ -50,6 +50,7 @@ org.apache.atlas:atlas-client-v2:jar:${atlas.version} org.apache.atlas:atlas-common:jar:${atlas.version} org.apache.atlas:atlas-intg:jar:${atlas.version} + org.apache.commons:commons-compress:jar:${atlas.commons.compress.version} org.apache.hadoop:hadoop-auth org.apache.hadoop:hadoop-common org.apache.kafka:kafka_${scala.binary.version}:jar:${kafka.version} diff --git a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasNotificationMapper.java b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasNotificationMapper.java index 8641d609dd..1c7f063a5b 100644 --- a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasNotificationMapper.java +++ b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasNotificationMapper.java @@ -51,7 +51,6 @@ public class AtlasNotificationMapper { private static final Log LOG = LogFactory.getLog(AtlasNotificationMapper.class); - private static Map unhandledEventTypes = new HashMap(); private static final ThreadLocal DATE_FORMATTER = new ThreadLocal() { @@ -140,8 +139,20 @@ static private boolean isNotificationHandled(EntityNotification entityNotificati switch (opType) { case ENTITY_CREATE: ret = CollectionUtils.isNotEmpty(entityNotification.getAllTraits()); + if (!ret) { + if (LOG.isDebugEnabled()) { + LOG.debug("ENTITY_CREATE notification is ignored, as there are no traits associated with the entity. Ranger will get necessary information from any subsequent TRAIT_ADDED notification"); + } + } break; case ENTITY_UPDATE: + ret = CollectionUtils.isNotEmpty(entityNotification.getAllTraits()); + if (!ret) { + if (LOG.isDebugEnabled()) { + LOG.debug("ENTITY_UPDATE notification is ignored, as there are no traits associated with the entity."); + } + } + break; case ENTITY_DELETE: case TRAIT_ADD: case TRAIT_UPDATE: @@ -278,9 +289,8 @@ static private List getTags(AtlasEntityWithTraits entityWithTraits) { List ret = new ArrayList(); IReferenceableInstance entity = entityWithTraits != null ? entityWithTraits.getEntity() : null; - if(entity != null && CollectionUtils.isNotEmpty(entity.getTraits())) { - for (String traitName : entity.getTraits()) { - IStruct trait = entity.getTrait(traitName); + if(entity != null && CollectionUtils.isNotEmpty(entityWithTraits.getAllTraits())) { + for (IStruct trait : entityWithTraits.getAllTraits()) { Map tagAttrs = new HashMap(); try { @@ -310,9 +320,8 @@ static private List getTagDefs(AtlasEntityWithTraits entityWithTra List ret = new ArrayList(); IReferenceableInstance entity = entityWithTraits != null ? entityWithTraits.getEntity() : null; - if(entity != null && CollectionUtils.isNotEmpty(entity.getTraits())) { - for (String traitName : entity.getTraits()) { - IStruct trait = entity.getTrait(traitName); + if(entity != null && CollectionUtils.isNotEmpty(entityWithTraits.getAllTraits())) { + for (IStruct trait : entityWithTraits.getAllTraits()) { RangerTagDef tagDef = new RangerTagDef(trait.getTypeName(), "Atlas"); try { @@ -415,7 +424,7 @@ static private ServiceTags buildServiceTags(AtlasEntityHeader entity, AtlasTypeR if (serviceResource != null) { List tags = getTags(entity, typeRegistry); - List tagDefs = getTagDefs(entity); + List tagDefs = getTagDefs(entity, typeRegistry); String serviceName = serviceResource.getServiceName(); ret = createOrGetServiceTags(serviceTagsMap, serviceName); @@ -477,28 +486,38 @@ static private List getTags(AtlasEntityHeader entity, AtlasTypeRegist return ret; } - static private List getTagDefs(AtlasEntityHeader entity) { + static private List getTagDefs(AtlasEntityHeader entity, AtlasTypeRegistry typeRegistry) { List ret = new ArrayList<>(); if(entity != null && CollectionUtils.isNotEmpty(entity.getClassificationNames())) { - List traits = entity.getClassifications(); + List classifications = entity.getClassifications(); - for (AtlasClassification trait : traits) { - RangerTagDef tagDef = new RangerTagDef(trait.getTypeName(), "Atlas"); + for (AtlasClassification classification : classifications) { + ret.add(getTagDef(classification)); - if(MapUtils.isNotEmpty(trait.getAttributes())) { - for (String attrName : trait.getAttributes().keySet()) { - tagDef.getAttributeDefs().add(new RangerTagAttributeDef(attrName, "string")); + List superClassifications = getSuperClassifications(classification, typeRegistry); + + if (CollectionUtils.isNotEmpty(superClassifications)) { + for (AtlasClassification superClassification : superClassifications) { + ret.add(getTagDef(superClassification)); } } - - ret.add(tagDef); } } return ret; } + static private RangerTagDef getTagDef(AtlasClassification classification) { + RangerTagDef tagDef = new RangerTagDef(classification.getTypeName(), "Atlas"); + if(MapUtils.isNotEmpty(classification.getAttributes())) { + for (String attrName : classification.getAttributes().keySet()) { + tagDef.getAttributeDefs().add(new RangerTagAttributeDef(attrName, "string")); + } + } + return tagDef; + } + static private List getSuperClassifications(AtlasClassification classification, AtlasTypeRegistry typeRegistry) { List ret = null; AtlasClassificationType classificationType = typeRegistry.getClassificationTypeByName(classification.getTypeName()); diff --git a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasTagSource.java b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasTagSource.java index 95ff8ec002..3810442764 100644 --- a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasTagSource.java +++ b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasTagSource.java @@ -160,32 +160,72 @@ public void run() { if (LOG.isDebugEnabled()) { LOG.debug("==> ConsumerRunnable.run()"); } + boolean seenCommitException = false; + long offsetOfLastMessageDeliveredToRanger = -1L; + while (true) { try { List> messages = consumer.receive(1000L); + int index = 0; + + if (seenCommitException) { + for (; index < messages.size(); index++) { + AtlasKafkaMessage message = messages.get(index); + if (message.getOffset() <= offsetOfLastMessageDeliveredToRanger) { + // Already delivered to Ranger + TopicPartition partition = new TopicPartition("ATLAS_ENTITIES", message.getPartition()); + try { + consumer.commit(partition, message.getOffset()); + } catch (Exception commitException) { + LOG.warn("Ranger tagsync already processed message at offset " + message.getOffset() + ". Ignoring failure in committing this message and continuing to process next message", commitException); + LOG.warn("This will cause Kafka to deliver this message:[" + message.getOffset() + "] repeatedly!! This may be unrecoverable error!!"); + } + } else { + seenCommitException = false; + offsetOfLastMessageDeliveredToRanger = -1L; + break; + } + } + } - for (AtlasKafkaMessage message : messages) { + for (; index < messages.size(); index++) { + AtlasKafkaMessage message = messages.get(index); EntityNotification notification = message != null ? message.getMessage() : null; if (notification != null) { if (LOG.isDebugEnabled()) { - LOG.debug("Notification=" + getPrintableEntityNotification(notification)); + LOG.debug("Message-offset=" + message.getOffset() + ", Notification=" + getPrintableEntityNotification(notification)); } ServiceTags serviceTags = AtlasNotificationMapper.processEntityNotification(notification); if (serviceTags != null) { updateSink(serviceTags); } - - TopicPartition partition = new TopicPartition("ATLAS_ENTITIES", message.getPartition()); - consumer.commit(partition, message.getOffset()); + offsetOfLastMessageDeliveredToRanger = message.getOffset(); + + if (!seenCommitException) { + TopicPartition partition = new TopicPartition("ATLAS_ENTITIES", message.getPartition()); + try { + consumer.commit(partition, message.getOffset()); + } catch (Exception commitException) { + seenCommitException = true; + LOG.warn("Ranger tagsync processed message at offset " + message.getOffset() + ". Ignoring failure in committing this message and continuing to process next message", commitException); + } + } } else { LOG.error("Null entityNotification received from Kafka!! Ignoring.."); } } } catch (Exception exception) { - LOG.error("Caught exception..: ", exception); - return; + LOG.error("Caught exception : ", exception); + // If transient error, retry after short interval + try { + Thread.sleep(100); + } catch (InterruptedException interrupted) { + LOG.error("Interrupted: ", interrupted); + LOG.error("Returning from thread. May cause process to be up but not processing events!!"); + return; + } } } } From 28733f047dcc6b5443f472e8fa9dfdccba631121 Mon Sep 17 00:00:00 2001 From: Abhay Kulkarni Date: Thu, 27 Apr 2017 08:58:10 +0530 Subject: [PATCH 086/151] RANGER-1436: Turn Ranger deny policy & except condition blocks ON by default (cherry picked from commit faf5bf177f2f145f40f667a598929a6dbd7e81df) --- .../ranger/plugin/model/RangerServiceDef.java | 1 + .../ranger/plugin/util/ServiceDefUtil.java | 4 +++- .../service/RangerServiceDefService.java | 19 ++++++++++++++++++- 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerServiceDef.java b/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerServiceDef.java index ad5108bd43..3803c58863 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerServiceDef.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerServiceDef.java @@ -113,6 +113,7 @@ public void updateFrom(RangerServiceDef other) { setLabel(other.getLabel()); setDescription(other.getDescription()); setConfigs(other.getConfigs()); + setOptions(other.getOptions()); setResources(other.getResources()); setAccessTypes(other.getAccessTypes()); setPolicyConditions(other.getPolicyConditions()); diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/ServiceDefUtil.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/ServiceDefUtil.java index dbdc935015..b0090d4bb6 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/util/ServiceDefUtil.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/ServiceDefUtil.java @@ -22,6 +22,7 @@ import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.MapUtils; import org.apache.commons.lang.StringUtils; +import org.apache.ranger.authorization.hadoop.config.RangerConfiguration; import org.apache.ranger.plugin.model.RangerPolicy; import org.apache.ranger.plugin.model.RangerServiceDef; import org.apache.ranger.plugin.model.RangerServiceDef.RangerAccessTypeDef; @@ -40,7 +41,8 @@ public static boolean getOption_enableDenyAndExceptionsInPolicies(RangerServiceD boolean ret = false; if(serviceDef != null) { - boolean defaultValue = StringUtils.equalsIgnoreCase(serviceDef.getName(), EmbeddedServiceDefsUtil.EMBEDDED_SERVICEDEF_TAG_NAME); + boolean enableDenyAndExceptionsInPoliciesHiddenOption = RangerConfiguration.getInstance().getBoolean("ranger.servicedef.enableDenyAndExceptionsInPolicies", true); + boolean defaultValue = enableDenyAndExceptionsInPoliciesHiddenOption || StringUtils.equalsIgnoreCase(serviceDef.getName(), EmbeddedServiceDefsUtil.EMBEDDED_SERVICEDEF_TAG_NAME); ret = ServiceDefUtil.getBooleanValue(serviceDef.getOptions(), RangerServiceDef.OPTION_ENABLE_DENY_AND_EXCEPTIONS_IN_POLICIES, defaultValue); } diff --git a/security-admin/src/main/java/org/apache/ranger/service/RangerServiceDefService.java b/security-admin/src/main/java/org/apache/ranger/service/RangerServiceDefService.java index 53b12d83a0..1e385a0246 100644 --- a/security-admin/src/main/java/org/apache/ranger/service/RangerServiceDefService.java +++ b/security-admin/src/main/java/org/apache/ranger/service/RangerServiceDefService.java @@ -19,9 +19,13 @@ import java.util.ArrayList; import java.util.List; +import java.util.Map; +import org.apache.commons.lang.StringUtils; +import org.apache.ranger.authorization.hadoop.config.RangerConfiguration; import org.apache.ranger.entity.XXServiceDef; import org.apache.ranger.plugin.model.RangerServiceDef; +import org.apache.ranger.plugin.store.EmbeddedServiceDefsUtil; import org.springframework.context.annotation.Scope; import org.springframework.stereotype.Service; @@ -50,7 +54,20 @@ protected XXServiceDef mapViewToEntityBean(RangerServiceDef vObj, XXServiceDef x @Override protected RangerServiceDef mapEntityToViewBean(RangerServiceDef vObj, XXServiceDef xObj) { - return super.mapEntityToViewBean(vObj, xObj); + RangerServiceDef ret = super.mapEntityToViewBean(vObj, xObj); + + Map serviceDefOptions = ret.getOptions(); + + if (serviceDefOptions.get(RangerServiceDef.OPTION_ENABLE_DENY_AND_EXCEPTIONS_IN_POLICIES) == null) { + boolean enableDenyAndExceptionsInPoliciesHiddenOption = RangerConfiguration.getInstance().getBoolean("ranger.servicedef.enableDenyAndExceptionsInPolicies", true); + if (enableDenyAndExceptionsInPoliciesHiddenOption || StringUtils.equalsIgnoreCase(ret.getName(), EmbeddedServiceDefsUtil.EMBEDDED_SERVICEDEF_TAG_NAME)) { + serviceDefOptions.put(RangerServiceDef.OPTION_ENABLE_DENY_AND_EXCEPTIONS_IN_POLICIES, "true"); + } else { + serviceDefOptions.put(RangerServiceDef.OPTION_ENABLE_DENY_AND_EXCEPTIONS_IN_POLICIES, "false"); + } + ret.setOptions(serviceDefOptions); + } + return ret; } public List getAllServiceDefs() { From 49d2962da60c896d1ceeadff39749e4efd2cd8e0 Mon Sep 17 00:00:00 2001 From: Abhay Kulkarni Date: Wed, 3 May 2017 00:31:10 +0530 Subject: [PATCH 087/151] RANGER-1553: Audit log record for 'show databases' hive command contains all tags (cherry picked from commit cd9d1a49105d42eeb3a7aa65205b6f1d3018a180) --- .../authorization/hive/authorizer/RangerHiveAuditHandler.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hive-agent/src/main/java/org/apache/ranger/authorization/hive/authorizer/RangerHiveAuditHandler.java b/hive-agent/src/main/java/org/apache/ranger/authorization/hive/authorizer/RangerHiveAuditHandler.java index 89bc0d8761..24a71fa695 100644 --- a/hive-agent/src/main/java/org/apache/ranger/authorization/hive/authorizer/RangerHiveAuditHandler.java +++ b/hive-agent/src/main/java/org/apache/ranger/authorization/hive/authorizer/RangerHiveAuditHandler.java @@ -58,8 +58,8 @@ AuthzAuditEvent createAuditEvent(RangerAccessResult result, String accessType, S RangerHiveAccessRequest hiveAccessRequest = (RangerHiveAccessRequest) request; RangerHiveResource hiveResource = (RangerHiveResource) resource; - if (hiveAccessRequest.getHiveAccessType() == HiveAccessType.USE && hiveResource.getObjectType() == HiveObjectType.DATABASE) { - // this should happen only for SHOWDATABASES and USE commands + if (hiveAccessRequest.getHiveAccessType() == HiveAccessType.USE && hiveResource.getObjectType() == HiveObjectType.DATABASE && StringUtils.isBlank(hiveResource.getDatabase())) { + // this should happen only for SHOWDATABASES auditEvent.setTags(null); } } From 1c68d4fe889520f17546eb0e925e06b85843b23c Mon Sep 17 00:00:00 2001 From: rmani Date: Sat, 17 Jun 2017 04:56:52 +0530 Subject: [PATCH 088/151] RANGER-1648 : Ranger Kafka Plugin now should use the Short name from Kafka Session Object (cherry picked from commit fdd1d294c6b3c0e3cd8dd4219d034ad63210b380) --- .../authorization/kafka/authorizer/RangerKafkaAuthorizer.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/plugin-kafka/src/main/java/org/apache/ranger/authorization/kafka/authorizer/RangerKafkaAuthorizer.java b/plugin-kafka/src/main/java/org/apache/ranger/authorization/kafka/authorizer/RangerKafkaAuthorizer.java index 8425fd393e..b3d5a74d59 100644 --- a/plugin-kafka/src/main/java/org/apache/ranger/authorization/kafka/authorizer/RangerKafkaAuthorizer.java +++ b/plugin-kafka/src/main/java/org/apache/ranger/authorization/kafka/authorizer/RangerKafkaAuthorizer.java @@ -140,8 +140,6 @@ public boolean authorize(Session session, Operation operation, String userName = null; if (session.principal() != null) { userName = session.principal().getName(); - userName = StringUtils.substringBefore(userName, "/"); - userName = StringUtils.substringBefore(userName, "@"); } java.util.Set userGroups = MiscUtil .getGroupsForRequestUser(userName); From fdf8551a05cd30bd451d7aa5ee2d0a3f2cce1d19 Mon Sep 17 00:00:00 2001 From: rmani Date: Mon, 5 Jun 2017 23:07:57 +0530 Subject: [PATCH 089/151] RANGER-1631 : create temp function failing with permission issues (cherry picked from commit 2e193e124399cf685c17798b8243e1d62f223315) --- .../authorization/hive/authorizer/RangerHiveResource.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/hive-agent/src/main/java/org/apache/ranger/authorization/hive/authorizer/RangerHiveResource.java b/hive-agent/src/main/java/org/apache/ranger/authorization/hive/authorizer/RangerHiveResource.java index 3f1279faee..d04d3bface 100644 --- a/hive-agent/src/main/java/org/apache/ranger/authorization/hive/authorizer/RangerHiveResource.java +++ b/hive-agent/src/main/java/org/apache/ranger/authorization/hive/authorizer/RangerHiveResource.java @@ -50,6 +50,9 @@ public RangerHiveResource(HiveObjectType objectType, String databaseorUrl, Strin break; case FUNCTION: + if (databaseorUrl == null) { + databaseorUrl = ""; + } setValue(KEY_DATABASE, databaseorUrl); setValue(KEY_UDF, tableOrUdf); break; From 73b8c6eee1d96dee91193035043a389784bcd34b Mon Sep 17 00:00:00 2001 From: Vishal Suvagia Date: Wed, 26 Sep 2018 18:21:36 +0530 Subject: [PATCH 090/151] RANGER-1580 - Update Kafka tests to work with 0.10.1.1 Signed-off-by: Colm O hEigeartaigh (cherry picked from commit fd931f20270d4db8187b885b9e7f95c8261fdd82) --- plugin-kafka/pom.xml | 7 ++++- .../KafkaRangerAuthorizerSASLSSLTest.java | 3 +- .../authorizer/KafkaRangerAuthorizerTest.java | 22 ++++++-------- .../src/test/resources/kafka-policies.json | 30 +++++++++++++++++-- 4 files changed, 45 insertions(+), 17 deletions(-) diff --git a/plugin-kafka/pom.xml b/plugin-kafka/pom.xml index fb0be1819c..61bdfd510d 100644 --- a/plugin-kafka/pom.xml +++ b/plugin-kafka/pom.xml @@ -84,7 +84,12 @@ ${bouncycastle.version} test - + + junit + junit + ${junit.version} + test + diff --git a/plugin-kafka/src/test/java/org/apache/ranger/authorization/kafka/authorizer/KafkaRangerAuthorizerSASLSSLTest.java b/plugin-kafka/src/test/java/org/apache/ranger/authorization/kafka/authorizer/KafkaRangerAuthorizerSASLSSLTest.java index 277474f565..22dce3a993 100644 --- a/plugin-kafka/src/test/java/org/apache/ranger/authorization/kafka/authorizer/KafkaRangerAuthorizerSASLSSLTest.java +++ b/plugin-kafka/src/test/java/org/apache/ranger/authorization/kafka/authorizer/KafkaRangerAuthorizerSASLSSLTest.java @@ -56,7 +56,8 @@ * CustomAuthorizer that enforces some authorization rules: * * - The "IT" group can do anything - * - The "public" group can only "read/describe" on the "test" topic, not "write". + * - The "public" group can "read/describe/write" on the "test" topic. + * - The "public" group can only "read/describe" on the "dev" topic, but not write. * * Policies available from admin via: * diff --git a/plugin-kafka/src/test/java/org/apache/ranger/authorization/kafka/authorizer/KafkaRangerAuthorizerTest.java b/plugin-kafka/src/test/java/org/apache/ranger/authorization/kafka/authorizer/KafkaRangerAuthorizerTest.java index 898c785bb3..abc03fa526 100644 --- a/plugin-kafka/src/test/java/org/apache/ranger/authorization/kafka/authorizer/KafkaRangerAuthorizerTest.java +++ b/plugin-kafka/src/test/java/org/apache/ranger/authorization/kafka/authorizer/KafkaRangerAuthorizerTest.java @@ -58,7 +58,8 @@ * CustomAuthorizer that enforces some authorization rules: * * - The "IT" group can do anything - * - The "public" group can only "read/describe" on the "test" topic, not "write". + * - The "public" group can "read/describe/write" on the "test" topic. + * - The "public" group can only "read/describe" on the "dev" topic, but not write. * * Policies available from admin via: * @@ -259,8 +260,8 @@ public void testAuthorizedWrite() throws Exception { producer.close(); } - - // The "public" group can't write to "test" or "dev" + + // The "public" group can write to "test" but not "dev" @Test public void testUnauthorizedWrite() throws Exception { // Create the Producer @@ -280,18 +281,13 @@ public void testUnauthorizedWrite() throws Exception { final Producer producer = new KafkaProducer<>(producerProps); // Send a message - try { - Future record = - producer.send(new ProducerRecord("test", "somekey", "somevalue")); - producer.flush(); - record.get(); - Assert.fail("Authorization failure expected"); - } catch (Exception ex) { - Assert.assertTrue(ex.getMessage().contains("Not authorized to access topics")); - } + Future record = + producer.send(new ProducerRecord("test", "somekey", "somevalue")); + producer.flush(); + record.get(); try { - Future record = + record = producer.send(new ProducerRecord("dev", "somekey", "somevalue")); producer.flush(); record.get(); diff --git a/plugin-kafka/src/test/resources/kafka-policies.json b/plugin-kafka/src/test/resources/kafka-policies.json index cc1c927ef8..dd9b802499 100644 --- a/plugin-kafka/src/test/resources/kafka-policies.json +++ b/plugin-kafka/src/test/resources/kafka-policies.json @@ -124,11 +124,37 @@ ], "users": [], "groups": [ - "IT" + "public" ], "conditions": [], "delegateAdmin": false - }, + } + ], + "denyPolicyItems": [], + "allowExceptions": [], + "denyExceptions": [], + "dataMaskPolicyItems": [], + "rowFilterPolicyItems": [], + "id": 19, + "isEnabled": true, + "version": 1 + }, + { + "service": "cl1_kafka", + "name": "DevPolicy", + "policyType": 0, + "description": "", + "isAuditEnabled": true, + "resources": { + "topic": { + "values": [ + "dev" + ], + "isExcludes": false, + "isRecursive": false + } + }, + "policyItems": [ { "accesses": [ { From 302c20a595b93c5af7824f56b7b922fa494d5986 Mon Sep 17 00:00:00 2001 From: yzhou2001 Date: Sat, 8 Apr 2017 03:10:54 +0530 Subject: [PATCH 091/151] RANGER-1502: Solr shutdown does not cause the audit log file to be flushed and closed Signed-off-by: rmani (cherry picked from commit b6c631dbdba2329b760eedeb58c983555648f504) --- .../audit/provider/AuditProviderFactory.java | 19 ++++++++++++++++++- .../solr/authorizer/RangerSolrAuthorizer.java | 6 ++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/provider/AuditProviderFactory.java b/agents-audit/src/main/java/org/apache/ranger/audit/provider/AuditProviderFactory.java index b37011e6f2..43107ba5e4 100644 --- a/agents-audit/src/main/java/org/apache/ranger/audit/provider/AuditProviderFactory.java +++ b/agents-audit/src/main/java/org/apache/ranger/audit/provider/AuditProviderFactory.java @@ -23,6 +23,7 @@ import java.util.Properties; import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -73,6 +74,7 @@ public class AuditProviderFactory { private AuditHandler mProvider = null; private String componentAppType = ""; private boolean mInitDone = false; + private JVMShutdownHook jvmShutdownHook = null; private AuditProviderFactory() { LOG.info("AuditProviderFactory: creating.."); @@ -106,6 +108,16 @@ public boolean isInitDone() { return mInitDone; } + /** + * call shutdown hook to provide a way to + * shutdown gracefully in addition to the ShutdownHook mechanism + */ + public void shutdown() { + if (isInitDone() && jvmShutdownHook != null) { + jvmShutdownHook.run(); + } + } + public synchronized void init(Properties props, String appType) { LOG.info("AuditProviderFactory: initializing.."); @@ -463,7 +475,7 @@ private AuditHandler getDefaultProvider() { private void installJvmSutdownHook(Properties props) { int shutdownHookMaxWaitSeconds = MiscUtil.getIntProperty(props, AUDIT_SHUTDOWN_HOOK_MAX_WAIT_SEC, AUDIT_SHUTDOWN_HOOK_MAX_WAIT_SEC_DEFAULT); - JVMShutdownHook jvmShutdownHook = new JVMShutdownHook(mProvider, shutdownHookMaxWaitSeconds); + jvmShutdownHook = new JVMShutdownHook(mProvider, shutdownHookMaxWaitSeconds); ShutdownHookManager.get().addShutdownHook(jvmShutdownHook, RANGER_AUDIT_SHUTDOWN_HOOK_PRIORITY); } @@ -503,6 +515,7 @@ private static class JVMShutdownHook extends Thread { final Semaphore doneCleanup = new Semaphore(0); final Thread cleanupThread; final int maxWait; + final AtomicBoolean done = new AtomicBoolean(false); public JVMShutdownHook(AuditHandler provider, int maxWait) { this.maxWait = maxWait; @@ -513,6 +526,10 @@ public JVMShutdownHook(AuditHandler provider, int maxWait) { } public void run() { + if (!done.compareAndSet(false, true)) { + LOG.info("==> JVMShutdownHook.run() already done by another thread"); + return; + } LOG.info("==> JVMShutdownHook.run()"); LOG.info("JVMShutdownHook: Signalling async audit cleanup to start."); startCleanup.release(); diff --git a/plugin-solr/src/main/java/org/apache/ranger/authorization/solr/authorizer/RangerSolrAuthorizer.java b/plugin-solr/src/main/java/org/apache/ranger/authorization/solr/authorizer/RangerSolrAuthorizer.java index 78b4ac4f3c..068ac27b3b 100644 --- a/plugin-solr/src/main/java/org/apache/ranger/authorization/solr/authorizer/RangerSolrAuthorizer.java +++ b/plugin-solr/src/main/java/org/apache/ranger/authorization/solr/authorizer/RangerSolrAuthorizer.java @@ -34,6 +34,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.ranger.audit.provider.AuditProviderFactory; import org.apache.ranger.audit.provider.MiscUtil; import org.apache.ranger.authorization.hadoop.config.RangerConfiguration; import org.apache.ranger.plugin.audit.RangerMultiResourceAuditHandler; @@ -148,6 +149,11 @@ public void close() throws IOException { logger.info("close() called"); try { solrPlugin.cleanup(); + /* Solr shutdown is not graceful so that JVM shutdown hooks + * are not always invoked and the audit store are not flushed. So + * we are forcing a cleanup here. + */ + AuditProviderFactory.getInstance().shutdown(); } catch (Throwable t) { logger.error("Error cleaning up Ranger plugin. Ignoring error", t); } From a3be2390eeae26fa41147270d358e51a552fa55c Mon Sep 17 00:00:00 2001 From: Vishal Suvagia Date: Wed, 26 Sep 2018 19:41:08 +0530 Subject: [PATCH 092/151] RANGER-1501: Audit Flush to HDFS does not actually cause the audit logs to be flushed to HDFS - improvement patch Signed-off-by: rmani (cherry picked from commit 1befffcc1c18679cdf2c6413496b2078f6d2a838) --- .../destination/HDFSAuditDestination.java | 26 +++++++++++++++---- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/destination/HDFSAuditDestination.java b/agents-audit/src/main/java/org/apache/ranger/audit/destination/HDFSAuditDestination.java index 889b6ffdc7..8b17fc510d 100644 --- a/agents-audit/src/main/java/org/apache/ranger/audit/destination/HDFSAuditDestination.java +++ b/agents-audit/src/main/java/org/apache/ranger/audit/destination/HDFSAuditDestination.java @@ -63,6 +63,7 @@ public class HDFSAuditDestination extends AuditDestination { private String logFolder; private PrintWriter logWriter = null; + volatile FSDataOutputStream ostream = null; // output stream wrapped in logWriter private String currentFileName; @@ -169,6 +170,7 @@ public PrintWriter run() throws Exception { addDeferredCount(events.size()); out.close(); logWriter = null; + ostream = null; return false; } } catch (Throwable t) { @@ -187,10 +189,22 @@ public PrintWriter run() throws Exception { @Override public void flush() { - if ( logWriter != null) { - logWriter.flush(); - logger.info("Flush HDFS audit logs completed....."); - } + logger.info("Flush called. name=" + getName()); + if (ostream != null) { + try { + synchronized (this) { + if (ostream != null) + // 1) PrinterWriter does not have bufferring of its own so + // we need to flush its underlying stream + // 2) HDFS flush() does not really flush all the way to disk. + ostream.hflush(); + logger.info("Flush HDFS audit logs completed....."); + } + } catch (IOException e) { + logger.error("Error on flushing log writer: " + e.getMessage() + + "\nException will be ignored. name=" + getName() + ", fileName=" + currentFileName); + } + } } /* @@ -246,6 +260,7 @@ synchronized public void stop() { + getName() + ", fileName=" + currentFileName); } logWriter = null; + ostream = null; } logStatus(); } @@ -290,7 +305,7 @@ synchronized private PrintWriter getLogFileStream() throws Exception { // Create the file to write logger.info("Creating new log file. hdfPath=" + fullPath); - FSDataOutputStream ostream = fileSystem.create(hdfPath); + ostream = fileSystem.create(hdfPath); logWriter = new PrintWriter(ostream); currentFileName = fullPath; } @@ -341,6 +356,7 @@ private void closeFileIfNeeded() throws FileNotFoundException, IOException { } logWriter = null; + ostream = null; currentFileName = null; if (!rollOverByDuration) { From 637f01a6c12ac53c6b3bee811d4538bdea1c5598 Mon Sep 17 00:00:00 2001 From: "peng.jianhua" Date: Thu, 16 Mar 2017 03:44:09 -0400 Subject: [PATCH 093/151] RANGER-1415:The ranger can be opened when the user enters http://localhost:6080/ in the browser address bar. But request policy from hadoop to ranger will failed after installing hdfs plugin if we set POLICY_MGR_URL equal to http://localhost:6080/. Signed-off-by: zhangqiang2 --- .../ranger/admin/client/RangerAdminRESTClient.java | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/agents-common/src/main/java/org/apache/ranger/admin/client/RangerAdminRESTClient.java b/agents-common/src/main/java/org/apache/ranger/admin/client/RangerAdminRESTClient.java index a32db46d4a..974634792c 100644 --- a/agents-common/src/main/java/org/apache/ranger/admin/client/RangerAdminRESTClient.java +++ b/agents-common/src/main/java/org/apache/ranger/admin/client/RangerAdminRESTClient.java @@ -31,6 +31,7 @@ import org.apache.ranger.admin.client.datatype.RESTResponse; import org.apache.ranger.audit.provider.MiscUtil; import org.apache.ranger.authorization.hadoop.config.RangerConfiguration; +import org.apache.ranger.authorization.utils.StringUtil; import org.apache.ranger.plugin.util.GrantRevokeRequest; import org.apache.ranger.plugin.util.RangerRESTClient; import org.apache.ranger.plugin.util.RangerRESTUtils; @@ -80,11 +81,18 @@ public void init(String serviceName, String appId, String propertyPrefix) { this.serviceName = serviceName; this.pluginId = restUtils.getPluginId(serviceName, appId); - String url = RangerConfiguration.getInstance().get(propertyPrefix + ".policy.rest.url"); + String url = ""; + String tmpUrl = RangerConfiguration.getInstance().get(propertyPrefix + ".policy.rest.url"); String sslConfigFileName = RangerConfiguration.getInstance().get(propertyPrefix + ".policy.rest.ssl.config.file"); clusterName = RangerConfiguration.getInstance().get(propertyPrefix + ".ambari.cluster.name", ""); int restClientConnTimeOutMs = RangerConfiguration.getInstance().getInt(propertyPrefix + ".policy.rest.client.connection.timeoutMs", 120 * 1000); int restClientReadTimeOutMs = RangerConfiguration.getInstance().getInt(propertyPrefix + ".policy.rest.client.read.timeoutMs", 30 * 1000); + if (!StringUtil.isEmpty(tmpUrl)) { + url = tmpUrl.trim(); + } + if (url.endsWith("/")) { + url = url.substring(0, url.length() - 1); + } init(url, sslConfigFileName, restClientConnTimeOutMs , restClientReadTimeOutMs); } From 48fd2586e96bc76216c20f260dd29da6d914b9a5 Mon Sep 17 00:00:00 2001 From: zhangqiang2 Date: Mon, 27 Feb 2017 07:10:35 -0500 Subject: [PATCH 094/151] RANGER-1408:When the error occurs, the system does not record the error message in RangerServiceService class Signed-off-by: Colm O hEigeartaigh --- .../apache/ranger/service/RangerServiceService.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/security-admin/src/main/java/org/apache/ranger/service/RangerServiceService.java b/security-admin/src/main/java/org/apache/ranger/service/RangerServiceService.java index ab44016816..30fc2131e6 100644 --- a/security-admin/src/main/java/org/apache/ranger/service/RangerServiceService.java +++ b/security-admin/src/main/java/org/apache/ranger/service/RangerServiceService.java @@ -25,6 +25,7 @@ import java.util.List; import java.util.Map; import java.util.Map.Entry; +import java.util.logging.Logger; import org.apache.commons.lang.StringUtils; import org.apache.ranger.biz.ServiceDBStore; @@ -35,7 +36,6 @@ import org.apache.ranger.common.view.VTrxLogAttr; import org.apache.ranger.db.XXServiceVersionInfoDao; import org.apache.ranger.entity.XXService; -import org.apache.ranger.entity.XXServiceBase; import org.apache.ranger.entity.XXServiceConfigMap; import org.apache.ranger.entity.XXServiceDef; import org.apache.ranger.entity.XXServiceVersionInfo; @@ -48,7 +48,7 @@ @Service @Scope("singleton") public class RangerServiceService extends RangerServiceServiceBase { - + private static final Logger LOG = Logger.getLogger(RangerServiceService.class.getName()); @Autowired JSONUtil jsonUtil; @@ -166,9 +166,9 @@ public List getTransactionLog(RangerService vObj, XXService mObj, int } } } catch (IllegalAccessException e) { - e.printStackTrace(); + LOG.info("Get transaction log failure." + e); } catch (NoSuchFieldException e) { - e.printStackTrace(); + LOG.info("Get transaction log failure." + e); } return trxLogList; } @@ -275,7 +275,7 @@ private XXTrxLog processFieldToCreateTrxLog(Field field, String objectName, xTrxLog.setNewValue(value); } } catch (IllegalArgumentException | IllegalAccessException e) { - e.printStackTrace(); + LOG.info("Process field to create trx log failure." + e); } xTrxLog.setAction(actionString); From 13f17952d9a6869307b10b6dba73001ffd33ee8e Mon Sep 17 00:00:00 2001 From: Colm O hEigeartaigh Date: Mon, 10 Apr 2017 12:23:38 +0100 Subject: [PATCH 095/151] RANGER-1505 - Remove KeyProtector code in KMS Signed-off-by: Colm O hEigeartaigh --- .../hadoop/crypto/key/RangerKeyStore.java | 104 +++++++++++++----- 1 file changed, 79 insertions(+), 25 deletions(-) diff --git a/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKeyStore.java b/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKeyStore.java index 4b1b9bb069..018ead5945 100644 --- a/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKeyStore.java +++ b/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKeyStore.java @@ -26,11 +26,13 @@ import java.io.ObjectInputStream; import java.io.ObjectOutputStream; import java.io.OutputStream; +import java.io.Serializable; import java.io.UnsupportedEncodingException; import java.lang.reflect.Constructor; import java.lang.reflect.Field; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; +import java.security.AlgorithmParameters; import java.security.DigestInputStream; import java.security.DigestOutputStream; import java.security.Key; @@ -39,6 +41,7 @@ import java.security.KeyStoreSpi; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; +import java.security.SecureRandom; import java.security.UnrecoverableKeyException; import java.security.cert.Certificate; import java.security.cert.CertificateException; @@ -49,7 +52,13 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; +import javax.crypto.Cipher; +import javax.crypto.IllegalBlockSizeException; import javax.crypto.SealedObject; +import javax.crypto.SecretKey; +import javax.crypto.SecretKeyFactory; +import javax.crypto.spec.PBEKeySpec; +import javax.crypto.spec.PBEParameterSpec; import javax.xml.bind.DatatypeConverter; import org.apache.hadoop.crypto.key.KeyProvider.Metadata; @@ -111,17 +120,9 @@ public Key engineGetKey(String alias, char[] password)throws NoSuchAlgorithmExce return null; } - Class c = null; - Object o = null; try { - c = Class.forName("com.sun.crypto.provider.KeyProtector"); - Constructor constructor = c.getDeclaredConstructor(char[].class); - constructor.setAccessible(true); - o = constructor.newInstance(password); - Method m = c.getDeclaredMethod("unseal", SealedObject.class); - m.setAccessible(true); - key = (Key) m.invoke(o, ((SecretKeyEntry)entry).sealedKey); - } catch (ClassNotFoundException | NoSuchMethodException | SecurityException | InstantiationException | IllegalAccessException | IllegalArgumentException | InvocationTargetException e) { + key = unsealKey(((SecretKeyEntry)entry).sealedKey, password); + } catch (Exception e) { logger.error(e.getMessage()); } return key; @@ -147,22 +148,9 @@ public void addKeyEntry(String alias, Key key, char[] password, String cipher, i SecretKeyEntry entry = new SecretKeyEntry(); synchronized(deltaEntries) { try { - Class c = null; - Object o = null; - try { - c = Class.forName("com.sun.crypto.provider.KeyProtector"); - Constructor constructor = c.getDeclaredConstructor(char[].class); - constructor.setAccessible(true); - o = constructor.newInstance(password); - } catch (ClassNotFoundException | NoSuchMethodException | SecurityException | InstantiationException | IllegalAccessException | IllegalArgumentException | InvocationTargetException e) { - logger.error(e.getMessage()); - throw new KeyStoreException(e.getMessage()); - } entry.date = new Date(); // seal and store the key - Method m = c.getDeclaredMethod("seal", Key.class); - m.setAccessible(true); - entry.sealedKey = (SealedObject) m.invoke(o, key); + entry.sealedKey = sealKey(key, password); entry.cipher_field = cipher; entry.bit_length = bitLength; @@ -185,6 +173,47 @@ public void addKeyEntry(String alias, Key key, char[] password, String cipher, i } } + private SealedObject sealKey(Key key, char[] password) throws Exception { + // Create SecretKey + SecretKeyFactory secretKeyFactory = SecretKeyFactory.getInstance("PBEWithMD5AndTripleDES"); + PBEKeySpec pbeKeySpec = new PBEKeySpec(password); + SecretKey secretKey = secretKeyFactory.generateSecret(pbeKeySpec); + pbeKeySpec.clearPassword(); + + // Generate random bytes + set up the PBEParameterSpec + SecureRandom random = new SecureRandom(); + byte[] salt = new byte[8]; + random.nextBytes(salt); + PBEParameterSpec pbeSpec = new PBEParameterSpec(salt, 20); + + // Seal the Key + Cipher cipher = Cipher.getInstance("PBEWithMD5AndTripleDES"); + cipher.init(Cipher.ENCRYPT_MODE, secretKey, pbeSpec); + return new RangerSealedObject(key, cipher); + } + + private Key unsealKey(SealedObject sealedKey, char[] password) throws Exception { + // Create SecretKey + SecretKeyFactory secretKeyFactory = SecretKeyFactory.getInstance("PBEWithMD5AndTripleDES"); + PBEKeySpec pbeKeySpec = new PBEKeySpec(password); + SecretKey secretKey = secretKeyFactory.generateSecret(pbeKeySpec); + pbeKeySpec.clearPassword(); + + // Get the AlgorithmParameters from RangerSealedObject + AlgorithmParameters algorithmParameters = null; + if (sealedKey instanceof RangerSealedObject) { + algorithmParameters = ((RangerSealedObject)sealedKey).getParameters(); + } else { + algorithmParameters = new RangerSealedObject(sealedKey).getParameters(); + } + + // Unseal the Key + Cipher cipher = Cipher.getInstance("PBEWithMD5AndTripleDES"); + cipher.init(Cipher.DECRYPT_MODE, secretKey, algorithmParameters); + + return (Key)sealedKey.getObject(cipher); + } + @Override public void engineDeleteEntry(String alias) throws KeyStoreException @@ -599,5 +628,30 @@ private void validateKeyName(String name) { public void clearDeltaEntires(){ deltaEntries.clear(); } - + + /** + * Encapsulate the encrypted key, so that we can retrieve the AlgorithmParameters object on the decryption side + */ + private static class RangerSealedObject extends SealedObject { + + /** + * + */ + private static final long serialVersionUID = -7551578543434362070L; + + protected RangerSealedObject(SealedObject so) { + super(so); + } + + protected RangerSealedObject(Serializable object, Cipher cipher) throws IllegalBlockSizeException, IOException { + super(object, cipher); + } + + public AlgorithmParameters getParameters() throws NoSuchAlgorithmException, IOException { + AlgorithmParameters algorithmParameters = AlgorithmParameters.getInstance("PBEWithMD5AndTripleDES"); + algorithmParameters.init(super.encodedParams); + return algorithmParameters; + } + + } } From 8a8bcd195e92f48c9392fc351cb9ee96e776f38a Mon Sep 17 00:00:00 2001 From: Colm O hEigeartaigh Date: Tue, 21 Feb 2017 12:05:39 +0000 Subject: [PATCH 096/151] RANGER-1402 - NPE if there is a problem with the HiveClient driverClassName Signed-off-by: Colm O hEigeartaigh --- .../services/hive/client/HiveConnectionMgr.java | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/hive-agent/src/main/java/org/apache/ranger/services/hive/client/HiveConnectionMgr.java b/hive-agent/src/main/java/org/apache/ranger/services/hive/client/HiveConnectionMgr.java index b36d5da4d0..9376358725 100644 --- a/hive-agent/src/main/java/org/apache/ranger/services/hive/client/HiveConnectionMgr.java +++ b/hive-agent/src/main/java/org/apache/ranger/services/hive/client/HiveConnectionMgr.java @@ -64,10 +64,19 @@ public HiveClient call() throws Exception { LOG.error("Error connecting hive repository : "+ serviceName +" using config : "+ configs, e); } - HiveClient oldClient = hiveConnectionCache.putIfAbsent(serviceName, hiveClient); + + HiveClient oldClient = null; + if (hiveClient != null) { + oldClient = hiveConnectionCache.putIfAbsent(serviceName, hiveClient); + } else { + oldClient = hiveConnectionCache.get(serviceName); + } + if (oldClient != null) { // in the meantime someone else has put a valid client into the cache, let's use that instead. - hiveClient.close(); + if (hiveClient != null) { + hiveClient.close(); + } hiveClient = oldClient; } repoConnectStatusMap.put(serviceName, true); From 1de5cab130c4d3b85a56ee96ae33539ba7b2ad52 Mon Sep 17 00:00:00 2001 From: zhangqiang2 Date: Wed, 22 Feb 2017 02:11:15 -0500 Subject: [PATCH 097/151] RANGER-1403:There is a problem in buildks class when delete invalid keystore file. Signed-off-by: Colm O hEigeartaigh --- .../src/main/java/org/apache/ranger/credentialapi/buildks.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/credentialbuilder/src/main/java/org/apache/ranger/credentialapi/buildks.java b/credentialbuilder/src/main/java/org/apache/ranger/credentialapi/buildks.java index 043f44cfc2..eb385060e5 100644 --- a/credentialbuilder/src/main/java/org/apache/ranger/credentialapi/buildks.java +++ b/credentialbuilder/src/main/java/org/apache/ranger/credentialapi/buildks.java @@ -527,7 +527,7 @@ public void deleteInvalidKeystore(String providerPath){ } if(keystore!=null && !keystore.isEmpty()){ File file =new File(keystore); - if(file!=null && file.length()==0){ + if(file!=null && file.exists() && file.length()==0){ System.out.println("Provider file '"+keystore+"' is in invalid state or corrupt!! will try to delete first."); file.delete(); file=null; From 9f4381080524c8ab5a4298726c7e4d82b34c7c40 Mon Sep 17 00:00:00 2001 From: Abhay Kulkarni Date: Wed, 26 Sep 2018 12:38:39 -0700 Subject: [PATCH 098/151] RANGER-1731: Exclude old guava versions which could cause build problems --- security-admin/pom.xml | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/security-admin/pom.xml b/security-admin/pom.xml index f863d746ec..65762acf4c 100644 --- a/security-admin/pom.xml +++ b/security-admin/pom.xml @@ -281,6 +281,12 @@ com.googlecode.owasp-java-html-sanitizer owasp-java-html-sanitizer ${owasp-java-html-sanitizer.version} + + + com.google.guava + guava + + org.springframework.ldap @@ -395,12 +401,42 @@ tomcat jasper-runtime + + com.sun.jersey + jersey-core + + + com.sun.jersey + jersey-server + + + com.sun.jersey + jersey-json + + + com.google.guava + guava + org.apache.hadoop hadoop-hdfs ${hadoop.version} + + + com.sun.jersey + jersey-core + + + com.sun.jersey + jersey-server + + + com.google.guava + guava + + org.apache.ranger From a582316696f46a133d11af9bfb5cc61fcaa22ceb Mon Sep 17 00:00:00 2001 From: Nikhil P Date: Fri, 19 Jan 2018 12:31:48 +0530 Subject: [PATCH 099/151] RANGER-1953 : improvement on user-group page listing Signed-off-by: Mehul Parikh --- .../java/org/apache/ranger/biz/XUserMgr.java | 17 +++++++++++++++++ .../common/RangerServicePoliciesCache.java | 2 +- .../webapp/scripts/views/users/GroupCreate.js | 1 + .../org/apache/ranger/biz/TestXUserMgr.java | 8 +++++++- 4 files changed, 26 insertions(+), 2 deletions(-) diff --git a/security-admin/src/main/java/org/apache/ranger/biz/XUserMgr.java b/security-admin/src/main/java/org/apache/ranger/biz/XUserMgr.java index e6e82d1d2e..4c01d57c43 100644 --- a/security-admin/src/main/java/org/apache/ranger/biz/XUserMgr.java +++ b/security-admin/src/main/java/org/apache/ranger/biz/XUserMgr.java @@ -53,6 +53,7 @@ import org.apache.ranger.common.MessageEnums; import org.apache.ranger.common.PropertiesUtil; import org.apache.ranger.common.RangerConstants; +import org.apache.ranger.common.RangerServicePoliciesCache; import org.apache.ranger.common.SearchCriteria; import org.apache.ranger.common.UserSessionBase; import org.apache.ranger.db.RangerDaoManager; @@ -844,8 +845,24 @@ public VXGroup updateXGroup(VXGroup vXGroup) { xGroup, "update"); xaBizUtil.createTrxLog(trxLogList); vXGroup = (VXGroup) xGroupService.updateResource(vXGroup); + if (vXGroup != null) { + updateXgroupUserForGroupUpdate(vXGroup); + RangerServicePoliciesCache.sInstance=null; + } return vXGroup; } + + private void updateXgroupUserForGroupUpdate(VXGroup vXGroup) { + List grpUsers = daoManager.getXXGroupUser().findByGroupId(vXGroup.getId()); + if(CollectionUtils.isNotEmpty(grpUsers)){ + for (XXGroupUser grpUser : grpUsers) { + VXGroupUser vXGroupUser = xGroupUserService.populateViewBean(grpUser); + vXGroupUser.setName(vXGroup.getName()); + updateXGroupUser(vXGroupUser); + } + } + } + public VXGroupUser updateXGroupUser(VXGroupUser vXGroupUser) { checkAdminAccess(); return super.updateXGroupUser(vXGroupUser); diff --git a/security-admin/src/main/java/org/apache/ranger/common/RangerServicePoliciesCache.java b/security-admin/src/main/java/org/apache/ranger/common/RangerServicePoliciesCache.java index eb20f693e1..0d5689a034 100644 --- a/security-admin/src/main/java/org/apache/ranger/common/RangerServicePoliciesCache.java +++ b/security-admin/src/main/java/org/apache/ranger/common/RangerServicePoliciesCache.java @@ -42,7 +42,7 @@ public class RangerServicePoliciesCache { private static final int MAX_WAIT_TIME_FOR_UPDATE = 10; - private static volatile RangerServicePoliciesCache sInstance = null; + public static volatile RangerServicePoliciesCache sInstance = null; private final boolean useServicePoliciesCache; private final int waitTimeInSeconds; diff --git a/security-admin/src/main/webapp/scripts/views/users/GroupCreate.js b/security-admin/src/main/webapp/scripts/views/users/GroupCreate.js index d16cbe5fd3..b73f755284 100644 --- a/security-admin/src/main/webapp/scripts/views/users/GroupCreate.js +++ b/security-admin/src/main/webapp/scripts/views/users/GroupCreate.js @@ -113,6 +113,7 @@ define(function(require){ success: function () { XAUtil.blockUI('unblock'); XAUtil.allowNavigation(); + Backbone.fetchCache._cache = {} var msg = that.editGroup ? 'Group updated successfully' :'Group created successfully'; XAUtil.notifySuccess('Success', msg); if(that.editGroup){ diff --git a/security-admin/src/test/java/org/apache/ranger/biz/TestXUserMgr.java b/security-admin/src/test/java/org/apache/ranger/biz/TestXUserMgr.java index b6ef5725c7..02798833a8 100644 --- a/security-admin/src/test/java/org/apache/ranger/biz/TestXUserMgr.java +++ b/security-admin/src/test/java/org/apache/ranger/biz/TestXUserMgr.java @@ -49,6 +49,7 @@ import org.apache.ranger.entity.XXGroup; import org.apache.ranger.entity.XXGroupGroup; import org.apache.ranger.entity.XXGroupPermission; +import org.apache.ranger.entity.XXGroupUser; import org.apache.ranger.entity.XXModuleDef; import org.apache.ranger.entity.XXPolicy; import org.apache.ranger.entity.XXPortalUser; @@ -467,6 +468,8 @@ public void test16CreateXGroup() { @Test public void test17UpdateXGroup() { XXGroupDao xxGroupDao = Mockito.mock(XXGroupDao.class); + XXGroupUserDao xxGroupUserDao = Mockito.mock(XXGroupUserDao.class); + List grpUsers =new ArrayList(); setup(); VXGroup vXGroup = new VXGroup(); vXGroup.setId(userId); @@ -477,7 +480,8 @@ public void test17UpdateXGroup() { Mockito.when(daoManager.getXXGroup()).thenReturn(xxGroupDao); Mockito.when(xxGroupDao.getById(vXGroup.getId())).thenReturn(xxGroup); Mockito.when(xGroupService.updateResource(vXGroup)).thenReturn(vXGroup); - + Mockito.when(daoManager.getXXGroupUser()).thenReturn(xxGroupUserDao); + Mockito.when(xxGroupUserDao.findByGroupId(vXGroup.getId())).thenReturn(grpUsers); VXGroup dbvxGroup = xUserMgr.updateXGroup(vXGroup); Assert.assertNotNull(dbvxGroup); userId = dbvxGroup.getId(); @@ -486,7 +490,9 @@ public void test17UpdateXGroup() { dbvxGroup.getDescription()); Assert.assertEquals(vXGroup.getName(), dbvxGroup.getName()); Mockito.verify(daoManager).getXXGroup(); + Mockito.verify(daoManager).getXXGroupUser(); Mockito.verify(xGroupService).updateResource(vXGroup); + Mockito.verify(xxGroupUserDao).findByGroupId(vXGroup.getId()); } @Test From bb5e8590cc99c1d9f6fa2e6d91ab0f35a07f6a83 Mon Sep 17 00:00:00 2001 From: "peng.jianhua" Date: Tue, 5 Dec 2017 10:23:34 +0800 Subject: [PATCH 100/151] RANGER-1797:Tomcat Security Vulnerability Alert. The version of the tomcat for ranger should upgrade to 7.0.82. Signed-off-by: peng.jianhua --- embeddedwebserver/pom.xml | 5 +++++ pom.xml | 2 +- src/main/assembly/admin-web.xml | 1 + src/main/assembly/kms.xml | 1 + 4 files changed, 8 insertions(+), 1 deletion(-) diff --git a/embeddedwebserver/pom.xml b/embeddedwebserver/pom.xml index 149ed1766e..852f04ba81 100644 --- a/embeddedwebserver/pom.xml +++ b/embeddedwebserver/pom.xml @@ -33,6 +33,11 @@ tomcat-embed-core ${tomcat.embed.version} + + org.apache.tomcat + tomcat-annotations-api + ${tomcat.embed.version} + org.apache.tomcat.embed tomcat-embed-el diff --git a/pom.xml b/pom.xml index 2ec6768ff7..9d7bd95756 100644 --- a/pom.xml +++ b/pom.xml @@ -220,7 +220,7 @@ 1.4 1.4 0.5.2 - 7.0.77 + 7.0.82 1.7 0.52 1.0 diff --git a/src/main/assembly/admin-web.xml b/src/main/assembly/admin-web.xml index 4dc52fd41b..0d541cdaf3 100644 --- a/src/main/assembly/admin-web.xml +++ b/src/main/assembly/admin-web.xml @@ -200,6 +200,7 @@ /ews/lib org.apache.tomcat.embed:tomcat-embed* + org.apache.tomcat:tomcat-annotations-api* org.eclipse.jdt.core.compiler:ecj:jar:P20140317-1600 log4j:log4j org.apache.hadoop:hadoop-auth:jar:${hadoop.version} diff --git a/src/main/assembly/kms.xml b/src/main/assembly/kms.xml index 1709859acf..49aa6470eb 100755 --- a/src/main/assembly/kms.xml +++ b/src/main/assembly/kms.xml @@ -137,6 +137,7 @@ /ews/webapp/WEB-INF/classes/lib org.apache.tomcat.embed:tomcat-embed* + org.apache.tomcat:tomcat-annotations-api* org.eclipse.jdt.core.compiler:ecj:jar:P20140317-1600 com.google.protobuf:protobuf-java:jar:${protobuf-java.version} org.apache.hadoop:hadoop-hdfs:jar:${hadoop.version} From 8257bc94f0d73175da99fed65eec241bf7a4efe2 Mon Sep 17 00:00:00 2001 From: Colm O hEigeartaigh Date: Tue, 31 Oct 2017 12:39:06 +0000 Subject: [PATCH 101/151] RANGER-1867 - Update nimbus-jose-jwt to 4.41.2 Signed-off-by: Colm O hEigeartaigh --- security-admin/pom.xml | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/security-admin/pom.xml b/security-admin/pom.xml index 65762acf4c..d023c7bc26 100644 --- a/security-admin/pom.xml +++ b/security-admin/pom.xml @@ -456,14 +456,8 @@ com.nimbusds nimbus-jose-jwt - 3.9 + 4.41.2 compile - - - org.bouncycastle - bcprov-jdk15on - - com.google.inject From d1e8a31ffacda15adf017fd03442684539a9795a Mon Sep 17 00:00:00 2001 From: pradeep Date: Mon, 12 Feb 2018 15:15:00 +0530 Subject: [PATCH 102/151] RANGER-1976: Upgrade Apache POI to 3.15 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 9d7bd95756..40fd07d476 100644 --- a/pom.xml +++ b/pom.xml @@ -199,7 +199,7 @@ 0.6 r239 2.3 - 3.12 + 3.17 UTF-8 2.5.0 2.11.8 From 38a3157c9cce141cde701ebac986d4ed7a8580e9 Mon Sep 17 00:00:00 2001 From: pradeep Date: Mon, 12 Feb 2018 15:23:05 +0530 Subject: [PATCH 103/151] RANGER-1977: Upgrade Apache commons-beanutils to 1.9.3 --- pom.xml | 8 ++++++-- security-admin/pom.xml | 9 ++------- src/main/assembly/kms.xml | 1 - 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/pom.xml b/pom.xml index 40fd07d476..387d854ca2 100644 --- a/pom.xml +++ b/pom.xml @@ -136,8 +136,7 @@ 0.9.1.2 2.2.0-b23 1.9.13 - 1.8.3 - 1.8.3 + 1.9.3 1.2 1.9 3.2.2 @@ -359,6 +358,11 @@ enunciate-core-annotations 2.8.0 + + commons-beanutils + commons-beanutils + ${commons.beanutils.version} + com.sun.jersey.contribs jersey-multipart diff --git a/security-admin/pom.xml b/security-admin/pom.xml index d023c7bc26..fb76d8683e 100644 --- a/security-admin/pom.xml +++ b/security-admin/pom.xml @@ -54,13 +54,8 @@ commons-beanutils - commons-beanutils-bean-collections - ${commons.beanutils.collections.version} - - - commons-beanutils - commons-beanutils-core - ${commons.beanutils.core.version} + commons-beanutils + ${commons.beanutils.version} commons-cli diff --git a/src/main/assembly/kms.xml b/src/main/assembly/kms.xml index 49aa6470eb..fca6a324e3 100755 --- a/src/main/assembly/kms.xml +++ b/src/main/assembly/kms.xml @@ -69,7 +69,6 @@ org.apache.directory.api:api-util org.apache.avro:avro commons-beanutils:commons-beanutils - commons-beanutils:commons-beanutils-core commons-cli:commons-cli commons-codec:commons-codec org.apache.commons:commons-compress From 20daa74ab9cf805baf2522d0fb890dbd531f7ab0 Mon Sep 17 00:00:00 2001 From: pradeep Date: Mon, 12 Feb 2018 15:55:51 +0530 Subject: [PATCH 104/151] RANGER-1979: Upgrade Spring-LDAP to 2.3.2 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 387d854ca2..214df89763 100644 --- a/pom.xml +++ b/pom.xml @@ -209,7 +209,7 @@ 1.7.5 1.0.4.1 5.5.1 - 1.3.1.RELEASE + 2.3.2.RELEASE 3.2.10.RELEASE 3.2.10.RELEASE 3.2.10.RELEASE From a6ea029b99cd7b0b68ebb63fc8b8a83978678289 Mon Sep 17 00:00:00 2001 From: Vishal Suvagia Date: Mon, 9 Apr 2018 16:27:21 +0530 Subject: [PATCH 105/151] RANGER-2060 : Knox proxy with knox-sso is not working for ranger. Signed-off-by: Mehul Parikh --- .../web/filter/RangerKRBAuthenticationFilter.java | 10 +++++----- .../web/filter/RangerSSOAuthenticationFilter.java | 6 ++++++ 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/security-admin/src/main/java/org/apache/ranger/security/web/filter/RangerKRBAuthenticationFilter.java b/security-admin/src/main/java/org/apache/ranger/security/web/filter/RangerKRBAuthenticationFilter.java index c8d22aa45f..7af59888b8 100644 --- a/security-admin/src/main/java/org/apache/ranger/security/web/filter/RangerKRBAuthenticationFilter.java +++ b/security-admin/src/main/java/org/apache/ranger/security/web/filter/RangerKRBAuthenticationFilter.java @@ -215,7 +215,7 @@ protected void doFilter(FilterChain filterChain, RangerAuthenticationProvider authenticationProvider = new RangerAuthenticationProvider(); Authentication authentication = authenticationProvider.authenticate(finalAuthentication); authentication = getGrantedAuthority(authentication); - SecurityContextHolder.getContext().setAuthentication(authentication); + SecurityContextHolder.getContext().setAuthentication(authentication); request.setAttribute("spnegoEnabled", true); LOG.info("Logged into Ranger as = "+userName); filterChain.doFilter(request, response); @@ -236,9 +236,9 @@ public void doFilter(ServletRequest request, ServletResponse response, FilterChain filterChain) throws IOException, ServletException { String authtype = PropertiesUtil.getProperty(RANGER_AUTH_TYPE); HttpServletRequest httpRequest = (HttpServletRequest)request; - if(isSpnegoEnable(authtype)){ + Authentication existingAuth = SecurityContextHolder.getContext().getAuthentication(); + if(isSpnegoEnable(authtype) && (existingAuth == null || !existingAuth.isAuthenticated())){ KerberosName.setRules(PropertiesUtil.getProperty(NAME_RULES, "DEFAULT")); - Authentication existingAuth = SecurityContextHolder.getContext().getAuthentication(); String userName = null; Cookie[] cookie = httpRequest.getCookies(); if(cookie != null){ @@ -261,8 +261,8 @@ public void doFilter(ServletRequest request, ServletResponse response, userName = cname.substring(ustr+2, andStr); } } - } - } + } + } } if((existingAuth == null || !existingAuth.isAuthenticated()) && (!StringUtils.isEmpty(userName))){ //--------------------------- To Create Ranger Session -------------------------------------- diff --git a/security-admin/src/main/java/org/apache/ranger/security/web/filter/RangerSSOAuthenticationFilter.java b/security-admin/src/main/java/org/apache/ranger/security/web/filter/RangerSSOAuthenticationFilter.java index d852667ca9..1843d698e5 100644 --- a/security-admin/src/main/java/org/apache/ranger/security/web/filter/RangerSSOAuthenticationFilter.java +++ b/security-admin/src/main/java/org/apache/ranger/security/web/filter/RangerSSOAuthenticationFilter.java @@ -252,6 +252,12 @@ private String constructForwardableURL(HttpServletRequest httpRequest){ } } } + if (xForwardedHost.contains(",")) { + if(LOG.isDebugEnabled()) { + LOG.debug("xForwardedHost value is " + xForwardedHost + " it contains multiple hosts, selecting the first host."); + } + xForwardedHost = xForwardedHost.split(",")[0].trim(); + } String xForwardedURL = ""; if (StringUtils.trimToNull(xForwardedProto) != null && StringUtils.trimToNull(xForwardedHost) != null && StringUtils.trimToNull(xForwardedContext) != null) { xForwardedURL = xForwardedProto + "://" + xForwardedHost From 10d3513f37426a24e9084bd41fa6334849461d05 Mon Sep 17 00:00:00 2001 From: Abhay Kulkarni Date: Tue, 20 Feb 2018 16:32:19 -0800 Subject: [PATCH 106/151] RANGER-1988: Fix insecure randomness --- .../hadoop/RangerHdfsAuthorizer.java | 30 +++++++++++++------ 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/hdfs-agent/src/main/java/org/apache/ranger/authorization/hadoop/RangerHdfsAuthorizer.java b/hdfs-agent/src/main/java/org/apache/ranger/authorization/hadoop/RangerHdfsAuthorizer.java index 97fd5cd5a1..b37d0ff147 100644 --- a/hdfs-agent/src/main/java/org/apache/ranger/authorization/hadoop/RangerHdfsAuthorizer.java +++ b/hdfs-agent/src/main/java/org/apache/ranger/authorization/hadoop/RangerHdfsAuthorizer.java @@ -24,16 +24,15 @@ import static org.apache.ranger.authorization.hadoop.constants.RangerHadoopConstants.WRITE_ACCCESS_TYPE; import java.net.InetAddress; +import java.security.SecureRandom; import java.util.Date; import java.util.HashMap; import java.util.HashSet; import java.util.Map; -import java.util.Random; import java.util.Set; import java.util.Stack; import org.apache.commons.lang.ArrayUtils; -import org.apache.commons.lang.RandomStringUtils; import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -616,15 +615,28 @@ public void init() { RangerHdfsPlugin.fileNameExtensionSeparator = RangerConfiguration.getInstance().get(RangerHdfsAuthorizer.RANGER_FILENAME_EXTENSION_SEPARATOR_PROP, RangerHdfsAuthorizer.DEFAULT_FILENAME_EXTENSION_SEPARATOR); RangerHdfsPlugin.optimizeSubAccessAuthEnabled = RangerConfiguration.getInstance().getBoolean(RangerHadoopConstants.RANGER_OPTIMIZE_SUBACCESS_AUTHORIZATION_PROP, RangerHadoopConstants.RANGER_OPTIMIZE_SUBACCESS_AUTHORIZATION_DEFAULT); - // Build random string of random length + String random = generateString("^&#@!%()-_+=@:;'<>`~abcdefghijklmnopqrstuvwxyz01234567890"); + randomizedWildcardPathName = RangerPathResourceMatcher.WILDCARD_ASTERISK + random + RangerPathResourceMatcher.WILDCARD_ASTERISK; + } + + // Build random string of length between 56 and 112 characters + private static String generateString(String source) + { + SecureRandom rng = new SecureRandom(); + byte[] bytes = new byte[1]; - new Random().nextBytes(bytes); - int count = bytes[0]; - count = count < 56 ? 56 : count; - count = count > 112 ? 112 : count; + rng.nextBytes(bytes); + int length = bytes[0]; + length = length < 56 ? 56 : length; + length = length > 112 ? 112 : length; - String random = RandomStringUtils.random(count, "^&#@!%()-_+=@:;'<>`~abcdefghijklmnopqrstuvwxyz01234567890"); - randomizedWildcardPathName = RangerPathResourceMatcher.WILDCARD_ASTERISK + random + RangerPathResourceMatcher.WILDCARD_ASTERISK; + char[] text = new char[length]; + + for (int i = 0; i < length; i++) + { + text[i] = source.charAt(rng.nextInt(source.length())); + } + return new String(text); } public static boolean isHadoopAuthEnabled() { From 977a1611c6cfb9e8f8d4d26cf7f242c03efccc82 Mon Sep 17 00:00:00 2001 From: Abhay Kulkarni Date: Wed, 26 Sep 2018 16:19:12 -0700 Subject: [PATCH 107/151] RANGER-1984: Hbase audit log records may not show all tags associated with accessed column --- .../contextenricher/RangerTagForEval.java | 2 +- .../RangerDefaultPolicyEvaluator.java | 2 +- .../hbase/RangerAuthorizationCoprocessor.java | 49 +++++++++---------- .../hbase/HBaseRangerAuthorizationTest.java | 10 ++-- 4 files changed, 31 insertions(+), 32 deletions(-) diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagForEval.java b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagForEval.java index b8f5b429c2..e31efa3c54 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagForEval.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagForEval.java @@ -33,7 +33,7 @@ @JsonAutoDetect(fieldVisibility=JsonAutoDetect.Visibility.ANY) @JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL) -@JsonIgnoreProperties(ignoreUnknown=true, value="matchType") +@JsonIgnoreProperties(ignoreUnknown=true) @XmlRootElement @XmlAccessorType(XmlAccessType.FIELD) diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerDefaultPolicyEvaluator.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerDefaultPolicyEvaluator.java index 6126f603b9..213a25c498 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerDefaultPolicyEvaluator.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerDefaultPolicyEvaluator.java @@ -184,7 +184,7 @@ public void evaluate(RangerAccessRequest request, RangerAccessResult result) { } else if (request.getResourceMatchingScope() == RangerAccessRequest.ResourceMatchingScope.SELF_OR_DESCENDANTS) { isMatched = matchType != RangerPolicyResourceMatcher.MatchType.NONE; } else { - isMatched = matchType == RangerPolicyResourceMatcher.MatchType.SELF || matchType == RangerPolicyResourceMatcher.MatchType.ANCESTOR; + isMatched = matchType == RangerPolicyResourceMatcher.MatchType.SELF || matchType == RangerPolicyResourceMatcher.MatchType.ANCESTOR; } if (isMatched) { diff --git a/hbase-agent/src/main/java/org/apache/ranger/authorization/hbase/RangerAuthorizationCoprocessor.java b/hbase-agent/src/main/java/org/apache/ranger/authorization/hbase/RangerAuthorizationCoprocessor.java index 038bf08eee..4c863c0509 100644 --- a/hbase-agent/src/main/java/org/apache/ranger/authorization/hbase/RangerAuthorizationCoprocessor.java +++ b/hbase-agent/src/main/java/org/apache/ranger/authorization/hbase/RangerAuthorizationCoprocessor.java @@ -404,47 +404,33 @@ ColumnFamilyAccessResult evaluateAccess(String operation, Action action, final R if (columns == null || columns.isEmpty()) { LOG.debug("evaluateAccess: columns collection null or empty, ok. Family level access is desired."); session.column(null) // zap stale column from prior iteration of this loop, if any - .buildRequest() - .authorize(); + .buildRequest() + .authorize(); AuthzAuditEvent auditEvent = auditHandler.getAndDiscardMostRecentEvent(); // capture it only for success if (session.isAuthorized()) { - if (LOG.isDebugEnabled()) { - LOG.debug("evaluateAccess: has family level access [" + family + "]"); - } - // we need to do 3 things: housekeeping, decide about audit events, building the results cache for filter somethingIsAccessible = true; - familesAccessAllowed.add(family); - if (auditEvent != null) { - LOG.debug("evaluateAccess: adding to family-level-access-granted-event-set"); - familyLevelAccessEvents.add(auditEvent); - } - } else { - everythingIsAccessible = false; - if (auditEvent != null && deniedEvent == null) { // we need to capture just one denial event - LOG.debug("evaluateAccess: Setting denied access audit event with last auth failure audit event."); - deniedEvent = auditEvent; - } if (LOG.isDebugEnabled()) { - LOG.debug("evaluateAccess: no family level access [" + family + "]. Checking if has partial access (of any type)..."); + LOG.debug("evaluateAccess: has family level access [" + family + "]. Checking if [" + family + "] descendants have access."); } - session.resourceMatchingScope(RangerAccessRequest.ResourceMatchingScope.SELF_OR_DESCENDANTS) .buildRequest() .authorize(); auditEvent = auditHandler.getAndDiscardMostRecentEvent(); // capture it only for failure if (session.isAuthorized()) { if (LOG.isDebugEnabled()) { - LOG.debug("evaluateAccess: has partial access (of some type) in family [" + family + "]"); + LOG.debug("evaluateAccess: [" + family + "] descendants have access"); + } + familesAccessAllowed.add(family); + if (auditEvent != null) { + LOG.debug("evaluateAccess: adding to family-level-access-granted-event-set"); + familyLevelAccessEvents.add(auditEvent); } - // we need to do 3 things: housekeeping, decide about audit events, building the results cache for filter - somethingIsAccessible = true; - familesAccessIndeterminate.add(family); } else { if (LOG.isDebugEnabled()) { - LOG.debug("evaluateAccess: has no access of ["+ access + "] type in family [" + family + "]"); + LOG.debug("evaluateAccess: has partial access (of some type) in family [" + family + "]"); } - familesAccessDenied.add(family); - denialReason = String.format("Insufficient permissions for user ‘%s',action: %s, tableName:%s, family:%s.", user.getName(), operation, table, family); + everythingIsAccessible = false; + familesAccessIndeterminate.add(family); if (auditEvent != null && deniedEvent == null) { // we need to capture just one denial event LOG.debug("evaluateAccess: Setting denied access audit event with last auth failure audit event."); deniedEvent = auditEvent; @@ -452,6 +438,17 @@ ColumnFamilyAccessResult evaluateAccess(String operation, Action action, final R } // Restore the headMatch setting session.resourceMatchingScope(RangerAccessRequest.ResourceMatchingScope.SELF); + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("evaluateAccess: has no access of [" + access + "] type in family [" + family + "]"); + } + everythingIsAccessible = false; + familesAccessDenied.add(family); + denialReason = String.format("Insufficient permissions for user ‘%s',action: %s, tableName:%s, family:%s.", user.getName(), operation, table, family); + if (auditEvent != null && deniedEvent == null) { // we need to capture just one denial event + LOG.debug("evaluateAccess: Setting denied access audit event with last auth failure audit event."); + deniedEvent = auditEvent; + } } } else { LOG.debug("evaluateAccess: columns collection not empty. Skipping Family level check, will do finer level access check."); diff --git a/hbase-agent/src/test/java/org/apache/ranger/authorization/hbase/HBaseRangerAuthorizationTest.java b/hbase-agent/src/test/java/org/apache/ranger/authorization/hbase/HBaseRangerAuthorizationTest.java index 7aec352bec..90e09ad8fe 100644 --- a/hbase-agent/src/test/java/org/apache/ranger/authorization/hbase/HBaseRangerAuthorizationTest.java +++ b/hbase-agent/src/test/java/org/apache/ranger/authorization/hbase/HBaseRangerAuthorizationTest.java @@ -317,8 +317,9 @@ public Void run() throws Exception { // Read a row try { Get get = new Get(Bytes.toBytes("row1")); - table.get(get); - Assert.fail("Failure expected on an unauthorized user"); + Result result = table.get(get); + byte[] valResult = result.getValue(Bytes.toBytes("colfam1"), Bytes.toBytes("col1")); + Assert.assertNull("Failure expected on an unauthorized user", valResult); } catch (IOException ex) { // expected } @@ -526,8 +527,9 @@ public Void run() throws Exception { // Read a row try { Get get = new Get(Bytes.toBytes("row1")); - table.get(get); - Assert.fail("Failure expected on an unauthorized user"); + Result result = table.get(get); + byte[] valResult = result.getValue(Bytes.toBytes("colfam2"), Bytes.toBytes("col1")); + Assert.assertNull("Failure expected on an unauthorized user", valResult); } catch (IOException ex) { // expected } From c3449185227241b465733795550b32e8b7d0218b Mon Sep 17 00:00:00 2001 From: fatimaawez Date: Wed, 21 Feb 2018 15:49:44 +0530 Subject: [PATCH 108/151] RANGER-1982i:Error Improvement for Analytics Metric of Ranger Admin and Ranger Kms Signed-off-by: Mehul Parikh --- embeddedwebserver/scripts/ranger-admin-services.sh | 6 +++++- kms/scripts/ranger-kms | 4 ++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/embeddedwebserver/scripts/ranger-admin-services.sh b/embeddedwebserver/scripts/ranger-admin-services.sh index f2d2bf5680..5780319d3a 100755 --- a/embeddedwebserver/scripts/ranger-admin-services.sh +++ b/embeddedwebserver/scripts/ranger-admin-services.sh @@ -137,7 +137,11 @@ stop(){ } metric(){ - java ${JAVA_OPTS} -Dlogdir=${RANGER_ADMIN_LOG_DIR} -cp "${XAPOLICYMGR_EWS_DIR}/webapp/WEB-INF/classes/conf:${XAPOLICYMGR_EWS_DIR}/webapp/WEB-INF/classes/:${XAPOLICYMGR_EWS_DIR}/webapp/WEB-INF/classes/lib/*:${XAPOLICYMGR_EWS_DIR}/webapp/WEB-INF/classes/META-INF:${XAPOLICYMGR_EWS_DIR}/webapp/WEB-INF/lib/*:${XAPOLICYMGR_EWS_DIR}/webapp/META-INF:${XAPOLICYMGR_EWS_DIR}/lib/*:${RANGER_JAAS_LIB_DIR}/*:${RANGER_JAAS_CONF_DIR}:${JAVA_HOME}/lib/*:${RANGER_HADOOP_CONF_DIR}/*:$CLASSPATH" org.apache.ranger.patch.cliutil.MetricUtil ${arg2} ${arg3} 2>/dev/null + if [ "$JAVA_HOME" == "" ]; then + echo "[E] JAVA_HOME environment variable not defined, aborting Apache Ranger Admin metric collection" + exit 1; + fi + java ${JAVA_OPTS} -Duser=${USER} -Dhostname=${HOSTNAME} -Dlogdir=${RANGER_ADMIN_LOG_DIR} -cp "${XAPOLICYMGR_EWS_DIR}/webapp/WEB-INF/classes/conf:${XAPOLICYMGR_EWS_DIR}/webapp/WEB-INF/classes/:${XAPOLICYMGR_EWS_DIR}/webapp/WEB-INF/classes/lib/*:${XAPOLICYMGR_EWS_DIR}/webapp/WEB-INF/classes/META-INF:${XAPOLICYMGR_EWS_DIR}/webapp/WEB-INF/lib/*:${XAPOLICYMGR_EWS_DIR}/webapp/META-INF:${XAPOLICYMGR_EWS_DIR}/lib/*:${RANGER_JAAS_LIB_DIR}/*:${RANGER_JAAS_CONF_DIR}:${JAVA_HOME}/lib/*:${RANGER_HADOOP_CONF_DIR}/*:$CLASSPATH" org.apache.ranger.patch.cliutil.MetricUtil ${arg2} ${arg3} 2>/dev/null } if [ "${action}" == "START" ]; then diff --git a/kms/scripts/ranger-kms b/kms/scripts/ranger-kms index 11dc4ff0c3..54499468f1 100755 --- a/kms/scripts/ranger-kms +++ b/kms/scripts/ranger-kms @@ -142,6 +142,10 @@ killRangerKMSPid () { fi } metric(){ + if [ "$JAVA_HOME" == "" ]; then + echo "[E] JAVA_HOME environment variable not defined, aborting Apache Ranger KMS metric collection" + exit 1; + fi java ${JAVA_OPTS} org.apache.hadoop.crypto.key.kms.server.KMSMetricUtil ${arg2} ${arg3} 2>/dev/null } if [ "${action}" == "START" ]; then From 508d347bcab294a891b18a3179f52ad05bcac548 Mon Sep 17 00:00:00 2001 From: fatimaawez Date: Mon, 26 Feb 2018 18:17:04 +0530 Subject: [PATCH 109/151] RANGER-1982 : Error Improvement for Analytics Metric of Ranger Admin and Ranger Kms Signed-off-by: Mehul Parikh --- embeddedwebserver/scripts/ranger-admin-services.sh | 2 +- kms/scripts/ranger-kms | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/embeddedwebserver/scripts/ranger-admin-services.sh b/embeddedwebserver/scripts/ranger-admin-services.sh index 5780319d3a..19dab2a9d4 100755 --- a/embeddedwebserver/scripts/ranger-admin-services.sh +++ b/embeddedwebserver/scripts/ranger-admin-services.sh @@ -138,7 +138,7 @@ stop(){ metric(){ if [ "$JAVA_HOME" == "" ]; then - echo "[E] JAVA_HOME environment variable not defined, aborting Apache Ranger Admin metric collection" + echo "[E] JAVA_HOME environment variable not defined, aborting Apache Ranger Admin metric collection" 1>&2; exit 1; fi java ${JAVA_OPTS} -Duser=${USER} -Dhostname=${HOSTNAME} -Dlogdir=${RANGER_ADMIN_LOG_DIR} -cp "${XAPOLICYMGR_EWS_DIR}/webapp/WEB-INF/classes/conf:${XAPOLICYMGR_EWS_DIR}/webapp/WEB-INF/classes/:${XAPOLICYMGR_EWS_DIR}/webapp/WEB-INF/classes/lib/*:${XAPOLICYMGR_EWS_DIR}/webapp/WEB-INF/classes/META-INF:${XAPOLICYMGR_EWS_DIR}/webapp/WEB-INF/lib/*:${XAPOLICYMGR_EWS_DIR}/webapp/META-INF:${XAPOLICYMGR_EWS_DIR}/lib/*:${RANGER_JAAS_LIB_DIR}/*:${RANGER_JAAS_CONF_DIR}:${JAVA_HOME}/lib/*:${RANGER_HADOOP_CONF_DIR}/*:$CLASSPATH" org.apache.ranger.patch.cliutil.MetricUtil ${arg2} ${arg3} 2>/dev/null diff --git a/kms/scripts/ranger-kms b/kms/scripts/ranger-kms index 54499468f1..dd14639bef 100755 --- a/kms/scripts/ranger-kms +++ b/kms/scripts/ranger-kms @@ -143,7 +143,7 @@ killRangerKMSPid () { } metric(){ if [ "$JAVA_HOME" == "" ]; then - echo "[E] JAVA_HOME environment variable not defined, aborting Apache Ranger KMS metric collection" + echo "[E] JAVA_HOME environment variable not defined, aborting Apache Ranger KMS metric collection" 1>&2; exit 1; fi java ${JAVA_OPTS} org.apache.hadoop.crypto.key.kms.server.KMSMetricUtil ${arg2} ${arg3} 2>/dev/null From 3d430201d2e90708da3bae8ee0034f32e1c925f7 Mon Sep 17 00:00:00 2001 From: ni3galave Date: Tue, 6 Mar 2018 15:10:49 +0530 Subject: [PATCH 110/151] RANGER-2008: Policy evaluation is failing for multiline policy conditions. Signed-off-by: pradeep --- .../scripts/views/policies/PermissionList.js | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/security-admin/src/main/webapp/scripts/views/policies/PermissionList.js b/security-admin/src/main/webapp/scripts/views/policies/PermissionList.js index c4aad36831..9184675144 100644 --- a/security-admin/src/main/webapp/scripts/views/policies/PermissionList.js +++ b/security-admin/src/main/webapp/scripts/views/policies/PermissionList.js @@ -530,7 +530,7 @@ define(function(require) { emptytext : 'Add Conditions', value : this.conditions, display: function(value) { - var continue_ = false, i = 0; + var continue_ = false, i = 0, cond = []; if(!value) { $(this).empty(); return; @@ -545,19 +545,18 @@ define(function(require) { return ''; } //Add label for policy condition - var pcond = _.findWhere(that.multiLinecond, { 'name': name}) + var pcond = _.findWhere(that.multiLinecond, { 'name': name}); if(!_.isUndefined(pcond) && !_.isUndefined(pcond['evaluatorOptions']) && ! _.isUndefined(pcond['evaluatorOptions']["ui.isMultiline"]) && ! _.isUndefined(pcond['evaluatorOptions']['engineName'])){ - val = pcond['evaluatorOptions']['engineName'] + ' Condition' + cond.push({ 'type' : name, 'values' : !_.isArray(val) ? [val] : val }); + val = pcond['evaluatorOptions']['engineName'] + ' Condition'; + } else { + cond.push({ 'type' : name, 'values' : !_.isArray(val) ? val.split(',') : val }); } i++; - return ''+name+' : '+ _.escape(val) + ''; + return ''+name+' : '+ _.escape(val) + ''; }); - var cond = _.map(value, function(val, name) { - return {'type' : name, 'values' : !_.isArray(val) ? val.split(',') : val}; - }); - that.model.set('conditions', cond); $(this).html(html); that.ui.addConditionsSpan.find('i').attr('class', 'icon-pencil'); From 92bdc78f3c05d92316f9d1bb1c304f374a790b89 Mon Sep 17 00:00:00 2001 From: Abhay Kulkarni Date: Mon, 5 Mar 2018 17:02:57 -0800 Subject: [PATCH 111/151] RANGER-2007: ranger-tagsync's Kerberos ticket fails to renew --- .../tagsync/process/TagSynchronizer.java | 33 +++++++------------ 1 file changed, 12 insertions(+), 21 deletions(-) diff --git a/tagsync/src/main/java/org/apache/ranger/tagsync/process/TagSynchronizer.java b/tagsync/src/main/java/org/apache/ranger/tagsync/process/TagSynchronizer.java index b07cd34ba5..612dd64da0 100644 --- a/tagsync/src/main/java/org/apache/ranger/tagsync/process/TagSynchronizer.java +++ b/tagsync/src/main/java/org/apache/ranger/tagsync/process/TagSynchronizer.java @@ -28,7 +28,6 @@ import org.apache.ranger.tagsync.model.TagSink; import org.apache.ranger.tagsync.model.TagSource; -import javax.security.auth.Subject; import java.io.IOException; import java.util.ArrayList; import java.util.List; @@ -380,31 +379,23 @@ private static boolean initializeKerberosIdentity(Properties props) { if (LOG.isDebugEnabled()) { LOG.debug("Trying to get kerberos identitiy"); } - Subject subject = null; - try { - subject = SecureClientLogin.loginUserFromKeytab(principal, keytab, nameRules); - } catch(IOException exception) { - LOG.error("Could not get Subject from principal:[" + principal + "], keytab:[" + keytab + "], nameRules:[" + nameRules + "]", exception); - } UserGroupInformation kerberosIdentity; - if (subject != null) { - try { - UserGroupInformation.loginUserFromSubject(subject); - kerberosIdentity = UserGroupInformation.getLoginUser(); - if (kerberosIdentity != null) { - props.put(TagSyncConfig.TAGSYNC_KERBEROS_IDENTITY, kerberosIdentity.getUserName()); - if (LOG.isDebugEnabled()) { - LOG.debug("Got UGI, user:[" + kerberosIdentity.getUserName() + "]"); - } - ret = true; - } else { - LOG.error("KerberosIdentity is null!"); + try { + UserGroupInformation.loginUserFromKeytab(principal, keytab); + kerberosIdentity = UserGroupInformation.getLoginUser(); + if (kerberosIdentity != null) { + props.put(TagSyncConfig.TAGSYNC_KERBEROS_IDENTITY, kerberosIdentity.getUserName()); + if (LOG.isDebugEnabled()) { + LOG.debug("Got UGI, user:[" + kerberosIdentity.getUserName() + "]"); } - } catch (IOException exception) { - LOG.error("Failed to get UGI from Subject:[" + subject + "]", exception); + ret = true; + } else { + LOG.error("KerberosIdentity is null!"); } + } catch (IOException exception) { + LOG.error("Failed to get UGI from principal:[" + principal + "], and keytab:[" + keytab + "]", exception); } } else { if (LOG.isDebugEnabled()) { From 00f0ac25547dd9bc1e6b2d3d394d134c740303b3 Mon Sep 17 00:00:00 2001 From: Mehul Parikh Date: Mon, 14 May 2018 10:32:43 +0530 Subject: [PATCH 112/151] RANGER-2076 : Handle proxy users for Kerberos based authentication --- .../filter/RangerKRBAuthenticationFilter.java | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/security-admin/src/main/java/org/apache/ranger/security/web/filter/RangerKRBAuthenticationFilter.java b/security-admin/src/main/java/org/apache/ranger/security/web/filter/RangerKRBAuthenticationFilter.java index 7af59888b8..11bc9e294c 100644 --- a/security-admin/src/main/java/org/apache/ranger/security/web/filter/RangerKRBAuthenticationFilter.java +++ b/security-admin/src/main/java/org/apache/ranger/security/web/filter/RangerKRBAuthenticationFilter.java @@ -215,6 +215,28 @@ protected void doFilter(FilterChain filterChain, RangerAuthenticationProvider authenticationProvider = new RangerAuthenticationProvider(); Authentication authentication = authenticationProvider.authenticate(finalAuthentication); authentication = getGrantedAuthority(authentication); + if(authentication != null && authentication.isAuthenticated()) { + if (request.getParameterMap().containsKey("doAs")) { + if(!response.isCommitted()) { + if(LOG.isDebugEnabled()) { + LOG.debug("Request contains unsupported parameter, doAs."); + } + request.setAttribute("spnegoenabled", false); + response.sendError(HttpServletResponse.SC_FORBIDDEN, "Missing authentication token."); + } + } + if(request.getParameterMap().containsKey("user.name")) { + if(!response.isCommitted()) { + if(LOG.isDebugEnabled()) { + LOG.debug("Request contains an unsupported parameter user.name"); + } + request.setAttribute("spnegoenabled", false); + response.sendError(HttpServletResponse.SC_FORBIDDEN, "Missing authentication token."); + } else { + LOG.info("Response seems to be already committed for user.name."); + } + } + } SecurityContextHolder.getContext().setAuthentication(authentication); request.setAttribute("spnegoEnabled", true); LOG.info("Logged into Ranger as = "+userName); From 2b1b92373160fa3215ae5fb1a9caf90161c2ba3e Mon Sep 17 00:00:00 2001 From: Velmurugan Periasamy Date: Tue, 24 Apr 2018 14:46:38 -0400 Subject: [PATCH 113/151] RANGER-2080: Add json-smart explicitly to security-admin/pom.xml Signed-off-by: Velmurugan Periasamy --- security-admin/pom.xml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/security-admin/pom.xml b/security-admin/pom.xml index fb76d8683e..28f2b9ca28 100644 --- a/security-admin/pom.xml +++ b/security-admin/pom.xml @@ -454,6 +454,15 @@ 4.41.2 compile + + + net.minidev + json-smart + 2.3 + com.google.inject guice From b47faac6f5fb3fffdf1c3760146fc397653523b5 Mon Sep 17 00:00:00 2001 From: pradeep Date: Thu, 26 Jul 2018 13:37:41 +0530 Subject: [PATCH 114/151] RANGER-2162: Upgrade c3p0 libraries --- pom.xml | 2 +- security-admin/pom.xml | 2 +- .../src/main/webapp/META-INF/applicationContext.xml | 7 +++++-- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/pom.xml b/pom.xml index 214df89763..c415e538d7 100644 --- a/pom.xml +++ b/pom.xml @@ -133,7 +133,7 @@ 1.1.3 1.7.1 1.55 - 0.9.1.2 + 0.9.5.2 2.2.0-b23 1.9.13 1.9.3 diff --git a/security-admin/pom.xml b/security-admin/pom.xml index 28f2b9ca28..8283e31594 100644 --- a/security-admin/pom.xml +++ b/security-admin/pom.xml @@ -43,7 +43,7 @@ ${asm.version} - c3p0 + com.mchange c3p0 ${c3p0.version} diff --git a/security-admin/src/main/webapp/META-INF/applicationContext.xml b/security-admin/src/main/webapp/META-INF/applicationContext.xml index 79d20d3aee..aff0d6b6f9 100644 --- a/security-admin/src/main/webapp/META-INF/applicationContext.xml +++ b/security-admin/src/main/webapp/META-INF/applicationContext.xml @@ -170,8 +170,11 @@ http://www.springframework.org/schema/util/spring-util.xsd"> ${ranger.jpa.jdbc.idleconnectiontestperiod} - - + + 1 + + + ${ranger.jpa.jdbc.driver} From 475b5290aa1b78f8dbcd1d56d36af719660967ed Mon Sep 17 00:00:00 2001 From: Abhay Kulkarni Date: Wed, 26 Sep 2018 18:19:29 -0700 Subject: [PATCH 115/151] RANGER-2158: Performance improvement to REST API call to update policy --- .../validation/RangerPolicyValidator.java | 95 +++++++++---------- .../model/validation/RangerValidator.java | 22 ++--- .../ranger/plugin/store/ServiceStore.java | 2 + .../validation/TestRangerPolicyValidator.java | 53 ++++------- .../model/validation/TestRangerValidator.java | 38 -------- .../org/apache/ranger/biz/ServiceDBStore.java | 19 +++- .../ranger/service/RangerPolicyService.java | 15 ++- 7 files changed, 98 insertions(+), 146 deletions(-) diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerPolicyValidator.java b/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerPolicyValidator.java index 0c82b7edbb..e48e5e1e1b 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerPolicyValidator.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerPolicyValidator.java @@ -153,72 +153,67 @@ boolean isValid(RangerPolicy policy, Action action, boolean isAdmin, List policies = getPolicies(serviceName, policyName); - if (CollectionUtils.isNotEmpty(policies)) { - if (policies.size() > 1) { - ValidationErrorCode error = ValidationErrorCode.POLICY_VALIDATION_ERR_POLICY_NAME_MULTIPLE_POLICIES_WITH_SAME_NAME; - failures.add(new ValidationFailureDetailsBuilder() - .field("name") - .isAnInternalError() - .becauseOf(error.getMessage(policyName)) - .errorCode(error.getErrorCode()) - .build()); - valid = false; - } else if (action == Action.CREATE) { // size == 1 - ValidationErrorCode error = ValidationErrorCode.POLICY_VALIDATION_ERR_POLICY_NAME_CONFLICT; - failures.add(new ValidationFailureDetailsBuilder() - .field("policy name") - .isSemanticallyIncorrect() - .becauseOf(error.getMessage(policies.iterator().next().getId(), serviceName)) - .errorCode(error.getErrorCode()) - .build()); - valid = false; - } else if (!policies.iterator().next().getId().equals(id)) { // size == 1 && action == UPDATE - ValidationErrorCode error = ValidationErrorCode.POLICY_VALIDATION_ERR_POLICY_NAME_CONFLICT; - failures.add(new ValidationFailureDetailsBuilder() - .field("id/name") + service = getService(serviceName); + if (service == null) { + ValidationErrorCode error = ValidationErrorCode.POLICY_VALIDATION_ERR_INVALID_SERVICE_NAME; + failures.add(new ValidationFailureDetailsBuilder() + .field("service name") .isSemanticallyIncorrect() - .becauseOf(error.getMessage(policies.iterator().next().getId(), serviceName)) + .becauseOf(error.getMessage(serviceName)) .errorCode(error.getErrorCode()) .build()); - valid = false; - } + valid = false; + } else { + serviceNameValid = true; } } - RangerService service = null; - boolean serviceNameValid = false; - if (StringUtils.isBlank(serviceName)) { + + if (StringUtils.isBlank(policyName)) { ValidationErrorCode error = ValidationErrorCode.POLICY_VALIDATION_ERR_MISSING_FIELD; failures.add(new ValidationFailureDetailsBuilder() - .field("service name") + .field("name") .isMissing() - .becauseOf(error.getMessage("service name")) + .becauseOf(error.getMessage("name")) .errorCode(error.getErrorCode()) .build()); valid = false; } else { - service = getService(serviceName); - if (service == null) { - ValidationErrorCode error = ValidationErrorCode.POLICY_VALIDATION_ERR_INVALID_SERVICE_NAME; - failures.add(new ValidationFailureDetailsBuilder() - .field("service name") - .isSemanticallyIncorrect() - .becauseOf(error.getMessage(serviceName)) - .errorCode(error.getErrorCode()) - .build()); - valid = false; - } else { - serviceNameValid = true; + if (service != null) { + Long policyId = getPolicyId(service.getId(), policyName); + if (policyId != null) { + if (action == Action.CREATE) { + ValidationErrorCode error = ValidationErrorCode.POLICY_VALIDATION_ERR_POLICY_NAME_CONFLICT; + failures.add(new ValidationFailureDetailsBuilder() + .field("policy name") + .isSemanticallyIncorrect() + .becauseOf(error.getMessage(policyId, serviceName)) + .errorCode(error.getErrorCode()) + .build()); + valid = false; + } else if (!policyId.equals(id)) { // action == UPDATE + ValidationErrorCode error = ValidationErrorCode.POLICY_VALIDATION_ERR_POLICY_NAME_CONFLICT; + failures.add(new ValidationFailureDetailsBuilder() + .field("id/name") + .isSemanticallyIncorrect() + .becauseOf(error.getMessage(policyId, serviceName)) + .errorCode(error.getErrorCode()) + .build()); + valid = false; + } + } } } diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerValidator.java b/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerValidator.java index 3400d816ff..3ae02bfd60 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerValidator.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerValidator.java @@ -246,29 +246,23 @@ RangerPolicy getPolicy(Long id) { return result; } - List getPolicies(final String serviceName, final String policyName) { + Long getPolicyId(final Long serviceId, final String policyName) { if(LOG.isDebugEnabled()) { - LOG.debug("==> RangerValidator.getPolicies(" + serviceName + ", " + policyName + ")"); + LOG.debug("==> RangerValidator.getPolicyId(" + serviceId + ", " + policyName + ")"); } - List policies = null; + Long policyId = null; try { - SearchFilter filter = new SearchFilter(); - if (StringUtils.isNotBlank(policyName)) { - filter.setParam(SearchFilter.POLICY_NAME, policyName); - } - filter.setParam(SearchFilter.SERVICE_NAME, serviceName); - - policies = _store.getPolicies(filter); + policyId = _store.getPolicyId(serviceId, policyName); + } catch (Exception e) { LOG.debug("Encountred exception while retrieving service from service store!", e); } - + if(LOG.isDebugEnabled()) { - int count = policies == null ? 0 : policies.size(); - LOG.debug("<== RangerValidator.getPolicies(" + serviceName + ", " + policyName + "): count[" + count + "], " + policies); + LOG.debug("<== RangerValidator.getPolicyId(" + serviceId + ", " + policyName + "): policy-id[" + policyId + "]"); } - return policies; + return policyId; } List getPoliciesForResourceSignature(String serviceName, String policySignature) { diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/store/ServiceStore.java b/agents-common/src/main/java/org/apache/ranger/plugin/store/ServiceStore.java index 2c57a6fd8d..9924cb4c40 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/store/ServiceStore.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/store/ServiceStore.java @@ -74,6 +74,8 @@ public interface ServiceStore { List getPolicies(SearchFilter filter) throws Exception; + Long getPolicyId(final Long serviceId, final String policyName); + PList getPaginatedPolicies(SearchFilter filter) throws Exception; List getPoliciesByResourceSignature(String serviceName, String policySignature, Boolean isPolicyEnabled) throws Exception; diff --git a/agents-common/src/test/java/org/apache/ranger/plugin/model/validation/TestRangerPolicyValidator.java b/agents-common/src/test/java/org/apache/ranger/plugin/model/validation/TestRangerPolicyValidator.java index caa8e35438..97a3ea75f9 100644 --- a/agents-common/src/test/java/org/apache/ranger/plugin/model/validation/TestRangerPolicyValidator.java +++ b/agents-common/src/test/java/org/apache/ranger/plugin/model/validation/TestRangerPolicyValidator.java @@ -42,7 +42,6 @@ import org.apache.ranger.plugin.model.validation.RangerValidator.Action; import org.apache.ranger.plugin.store.ServiceStore; import org.apache.ranger.plugin.util.RangerObjectFactory; -import org.apache.ranger.plugin.util.SearchFilter; import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -229,6 +228,7 @@ public final void testIsValid_happyPath() throws Exception { // service name exists RangerService service = mock(RangerService.class); when(service.getType()).thenReturn("service-type"); + when(service.getId()).thenReturn(2L); when(_store.getServiceByName("service-name")).thenReturn(service); // service points to a valid service-def _serviceDef = _utils.createServiceDefWithAccessTypes(accessTypes); @@ -240,17 +240,7 @@ public final void testIsValid_happyPath() throws Exception { when(existingPolicy.getId()).thenReturn(8L); when(existingPolicy.getService()).thenReturn("service-name"); when(_store.getPolicy(8L)).thenReturn(existingPolicy); - SearchFilter createFilter = new SearchFilter(); - createFilter.setParam(SearchFilter.SERVICE_TYPE, "service-type"); - createFilter.setParam(SearchFilter.POLICY_NAME, "policy-name-1"); // this name would be used for create - when(_store.getPolicies(createFilter)).thenReturn(new ArrayList()); // a matching policy should not exist for update. - SearchFilter updateFilter = new SearchFilter(); - updateFilter.setParam(SearchFilter.SERVICE_TYPE, "service-type"); - updateFilter.setParam(SearchFilter.POLICY_NAME, "policy-name-2"); // this name would be used for update - List existingPolicies = new ArrayList(); - existingPolicies.add(existingPolicy); - when(_store.getPolicies(updateFilter)).thenReturn(existingPolicies); // valid policy can have empty set of policy items if audit is turned on // null value for audit is treated as audit on. // for now we want to turn any resource related checking off @@ -262,6 +252,7 @@ public final void testIsValid_happyPath() throws Exception { if (action == Action.CREATE) { when(_policy.getId()).thenReturn(7L); when(_policy.getName()).thenReturn("policy-name-1"); + when(_store.getPolicyId(service.getId(), _policy.getName())).thenReturn(null); Assert.assertTrue("" + action + ", " + auditEnabled, _validator.isValid(_policy, action, isAdmin, _failures)); Assert.assertTrue(_failures.isEmpty()); } else { @@ -272,6 +263,7 @@ public final void testIsValid_happyPath() throws Exception { Assert.assertTrue(_failures.isEmpty()); when(_policy.getName()).thenReturn("policy-name-2"); + when(_store.getPolicyId(service.getId(), _policy.getName())).thenReturn(null); Assert.assertTrue("" + action + ", " + auditEnabled, _validator.isValid(_policy, action, isAdmin, _failures)); Assert.assertTrue(_failures.isEmpty()); } @@ -370,20 +362,22 @@ public final void testIsValid_failures() throws Exception { checkFailure_isValid(action, "missing", "id"); } } + RangerService service = mock(RangerService.class); /* * Id is ignored for Create but name should not belong to an existing policy. For update, policy should exist for its id and should match its name. */ when(_policy.getName()).thenReturn("policy-name"); when(_policy.getService()).thenReturn("service-name"); + when(_store.getServiceByName("service-name")).thenReturn(service); + when(service.getId()).thenReturn(2L); + RangerPolicy existingPolicy = mock(RangerPolicy.class); when(existingPolicy.getId()).thenReturn(7L); - List existingPolicies = new ArrayList(); - existingPolicies.add(existingPolicy); - SearchFilter filter = new SearchFilter(); - filter.setParam(SearchFilter.SERVICE_NAME, "service-name"); - filter.setParam(SearchFilter.POLICY_NAME, "policy-name"); - when(_store.getPolicies(filter)).thenReturn(existingPolicies); + when(existingPolicy.getService()).thenReturn("service-name"); + List existingPolicies = new ArrayList<>(); + + when(_store.getPolicyId(service.getId(), "policy-name")).thenReturn(7L); checkFailure_isValid(Action.CREATE, "semantic", "policy name"); // update : does not exist for id @@ -395,21 +389,11 @@ public final void testIsValid_failures() throws Exception { when(_store.getPolicy(7L)).thenReturn(existingPolicy); RangerPolicy anotherExistingPolicy = mock(RangerPolicy.class); when(anotherExistingPolicy.getId()).thenReturn(8L); - existingPolicies.clear(); + when(anotherExistingPolicy.getService()).thenReturn("service-name"); + existingPolicies.add(anotherExistingPolicy); - when(_store.getPolicies(filter)).thenReturn(existingPolicies); + when(_store.getPolicyId(service.getId(), "policy-name")).thenReturn(8L); checkFailure_isValid(Action.UPDATE, "semantic", "id/name"); - - // more than one policies with same name is also an internal error - when(_policy.getName()).thenReturn("policy-name"); - when(_store.getPolicies(filter)).thenReturn(existingPolicies); - existingPolicies.add(existingPolicy); - existingPolicy = mock(RangerPolicy.class); - existingPolicies.add(existingPolicy); - for (boolean isAdmin : new boolean[] { true, false }) { - _failures.clear(); Assert.assertFalse(_validator.isValid(_policy, Action.UPDATE, isAdmin, _failures)); - _utils.checkFailureForInternalError(_failures); - } // policy must have service name on it and it should be valid when(_policy.getName()).thenReturn("policy-name"); @@ -449,10 +433,7 @@ public final void testIsValid_failures() throws Exception { } // policy must contain at least one policy item - List policyItems = new ArrayList(); - when(_policy.getService()).thenReturn("service-name"); - RangerService service = mock(RangerService.class); - when(_store.getServiceByName("service-name")).thenReturn(service); + List policyItems = new ArrayList<>(); for (Action action : cu) { for (boolean isAdmin : new boolean[] { true, false }) { // when it is null @@ -474,6 +455,8 @@ public final void testIsValid_failures() throws Exception { when(_store.getServiceDefByName("service-type")).thenReturn(null); for (Action action : cu) { for (boolean isAdmin : new boolean[] { true, false }) { + when(_policy.getService()).thenReturn("service-name"); + when(_store.getServiceByName("service-name")).thenReturn(service); _failures.clear(); Assert.assertFalse(_validator.isValid(_policy, action, isAdmin, _failures)); _utils.checkFailureForInternalError(_failures, "policy service def"); } @@ -491,7 +474,7 @@ public final void testIsValid_failures() throws Exception { // create the right service def with right resource defs - this is the same as in the happypath test above. _serviceDef = _utils.createServiceDefWithAccessTypes(accessTypes, "service-type"); - when(_store.getPolicies(filter)).thenReturn(null); + when(_store.getPolicyId(service.getId(), "policy-name")).thenReturn(null); List resourceDefs = _utils.createResourceDefs(resourceDefData); when(_serviceDef.getResources()).thenReturn(resourceDefs); when(_store.getServiceDefByName("service-type")).thenReturn(_serviceDef); diff --git a/agents-common/src/test/java/org/apache/ranger/plugin/model/validation/TestRangerValidator.java b/agents-common/src/test/java/org/apache/ranger/plugin/model/validation/TestRangerValidator.java index 5519a2c060..fb8073fc47 100644 --- a/agents-common/src/test/java/org/apache/ranger/plugin/model/validation/TestRangerValidator.java +++ b/agents-common/src/test/java/org/apache/ranger/plugin/model/validation/TestRangerValidator.java @@ -21,7 +21,6 @@ import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import java.util.ArrayList; @@ -43,7 +42,6 @@ import org.apache.ranger.plugin.model.RangerServiceDef.RangerServiceConfigDef; import org.apache.ranger.plugin.model.validation.RangerValidator.Action; import org.apache.ranger.plugin.store.ServiceStore; -import org.apache.ranger.plugin.util.SearchFilter; import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -410,42 +408,6 @@ public void test_getIsAuditEnabled() { result = _validator.getIsAuditEnabled(policy); Assert.assertTrue(result); } - - @Test - public void test_getPolicies() throws Exception { - - // returns null when store returns null - String policyName = "aPolicy"; - String serviceName = "aService"; - SearchFilter filter = new SearchFilter(); - filter.setParam(SearchFilter.POLICY_NAME, policyName); - filter.setParam(SearchFilter.SERVICE_NAME, serviceName); - - when(_store.getPolicies(filter)).thenReturn(null); - List result = _validator.getPolicies(serviceName, policyName); - // validate store is queried with both parameters - verify(_store).getPolicies(filter); - Assert.assertNull(result); - - // returns null if store throws an exception - when(_store.getPolicies(filter)).thenThrow(new Exception()); - result = _validator.getPolicies(serviceName, policyName); - Assert.assertNull(result); - - // does not shove policy into search filter if policy name passed in is "blank" - filter = new SearchFilter(); - filter.setParam(SearchFilter.SERVICE_NAME, serviceName); - - List policies = new ArrayList(); - RangerPolicy policy = mock(RangerPolicy.class); - policies.add(policy); - - when(_store.getPolicies(filter)).thenReturn(policies); - for (String aName : new String[]{ null, "", " "}) { - result = _validator.getPolicies(serviceName, aName); - Assert.assertTrue(result.iterator().next() == policy); - } - } @Test public void test_getServiceDef_byId() throws Exception { diff --git a/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java b/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java index 6a1ef09771..ed6ddac671 100644 --- a/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java +++ b/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java @@ -1948,7 +1948,7 @@ public RangerPolicy updatePolicy(RangerPolicy policy) throws Exception { policy.setGuid(xxExisting.getGuid()); policy.setVersion(xxExisting.getVersion()); - List trxLogList = policyService.getTransactionLog(policy, xxExisting, RangerPolicyService.OPERATION_UPDATE_CONTEXT); + List trxLogList = policyService.getTransactionLog(policy, xxExisting, existing, RangerPolicyService.OPERATION_UPDATE_CONTEXT); updatePolicySignature(policy); @@ -2070,6 +2070,23 @@ public List getPolicies(SearchFilter filter) throws Exception { return ret; } + @Override + public Long getPolicyId(final Long serviceId, final String policyName) { + if(LOG.isDebugEnabled()) { + LOG.debug("==> ServiceDBStore.getPolicyId()"); + } + Long ret = null; + XXPolicy xxPolicy = daoMgr.getXXPolicy().findByNameAndServiceId(policyName, serviceId); + if (xxPolicy != null) { + ret = xxPolicy.getId(); + } + if(LOG.isDebugEnabled()) { + LOG.debug("<== ServiceDBStore.getPolicyId()"); + } + return ret; + } + + public void getPoliciesInExcel(List policies, HttpServletResponse response) throws Exception { if (LOG.isDebugEnabled()) { LOG.debug("==> ServiceDBStore.getPoliciesInExcel()"); diff --git a/security-admin/src/main/java/org/apache/ranger/service/RangerPolicyService.java b/security-admin/src/main/java/org/apache/ranger/service/RangerPolicyService.java index ecefc4bba5..e8b43feca3 100644 --- a/security-admin/src/main/java/org/apache/ranger/service/RangerPolicyService.java +++ b/security-admin/src/main/java/org/apache/ranger/service/RangerPolicyService.java @@ -128,10 +128,10 @@ public RangerPolicy getPopulatedViewObject(XXPolicy xPolicy) { } public List getTransactionLog(RangerPolicy vPolicy, int action) { - return getTransactionLog(vPolicy, null, action); + return getTransactionLog(vPolicy, null, null, action); } - public List getTransactionLog(RangerPolicy vObj, XXPolicy mObj, int action) { + public List getTransactionLog(RangerPolicy vObj, XXPolicy mObj, RangerPolicy oldPolicy, int action) { if (vObj == null || action == 0 || (action == OPERATION_UPDATE_CONTEXT && mObj == null)) { return null; } @@ -147,7 +147,7 @@ public List getTransactionLog(RangerPolicy vObj, XXPolicy mObj, int ac if (!trxLogAttrs.containsKey(field.getName())) { continue; } - XXTrxLog xTrxLog = processFieldToCreateTrxLog(field, objectName, nameField, vObj, mObj, action); + XXTrxLog xTrxLog = processFieldToCreateTrxLog(field, objectName, nameField, vObj, mObj, oldPolicy, action); if (xTrxLog != null) { trxLogList.add(xTrxLog); } @@ -156,8 +156,8 @@ public List getTransactionLog(RangerPolicy vObj, XXPolicy mObj, int ac Field[] superClassFields = vObj.getClass().getSuperclass() .getDeclaredFields(); for (Field field : superClassFields) { - if (field.getName().equalsIgnoreCase("isEnabled")) { - XXTrxLog xTrx = processFieldToCreateTrxLog(field, objectName, nameField, vObj, mObj, action); + if ("isEnabled".equalsIgnoreCase(field.getName())) { + XXTrxLog xTrx = processFieldToCreateTrxLog(field, objectName, nameField, vObj, mObj, oldPolicy, action); if (xTrx != null) { trxLogList.add(xTrx); } @@ -174,7 +174,7 @@ public List getTransactionLog(RangerPolicy vObj, XXPolicy mObj, int ac } private XXTrxLog processFieldToCreateTrxLog(Field field, String objectName, - Field nameField, RangerPolicy vObj, XXPolicy mObj, int action) { + Field nameField, RangerPolicy vObj, XXPolicy mObj, RangerPolicy oldPolicy, int action) { String actionString = ""; @@ -260,8 +260,7 @@ private XXTrxLog processFieldToCreateTrxLog(Field field, String objectName, break; } } - RangerPolicy oldPolicy = populateViewBean(mObj); - if (fieldName.equalsIgnoreCase(POLICY_RESOURCE_CLASS_FIELD_NAME)) { + if (POLICY_RESOURCE_CLASS_FIELD_NAME.equalsIgnoreCase(fieldName)) { if (oldPolicy != null) { oldValue = processPolicyResourcesForTrxLog(oldPolicy.getResources()); } From 320169a8a796d2f7bf10a45fff4a39fcb80cec9a Mon Sep 17 00:00:00 2001 From: Abhay Kulkarni Date: Tue, 31 Jul 2018 15:15:33 -0700 Subject: [PATCH 116/151] RANGER-2165: Address JPA Cache issue when policies Create, Update and Delete are done via REST API in Apache Ranger admin --- .../ranger/biz/RangerPolicyRetriever.java | 83 ++++++++++++++-- .../ranger/biz/RangerTagDBRetriever.java | 99 +++++++++++++++++-- .../org/apache/ranger/biz/ServiceDBStore.java | 2 +- .../org/apache/ranger/biz/TagDBStore.java | 8 +- 4 files changed, 175 insertions(+), 17 deletions(-) diff --git a/security-admin/src/main/java/org/apache/ranger/biz/RangerPolicyRetriever.java b/security-admin/src/main/java/org/apache/ranger/biz/RangerPolicyRetriever.java index 1b6f4407f3..e867cf48b6 100644 --- a/security-admin/src/main/java/org/apache/ranger/biz/RangerPolicyRetriever.java +++ b/security-admin/src/main/java/org/apache/ranger/biz/RangerPolicyRetriever.java @@ -43,18 +43,36 @@ import org.apache.ranger.plugin.model.RangerPolicy.RangerRowFilterPolicyItem; import org.apache.ranger.plugin.policyevaluator.RangerPolicyItemEvaluator; import org.apache.ranger.plugin.util.RangerPerfTracer; - +import org.springframework.transaction.PlatformTransactionManager; +import org.springframework.transaction.TransactionStatus; +import org.springframework.transaction.support.TransactionCallback; +import org.springframework.transaction.support.TransactionTemplate; public class RangerPolicyRetriever { static final Log LOG = LogFactory.getLog(RangerPolicyRetriever.class); static final Log PERF_LOG = RangerPerfTracer.getPerfLogger("db.RangerPolicyRetriever"); - final RangerDaoManager daoMgr; - final LookupCache lookupCache; + private final RangerDaoManager daoMgr; + private final LookupCache lookupCache = new LookupCache(); + + private final PlatformTransactionManager txManager; + private final TransactionTemplate txTemplate; + + public RangerPolicyRetriever(RangerDaoManager daoMgr, PlatformTransactionManager txManager) { + this.daoMgr = daoMgr; + this.txManager = txManager; + if (this.txManager != null) { + this.txTemplate = new TransactionTemplate(this.txManager); + this.txTemplate.setReadOnly(true); + } else { + this.txTemplate = null; + } + } public RangerPolicyRetriever(RangerDaoManager daoMgr) { this.daoMgr = daoMgr; - this.lookupCache = new LookupCache(); + this.txManager = null; + this.txTemplate = null; } public List getServicePolicies(Long serviceId) { @@ -93,7 +111,41 @@ public List getServicePolicies(String serviceName) { return ret; } - public List getServicePolicies(XXService xService) { + private class PolicyLoaderThread extends Thread { + final TransactionTemplate txTemplate; + final XXService xService; + List policies; + + PolicyLoaderThread(TransactionTemplate txTemplate, final XXService xService) { + this.txTemplate = txTemplate; + this.xService = xService; + } + + public List getPolicies() { return policies; } + + @Override + public void run() { + try { + policies = txTemplate.execute(new TransactionCallback>() { + @Override + public List doInTransaction(TransactionStatus status) { + try { + RetrieverContext ctx = new RetrieverContext(xService); + return ctx.getAllPolicies(); + } catch (Exception ex) { + LOG.error("RangerPolicyRetriever.getServicePolicies(): Failed to get policies for service:[" + xService.getName() + "] in a new transaction", ex); + status.setRollbackOnly(); + return null; + } + } + }); + } catch (Throwable ex) { + LOG.error("RangerPolicyRetriever.getServicePolicies(): Failed to get policies for service:[" + xService.getName() + "] in a new transaction", ex); + } + } + } + + public List getServicePolicies(final XXService xService) { String serviceName = xService == null ? null : xService.getName(); Long serviceId = xService == null ? null : xService.getId(); @@ -109,9 +161,26 @@ public List getServicePolicies(XXService xService) { } if(xService != null) { - RetrieverContext ctx = new RetrieverContext(xService); + if (txTemplate == null) { + if (LOG.isDebugEnabled()) { + LOG.debug("Transaction Manager is null; Retrieving policies in the existing transaction"); + } + RetrieverContext ctx = new RetrieverContext(xService); + ret = ctx.getAllPolicies(); + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("Retrieving policies in a new, read-only transaction"); + } - ret = ctx.getAllPolicies(); + PolicyLoaderThread t = new PolicyLoaderThread(txTemplate, xService); + t.start(); + try { + t.join(); + ret = t.getPolicies(); + } catch (InterruptedException ie) { + LOG.error("Failed to retrieve policies in a new, read-only thread.", ie); + } + } } else { if(LOG.isDebugEnabled()) { LOG.debug("RangerPolicyRetriever.getServicePolicies(xService=" + xService + "): invalid parameter"); diff --git a/security-admin/src/main/java/org/apache/ranger/biz/RangerTagDBRetriever.java b/security-admin/src/main/java/org/apache/ranger/biz/RangerTagDBRetriever.java index 52c12882c4..26d8d37942 100644 --- a/security-admin/src/main/java/org/apache/ranger/biz/RangerTagDBRetriever.java +++ b/security-admin/src/main/java/org/apache/ranger/biz/RangerTagDBRetriever.java @@ -35,6 +35,10 @@ import org.apache.ranger.plugin.model.*; import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyResource; import org.apache.ranger.plugin.util.RangerPerfTracer; +import org.springframework.transaction.PlatformTransactionManager; +import org.springframework.transaction.TransactionStatus; +import org.springframework.transaction.support.TransactionCallback; +import org.springframework.transaction.support.TransactionTemplate; public class RangerTagDBRetriever { @@ -46,6 +50,9 @@ public class RangerTagDBRetriever { private final XXService xService; private final LookupCache lookupCache; + private final PlatformTransactionManager txManager; + private final TransactionTemplate txTemplate; + private List serviceResources; private Map tagDefs; private Map tags; @@ -53,8 +60,15 @@ public class RangerTagDBRetriever { private boolean filterForServicePlugin; - public RangerTagDBRetriever(final RangerDaoManager daoMgr, final XXService xService) { + public RangerTagDBRetriever(final RangerDaoManager daoMgr, final PlatformTransactionManager txManager, final XXService xService) { this.daoMgr = daoMgr; + this.txManager = txManager; + if (this.txManager != null) { + this.txTemplate = new TransactionTemplate(this.txManager); + this.txTemplate.setReadOnly(true); + } else { + this.txTemplate = null; + } this.xService = xService; this.lookupCache = new LookupCache(); @@ -68,14 +82,27 @@ public RangerTagDBRetriever(final RangerDaoManager daoMgr, final XXService xServ } filterForServicePlugin = RangerConfiguration.getInstance().getBoolean(OPTION_RANGER_FILTER_TAGS_FOR_SERVICE_PLUGIN, false); - TagRetrieverServiceResourceContext serviceResourceContext = new TagRetrieverServiceResourceContext(xService); - TagRetrieverTagDefContext tagDefContext = new TagRetrieverTagDefContext(xService); - TagRetrieverTagContext tagContext = new TagRetrieverTagContext(xService); - serviceResources = serviceResourceContext.getAllServiceResources(); - tagDefs = tagDefContext.getAllTagDefs(); - tags = tagContext.getAllTags(); - tagResourceMaps = getAllTagResourceMaps(); + if (this.txTemplate == null) { + if (LOG.isDebugEnabled()) { + LOG.debug("Load Tags in the same thread and using an existing transaction"); + } + if (initializeTagCache(xService) == false) { + LOG.error("Failed to get tags for service:[" + xService.getName() + "] in the same thread and using an existing transaction"); + } + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("Load Tags in a separate thread and using a new transaction"); + } + + TagLoaderThread t = new TagLoaderThread(txTemplate, xService); + t.start(); + try { + t.join(); + } catch (InterruptedException ie) { + LOG.error("Failed to get Tags in a separate thread and using a new transaction", ie); + } + } RangerPerfTracer.log(perf); @@ -98,6 +125,30 @@ public Map getTags() { return tags; } + private boolean initializeTagCache(XXService xService) { + boolean ret; + try { + TagRetrieverServiceResourceContext serviceResourceContext = new TagRetrieverServiceResourceContext(xService); + TagRetrieverTagDefContext tagDefContext = new TagRetrieverTagDefContext(xService); + TagRetrieverTagContext tagContext = new TagRetrieverTagContext(xService); + + serviceResources = serviceResourceContext.getAllServiceResources(); + tagDefs = tagDefContext.getAllTagDefs(); + tags = tagContext.getAllTags(); + + tagResourceMaps = getAllTagResourceMaps(); + + ret = true; + } catch (Exception ex) { + LOG.error("Failed to get tags for service:[" + xService.getName() + "]"); + serviceResources = null; + tagDefs = null; + tags = null; + tagResourceMaps = null; + ret = false; + } + return ret; + } private List getAllTagResourceMaps() { List xTagResourceMaps = filterForServicePlugin ? daoMgr.getXXTagResourceMap().findForServicePlugin(xService.getId()) : daoMgr.getXXTagResourceMap().findByServiceId(xService.getId()); @@ -198,6 +249,38 @@ String getResourceName(Long resourceDefId) { } } + private class TagLoaderThread extends Thread { + final TransactionTemplate txTemplate; + final XXService xService; + + TagLoaderThread(TransactionTemplate txTemplate, final XXService xService) { + this.txTemplate = txTemplate; + this.xService = xService; + } + + @Override + public void run() { + try { + Boolean result = txTemplate.execute(new TransactionCallback() { + @Override + public Boolean doInTransaction(TransactionStatus status) { + boolean ret = initializeTagCache(xService); + if (!ret) { + status.setRollbackOnly(); + LOG.error("Failed to get tags for service:[" + xService.getName() + "] in a new transaction"); + } + return ret; + } + }); + if (LOG.isDebugEnabled()) { + LOG.debug("transaction result:[" + result +"]"); + } + } catch (Throwable ex) { + LOG.error("Failed to get tags for service:[" + xService.getName() + "] in a new transaction", ex); + } + } + } + private class TagRetrieverServiceResourceContext { final XXService service; diff --git a/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java b/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java index ed6ddac671..64cf043b76 100644 --- a/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java +++ b/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java @@ -2407,7 +2407,7 @@ private List getServicePoliciesFromDb(XXService service) throws Ex LOG.debug("==> ServiceDBStore.getServicePoliciesFromDb(" + service.getName() + ")"); } - RangerPolicyRetriever policyRetriever = new RangerPolicyRetriever(daoMgr); + RangerPolicyRetriever policyRetriever = new RangerPolicyRetriever(daoMgr, txManager); List ret = policyRetriever.getServicePolicies(service); diff --git a/security-admin/src/main/java/org/apache/ranger/biz/TagDBStore.java b/security-admin/src/main/java/org/apache/ranger/biz/TagDBStore.java index a115bb7443..3234be6a29 100644 --- a/security-admin/src/main/java/org/apache/ranger/biz/TagDBStore.java +++ b/security-admin/src/main/java/org/apache/ranger/biz/TagDBStore.java @@ -62,7 +62,9 @@ import org.apache.ranger.service.RangerTagService; import org.apache.ranger.service.RangerServiceResourceService; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Component; +import org.springframework.transaction.PlatformTransactionManager; import javax.annotation.PostConstruct; import javax.servlet.http.HttpServletResponse; @@ -86,6 +88,10 @@ public class TagDBStore extends AbstractTagStore { @Autowired RangerDaoManager daoManager; + @Autowired + @Qualifier(value = "transactionManager") + PlatformTransactionManager txManager; + @Autowired RESTErrorUtil errorUtil; @@ -985,7 +991,7 @@ public ServiceTags getServiceTags(String serviceName) throws Exception { throw new Exception("service-def does not exist. id=" + xxService.getType()); } - RangerTagDBRetriever tagDBRetriever = new RangerTagDBRetriever(daoManager, xxService); + RangerTagDBRetriever tagDBRetriever = new RangerTagDBRetriever(daoManager, txManager, xxService); Map tagDefMap = tagDBRetriever.getTagDefs(); Map tagMap = tagDBRetriever.getTags(); From dddcf0155f63c35a347755c41e7063ccef93d308 Mon Sep 17 00:00:00 2001 From: Abhay Kulkarni Date: Tue, 14 Aug 2018 10:49:34 -0700 Subject: [PATCH 117/151] RANGER-2165: Address JPA Cache issue when policies Create, Update and Delete are done via REST API in Apache Ranger admin --- .../main/java/org/apache/ranger/biz/RangerPolicyRetriever.java | 1 + .../main/java/org/apache/ranger/biz/RangerTagDBRetriever.java | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/security-admin/src/main/java/org/apache/ranger/biz/RangerPolicyRetriever.java b/security-admin/src/main/java/org/apache/ranger/biz/RangerPolicyRetriever.java index e867cf48b6..7d1e15b002 100644 --- a/security-admin/src/main/java/org/apache/ranger/biz/RangerPolicyRetriever.java +++ b/security-admin/src/main/java/org/apache/ranger/biz/RangerPolicyRetriever.java @@ -126,6 +126,7 @@ private class PolicyLoaderThread extends Thread { @Override public void run() { try { + txTemplate.setReadOnly(true); policies = txTemplate.execute(new TransactionCallback>() { @Override public List doInTransaction(TransactionStatus status) { diff --git a/security-admin/src/main/java/org/apache/ranger/biz/RangerTagDBRetriever.java b/security-admin/src/main/java/org/apache/ranger/biz/RangerTagDBRetriever.java index 26d8d37942..27856ae795 100644 --- a/security-admin/src/main/java/org/apache/ranger/biz/RangerTagDBRetriever.java +++ b/security-admin/src/main/java/org/apache/ranger/biz/RangerTagDBRetriever.java @@ -261,7 +261,8 @@ private class TagLoaderThread extends Thread { @Override public void run() { try { - Boolean result = txTemplate.execute(new TransactionCallback() { + txTemplate.setReadOnly(true); + Boolean result = txTemplate.execute(new TransactionCallback() { @Override public Boolean doInTransaction(TransactionStatus status) { boolean ret = initializeTagCache(xService); From 6a6e955b3af41253b1a949e42f202e90875a4d06 Mon Sep 17 00:00:00 2001 From: Abhay Kulkarni Date: Tue, 31 Jul 2018 16:30:47 -0700 Subject: [PATCH 118/151] RANGER-2173: Optimize Trie constuction and Policy lookup --- .../plugin/util/RangerResourceTrie.java | 539 +++++++++++------- agents-common/src/test/resources/log4j.xml | 14 + 2 files changed, 348 insertions(+), 205 deletions(-) diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerResourceTrie.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerResourceTrie.java index 189a72b29c..a255566ee2 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerResourceTrie.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerResourceTrie.java @@ -21,6 +21,7 @@ import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyResource; @@ -30,7 +31,7 @@ import org.apache.ranger.plugin.resourcematcher.RangerResourceMatcher; import java.util.ArrayList; -import java.util.Collections; +import java.util.Collection; import java.util.Comparator; import java.util.HashMap; import java.util.List; @@ -38,14 +39,17 @@ public class RangerResourceTrie { private static final Log LOG = LogFactory.getLog(RangerResourceTrie.class); + private static final Log PERF_TRIE_INIT_LOG = RangerPerfTracer.getPerfLogger("resourcetrie.init"); + private static final Log PERF_TRIE_OP_LOG = RangerPerfTracer.getPerfLogger("resourcetrie.op"); private static final String DEFAULT_WILDCARD_CHARS = "*?"; - private final String resourceName; - private final boolean optIgnoreCase; - private final boolean optWildcard; - private final String wildcardChars; - private final TrieNode root; + private final String resourceName; + private final boolean optIgnoreCase; + private final boolean optWildcard; + private final String wildcardChars; + private final TrieNode root; + private final Comparator comparator; public RangerResourceTrie(RangerServiceDef.RangerResourceDef resourceDef, List evaluators) { this(resourceDef, evaluators, null); @@ -56,6 +60,12 @@ public RangerResourceTrie(RangerServiceDef.RangerResourceDef resourceDef, List RangerResourceTrie(" + resourceDef.getName() + ", evaluatorCount=" + evaluators.size() + ")"); } + RangerPerfTracer perf = null; + + if(RangerPerfTracer.isPerfTraceEnabled(PERF_TRIE_INIT_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_TRIE_INIT_LOG, "RangerResourceTrie(name=" + resourceDef.getName() + ")"); + } + Map matcherOptions = resourceDef.getMatcherOptions(); boolean optReplaceTokens = RangerAbstractResourceMatcher.getOptionReplaceTokens(matcherOptions); @@ -76,7 +86,8 @@ public RangerResourceTrie(RangerServiceDef.RangerResourceDef resourceDef, List(null); + this.comparator = comparator; for(T evaluator : evaluators) { Map policyResources = evaluator.getPolicyResource(); @@ -109,7 +120,15 @@ public RangerResourceTrie(RangerServiceDef.RangerResourceDef resourceDef, List curr = root; + + final String prefix = getNonWildcardPrefix(resource); + final boolean isWildcard = prefix.length() != resource.length(); + + if (StringUtils.isNotEmpty(prefix)) { + curr = curr.getOrCreateChild(prefix); + } + + if(isWildcard || isRecursive) { + curr.addWildcardEvaluator(evaluator); + } else { + curr.addEvaluator(evaluator); + } + + RangerPerfTracer.logAlways(perf); + } + + private String getNonWildcardPrefix(String str) { + if (!optWildcard) return str; + int minIndex = str.length(); + for (int i = 0; i < wildcardChars.length(); i++) { + int index = str.indexOf(wildcardChars.charAt(i)); + if (index != -1 && index < minIndex) { + minIndex = index; + } + } + return str.substring(0, minIndex); + } + public List getEvaluatorsForResource(String resource) { if(LOG.isDebugEnabled()) { LOG.debug("==> RangerResourceTrie.getEvaluatorsForResource(" + resource + ")"); } - List ret = null; + RangerPerfTracer perf = null; + + if(RangerPerfTracer.isPerfTraceEnabled(PERF_TRIE_OP_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_TRIE_OP_LOG, "RangerResourceTrie.getEvaluatorsForResource(resource=" + resource + ")"); + } + + TrieNode curr = root; - TrieNode curr = root; + final int len = resource.length(); + int i = 0; - final int len = resource.length(); - for(int i = 0; i < len; i++) { - Character ch = getLookupChar(resource.charAt(i)); - TrieNode child = curr.getChild(ch); + while (i < len) { + final TrieNode child = curr.getChild(getLookupChar(resource, i)); - if(child == null) { - ret = curr.getWildcardEvaluators(); - curr = null; // so that curr.getEvaluators() will not be called below + if (child == null) { break; } - curr = child; - } + final String childStr = child.getStr(); - if(ret == null) { - if(curr != null) { - ret = curr.getEvaluators(); + if (!resource.regionMatches(optIgnoreCase, i, childStr, 0, childStr.length())) { + break; } + + curr = child; + i += childStr.length(); } + List ret = i == len ? curr.getEvaluators() : curr.getWildcardEvaluators(); + + RangerPerfTracer.logAlways(perf); + if(LOG.isDebugEnabled()) { LOG.debug("<== RangerResourceTrie.getEvaluatorsForResource(" + resource + "): evaluatorCount=" + (ret == null ? 0 : ret.size())); } @@ -156,50 +242,55 @@ public List getEvaluatorsForResource(String resource) { return ret; } - public TrieData getTrieData() { - TrieData ret = new TrieData(); - - root.populateTrieData(ret); - ret.maxDepth = getMaxDepth(); + private List getEvaluatorsForResources(Collection resources) { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerResourceTrie.getEvaluatorsForResources(" + resources + ")"); + } - return ret; - } + List ret = null; + Map evaluatorsMap = null; - public int getMaxDepth() { - return root.getMaxDepth(); - } + for (String resource : resources) { + List resourceEvaluators = getEvaluatorsForResource(resource); - private final Character getLookupChar(char ch) { - if(optIgnoreCase) { - ch = Character.toLowerCase(ch); - } + if (CollectionUtils.isEmpty(resourceEvaluators)) { + continue; + } - return Character.valueOf(ch); - } + if (evaluatorsMap == null) { + if (ret == null) { // first resource: don't create map yet + ret = resourceEvaluators; + } else if (ret != resourceEvaluators) { // if evaluator list is same as earlier resources, retain the list, else create a map + evaluatorsMap = new HashMap<>(); - private void insert(String resource, boolean isRecursive, T evaluator) { - TrieNode curr = root; - boolean isWildcard = false; + for (T evaluator : ret) { + evaluatorsMap.put(evaluator.getId(), evaluator); + } - final int len = resource.length(); - for(int i = 0; i < len; i++) { - Character ch = getLookupChar(resource.charAt(i)); + ret = null; + } + } - if(optWildcard) { - if (wildcardChars.indexOf(ch) != -1) { - isWildcard = true; - break; + if (evaluatorsMap != null) { + for (T evaluator : resourceEvaluators) { + evaluatorsMap.put(evaluator.getId(), evaluator); } } + } - curr = curr.getOrCreateChild(ch); + if (ret == null && evaluatorsMap != null) { + ret = new ArrayList<>(evaluatorsMap.values()); + + if (comparator != null) { + ret.sort(comparator); + } } - if(isWildcard || isRecursive) { - curr.addWildcardEvaluator(evaluator); - } else { - curr.addEvaluator(evaluator); + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerResourceTrie.getEvaluatorsForResources(" + resources + "): evaluatorCount=" + (ret == null ? 0 : ret.size())); } + + return ret; } @Override @@ -224,219 +315,257 @@ public String toString() { return sb.toString(); } - public class TrieData { - int nodeCount = 0; - int leafNodeCount = 0; - int singleChildNodeCount = 0; - int maxDepth = 0; - int evaluatorListCount = 0; - int wildcardEvaluatorListCount = 0; - int evaluatorListRefCount = 0; - int wildcardEvaluatorListRefCount = 0; + class TrieData { + int nodeCount; + int leafNodeCount; + int singleChildNodeCount; + int maxDepth; + int evaluatorListCount; + int wildcardEvaluatorListCount; + int evaluatorListRefCount; + int wildcardEvaluatorListRefCount; } -} -class TrieNode { - private final Character c; - private Map children = null; - private List evaluators = null; - private List wildcardEvaluators = null; - private boolean isSharingParentWildcardEvaluators = false; + class TrieNode { + private String str; + private Map> children = new HashMap<>(); + private List evaluators; + private List wildcardEvaluators; + private boolean isSharingParentWildcardEvaluators; - TrieNode(Character c) { - this.c = c; - } + TrieNode(String str) { + this.str = str; + } - Character getChar() { - return c; - } + String getStr() { + return str; + } - Map getChildren() { - return children; - } + void setStr(String str) { + this.str = str; + } - List getEvaluators() { - return evaluators; - } + Map> getChildren() { + return children; + } - List getWildcardEvaluators() { - return wildcardEvaluators; - } + List getEvaluators() { + return evaluators; + } - TrieNode getChild(Character c) { - TrieNode ret = children == null ? null : children.get(c); + List getWildcardEvaluators() { + return wildcardEvaluators; + } - return ret; - } + TrieNode getChild(Character ch) { + return children == null ? null : children.get(ch); + } - void populateTrieData(RangerResourceTrie.TrieData trieData) { - trieData.nodeCount++; + void populateTrieData(RangerResourceTrie.TrieData trieData) { + trieData.nodeCount++; - if(wildcardEvaluators != null) { - if(isSharingParentWildcardEvaluators) { - trieData.wildcardEvaluatorListRefCount++; - } else { - trieData.wildcardEvaluatorListCount++; + if (wildcardEvaluators != null) { + if (isSharingParentWildcardEvaluators) { + trieData.wildcardEvaluatorListRefCount++; + } else { + trieData.wildcardEvaluatorListCount++; + } } - } - if(evaluators != null) { - if(evaluators == wildcardEvaluators) { - trieData.evaluatorListRefCount++; - } else { - trieData.evaluatorListCount++; + if (evaluators != null) { + if (evaluators == wildcardEvaluators) { + trieData.evaluatorListRefCount++; + } else { + trieData.evaluatorListCount++; + } } - } - if(children != null && children.size() > 0) { - if(children.size() == 1) { - trieData.singleChildNodeCount++; - } + if (children != null && !children.isEmpty()) { + if (children.size() == 1) { + trieData.singleChildNodeCount++; + } - for(Map.Entry entry : children.entrySet()) { - TrieNode child = entry.getValue(); + for (Map.Entry> entry : children.entrySet()) { + TrieNode child = entry.getValue(); - child.populateTrieData(trieData); + child.populateTrieData(trieData); + } + } else { + trieData.leafNodeCount++; } - } else { - trieData.leafNodeCount++; } - } - int getMaxDepth() { - int ret = 0; + int getMaxDepth() { + int ret = 0; - if(children != null) { - for(Map.Entry entry : children.entrySet()) { - TrieNode child = entry.getValue(); + if (children != null) { + for (Map.Entry> entry : children.entrySet()) { + TrieNode child = entry.getValue(); - int maxChildDepth = child.getMaxDepth(); + int maxChildDepth = child.getMaxDepth(); - if(maxChildDepth > ret) { - ret = maxChildDepth; + if (maxChildDepth > ret) { + ret = maxChildDepth; + } } } - } - return ret + 1; - } - - TrieNode getOrCreateChild(Character c) { - if(children == null) { - children = new HashMap(); + return ret + 1; } - TrieNode child = children.get(c); + TrieNode getOrCreateChild(String str) { + int len = str.length(); - if(child == null) { - child = new TrieNode(c); - children.put(c, child); - } + TrieNode child = children.get(getLookupChar(str, 0)); - return child; - } + if (child == null) { + child = new TrieNode<>(str); + addChild(child); + } else { + final String childStr = child.getStr(); + final int childStrLen = childStr.length(); + + if (!StringUtils.equals(childStr, str)) { + final int numOfCharactersToMatch = childStrLen < len ? childStrLen : len; + int index = 1; + for (; index < numOfCharactersToMatch; index++) { + if (getLookupChar(childStr, index) != getLookupChar(str, index)) { + break; + } + } + if (index == numOfCharactersToMatch) { + // Matched all + if (childStrLen > len) { + // Existing node has longer string, need to break up this node + TrieNode newChild = new TrieNode<>(str); + this.addChild(newChild); + child.setStr(childStr.substring(index)); + newChild.addChild(child); + child = newChild; + } else { + // This is a longer string, build a child with leftover string + child = child.getOrCreateChild(str.substring(index)); + } + } else { + // Partial match for both; both have leftovers + String matchedPart = str.substring(0, index); + TrieNode newChild = new TrieNode<>(matchedPart); + this.addChild(newChild); + child.setStr(childStr.substring(index)); + newChild.addChild(child); + child = newChild.getOrCreateChild(str.substring(index)); + } + } + } - void addEvaluator(T evaluator) { - if(evaluators == null) { - evaluators = new ArrayList(); + return child; } - if(!evaluators.contains(evaluator)) { - evaluators.add(evaluator); + private void addChild(TrieNode child) { + children.put(getLookupChar(child.getStr(), 0), child); } - } - void addWildcardEvaluator(T evaluator) { - if(wildcardEvaluators == null) { - wildcardEvaluators = new ArrayList(); - } + void addEvaluator(U evaluator) { + if (evaluators == null) { + evaluators = new ArrayList<>(); + } - if(!wildcardEvaluators.contains(evaluator)) { - wildcardEvaluators.add(evaluator); + if (!evaluators.contains(evaluator)) { + evaluators.add(evaluator); + } } - } - void postSetup(List parentWildcardEvaluators, Comparator comparator) { - // finalize wildcard-evaluators list by including parent's wildcard evaluators - if(parentWildcardEvaluators != null) { - if(CollectionUtils.isEmpty(this.wildcardEvaluators)) { - this.wildcardEvaluators = parentWildcardEvaluators; - } else { - for (T evaluator : parentWildcardEvaluators) { - addWildcardEvaluator(evaluator); - } + void addWildcardEvaluator(U evaluator) { + if (wildcardEvaluators == null) { + wildcardEvaluators = new ArrayList<>(); + } + + if (!wildcardEvaluators.contains(evaluator)) { + wildcardEvaluators.add(evaluator); } } - this.isSharingParentWildcardEvaluators = wildcardEvaluators == parentWildcardEvaluators; - // finalize evaluators list by including wildcard evaluators - if(wildcardEvaluators != null) { - if(CollectionUtils.isEmpty(this.evaluators)) { - this.evaluators = wildcardEvaluators; - } else { - for (T evaluator : wildcardEvaluators) { - addEvaluator(evaluator); + void postSetup(List parentWildcardEvaluators, Comparator comparator) { + // finalize wildcard-evaluators list by including parent's wildcard evaluators + if (parentWildcardEvaluators != null) { + if (CollectionUtils.isEmpty(this.wildcardEvaluators)) { + this.wildcardEvaluators = parentWildcardEvaluators; + } else { + for (U evaluator : parentWildcardEvaluators) { + addWildcardEvaluator(evaluator); + } } } - } + this.isSharingParentWildcardEvaluators = wildcardEvaluators == parentWildcardEvaluators; - if (comparator != null) { - if (!isSharingParentWildcardEvaluators && CollectionUtils.isNotEmpty(wildcardEvaluators)) { - Collections.sort(wildcardEvaluators, comparator); + // finalize evaluators list by including wildcard evaluators + if (wildcardEvaluators != null) { + if (CollectionUtils.isEmpty(this.evaluators)) { + this.evaluators = wildcardEvaluators; + } else { + for (U evaluator : wildcardEvaluators) { + addEvaluator(evaluator); + } + } } - if (evaluators != wildcardEvaluators && CollectionUtils.isNotEmpty(evaluators)) { - Collections.sort(evaluators, comparator); + if (comparator != null) { + if (!isSharingParentWildcardEvaluators && CollectionUtils.isNotEmpty(wildcardEvaluators)) { + wildcardEvaluators.sort(comparator); + } + + if (evaluators != wildcardEvaluators && CollectionUtils.isNotEmpty(evaluators)) { + evaluators.sort(comparator); + } } - } - if(children != null) { - for(Map.Entry entry : children.entrySet()) { - TrieNode child = entry.getValue(); + if (children != null) { + for (Map.Entry> entry : children.entrySet()) { + TrieNode child = entry.getValue(); - child.postSetup(wildcardEvaluators, comparator); + child.postSetup(wildcardEvaluators, comparator); + } } } - } - public void toString(String prefix, StringBuilder sb) { - String nodeValue = prefix; + public void toString(String prefix, StringBuilder sb) { + String nodeValue = prefix; - if(c != 0) { - nodeValue += c; - } + if (str != null) { + nodeValue += str; + } - sb.append("nodeValue=").append(nodeValue); - sb.append("; childCount=").append(children == null ? 0 : children.size()); - sb.append("; evaluators=[ "); - if(evaluators != null) { - for(T evaluator : evaluators) { - sb.append(evaluator.getId()).append(" "); + sb.append("nodeValue=").append(nodeValue); + sb.append("; childCount=").append(children == null ? 0 : children.size()); + sb.append("; evaluators=[ "); + if (evaluators != null) { + for (U evaluator : evaluators) { + sb.append(evaluator.getId()).append(" "); + } } - } - sb.append("]"); + sb.append("]"); - sb.append("; wildcardEvaluators=[ "); - if(wildcardEvaluators != null) { - for(T evaluator : wildcardEvaluators) { - sb.append(evaluator.getId()).append(" "); + sb.append("; wildcardEvaluators=[ "); + if (wildcardEvaluators != null) { + for (U evaluator : wildcardEvaluators) { + sb.append(evaluator.getId()).append(" "); + } } - } - sb.append("]"); - sb.append(Character.LINE_SEPARATOR); + sb.append("]\n"); - if(children != null) { - for(Map.Entry entry : children.entrySet()) { - TrieNode child = entry.getValue(); + if (children != null) { + for (Map.Entry> entry : children.entrySet()) { + TrieNode child = entry.getValue(); - child.toString(nodeValue, sb); + child.toString(nodeValue, sb); + } } } - } - public void clear() { - children = null; - evaluators = null; - wildcardEvaluators = null; + public void clear() { + children = null; + evaluators = null; + wildcardEvaluators = null; + } } } diff --git a/agents-common/src/test/resources/log4j.xml b/agents-common/src/test/resources/log4j.xml index d863cf1d69..802c308bfb 100644 --- a/agents-common/src/test/resources/log4j.xml +++ b/agents-common/src/test/resources/log4j.xml @@ -35,6 +35,20 @@ + + + + + ranger.policyengine.trie.builder.thread.count + 1 + + \ No newline at end of file diff --git a/ranger-tools/testdata/ranger-config.xml b/ranger-tools/testdata/ranger-config.xml new file mode 100644 index 0000000000..933d6705a8 --- /dev/null +++ b/ranger-tools/testdata/ranger-config.xml @@ -0,0 +1,18 @@ + + + + + + ranger.policyengine.trie.builder.thread.count + 1 + + \ No newline at end of file From 822e76472c787aa1d18985430ecb5d5f0bc6457b Mon Sep 17 00:00:00 2001 From: Vishal Suvagia Date: Thu, 27 Sep 2018 14:28:02 +0530 Subject: [PATCH 127/151] RANGER-2172: Good coding practices for unix authentication Service in Ranger Signed-off-by: Sailaja Polavarapu (cherry picked from commit 1d47302f937e3fe1d565914c578faaf989e54424) --- src/main/assembly/usersync.xml | 4 ++-- unixauthnative/src/main/c/credValidator.c | 7 +++++-- unixauthpam/src/main/c/pamCredValidator.c | 9 ++++++--- 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/src/main/assembly/usersync.xml b/src/main/assembly/usersync.xml index 66cf3dd2c3..5d1efe869f 100644 --- a/src/main/assembly/usersync.xml +++ b/src/main/assembly/usersync.xml @@ -113,7 +113,7 @@ 755 - 755 + 750 /native unixauthnative/target @@ -122,7 +122,7 @@ 755 - 755 + 750 /native unixauthpam/target diff --git a/unixauthnative/src/main/c/credValidator.c b/unixauthnative/src/main/c/credValidator.c index 1c17e927a5..e426bdd2fd 100644 --- a/unixauthnative/src/main/c/credValidator.c +++ b/unixauthnative/src/main/c/credValidator.c @@ -23,11 +23,14 @@ #include #include +#define STRLEN 64 + int main(int ac, char **av, char **ev) { - char username[64] ; - char password[64] ; + char username[STRLEN] ; + char password[STRLEN] ; char line[512] ; + char format[20]; struct passwd *pwp; struct spwd *spwd ; diff --git a/unixauthpam/src/main/c/pamCredValidator.c b/unixauthpam/src/main/c/pamCredValidator.c index 0b45a6f32c..60d38aebdb 100644 --- a/unixauthpam/src/main/c/pamCredValidator.c +++ b/unixauthpam/src/main/c/pamCredValidator.c @@ -32,6 +32,8 @@ #include #include +#define STRLEN 64 + int pamconv(int num_msg, const struct pam_message **msg, struct pam_response **resp, void *appdata_ptr) { if (num_msg != 1 || msg[0]->msg_style != PAM_PROMPT_ECHO_OFF) { fprintf(stderr, "ERROR: Unexpected PAM conversation '%d/%s'\n", msg[0]->msg_style, msg[0]->msg); @@ -56,16 +58,17 @@ struct pam_conv conv = { pamconv, NULL }; int main(int ac, char **av, char **ev) { - char username[64] ; - char password[64] ; + char username[STRLEN] ; + char password[STRLEN] ; char line[512] ; + char format[20]; int retval; pam_handle_t *pamh = NULL; sprintf(format, "LOGIN:%%%ds %%%ds", STRLEN-1, STRLEN-1); fgets(line,512,stdin) ; - sscanf(line, "LOGIN:%s %s",username,password) ; + sscanf(line, format, username,password) ; conv.appdata_ptr = (char *) password; retval = pam_start("ranger-remote", username, &conv, &pamh); From 678bf58dd6ab2ac9710497385a444142f505084f Mon Sep 17 00:00:00 2001 From: pradeep Date: Thu, 22 Feb 2018 19:37:20 +0530 Subject: [PATCH 128/151] RANGER-1990: Support one-way SSL connection to DB (cherry picked from commit 625cd35a49c772a7df44ae65ba02b0129e98c9f9) --- kms/config/kms-webapp/dbks-site.xml | 4 +++ kms/scripts/db_setup.py | 34 +++++++++++------- kms/scripts/dba_script.py | 35 +++++++++++------- kms/scripts/install.properties | 2 ++ kms/scripts/ranger-kms | 2 +- kms/scripts/setup.sh | 24 ++++++++++--- .../apache/hadoop/crypto/key/RangerKMSDB.java | 27 ++++++++------ security-admin/scripts/db_setup.py | 36 +++++++++++-------- security-admin/scripts/dba_script.py | 36 +++++++++++-------- security-admin/scripts/install.properties | 2 ++ security-admin/scripts/setup.sh | 18 +++++++++- .../apache/ranger/common/PropertiesUtil.java | 6 ++++ .../conf.dist/ranger-admin-default-site.xml | 4 +++ 13 files changed, 159 insertions(+), 71 deletions(-) diff --git a/kms/config/kms-webapp/dbks-site.xml b/kms/config/kms-webapp/dbks-site.xml index a098db1106..0e0f2eca9b 100755 --- a/kms/config/kms-webapp/dbks-site.xml +++ b/kms/config/kms-webapp/dbks-site.xml @@ -167,4 +167,8 @@ ranger.ks.db.ssl.verifyServerCertificate false + + ranger.ks.db.ssl.auth.type + 2-way + diff --git a/kms/scripts/db_setup.py b/kms/scripts/db_setup.py index d8b4b635ab..090e551407 100644 --- a/kms/scripts/db_setup.py +++ b/kms/scripts/db_setup.py @@ -102,13 +102,14 @@ def import_db_file(self, db_name, db_user, db_password, file_name): class MysqlConf(BaseDB): # Constructor - def __init__(self, host,SQL_CONNECTOR_JAR,JAVA_BIN,db_ssl_enabled,db_ssl_required,db_ssl_verifyServerCertificate,javax_net_ssl_keyStore,javax_net_ssl_keyStorePassword,javax_net_ssl_trustStore,javax_net_ssl_trustStorePassword): + def __init__(self, host,SQL_CONNECTOR_JAR,JAVA_BIN,db_ssl_enabled,db_ssl_required,db_ssl_verifyServerCertificate,javax_net_ssl_keyStore,javax_net_ssl_keyStorePassword,javax_net_ssl_trustStore,javax_net_ssl_trustStorePassword,db_ssl_auth_type): self.host = host self.SQL_CONNECTOR_JAR = SQL_CONNECTOR_JAR self.JAVA_BIN = JAVA_BIN self.db_ssl_enabled=db_ssl_enabled.lower() self.db_ssl_required=db_ssl_required.lower() self.db_ssl_verifyServerCertificate=db_ssl_verifyServerCertificate.lower() + self.db_ssl_auth_type=db_ssl_auth_type.lower() self.javax_net_ssl_keyStore=javax_net_ssl_keyStore self.javax_net_ssl_keyStorePassword=javax_net_ssl_keyStorePassword self.javax_net_ssl_trustStore=javax_net_ssl_trustStore @@ -121,7 +122,10 @@ def get_jisql_cmd(self, user, password ,db_name): if self.db_ssl_enabled == 'true': db_ssl_param="?useSSL=%s&requireSSL=%s&verifyServerCertificate=%s" %(self.db_ssl_enabled,self.db_ssl_required,self.db_ssl_verifyServerCertificate) if self.db_ssl_verifyServerCertificate == 'true': - db_ssl_cert_param=" -Djavax.net.ssl.keyStore=%s -Djavax.net.ssl.keyStorePassword=%s -Djavax.net.ssl.trustStore=%s -Djavax.net.ssl.trustStorePassword=%s " %(self.javax_net_ssl_keyStore,self.javax_net_ssl_keyStorePassword,self.javax_net_ssl_trustStore,self.javax_net_ssl_trustStorePassword) + if self.db_ssl_auth_type == '1-way': + db_ssl_cert_param=" -Djavax.net.ssl.trustStore=%s -Djavax.net.ssl.trustStorePassword=%s " %(self.javax_net_ssl_trustStore,self.javax_net_ssl_trustStorePassword) + else: + db_ssl_cert_param=" -Djavax.net.ssl.keyStore=%s -Djavax.net.ssl.keyStorePassword=%s -Djavax.net.ssl.trustStore=%s -Djavax.net.ssl.trustStorePassword=%s " %(self.javax_net_ssl_keyStore,self.javax_net_ssl_keyStorePassword,self.javax_net_ssl_trustStore,self.javax_net_ssl_trustStorePassword) self.JAVA_BIN = self.JAVA_BIN.strip("'") if is_unix: jisql_cmd = "%s %s -cp %s:%s/jisql/lib/* org.apache.util.sql.Jisql -driver mysqlconj -cstring jdbc:mysql://%s/%s%s -u '%s' -p '%s' -noheader -trim -c \;" %(self.JAVA_BIN,db_ssl_cert_param,self.SQL_CONNECTOR_JAR,path,self.host,db_name,db_ssl_param,user,password) @@ -572,6 +576,7 @@ def main(argv): db_ssl_enabled='false' db_ssl_required='false' db_ssl_verifyServerCertificate='false' + db_ssl_auth_type='2-way' javax_net_ssl_keyStore='' javax_net_ssl_keyStorePassword='' javax_net_ssl_trustStore='' @@ -585,30 +590,33 @@ def main(argv): db_ssl_required=globalDict['db_ssl_required'].lower() if 'db_ssl_verifyServerCertificate' in globalDict: db_ssl_verifyServerCertificate=globalDict['db_ssl_verifyServerCertificate'].lower() + if 'db_ssl_auth_type' in globalDict: + db_ssl_auth_type=globalDict['db_ssl_auth_type'].lower() if db_ssl_verifyServerCertificate == 'true': - if 'javax_net_ssl_keyStore' in globalDict: - javax_net_ssl_keyStore=globalDict['javax_net_ssl_keyStore'] - if 'javax_net_ssl_keyStorePassword' in globalDict: - javax_net_ssl_keyStorePassword=globalDict['javax_net_ssl_keyStorePassword'] if 'javax_net_ssl_trustStore' in globalDict: javax_net_ssl_trustStore=globalDict['javax_net_ssl_trustStore'] if 'javax_net_ssl_trustStorePassword' in globalDict: javax_net_ssl_trustStorePassword=globalDict['javax_net_ssl_trustStorePassword'] - if not os.path.exists(javax_net_ssl_keyStore): - log("[E] Invalid file Name! Unable to find keystore file:"+javax_net_ssl_keyStore,"error") - sys.exit(1) if not os.path.exists(javax_net_ssl_trustStore): log("[E] Invalid file Name! Unable to find truststore file:"+javax_net_ssl_trustStore,"error") sys.exit(1) - if javax_net_ssl_keyStorePassword is None or javax_net_ssl_keyStorePassword =="": - log("[E] Invalid ssl keystore password!","error") - sys.exit(1) if javax_net_ssl_trustStorePassword is None or javax_net_ssl_trustStorePassword =="": log("[E] Invalid ssl truststore password!","error") sys.exit(1) + if db_ssl_auth_type == '2-way': + if 'javax_net_ssl_keyStore' in globalDict: + javax_net_ssl_keyStore=globalDict['javax_net_ssl_keyStore'] + if 'javax_net_ssl_keyStorePassword' in globalDict: + javax_net_ssl_keyStorePassword=globalDict['javax_net_ssl_keyStorePassword'] + if not os.path.exists(javax_net_ssl_keyStore): + log("[E] Invalid file Name! Unable to find keystore file:"+javax_net_ssl_keyStore,"error") + sys.exit(1) + if javax_net_ssl_keyStorePassword is None or javax_net_ssl_keyStorePassword =="": + log("[E] Invalid ssl keystore password!","error") + sys.exit(1) MYSQL_CONNECTOR_JAR=globalDict['SQL_CONNECTOR_JAR'] - xa_sqlObj = MysqlConf(xa_db_host, MYSQL_CONNECTOR_JAR, JAVA_BIN,db_ssl_enabled,db_ssl_required,db_ssl_verifyServerCertificate,javax_net_ssl_keyStore,javax_net_ssl_keyStorePassword,javax_net_ssl_trustStore,javax_net_ssl_trustStorePassword) + xa_sqlObj = MysqlConf(xa_db_host, MYSQL_CONNECTOR_JAR, JAVA_BIN,db_ssl_enabled,db_ssl_required,db_ssl_verifyServerCertificate,javax_net_ssl_keyStore,javax_net_ssl_keyStorePassword,javax_net_ssl_trustStore,javax_net_ssl_trustStorePassword,db_ssl_auth_type) xa_db_core_file = os.path.join(RANGER_KMS_HOME , mysql_core_file) elif XA_DB_FLAVOR == "ORACLE": diff --git a/kms/scripts/dba_script.py b/kms/scripts/dba_script.py index 1e264cce80..6350d7da05 100755 --- a/kms/scripts/dba_script.py +++ b/kms/scripts/dba_script.py @@ -133,13 +133,14 @@ def create_db(self, root_user, db_root_password, db_name, db_user, db_password,d class MysqlConf(BaseDB): # Constructor - def __init__(self, host,SQL_CONNECTOR_JAR,JAVA_BIN,db_ssl_enabled,db_ssl_required,db_ssl_verifyServerCertificate,javax_net_ssl_keyStore,javax_net_ssl_keyStorePassword,javax_net_ssl_trustStore,javax_net_ssl_trustStorePassword): + def __init__(self, host,SQL_CONNECTOR_JAR,JAVA_BIN,db_ssl_enabled,db_ssl_required,db_ssl_verifyServerCertificate,javax_net_ssl_keyStore,javax_net_ssl_keyStorePassword,javax_net_ssl_trustStore,javax_net_ssl_trustStorePassword,db_ssl_auth_type): self.host = host self.SQL_CONNECTOR_JAR = SQL_CONNECTOR_JAR self.JAVA_BIN = JAVA_BIN self.db_ssl_enabled=db_ssl_enabled.lower() self.db_ssl_required=db_ssl_required.lower() self.db_ssl_verifyServerCertificate=db_ssl_verifyServerCertificate.lower() + self.db_ssl_auth_type=db_ssl_auth_type.lower() self.javax_net_ssl_keyStore=javax_net_ssl_keyStore self.javax_net_ssl_keyStorePassword=javax_net_ssl_keyStorePassword self.javax_net_ssl_trustStore=javax_net_ssl_trustStore @@ -153,7 +154,10 @@ def get_jisql_cmd(self, user, password ,db_name): if self.db_ssl_enabled == 'true': db_ssl_param="?useSSL=%s&requireSSL=%s&verifyServerCertificate=%s" %(self.db_ssl_enabled,self.db_ssl_required,self.db_ssl_verifyServerCertificate) if self.db_ssl_verifyServerCertificate == 'true': - db_ssl_cert_param=" -Djavax.net.ssl.keyStore=%s -Djavax.net.ssl.keyStorePassword=%s -Djavax.net.ssl.trustStore=%s -Djavax.net.ssl.trustStorePassword=%s " %(self.javax_net_ssl_keyStore,self.javax_net_ssl_keyStorePassword,self.javax_net_ssl_trustStore,self.javax_net_ssl_trustStorePassword) + if self.db_ssl_auth_type == '1-way': + db_ssl_cert_param=" -Djavax.net.ssl.trustStore=%s -Djavax.net.ssl.trustStorePassword=%s " %(self.javax_net_ssl_trustStore,self.javax_net_ssl_trustStorePassword) + else: + db_ssl_cert_param=" -Djavax.net.ssl.keyStore=%s -Djavax.net.ssl.keyStorePassword=%s -Djavax.net.ssl.trustStore=%s -Djavax.net.ssl.trustStorePassword=%s " %(self.javax_net_ssl_keyStore,self.javax_net_ssl_keyStorePassword,self.javax_net_ssl_trustStore,self.javax_net_ssl_trustStorePassword) if is_unix: jisql_cmd = "%s %s -cp %s:%s/jisql/lib/* org.apache.util.sql.Jisql -driver mysqlconj -cstring jdbc:mysql://%s/%s%s -u %s -p '%s' -noheader -trim -c \;" %(self.JAVA_BIN,db_ssl_cert_param,self.SQL_CONNECTOR_JAR,path,self.host,db_name,db_ssl_param,user,password) elif os_name == "WINDOWS": @@ -1363,6 +1367,7 @@ def main(argv): db_ssl_enabled='false' db_ssl_required='false' db_ssl_verifyServerCertificate='false' + db_ssl_auth_type='2-way' javax_net_ssl_keyStore='' javax_net_ssl_keyStorePassword='' javax_net_ssl_trustStore='' @@ -1375,30 +1380,34 @@ def main(argv): db_ssl_required=globalDict['db_ssl_required'].lower() if 'db_ssl_verifyServerCertificate' in globalDict: db_ssl_verifyServerCertificate=globalDict['db_ssl_verifyServerCertificate'].lower() + if 'db_ssl_auth_type' in globalDict: + db_ssl_auth_type=globalDict['db_ssl_auth_type'].lower() if db_ssl_verifyServerCertificate == 'true': - if 'javax_net_ssl_keyStore' in globalDict: - javax_net_ssl_keyStore=globalDict['javax_net_ssl_keyStore'] - if 'javax_net_ssl_keyStorePassword' in globalDict: - javax_net_ssl_keyStorePassword=globalDict['javax_net_ssl_keyStorePassword'] if 'javax_net_ssl_trustStore' in globalDict: javax_net_ssl_trustStore=globalDict['javax_net_ssl_trustStore'] if 'javax_net_ssl_trustStorePassword' in globalDict: javax_net_ssl_trustStorePassword=globalDict['javax_net_ssl_trustStorePassword'] - if not os.path.exists(javax_net_ssl_keyStore): - log("[E] Invalid file Name! Unable to find keystore file:"+javax_net_ssl_keyStore,"error") - sys.exit(1) if not os.path.exists(javax_net_ssl_trustStore): log("[E] Invalid file Name! Unable to find truststore file:"+javax_net_ssl_trustStore,"error") sys.exit(1) - if javax_net_ssl_keyStorePassword is None or javax_net_ssl_keyStorePassword =="": - log("[E] Invalid ssl keystore password!","error") - sys.exit(1) if javax_net_ssl_trustStorePassword is None or javax_net_ssl_trustStorePassword =="": log("[E] Invalid ssl truststore password!","error") sys.exit(1) + if db_ssl_auth_type == '2-way': + if 'javax_net_ssl_keyStore' in globalDict: + javax_net_ssl_keyStore=globalDict['javax_net_ssl_keyStore'] + if 'javax_net_ssl_keyStorePassword' in globalDict: + javax_net_ssl_keyStorePassword=globalDict['javax_net_ssl_keyStorePassword'] + if not os.path.exists(javax_net_ssl_keyStore): + log("[E] Invalid file Name! Unable to find keystore file:"+javax_net_ssl_keyStore,"error") + sys.exit(1) + if javax_net_ssl_keyStorePassword is None or javax_net_ssl_keyStorePassword =="": + log("[E] Invalid ssl keystore password!","error") + sys.exit(1) + MYSQL_CONNECTOR_JAR=CONNECTOR_JAR - xa_sqlObj = MysqlConf(xa_db_host, MYSQL_CONNECTOR_JAR, JAVA_BIN,db_ssl_enabled,db_ssl_required,db_ssl_verifyServerCertificate,javax_net_ssl_keyStore,javax_net_ssl_keyStorePassword,javax_net_ssl_trustStore,javax_net_ssl_trustStorePassword) + xa_sqlObj = MysqlConf(xa_db_host, MYSQL_CONNECTOR_JAR, JAVA_BIN,db_ssl_enabled,db_ssl_required,db_ssl_verifyServerCertificate,javax_net_ssl_keyStore,javax_net_ssl_keyStorePassword,javax_net_ssl_trustStore,javax_net_ssl_trustStorePassword,db_ssl_auth_type) xa_db_core_file = os.path.join(RANGER_KMS_HOME,mysql_core_file) elif XA_DB_FLAVOR == "ORACLE": diff --git a/kms/scripts/install.properties b/kms/scripts/install.properties index b173d13542..ddc779dad5 100755 --- a/kms/scripts/install.properties +++ b/kms/scripts/install.properties @@ -55,6 +55,8 @@ db_host=localhost db_ssl_enabled=false db_ssl_required=false db_ssl_verifyServerCertificate=false +#db_ssl_auth_type=1-way|2-way, where 1-way represents standard one way ssl authentication and 2-way represents mutual ssl authentication +db_ssl_auth_type=2-way javax_net_ssl_keyStore= javax_net_ssl_keyStorePassword= javax_net_ssl_trustStore= diff --git a/kms/scripts/ranger-kms b/kms/scripts/ranger-kms index dd14639bef..d1e33608f3 100755 --- a/kms/scripts/ranger-kms +++ b/kms/scripts/ranger-kms @@ -89,7 +89,7 @@ fi KMS_CONF_DIR=${RANGER_KMS_EWS_DIR}/webapp/WEB-INF/classes/conf SERVER_NAME=rangerkms -JAVA_OPTS="${JAVA_OPTS} -Dservername=${SERVER_NAME} -Dcatalina.base=${RANGER_KMS_EWS_DIR} -Dkms.config.dir=${KMS_CONF_DIR} -Dkms.log.dir=${TOMCAT_LOG_DIR} -cp ${RANGER_KMS_EWS_CONF_DIR}:${RANGER_KMS_EWS_LIB_DIR}/*:${RANGER_KMS_EWS_DIR}/webapp/lib/*:${JAVA_HOME}/lib/*:${RANGER_HADOOP_CONF_DIR}/*:$CLASSPATH " +JAVA_OPTS="${JAVA_OPTS} ${DB_SSL_PARAM} -Dservername=${SERVER_NAME} -Dcatalina.base=${RANGER_KMS_EWS_DIR} -Dkms.config.dir=${KMS_CONF_DIR} -Dkms.log.dir=${TOMCAT_LOG_DIR} -cp ${RANGER_KMS_EWS_CONF_DIR}:${RANGER_KMS_EWS_LIB_DIR}/*:${RANGER_KMS_EWS_DIR}/webapp/lib/*:${JAVA_HOME}/lib/*:${RANGER_HADOOP_CONF_DIR}/*:$CLASSPATH " createRangerKMSPid () { SLEEP_TIME_AFTER_START=5 nohup java -D${PROC_NAME} ${JAVA_OPTS} ${START_CLASS_NAME} ${KMS_CONFIG_FILENAME} > ${TOMCAT_LOG_FILE} 2>&1 & diff --git a/kms/scripts/setup.sh b/kms/scripts/setup.sh index c8d751926d..2db05b8628 100755 --- a/kms/scripts/setup.sh +++ b/kms/scripts/setup.sh @@ -66,6 +66,7 @@ db_password=$(get_prop 'db_password' $PROPFILE) db_ssl_enabled=$(get_prop 'db_ssl_enabled' $PROPFILE) db_ssl_required=$(get_prop 'db_ssl_required' $PROPFILE) db_ssl_verifyServerCertificate=$(get_prop 'db_ssl_verifyServerCertificate' $PROPFILE) +db_ssl_auth_type=$(get_prop 'db_ssl_auth_type' $PROPFILE) KMS_MASTER_KEY_PASSWD=$(get_prop 'KMS_MASTER_KEY_PASSWD' $PROPFILE) unix_user=$(get_prop 'unix_user' $PROPFILE) unix_group=$(get_prop 'unix_group' $PROPFILE) @@ -270,11 +271,13 @@ init_variables(){ db_ssl_enabled="false" db_ssl_required="false" db_ssl_verifyServerCertificate="false" + db_ssl_auth_type="2-way" fi if [ "${db_ssl_enabled}" == "true" ] then db_ssl_required=`echo $db_ssl_required | tr '[:upper:]' '[:lower:]'` db_ssl_verifyServerCertificate=`echo $db_ssl_verifyServerCertificate | tr '[:upper:]' '[:lower:]'` + db_ssl_auth_type=`echo $db_ssl_auth_type | tr '[:upper:]' '[:lower:]'` if [ "${db_ssl_required}" != "true" ] then db_ssl_required="false" @@ -283,6 +286,10 @@ init_variables(){ then db_ssl_verifyServerCertificate="false" fi + if [ "${db_ssl_auth_type}" != "1-way" ] + then + db_ssl_auth_type="2-way" + fi fi } @@ -448,17 +455,21 @@ update_properties() { if [ "${db_ssl_enabled}" != "" ] then - propertyName=ranger.db.ssl.enabled + propertyName=ranger.ks.db.ssl.enabled newPropertyValue="${db_ssl_enabled}" updatePropertyToFilePy $propertyName $newPropertyValue $to_file - propertyName=ranger.db.ssl.required + propertyName=ranger.ks.db.ssl.required newPropertyValue="${db_ssl_required}" updatePropertyToFilePy $propertyName $newPropertyValue $to_file - propertyName=ranger.db.ssl.verifyServerCertificate + propertyName=ranger.ks.db.ssl.verifyServerCertificate newPropertyValue="${db_ssl_verifyServerCertificate}" updatePropertyToFilePy $propertyName $newPropertyValue $to_file + + propertyName=ranger.ks.db.ssl.auth.type + newPropertyValue="${db_ssl_auth_type}" + updatePropertyToFilePy $propertyName $newPropertyValue $to_file fi if [ "${DB_FLAVOR}" == "MYSQL" ] @@ -891,7 +902,12 @@ setup_install_files(){ if [ "${db_ssl_verifyServerCertificate}" == "true" ] then - DB_SSL_PARAM="' -Djavax.net.ssl.keyStore=${javax_net_ssl_keyStore} -Djavax.net.ssl.keyStorePassword=${javax_net_ssl_keyStorePassword} -Djavax.net.ssl.trustStore=${javax_net_ssl_trustStore} -Djavax.net.ssl.trustStorePassword=${javax_net_ssl_trustStorePassword} '" + if [ "${db_ssl_auth_type}" == "1-way" ] + then + DB_SSL_PARAM="' -Djavax.net.ssl.trustStore=${javax_net_ssl_trustStore} -Djavax.net.ssl.trustStorePassword=${javax_net_ssl_trustStorePassword} '" + else + DB_SSL_PARAM="' -Djavax.net.ssl.keyStore=${javax_net_ssl_keyStore} -Djavax.net.ssl.keyStorePassword=${javax_net_ssl_keyStorePassword} -Djavax.net.ssl.trustStore=${javax_net_ssl_trustStore} -Djavax.net.ssl.trustStorePassword=${javax_net_ssl_trustStorePassword} '" + fi echo "export DB_SSL_PARAM=${DB_SSL_PARAM}" > ${WEBAPP_ROOT}/WEB-INF/classes/conf/ranger-kms-env-dbsslparam.sh chmod a+rx ${WEBAPP_ROOT}/WEB-INF/classes/conf/ranger-kms-env-dbsslparam.sh else diff --git a/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKMSDB.java b/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKMSDB.java index 649da30c59..c745438851 100755 --- a/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKMSDB.java +++ b/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKMSDB.java @@ -57,6 +57,7 @@ public class RangerKMSDB { private static final String DB_SSL_ENABLED="db.ssl.enabled"; private static final String DB_SSL_REQUIRED="db.ssl.required"; private static final String DB_SSL_VerifyServerCertificate="db.ssl.verifyServerCertificate"; + private static final String DB_SSL_AUTH_TYPE="db.ssl.auth.type"; private static final String DB_SSL_KEYSTORE="keystore.file"; private static final String DB_SSL_KEYSTORE_PASSWORD="keystore.password"; private static final String DB_SSL_TRUSTSTORE="truststore.file"; @@ -190,9 +191,11 @@ private void updateDBSSLURL(){ db_ssl_verifyServerCertificate="false"; } db_ssl_verifyServerCertificate=db_ssl_verifyServerCertificate.toLowerCase(); + String db_ssl_auth_type=conf.get(PROPERTY_PREFIX+DB_SSL_AUTH_TYPE,"2-way"); conf.set(PROPERTY_PREFIX+DB_SSL_ENABLED, db_ssl_enabled); conf.set(PROPERTY_PREFIX+DB_SSL_REQUIRED, db_ssl_required); conf.set(PROPERTY_PREFIX+DB_SSL_VerifyServerCertificate, db_ssl_verifyServerCertificate); + conf.set(PROPERTY_PREFIX+DB_SSL_AUTH_TYPE, db_ssl_auth_type); String ranger_jpa_jdbc_url=conf.get(PROPERTY_PREFIX+DB_URL); if(!StringUtils.isEmpty(ranger_jpa_jdbc_url)){ StringBuffer ranger_jpa_jdbc_url_ssl=new StringBuffer(ranger_jpa_jdbc_url); @@ -204,19 +207,21 @@ private void updateDBSSLURL(){ if("true".equalsIgnoreCase(db_ssl_verifyServerCertificate)){ if (conf!=null) { - // update system key store path with custom key store. - String keystore=conf.get(PROPERTY_PREFIX+DB_SSL_KEYSTORE); - if(!StringUtils.isEmpty(keystore)){ - Path path = Paths.get(keystore); - if (Files.exists(path) && Files.isReadable(path)) { - System.setProperty("javax.net.ssl.keyStore", conf.get(PROPERTY_PREFIX+DB_SSL_KEYSTORE)); - System.setProperty("javax.net.ssl.keyStorePassword", conf.get(PROPERTY_PREFIX+DB_SSL_KEYSTORE_PASSWORD)); - System.setProperty("javax.net.ssl.keyStoreType", KeyStore.getDefaultType()); + if(!"1-way".equalsIgnoreCase((db_ssl_auth_type))){ + // update system key store path with custom key store. + String keystore=conf.get(PROPERTY_PREFIX+DB_SSL_KEYSTORE); + if(!StringUtils.isEmpty(keystore)){ + Path path = Paths.get(keystore); + if (Files.exists(path) && Files.isReadable(path)) { + System.setProperty("javax.net.ssl.keyStore", conf.get(PROPERTY_PREFIX+DB_SSL_KEYSTORE)); + System.setProperty("javax.net.ssl.keyStorePassword", conf.get(PROPERTY_PREFIX+DB_SSL_KEYSTORE_PASSWORD)); + System.setProperty("javax.net.ssl.keyStoreType", KeyStore.getDefaultType()); + }else{ + logger.debug("Could not find or read keystore file '"+keystore+"'"); + } }else{ - logger.debug("Could not find or read keystore file '"+keystore+"'"); + logger.debug("keystore property '"+PROPERTY_PREFIX+DB_SSL_KEYSTORE+"' value not found!"); } - }else{ - logger.debug("keystore property '"+PROPERTY_PREFIX+DB_SSL_KEYSTORE+"' value not found!"); } // update system trust store path with custom trust store. String truststore=conf.get(PROPERTY_PREFIX+DB_SSL_TRUSTSTORE); diff --git a/security-admin/scripts/db_setup.py b/security-admin/scripts/db_setup.py index 6e791512b2..d4f37ed6fb 100644 --- a/security-admin/scripts/db_setup.py +++ b/security-admin/scripts/db_setup.py @@ -183,13 +183,14 @@ def import_core_db_schema(self, db_name, db_user, db_password, file_name,first_t class MysqlConf(BaseDB): # Constructor - def __init__(self, host,SQL_CONNECTOR_JAR,JAVA_BIN,db_ssl_enabled,db_ssl_required,db_ssl_verifyServerCertificate,javax_net_ssl_keyStore,javax_net_ssl_keyStorePassword,javax_net_ssl_trustStore,javax_net_ssl_trustStorePassword): + def __init__(self, host,SQL_CONNECTOR_JAR,JAVA_BIN,db_ssl_enabled,db_ssl_required,db_ssl_verifyServerCertificate,javax_net_ssl_keyStore,javax_net_ssl_keyStorePassword,javax_net_ssl_trustStore,javax_net_ssl_trustStorePassword,db_ssl_auth_type): self.host = host self.SQL_CONNECTOR_JAR = SQL_CONNECTOR_JAR self.JAVA_BIN = JAVA_BIN self.db_ssl_enabled=db_ssl_enabled.lower() self.db_ssl_required=db_ssl_required.lower() self.db_ssl_verifyServerCertificate=db_ssl_verifyServerCertificate.lower() + self.db_ssl_auth_type=db_ssl_auth_type.lower() self.javax_net_ssl_keyStore=javax_net_ssl_keyStore self.javax_net_ssl_keyStorePassword=javax_net_ssl_keyStorePassword self.javax_net_ssl_trustStore=javax_net_ssl_trustStore @@ -202,7 +203,10 @@ def get_jisql_cmd(self, user, password ,db_name): if self.db_ssl_enabled == 'true': db_ssl_param="?useSSL=%s&requireSSL=%s&verifyServerCertificate=%s" %(self.db_ssl_enabled,self.db_ssl_required,self.db_ssl_verifyServerCertificate) if self.db_ssl_verifyServerCertificate == 'true': - db_ssl_cert_param=" -Djavax.net.ssl.keyStore=%s -Djavax.net.ssl.keyStorePassword=%s -Djavax.net.ssl.trustStore=%s -Djavax.net.ssl.trustStorePassword=%s " %(self.javax_net_ssl_keyStore,self.javax_net_ssl_keyStorePassword,self.javax_net_ssl_trustStore,self.javax_net_ssl_trustStorePassword) + if self.db_ssl_auth_type == '1-way': + db_ssl_cert_param=" -Djavax.net.ssl.trustStore=%s -Djavax.net.ssl.trustStorePassword=%s " %(self.javax_net_ssl_trustStore,self.javax_net_ssl_trustStorePassword) + else: + db_ssl_cert_param=" -Djavax.net.ssl.keyStore=%s -Djavax.net.ssl.keyStorePassword=%s -Djavax.net.ssl.trustStore=%s -Djavax.net.ssl.trustStorePassword=%s " %(self.javax_net_ssl_keyStore,self.javax_net_ssl_keyStorePassword,self.javax_net_ssl_trustStore,self.javax_net_ssl_trustStorePassword) self.JAVA_BIN = self.JAVA_BIN.strip("'") if is_unix: jisql_cmd = "%s %s -cp %s:%s/jisql/lib/* org.apache.util.sql.Jisql -driver mysqlconj -cstring jdbc:mysql://%s/%s%s -u '%s' -p '%s' -noheader -trim -c \;" %(self.JAVA_BIN,db_ssl_cert_param,self.SQL_CONNECTOR_JAR,path,self.host,db_name,db_ssl_param,user,password) @@ -3604,6 +3608,7 @@ def main(argv): db_ssl_enabled='false' db_ssl_required='false' db_ssl_verifyServerCertificate='false' + db_ssl_auth_type='2-way' javax_net_ssl_keyStore='' javax_net_ssl_keyStorePassword='' javax_net_ssl_trustStore='' @@ -3617,30 +3622,33 @@ def main(argv): db_ssl_required=globalDict['db_ssl_required'].lower() if 'db_ssl_verifyServerCertificate' in globalDict: db_ssl_verifyServerCertificate=globalDict['db_ssl_verifyServerCertificate'].lower() + if 'db_ssl_auth_type' in globalDict: + db_ssl_auth_type=globalDict['db_ssl_auth_type'].lower() if db_ssl_verifyServerCertificate == 'true': - if 'javax_net_ssl_keyStore' in globalDict: - javax_net_ssl_keyStore=globalDict['javax_net_ssl_keyStore'] - if 'javax_net_ssl_keyStorePassword' in globalDict: - javax_net_ssl_keyStorePassword=globalDict['javax_net_ssl_keyStorePassword'] if 'javax_net_ssl_trustStore' in globalDict: javax_net_ssl_trustStore=globalDict['javax_net_ssl_trustStore'] if 'javax_net_ssl_trustStorePassword' in globalDict: javax_net_ssl_trustStorePassword=globalDict['javax_net_ssl_trustStorePassword'] - if not os.path.exists(javax_net_ssl_keyStore): - log("[E] Invalid file Name! Unable to find keystore file:"+javax_net_ssl_keyStore,"error") - sys.exit(1) if not os.path.exists(javax_net_ssl_trustStore): log("[E] Invalid file Name! Unable to find truststore file:"+javax_net_ssl_trustStore,"error") sys.exit(1) - if javax_net_ssl_keyStorePassword is None or javax_net_ssl_keyStorePassword =="": - log("[E] Invalid ssl keystore password!","error") - sys.exit(1) if javax_net_ssl_trustStorePassword is None or javax_net_ssl_trustStorePassword =="": log("[E] Invalid ssl truststore password!","error") sys.exit(1) + if db_ssl_auth_type == '2-way': + if 'javax_net_ssl_keyStore' in globalDict: + javax_net_ssl_keyStore=globalDict['javax_net_ssl_keyStore'] + if 'javax_net_ssl_keyStorePassword' in globalDict: + javax_net_ssl_keyStorePassword=globalDict['javax_net_ssl_keyStorePassword'] + if not os.path.exists(javax_net_ssl_keyStore): + log("[E] Invalid file Name! Unable to find keystore file:"+javax_net_ssl_keyStore,"error") + sys.exit(1) + if javax_net_ssl_keyStorePassword is None or javax_net_ssl_keyStorePassword =="": + log("[E] Invalid ssl keystore password!","error") + sys.exit(1) MYSQL_CONNECTOR_JAR=globalDict['SQL_CONNECTOR_JAR'] - xa_sqlObj = MysqlConf(xa_db_host, MYSQL_CONNECTOR_JAR, JAVA_BIN,db_ssl_enabled,db_ssl_required,db_ssl_verifyServerCertificate,javax_net_ssl_keyStore,javax_net_ssl_keyStorePassword,javax_net_ssl_trustStore,javax_net_ssl_trustStorePassword) + xa_sqlObj = MysqlConf(xa_db_host, MYSQL_CONNECTOR_JAR, JAVA_BIN,db_ssl_enabled,db_ssl_required,db_ssl_verifyServerCertificate,javax_net_ssl_keyStore,javax_net_ssl_keyStorePassword,javax_net_ssl_trustStore,javax_net_ssl_trustStorePassword,db_ssl_auth_type) xa_db_version_file = os.path.join(RANGER_ADMIN_HOME , mysql_dbversion_catalog) xa_db_core_file = os.path.join(RANGER_ADMIN_HOME , mysql_core_file) xa_patch_file = os.path.join(RANGER_ADMIN_HOME ,mysql_patches) @@ -3700,7 +3708,7 @@ def main(argv): if AUDIT_DB_FLAVOR == "MYSQL": MYSQL_CONNECTOR_JAR=globalDict['SQL_CONNECTOR_JAR'] - audit_sqlObj = MysqlConf(audit_db_host,MYSQL_CONNECTOR_JAR,JAVA_BIN,db_ssl_enabled,db_ssl_required,db_ssl_verifyServerCertificate,javax_net_ssl_keyStore,javax_net_ssl_keyStorePassword,javax_net_ssl_trustStore,javax_net_ssl_trustStorePassword) + audit_sqlObj = MysqlConf(audit_db_host,MYSQL_CONNECTOR_JAR,JAVA_BIN,db_ssl_enabled,db_ssl_required,db_ssl_verifyServerCertificate,javax_net_ssl_keyStore,javax_net_ssl_keyStorePassword,javax_net_ssl_trustStore,javax_net_ssl_trustStorePassword,db_ssl_auth_type) audit_db_file = os.path.join(RANGER_ADMIN_HOME ,mysql_audit_file) elif AUDIT_DB_FLAVOR == "ORACLE": diff --git a/security-admin/scripts/dba_script.py b/security-admin/scripts/dba_script.py index 83d6fe728b..6843aa80b6 100644 --- a/security-admin/scripts/dba_script.py +++ b/security-admin/scripts/dba_script.py @@ -157,13 +157,14 @@ def create_auditdb_user(self, xa_db_host, audit_db_host, db_name, audit_db_name, class MysqlConf(BaseDB): # Constructor - def __init__(self, host,SQL_CONNECTOR_JAR,JAVA_BIN,db_ssl_enabled,db_ssl_required,db_ssl_verifyServerCertificate,javax_net_ssl_keyStore,javax_net_ssl_keyStorePassword,javax_net_ssl_trustStore,javax_net_ssl_trustStorePassword): + def __init__(self, host,SQL_CONNECTOR_JAR,JAVA_BIN,db_ssl_enabled,db_ssl_required,db_ssl_verifyServerCertificate,javax_net_ssl_keyStore,javax_net_ssl_keyStorePassword,javax_net_ssl_trustStore,javax_net_ssl_trustStorePassword,db_ssl_auth_type): self.host = host self.SQL_CONNECTOR_JAR = SQL_CONNECTOR_JAR self.JAVA_BIN = JAVA_BIN self.db_ssl_enabled=db_ssl_enabled.lower() self.db_ssl_required=db_ssl_required.lower() self.db_ssl_verifyServerCertificate=db_ssl_verifyServerCertificate.lower() + self.db_ssl_auth_type=db_ssl_auth_type.lower() self.javax_net_ssl_keyStore=javax_net_ssl_keyStore self.javax_net_ssl_keyStorePassword=javax_net_ssl_keyStorePassword self.javax_net_ssl_trustStore=javax_net_ssl_trustStore @@ -177,7 +178,10 @@ def get_jisql_cmd(self, user, password ,db_name): if self.db_ssl_enabled == 'true': db_ssl_param="?useSSL=%s&requireSSL=%s&verifyServerCertificate=%s" %(self.db_ssl_enabled,self.db_ssl_required,self.db_ssl_verifyServerCertificate) if self.db_ssl_verifyServerCertificate == 'true': - db_ssl_cert_param=" -Djavax.net.ssl.keyStore=%s -Djavax.net.ssl.keyStorePassword=%s -Djavax.net.ssl.trustStore=%s -Djavax.net.ssl.trustStorePassword=%s " %(self.javax_net_ssl_keyStore,self.javax_net_ssl_keyStorePassword,self.javax_net_ssl_trustStore,self.javax_net_ssl_trustStorePassword) + if self.db_ssl_auth_type == '1-way': + db_ssl_cert_param=" -Djavax.net.ssl.trustStore=%s -Djavax.net.ssl.trustStorePassword=%s " %(self.javax_net_ssl_trustStore,self.javax_net_ssl_trustStorePassword) + else: + db_ssl_cert_param=" -Djavax.net.ssl.keyStore=%s -Djavax.net.ssl.keyStorePassword=%s -Djavax.net.ssl.trustStore=%s -Djavax.net.ssl.trustStorePassword=%s " %(self.javax_net_ssl_keyStore,self.javax_net_ssl_keyStorePassword,self.javax_net_ssl_trustStore,self.javax_net_ssl_trustStorePassword) if is_unix: jisql_cmd = "%s %s -cp %s:%s/jisql/lib/* org.apache.util.sql.Jisql -driver mysqlconj -cstring jdbc:mysql://%s/%s%s -u %s -p '%s' -noheader -trim -c \;" %(self.JAVA_BIN,db_ssl_cert_param,self.SQL_CONNECTOR_JAR,path,self.host,db_name,db_ssl_param,user,password) elif os_name == "WINDOWS": @@ -1644,6 +1648,7 @@ def main(argv): db_ssl_enabled='false' db_ssl_required='false' db_ssl_verifyServerCertificate='false' + db_ssl_auth_type='2-way' javax_net_ssl_keyStore='' javax_net_ssl_keyStorePassword='' javax_net_ssl_trustStore='' @@ -1656,30 +1661,33 @@ def main(argv): db_ssl_required=globalDict['db_ssl_required'].lower() if 'db_ssl_verifyServerCertificate' in globalDict: db_ssl_verifyServerCertificate=globalDict['db_ssl_verifyServerCertificate'].lower() + if 'db_ssl_auth_type' in globalDict: + db_ssl_auth_type=globalDict['db_ssl_auth_type'].lower() if db_ssl_verifyServerCertificate == 'true': - if 'javax_net_ssl_keyStore' in globalDict: - javax_net_ssl_keyStore=globalDict['javax_net_ssl_keyStore'] - if 'javax_net_ssl_keyStorePassword' in globalDict: - javax_net_ssl_keyStorePassword=globalDict['javax_net_ssl_keyStorePassword'] if 'javax_net_ssl_trustStore' in globalDict: javax_net_ssl_trustStore=globalDict['javax_net_ssl_trustStore'] if 'javax_net_ssl_trustStorePassword' in globalDict: javax_net_ssl_trustStorePassword=globalDict['javax_net_ssl_trustStorePassword'] - if not os.path.exists(javax_net_ssl_keyStore): - log("[E] Invalid file Name! Unable to find keystore file:"+javax_net_ssl_keyStore,"error") - sys.exit(1) if not os.path.exists(javax_net_ssl_trustStore): log("[E] Invalid file Name! Unable to find truststore file:"+javax_net_ssl_trustStore,"error") sys.exit(1) - if javax_net_ssl_keyStorePassword is None or javax_net_ssl_keyStorePassword =="": - log("[E] Invalid ssl keystore password!","error") - sys.exit(1) if javax_net_ssl_trustStorePassword is None or javax_net_ssl_trustStorePassword =="": log("[E] Invalid ssl truststore password!","error") sys.exit(1) + if db_ssl_auth_type == '2-way': + if 'javax_net_ssl_keyStore' in globalDict: + javax_net_ssl_keyStore=globalDict['javax_net_ssl_keyStore'] + if 'javax_net_ssl_keyStorePassword' in globalDict: + javax_net_ssl_keyStorePassword=globalDict['javax_net_ssl_keyStorePassword'] + if not os.path.exists(javax_net_ssl_keyStore): + log("[E] Invalid file Name! Unable to find keystore file:"+javax_net_ssl_keyStore,"error") + sys.exit(1) + if javax_net_ssl_keyStorePassword is None or javax_net_ssl_keyStorePassword =="": + log("[E] Invalid ssl keystore password!","error") + sys.exit(1) MYSQL_CONNECTOR_JAR=CONNECTOR_JAR - xa_sqlObj = MysqlConf(xa_db_host, MYSQL_CONNECTOR_JAR, JAVA_BIN,db_ssl_enabled,db_ssl_required,db_ssl_verifyServerCertificate,javax_net_ssl_keyStore,javax_net_ssl_keyStorePassword,javax_net_ssl_trustStore,javax_net_ssl_trustStorePassword) + xa_sqlObj = MysqlConf(xa_db_host, MYSQL_CONNECTOR_JAR, JAVA_BIN,db_ssl_enabled,db_ssl_required,db_ssl_verifyServerCertificate,javax_net_ssl_keyStore,javax_net_ssl_keyStorePassword,javax_net_ssl_trustStore,javax_net_ssl_trustStorePassword,db_ssl_auth_type) xa_db_version_file = os.path.join(RANGER_ADMIN_HOME,mysql_dbversion_catalog) xa_db_core_file = os.path.join(RANGER_ADMIN_HOME,mysql_core_file) xa_patch_file = os.path.join(RANGER_ADMIN_HOME,mysql_patches) @@ -1726,7 +1734,7 @@ def main(argv): if AUDIT_DB_FLAVOR == "MYSQL": MYSQL_CONNECTOR_JAR=CONNECTOR_JAR - audit_sqlObj = MysqlConf(audit_db_host,MYSQL_CONNECTOR_JAR,JAVA_BIN,db_ssl_enabled,db_ssl_required,db_ssl_verifyServerCertificate,javax_net_ssl_keyStore,javax_net_ssl_keyStorePassword,javax_net_ssl_trustStore,javax_net_ssl_trustStorePassword) + audit_sqlObj = MysqlConf(audit_db_host,MYSQL_CONNECTOR_JAR,JAVA_BIN,db_ssl_enabled,db_ssl_required,db_ssl_verifyServerCertificate,javax_net_ssl_keyStore,javax_net_ssl_keyStorePassword,javax_net_ssl_trustStore,javax_net_ssl_trustStorePassword,db_ssl_auth_type) audit_db_file = os.path.join(RANGER_ADMIN_HOME,mysql_audit_file) elif AUDIT_DB_FLAVOR == "ORACLE": diff --git a/security-admin/scripts/install.properties b/security-admin/scripts/install.properties index f323c955da..687bd99fd3 100644 --- a/security-admin/scripts/install.properties +++ b/security-admin/scripts/install.properties @@ -56,6 +56,8 @@ db_host=localhost db_ssl_enabled=false db_ssl_required=false db_ssl_verifyServerCertificate=false +#db_ssl_auth_type=1-way|2-way, where 1-way represents standard one way ssl authentication and 2-way represents mutual ssl authentication +db_ssl_auth_type=2-way javax_net_ssl_keyStore= javax_net_ssl_keyStorePassword= javax_net_ssl_trustStore= diff --git a/security-admin/scripts/setup.sh b/security-admin/scripts/setup.sh index 87be127589..633d363acc 100755 --- a/security-admin/scripts/setup.sh +++ b/security-admin/scripts/setup.sh @@ -68,6 +68,7 @@ db_password=$(get_prop 'db_password' $PROPFILE) db_ssl_enabled=$(get_prop 'db_ssl_enabled' $PROPFILE) db_ssl_required=$(get_prop 'db_ssl_required' $PROPFILE) db_ssl_verifyServerCertificate=$(get_prop 'db_ssl_verifyServerCertificate' $PROPFILE) +db_ssl_auth_type=$(get_prop 'db_ssl_auth_type' $PROPFILE) javax_net_ssl_keyStore=$(get_prop 'javax_net_ssl_keyStore' $PROPFILE) javax_net_ssl_keyStorePassword=$(get_prop 'javax_net_ssl_keyStorePassword' $PROPFILE) javax_net_ssl_trustStore=$(get_prop 'javax_net_ssl_trustStore' $PROPFILE) @@ -254,11 +255,13 @@ init_variables(){ db_ssl_enabled="false" db_ssl_required="false" db_ssl_verifyServerCertificate="false" + db_ssl_auth_type="2-way" fi if [ "${db_ssl_enabled}" == "true" ] then db_ssl_required=`echo $db_ssl_required | tr '[:upper:]' '[:lower:]'` db_ssl_verifyServerCertificate=`echo $db_ssl_verifyServerCertificate | tr '[:upper:]' '[:lower:]'` + db_ssl_auth_type=`echo $db_ssl_auth_type | tr '[:upper:]' '[:lower:]'` if [ "${db_ssl_required}" != "true" ] then db_ssl_required="false" @@ -267,6 +270,10 @@ init_variables(){ then db_ssl_verifyServerCertificate="false" fi + if [ "${db_ssl_auth_type}" != "1-way" ] + then + db_ssl_auth_type="2-way" + fi fi } @@ -485,6 +492,10 @@ update_properties() { propertyName=ranger.db.ssl.verifyServerCertificate newPropertyValue="${db_ssl_verifyServerCertificate}" updatePropertyToFilePy $propertyName $newPropertyValue $to_file_default + + propertyName=ranger.db.ssl.auth.type + newPropertyValue="${db_ssl_auth_type}" + updatePropertyToFilePy $propertyName $newPropertyValue $to_file_default fi if [ "${DB_FLAVOR}" == "MYSQL" ] @@ -1377,7 +1388,12 @@ setup_install_files(){ if [ "${db_ssl_verifyServerCertificate}" == "true" ] then - DB_SSL_PARAM="' -Djavax.net.ssl.keyStore=${javax_net_ssl_keyStore} -Djavax.net.ssl.keyStorePassword=${javax_net_ssl_keyStorePassword} -Djavax.net.ssl.trustStore=${javax_net_ssl_trustStore} -Djavax.net.ssl.trustStorePassword=${javax_net_ssl_trustStorePassword} '" + if [ "${db_ssl_auth_type}" == "1-way" ] + then + DB_SSL_PARAM="' -Djavax.net.ssl.trustStore=${javax_net_ssl_trustStore} -Djavax.net.ssl.trustStorePassword=${javax_net_ssl_trustStorePassword} '" + else + DB_SSL_PARAM="' -Djavax.net.ssl.keyStore=${javax_net_ssl_keyStore} -Djavax.net.ssl.keyStorePassword=${javax_net_ssl_keyStorePassword} -Djavax.net.ssl.trustStore=${javax_net_ssl_trustStore} -Djavax.net.ssl.trustStorePassword=${javax_net_ssl_trustStorePassword} '" + fi echo "export DB_SSL_PARAM=${DB_SSL_PARAM}" > ${WEBAPP_ROOT}/WEB-INF/classes/conf/ranger-admin-env-dbsslparam.sh chmod a+rx ${WEBAPP_ROOT}/WEB-INF/classes/conf/ranger-admin-env-dbsslparam.sh else diff --git a/security-admin/src/main/java/org/apache/ranger/common/PropertiesUtil.java b/security-admin/src/main/java/org/apache/ranger/common/PropertiesUtil.java index 537d556a20..0dc5df8112 100644 --- a/security-admin/src/main/java/org/apache/ranger/common/PropertiesUtil.java +++ b/security-admin/src/main/java/org/apache/ranger/common/PropertiesUtil.java @@ -268,12 +268,18 @@ protected void processProperties( db_ssl_verifyServerCertificate="false"; } db_ssl_verifyServerCertificate=db_ssl_verifyServerCertificate.toLowerCase(); + String db_ssl_auth_type=propertiesMap.get("ranger.db.ssl.auth.type"); + if(StringUtils.isEmpty(db_ssl_auth_type)|| !"1-way".equalsIgnoreCase(db_ssl_auth_type)){ + db_ssl_auth_type="2-way"; + } propertiesMap.put("ranger.db.ssl.enabled", db_ssl_enabled); props.put("ranger.db.ssl.enabled", db_ssl_enabled); propertiesMap.put("ranger.db.ssl.required", db_ssl_required); props.put("ranger.db.ssl.required", db_ssl_required); propertiesMap.put("ranger.db.ssl.verifyServerCertificate", db_ssl_verifyServerCertificate); props.put("ranger.db.ssl.verifyServerCertificate", db_ssl_verifyServerCertificate); + propertiesMap.put("ranger.db.ssl.auth.type", db_ssl_auth_type); + props.put("ranger.db.ssl.auth.type", db_ssl_auth_type); String ranger_jpa_jdbc_url=propertiesMap.get("ranger.jpa.jdbc.url"); if(!StringUtils.isEmpty(ranger_jpa_jdbc_url)){ StringBuffer ranger_jpa_jdbc_url_ssl=new StringBuffer(ranger_jpa_jdbc_url); diff --git a/security-admin/src/main/resources/conf.dist/ranger-admin-default-site.xml b/security-admin/src/main/resources/conf.dist/ranger-admin-default-site.xml index 9dfc03df11..1e52a442cc 100644 --- a/security-admin/src/main/resources/conf.dist/ranger-admin-default-site.xml +++ b/security-admin/src/main/resources/conf.dist/ranger-admin-default-site.xml @@ -507,6 +507,10 @@ ranger.db.ssl.verifyServerCertificate false + + ranger.db.ssl.auth.type + 2-way + ranger.keystore.file From 21c56088457382be34325732ec4a0cc05b28b9f6 Mon Sep 17 00:00:00 2001 From: Bhavik Patel Date: Thu, 16 Aug 2018 12:25:25 +0530 Subject: [PATCH 129/151] RANGER-2187 : External Group search fails on Ranger UI when installed with postgres Signed-off-by: Mehul Parikh (cherry picked from commit 49142eb1b8a31f7ff575eafe74a902bebaf854e3) --- .../src/main/java/org/apache/ranger/rest/XUserREST.java | 5 ++--- .../main/java/org/apache/ranger/service/XGroupService.java | 2 +- .../src/test/java/org/apache/ranger/rest/TestXUserREST.java | 4 ++-- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/security-admin/src/main/java/org/apache/ranger/rest/XUserREST.java b/security-admin/src/main/java/org/apache/ranger/rest/XUserREST.java index a07c243af2..3f25506e01 100644 --- a/security-admin/src/main/java/org/apache/ranger/rest/XUserREST.java +++ b/security-admin/src/main/java/org/apache/ranger/rest/XUserREST.java @@ -242,8 +242,7 @@ public VXGroupList searchXGroups(@Context HttpServletRequest request) { request, xGroupService.sortFields); searchUtil.extractString(request, searchCriteria, "name", "group name", null); searchUtil.extractInt(request, searchCriteria, "isVisible", "Group Visibility"); - searchUtil.extractString(request, searchCriteria, "groupSource", "group source", null); -// searchUtil.extractInt(request, searchCriteria, "groupSource", "group source"); + searchUtil.extractInt(request, searchCriteria, "groupSource", "group source"); return xUserMgr.searchXGroups(searchCriteria); } @@ -1152,4 +1151,4 @@ public void deleteSingleGroupByGroupId(@Context HttpServletRequest request, @Pat xUserMgr.deleteXGroup(groupId, forceDelete); } } -} \ No newline at end of file +} diff --git a/security-admin/src/main/java/org/apache/ranger/service/XGroupService.java b/security-admin/src/main/java/org/apache/ranger/service/XGroupService.java index e259eaeca2..fbb8a5c1f3 100644 --- a/security-admin/src/main/java/org/apache/ranger/service/XGroupService.java +++ b/security-admin/src/main/java/org/apache/ranger/service/XGroupService.java @@ -69,7 +69,7 @@ public XGroupService() { searchFields.add(new SearchField("name", "obj.name", SearchField.DATA_TYPE.STRING, SearchField.SEARCH_TYPE.PARTIAL)); searchFields.add(new SearchField("groupSource", "obj.groupSource", - SearchField.DATA_TYPE.STRING, SearchField.SEARCH_TYPE.FULL)); + SearchField.DATA_TYPE.INTEGER, SearchField.SEARCH_TYPE.FULL)); searchFields.add(new SearchField("isVisible", "obj.isVisible", SearchField.DATA_TYPE.INTEGER, SearchField.SEARCH_TYPE.FULL )); diff --git a/security-admin/src/test/java/org/apache/ranger/rest/TestXUserREST.java b/security-admin/src/test/java/org/apache/ranger/rest/TestXUserREST.java index 5068fdc799..fceda025dd 100644 --- a/security-admin/src/test/java/org/apache/ranger/rest/TestXUserREST.java +++ b/security-admin/src/test/java/org/apache/ranger/rest/TestXUserREST.java @@ -327,7 +327,7 @@ public void test11searchXGroups() { Mockito.when(searchUtil.extractString(request, testSearchCriteria, "name", "group name", null)).thenReturn(""); Mockito.when(searchUtil.extractInt(request, testSearchCriteria, "isVisible", "Group Visibility")).thenReturn(1); - Mockito.when(searchUtil.extractString(request, testSearchCriteria, "groupSource", "group source", null)).thenReturn(""); + Mockito.when(searchUtil.extractInt(request, testSearchCriteria, "groupSource", "group source")).thenReturn(0); VXGroupList testvXGroupList=createxGroupList(); Mockito.when(xUserMgr.searchXGroups(testSearchCriteria)).thenReturn(testvXGroupList); VXGroupList outputvXGroupList=xUserRest.searchXGroups(request); @@ -336,7 +336,7 @@ public void test11searchXGroups() { Mockito.verify(searchUtil).extractCommonCriterias((HttpServletRequest)Mockito.anyObject() ,(List)Mockito.anyObject()); Mockito.verify(searchUtil).extractString(request, testSearchCriteria, "name", "group name", null); Mockito.verify(searchUtil).extractInt(request, testSearchCriteria, "isVisible", "Group Visibility"); - Mockito.verify(searchUtil).extractString(request, testSearchCriteria, "groupSource", "group source", null); + Mockito.verify(searchUtil).extractInt(request, testSearchCriteria, "groupSource", "group source"); assertNotNull(outputvXGroupList); assertEquals(outputvXGroupList.getTotalCount(),testvXGroupList.getTotalCount()); assertEquals(outputvXGroupList.getClass(),testvXGroupList.getClass()); From 7d40b35a29a17acdd2d14d125726e38f83baa8ef Mon Sep 17 00:00:00 2001 From: Vishal Suvagia Date: Mon, 24 Sep 2018 11:27:32 +0530 Subject: [PATCH 130/151] RANGER-2204: Fixed issue where audit log values for policy service are not processed properly Signed-off-by: Sailaja Polavarapu (cherry picked from commit 9f13560204dc2c7e721ac73a4158f5d8ba8e86da) --- .../ranger/service/RangerPolicyService.java | 21 ++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/security-admin/src/main/java/org/apache/ranger/service/RangerPolicyService.java b/security-admin/src/main/java/org/apache/ranger/service/RangerPolicyService.java index e8b43feca3..cfcb56e3be 100644 --- a/security-admin/src/main/java/org/apache/ranger/service/RangerPolicyService.java +++ b/security-admin/src/main/java/org/apache/ranger/service/RangerPolicyService.java @@ -228,10 +228,10 @@ private XXTrxLog processFieldToCreateTrxLog(Field field, String objectName, } else if (fieldName.equalsIgnoreCase(ROWFILTER_POLICY_ITEM_CLASS_FIELD_NAME)) { value = processRowFilterPolicyItemForTrxLog(field.get(vObj)); } else if (fieldName.equalsIgnoreCase(IS_ENABLED_CLASS_FIELD_NAME)) { - value = String.valueOf(processIsEnabledClassFieldNameForTrxLog(field.get(vObj))); - - } - else { + value = processIsEnabledClassFieldNameForTrxLog(field.get(vObj)); + } else if (fieldName.equalsIgnoreCase(IS_AUDIT_ENABLED_CLASS_FIELD_NAME)) { + value = processIsAuditEnabledClassFieldNameForTrxLog(field.get(vObj)); + } else { value = "" + field.get(vObj); } @@ -318,7 +318,11 @@ private XXTrxLog processFieldToCreateTrxLog(Field field, String objectName, } }else if (fieldName.equalsIgnoreCase(IS_ENABLED_CLASS_FIELD_NAME)) { if (oldPolicy != null) { - oldValue = String.valueOf(processIsEnabledClassFieldNameForTrxLog(oldPolicy.getIsEnabled())); + oldValue = processIsEnabledClassFieldNameForTrxLog(oldPolicy.getIsEnabled()); + } + } else if(fieldName.equalsIgnoreCase(IS_AUDIT_ENABLED_CLASS_FIELD_NAME)) { + if (oldPolicy != null) { + oldValue = processIsAuditEnabledClassFieldNameForTrxLog(oldPolicy.getIsAuditEnabled()); } } if (oldValue == null || oldValue.equalsIgnoreCase(value)) { @@ -577,6 +581,13 @@ private String processIsEnabledClassFieldNameForTrxLog(Object value) { return isEnabled; } + private String processIsAuditEnabledClassFieldNameForTrxLog(Object value) { + if(value == null) + return null; + String isAuditEnabled = String.valueOf(value); + return isAuditEnabled; + } + private boolean compareTwoDataMaskingPolicyItemList(String value, String oldValue) { if (value == null && oldValue == null) { return true; From ac456e84c848cf3378ee97c95b4a381bbdd6f703 Mon Sep 17 00:00:00 2001 From: Abhay Kulkarni Date: Thu, 27 Sep 2018 17:03:50 -0700 Subject: [PATCH 131/151] RANGER-2218: Added validations for names duing service def updates --- .../validation/RangerServiceDefValidator.java | 123 ++++++++++++++++-- .../TestRangerServiceDefValidator.java | 36 ++--- .../java/org/apache/ranger/biz/XUserMgr.java | 11 +- .../org/apache/ranger/biz/TestXUserMgr.java | 1 + 4 files changed, 145 insertions(+), 26 deletions(-) diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerServiceDefValidator.java b/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerServiceDefValidator.java index 79ac6741e7..709c396a58 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerServiceDefValidator.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerServiceDefValidator.java @@ -26,6 +26,7 @@ import java.util.Map; import java.util.Objects; import java.util.Set; +import java.util.HashMap; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang.StringUtils; @@ -40,6 +41,7 @@ import org.apache.ranger.plugin.model.RangerServiceDef.RangerPolicyConditionDef; import org.apache.ranger.plugin.model.RangerServiceDef.RangerResourceDef; import org.apache.ranger.plugin.model.RangerServiceDef.RangerServiceConfigDef; +import org.apache.ranger.plugin.model.RangerServiceDef.RangerDataMaskTypeDef; import org.apache.ranger.plugin.store.ServiceStore; import com.google.common.collect.ImmutableSet; @@ -130,8 +132,8 @@ boolean isValid(final RangerServiceDef serviceDef, final Action action, final Li Long id = serviceDef.getId(); valid = isValidServiceDefId(id, action, failures) && valid; valid = isValidServiceDefName(serviceDef.getName(), id, action, failures) && valid; - valid = isValidAccessTypes(serviceDef.getAccessTypes(), failures) && valid; - if (isValidResources(serviceDef, failures)) { + valid = isValidAccessTypes(serviceDef.getId(), serviceDef.getAccessTypes(), failures, action) && valid; + if (isValidResources(serviceDef, failures, action)) { // Semantic check of resource graph can only be done if resources are "syntactically" valid valid = isValidResourceGraph(serviceDef, failures) && valid; } else { @@ -144,7 +146,8 @@ boolean isValid(final RangerServiceDef serviceDef, final Action action, final Li } else { valid = false; } - valid = isValidPolicyConditions(serviceDef.getPolicyConditions(), failures) && valid; + valid = isValidPolicyConditions(serviceDef.getId(), serviceDef.getPolicyConditions(), failures, action) && valid; + valid = isValidDataMaskTypes(serviceDef.getId(), serviceDef.getDataMaskDef().getMaskTypes(), failures, action) && valid; } if(LOG.isDebugEnabled()) { @@ -231,11 +234,10 @@ boolean isValidServiceDefName(String name, Long id, final Action action, final L return valid; } - boolean isValidAccessTypes(final List accessTypeDefs, final List failures) { + boolean isValidAccessTypes(final Long serviceDefId, final List accessTypeDefs, final List failures, final Action action) { if(LOG.isDebugEnabled()) { LOG.debug(String.format("==> RangerServiceDefValidator.isValidAccessTypes(%s, %s)", accessTypeDefs, failures)); } - boolean valid = true; if (CollectionUtils.isEmpty(accessTypeDefs)) { ValidationErrorCode error = ValidationErrorCode.SERVICE_DEF_VALIDATION_ERR_MISSING_FIELD; @@ -247,13 +249,32 @@ boolean isValidAccessTypes(final List accessTypeDefs, final .build()); valid = false; } else { + Map existingAccessTypeIDNameMap = new HashMap<>(); + if (action == Action.UPDATE) { + List existingAccessTypes = this.getServiceDef(serviceDefId).getAccessTypes(); + for (RangerAccessTypeDef existingAccessType : existingAccessTypes) { + existingAccessTypeIDNameMap.put(existingAccessType.getItemId(), existingAccessType.getName()); + } + } + LOG.debug("accessType names from db = " + existingAccessTypeIDNameMap.values()); + List defsWithImpliedGrants = new ArrayList(); Set accessNames = new HashSet(); Set ids = new HashSet(); for (RangerAccessTypeDef def : accessTypeDefs) { String name = def.getName(); + Long itemId = def.getItemId(); + LOG.debug("accessType name from input = " + name); valid = isUnique(name, accessNames, "access type name", "access types", failures) && valid; valid = isUnique(def.getItemId(), ids, "access type itemId", "access types", failures) && valid; + if (action == Action.UPDATE) { + if (existingAccessTypeIDNameMap.get(itemId) != null && !existingAccessTypeIDNameMap.get(itemId).equals(name)) { + ValidationErrorCode error; + error = ValidationErrorCode.SERVICE_DEF_VALIDATION_ERR_SERVICE_DEF_NAME_CONFICT; + failures.add((new ValidationFailureDetailsBuilder()).field("access type name").isSemanticallyIncorrect().errorCode(error.getErrorCode()).becauseOf(String.format("changing %s[%s] in %s is not supported", "access type name", name, "access types")).build()); + valid = false; + } + } if (CollectionUtils.isNotEmpty(def.getImpliedGrants())) { defsWithImpliedGrants.add(def); } @@ -295,7 +316,7 @@ boolean isValidAccessTypes(final List accessTypeDefs, final return valid; } - boolean isValidPolicyConditions(List policyConditions, List failures) { + boolean isValidPolicyConditions(Long serviceDefId, List policyConditions, List failures, final Action action) { if(LOG.isDebugEnabled()) { LOG.debug(String.format("==> RangerServiceDefValidator.isValidPolicyConditions(%s, %s)", policyConditions, failures)); } @@ -304,12 +325,31 @@ boolean isValidPolicyConditions(List policyConditions, if (CollectionUtils.isEmpty(policyConditions)) { LOG.debug("Configs collection was null/empty! ok"); } else { + Map existingPolicyCondIDNameMap = new HashMap<>(); + if (action == Action.UPDATE) { + List existingPolicyConditions = this.getServiceDef(serviceDefId).getPolicyConditions(); + for (RangerPolicyConditionDef existingPolicyCondition : existingPolicyConditions) { + existingPolicyCondIDNameMap.put(existingPolicyCondition.getItemId(), existingPolicyCondition.getName()); + } + } + LOG.debug("policy condition names from db = " + existingPolicyCondIDNameMap.values()); + Set ids = new HashSet(); Set names = new HashSet(); for (RangerPolicyConditionDef conditionDef : policyConditions) { - valid = isUnique(conditionDef.getItemId(), ids, "policy condition def itemId", "policy condition defs", failures) && valid; String name = conditionDef.getName(); + Long itemId = conditionDef.getItemId(); + LOG.debug("policy condition name from input = " + name); + valid = isUnique(itemId, ids, "policy condition def itemId", "policy condition defs", failures) && valid; valid = isUnique(name, names, "policy condition def name", "policy condition defs", failures) && valid; + if (action == Action.UPDATE) { + if (existingPolicyCondIDNameMap.get(itemId) != null && !existingPolicyCondIDNameMap.get(itemId).equals(name)) { + ValidationErrorCode error; + error = ValidationErrorCode.SERVICE_DEF_VALIDATION_ERR_SERVICE_DEF_NAME_CONFICT; + failures.add((new ValidationFailureDetailsBuilder()).field("policy condition def name").isSemanticallyIncorrect().errorCode(error.getErrorCode()).becauseOf(String.format("changing %s[%s] in %s is not supported", "policy condition def name", name, "policy condition defs")).build()); + valid = false; + } + } if (StringUtils.isBlank(conditionDef.getEvaluator())) { ValidationErrorCode error = ValidationErrorCode.SERVICE_DEF_VALIDATION_ERR_POLICY_CONDITION_NULL_EVALUATOR; failures.add(new ValidationFailureDetailsBuilder() @@ -445,7 +485,7 @@ boolean isValidConfigType(String type, String configName, List failures) { + boolean isValidResources(RangerServiceDef serviceDef, List failures, final Action action) { if(LOG.isDebugEnabled()) { LOG.debug(String.format("==> RangerServiceDefValidator.isValidResources(%s, %s)", serviceDef, failures)); } @@ -462,14 +502,34 @@ boolean isValidResources(RangerServiceDef serviceDef, List existingResourceIDNameMap = new HashMap<>(); + if (action == Action.UPDATE) { + List existingResources = this.getServiceDef(serviceDef.getId()).getResources(); + for (RangerResourceDef existingResource : existingResources) { + existingResourceIDNameMap.put(existingResource.getItemId(), existingResource.getName()); + } + } + LOG.debug("resource names from db = " + existingResourceIDNameMap.values()); + Set names = new HashSet(resources.size()); Set ids = new HashSet(resources.size()); for (RangerResourceDef resource : resources) { /* * While id is the natural key, name is a surrogate key. At several places code expects resource name to be unique within a service. */ - valid = isUnique(resource.getName(), names, "resource name", "resources", failures) && valid; - valid = isUnique(resource.getItemId(), ids, "resource itemId", "resources", failures) && valid; + String name = resource.getName(); + Long itemId = resource.getItemId(); + LOG.debug("resource name from input = " + name); + valid = isUnique(name, names, "resource name", "resources", failures) && valid; + valid = isUnique(itemId, ids, "resource itemId", "resources", failures) && valid; + if (action == Action.UPDATE) { + if (existingResourceIDNameMap.get(itemId) != null && !existingResourceIDNameMap.get(itemId).equals(name)) { + ValidationErrorCode error; + error = ValidationErrorCode.SERVICE_DEF_VALIDATION_ERR_SERVICE_DEF_NAME_CONFICT; + failures.add((new ValidationFailureDetailsBuilder()).field("resource name").isSemanticallyIncorrect().errorCode(error.getErrorCode()).becauseOf(String.format("changing %s[%s] in %s is not supported", "resource name", name, "resources")).build()); + valid = false; + } + } } } @@ -611,4 +671,47 @@ boolean isValidEnumElements(List enumElementsDefs, List dataMaskTypes, List failures, final Action action) { + if(LOG.isDebugEnabled()) { + LOG.debug(String.format("==> RangerServiceDefValidator.isValidDataMaskTypes(%s, %s)", dataMaskTypes, failures)); + } + boolean valid = true; + + if (CollectionUtils.isEmpty(dataMaskTypes)) { + LOG.debug("Configs collection was null/empty! ok"); + } else { + Map existingDataMaskTypeIDNameMap = new HashMap<>(); + if (action == Action.UPDATE) { + List existingDataMaskTypes = this.getServiceDef(serviceDefId).getDataMaskDef().getMaskTypes(); + for (RangerDataMaskTypeDef existingDataMaskType : existingDataMaskTypes) { + existingDataMaskTypeIDNameMap.put(existingDataMaskType.getItemId(), existingDataMaskType.getName()); + } + } + LOG.debug("data mask type names from db = " + existingDataMaskTypeIDNameMap.values()); + + Set ids = new HashSet(); + Set names = new HashSet(); + for (RangerDataMaskTypeDef dataMaskType : dataMaskTypes) { + String name = dataMaskType.getName(); + Long itemId = dataMaskType.getItemId(); + LOG.debug("data mask type name from input = " + name); + valid = isUnique(itemId, ids, "data mask type def itemId", "data mask type defs", failures) && valid; + valid = isUnique(name, names, "data mask type def name", "data mask type defs", failures) && valid; + if (action == Action.UPDATE) { + if (existingDataMaskTypeIDNameMap.get(itemId) != null && !existingDataMaskTypeIDNameMap.get(itemId).equals(name)) { + ValidationErrorCode error; + error = ValidationErrorCode.SERVICE_DEF_VALIDATION_ERR_SERVICE_DEF_NAME_CONFICT; + failures.add((new ValidationFailureDetailsBuilder()).field("data mask type def name").isSemanticallyIncorrect().errorCode(error.getErrorCode()).becauseOf(String.format("changing %s[%s] in %s is not supported", "data mask type def name", name, "data mask type defs")).build()); + valid = false; + } + } + } + } + + if(LOG.isDebugEnabled()) { + LOG.debug(String.format("<== RangerServiceDefValidator.isValidDataMaskTypes(%s, %s): %s", dataMaskTypes, failures, valid)); + } + return valid; + } } diff --git a/agents-common/src/test/java/org/apache/ranger/plugin/model/validation/TestRangerServiceDefValidator.java b/agents-common/src/test/java/org/apache/ranger/plugin/model/validation/TestRangerServiceDefValidator.java index 33e6f4a946..370f549a9d 100644 --- a/agents-common/src/test/java/org/apache/ranger/plugin/model/validation/TestRangerServiceDefValidator.java +++ b/agents-common/src/test/java/org/apache/ranger/plugin/model/validation/TestRangerServiceDefValidator.java @@ -211,46 +211,50 @@ public final void test_isValidName_update() throws Exception { @Test public final void test_isValidAccessTypes_happyPath() { + long id = 7; + when(_serviceDef.getId()).thenReturn(id); List input = _utils.createAccessTypeDefs(accessTypes_good); - assertTrue(_validator.isValidAccessTypes(input, _failures)); + assertTrue(_validator.isValidAccessTypes(id, input, _failures, Action.CREATE)); assertTrue(_failures.isEmpty()); } @Test public final void test_isValidAccessTypes_failures() { + long id = 7; + when(_serviceDef.getId()).thenReturn(id); // null or empty access type defs List accessTypeDefs = null; - _failures.clear(); assertFalse(_validator.isValidAccessTypes(accessTypeDefs, _failures)); + _failures.clear(); assertFalse(_validator.isValidAccessTypes(id, accessTypeDefs, _failures, Action.CREATE)); _utils.checkFailureForMissingValue(_failures, "access types"); accessTypeDefs = new ArrayList(); - _failures.clear(); assertFalse(_validator.isValidAccessTypes(accessTypeDefs, _failures)); + _failures.clear(); assertFalse(_validator.isValidAccessTypes(id, accessTypeDefs, _failures, Action.CREATE)); _utils.checkFailureForMissingValue(_failures, "access types"); // null/empty access types accessTypeDefs = _utils.createAccessTypeDefs(new String[] { null, "", " " }); - _failures.clear(); assertFalse(_validator.isValidAccessTypes(accessTypeDefs, _failures)); + _failures.clear(); assertFalse(_validator.isValidAccessTypes(id, accessTypeDefs, _failures, Action.CREATE)); _utils.checkFailureForMissingValue(_failures, "access type name"); // duplicate access types accessTypeDefs = _utils.createAccessTypeDefs(new String[] { "read", "write", "execute", "read" } ); - _failures.clear(); assertFalse(_validator.isValidAccessTypes(accessTypeDefs, _failures)); + _failures.clear(); assertFalse(_validator.isValidAccessTypes(id, accessTypeDefs, _failures, Action.CREATE)); _utils.checkFailureForSemanticError(_failures, "access type name", "read"); // duplicate access types - case-insensitive accessTypeDefs = _utils.createAccessTypeDefs(new String[] { "read", "write", "execute", "READ" } ); - _failures.clear(); assertFalse(_validator.isValidAccessTypes(accessTypeDefs, _failures)); + _failures.clear(); assertFalse(_validator.isValidAccessTypes(id, accessTypeDefs, _failures, Action.CREATE)); _utils.checkFailureForSemanticError(_failures, "access type name", "READ"); // unknown access type in implied grants list accessTypeDefs = _utils.createAccessTypeDefs(accessTypes_bad_unknownType); - _failures.clear(); assertFalse(_validator.isValidAccessTypes(accessTypeDefs, _failures)); + _failures.clear(); assertFalse(_validator.isValidAccessTypes(id, accessTypeDefs, _failures, Action.CREATE)); _utils.checkFailureForSemanticError(_failures, "implied grants", "execute"); _utils.checkFailureForSemanticError(_failures, "access type itemId", "1"); // id 1 is duplicated // access type with implied grant referring to itself accessTypeDefs = _utils.createAccessTypeDefs(accessTypes_bad_selfReference); - _failures.clear(); assertFalse(_validator.isValidAccessTypes(accessTypeDefs, _failures)); + _failures.clear(); assertFalse(_validator.isValidAccessTypes(id, accessTypeDefs, _failures, Action.CREATE)); _utils.checkFailureForSemanticError(_failures, "implied grants", "admin"); } @@ -396,16 +400,16 @@ public final void test_isValidEnumElements_failures() { public final void test_isValidResources() { // null/empty resources are an error when(_serviceDef.getResources()).thenReturn(null); - _failures.clear(); assertFalse(_validator.isValidResources(_serviceDef, _failures)); + _failures.clear(); assertFalse(_validator.isValidResources(_serviceDef, _failures, Action.CREATE)); _utils.checkFailureForMissingValue(_failures, "resources"); List resources = new ArrayList(); when(_serviceDef.getResources()).thenReturn(resources); - _failures.clear(); assertFalse(_validator.isValidResources(_serviceDef, _failures)); + _failures.clear(); assertFalse(_validator.isValidResources(_serviceDef, _failures, Action.CREATE)); _utils.checkFailureForMissingValue(_failures, "resources"); resources.addAll(_utils.createResourceDefsWithIds(invalidResources)); - _failures.clear(); assertFalse(_validator.isValidResources(_serviceDef, _failures)); + _failures.clear(); assertFalse(_validator.isValidResources(_serviceDef, _failures, Action.CREATE)); _utils.checkFailureForMissingValue(_failures, "resource name"); _utils.checkFailureForMissingValue(_failures, "resource itemId"); _utils.checkFailureForSemanticError(_failures, "resource itemId", "1"); // id 1 is duplicate @@ -456,7 +460,7 @@ public final void test_isValidResources_happyPath() { }; List resources = _utils.createResourceDefsWithIds(data); when(_serviceDef.getResources()).thenReturn(resources); - assertTrue(_validator.isValidResources(_serviceDef, _failures)); + assertTrue(_validator.isValidResources(_serviceDef, _failures, Action.CREATE)); assertTrue(_failures.isEmpty()); } @@ -490,11 +494,13 @@ public final void test_isValidConfigs_failures() { @Test public final void test_isValidPolicyConditions() { + long id = 7; + when(_serviceDef.getId()).thenReturn(id); // null/empty policy conditions are ok - assertTrue(_validator.isValidPolicyConditions(null, _failures)); + assertTrue(_validator.isValidPolicyConditions(id,null, _failures, Action.CREATE)); assertTrue(_failures.isEmpty()); List conditionDefs = new ArrayList(); - assertTrue(_validator.isValidPolicyConditions(conditionDefs, _failures)); + assertTrue(_validator.isValidPolicyConditions(id, conditionDefs, _failures, Action.CREATE)); assertTrue(_failures.isEmpty()); Object[][] policyCondition_data = { @@ -506,7 +512,7 @@ public final void test_isValidPolicyConditions() { }; conditionDefs.addAll(_utils.createPolicyConditionDefs(policyCondition_data)); - _failures.clear(); assertFalse(_validator.isValidPolicyConditions(conditionDefs, _failures)); + _failures.clear(); assertFalse(_validator.isValidPolicyConditions(id, conditionDefs, _failures, Action.CREATE)); _utils.checkFailureForMissingValue(_failures, "policy condition def itemId"); _utils.checkFailureForMissingValue(_failures, "policy condition def name"); _utils.checkFailureForMissingValue(_failures, "policy condition def evaluator"); diff --git a/security-admin/src/main/java/org/apache/ranger/biz/XUserMgr.java b/security-admin/src/main/java/org/apache/ranger/biz/XUserMgr.java index 4c01d57c43..410e3f852a 100644 --- a/security-admin/src/main/java/org/apache/ranger/biz/XUserMgr.java +++ b/security-admin/src/main/java/org/apache/ranger/biz/XUserMgr.java @@ -353,11 +353,15 @@ public VXUser updateXUser(VXUser vXUser) { checkAccess(vXUser.getName()); VXPortalUser oldUserProfile = userMgr.getUserProfileByLoginId(vXUser .getName()); + if (oldUserProfile == null) { + throw restErrorUtil.createRESTException( + "user " + vXUser.getName() + " does not exist.", + MessageEnums.INVALID_INPUT_DATA); + } VXPortalUser vXPortalUser = new VXPortalUser(); if (oldUserProfile != null && oldUserProfile.getId() != null) { vXPortalUser.setId(oldUserProfile.getId()); } - // TODO : There is a possibility that old user may not exist. vXPortalUser.setFirstName(vXUser.getFirstName()); if("null".equalsIgnoreCase(vXPortalUser.getFirstName())){ @@ -841,6 +845,11 @@ private void setUserDesc(VXUser vXUser) { public VXGroup updateXGroup(VXGroup vXGroup) { checkAdminAccess(); XXGroup xGroup = daoManager.getXXGroup().getById(vXGroup.getId()); + if (vXGroup != null && xGroup != null && !vXGroup.getName().equals(xGroup.getName())) { + throw restErrorUtil.createRESTException( + "group name updates are not allowed.", + MessageEnums.INVALID_INPUT_DATA); + } List trxLogList = xGroupService.getTransactionLog(vXGroup, xGroup, "update"); xaBizUtil.createTrxLog(trxLogList); diff --git a/security-admin/src/test/java/org/apache/ranger/biz/TestXUserMgr.java b/security-admin/src/test/java/org/apache/ranger/biz/TestXUserMgr.java index 02798833a8..826307e8ca 100644 --- a/security-admin/src/test/java/org/apache/ranger/biz/TestXUserMgr.java +++ b/security-admin/src/test/java/org/apache/ranger/biz/TestXUserMgr.java @@ -477,6 +477,7 @@ public void test17UpdateXGroup() { vXGroup.setName("grouptest"); XXGroup xxGroup = new XXGroup(); + xxGroup.setName("grouptest"); Mockito.when(daoManager.getXXGroup()).thenReturn(xxGroupDao); Mockito.when(xxGroupDao.getById(vXGroup.getId())).thenReturn(xxGroup); Mockito.when(xGroupService.updateResource(vXGroup)).thenReturn(vXGroup); From 46c6cf878026b1c2d7e76f838c95733271e1497b Mon Sep 17 00:00:00 2001 From: Pradeep Agrawal Date: Wed, 19 Sep 2018 12:33:11 +0530 Subject: [PATCH 132/151] RANGER-2168: Add service admin user through service config (cherry picked from commit 0ebc2d30eb803f61ff51656bbc1a00f148297a08) (cherry picked from commit a8c4c0091929fa26a6afcc2946617f5ba9eeca10) --- .../org/apache/ranger/biz/ServiceDBStore.java | 16 +++++++++++++++ .../ranger/db/XXServiceConfigMapDao.java | 14 +++++++++++++ .../org/apache/ranger/rest/ServiceREST.java | 20 +++++++++---------- .../resources/META-INF/jpa_named_queries.xml | 5 +++++ 4 files changed, 44 insertions(+), 11 deletions(-) diff --git a/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java b/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java index 64cf043b76..ceee8cef29 100644 --- a/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java +++ b/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java @@ -204,6 +204,7 @@ public class ServiceDBStore extends AbstractServiceStore { private static final String TIMESTAMP = "Export time"; private static final String AMBARI_SERVICE_CHECK_USER = "ambari.service.check.user"; + private static final String SERVICE_ADMIN_USERS = "service.admin.users"; public static final String CRYPT_ALGO = PropertiesUtil.getProperty("ranger.password.encryption.algorithm", PasswordUtils.DEFAULT_CRYPT_ALGO); public static final String ENCRYPT_KEY = PropertiesUtil.getProperty("ranger.password.encryption.key", PasswordUtils.DEFAULT_ENCRYPT_KEY); @@ -3993,4 +3994,19 @@ private void createGenericUsers() { genericUser.setDescription(RangerPolicyEngine.RESOURCE_OWNER); xUserService.createXUserWithOutLogin(genericUser); } + + public boolean isServiceAdminUser(String serviceName, String userName) { + boolean ret=false; + XXServiceConfigMap cfgSvcAdminUsers = daoMgr.getXXServiceConfigMap().findByServiceNameAndConfigKey(serviceName, SERVICE_ADMIN_USERS); + String svcAdminUsers = cfgSvcAdminUsers != null ? cfgSvcAdminUsers.getConfigvalue() : null; + if (svcAdminUsers != null) { + for (String svcAdminUser : svcAdminUsers.split(",")) { + if (userName.equals(svcAdminUser)) { + ret=true; + break; + } + } + } + return ret; + } } diff --git a/security-admin/src/main/java/org/apache/ranger/db/XXServiceConfigMapDao.java b/security-admin/src/main/java/org/apache/ranger/db/XXServiceConfigMapDao.java index 9f97b6073c..9559161a21 100644 --- a/security-admin/src/main/java/org/apache/ranger/db/XXServiceConfigMapDao.java +++ b/security-admin/src/main/java/org/apache/ranger/db/XXServiceConfigMapDao.java @@ -60,4 +60,18 @@ public XXServiceConfigMap findByServiceAndConfigKey(Long serviceId, } } + public XXServiceConfigMap findByServiceNameAndConfigKey(String serviceName, String configKey) { + if(serviceName == null || configKey == null) { + return null; + } + try { + return getEntityManager() + .createNamedQuery("XXServiceConfigMap.findByServiceNameAndConfigKey", tClass) + .setParameter("name", serviceName) + .setParameter("configKey", configKey).getSingleResult(); + } catch (NoResultException e) { + return null; + } + } + } diff --git a/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java b/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java index e2a0c29ab9..5e5e7dd92b 100644 --- a/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java +++ b/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java @@ -2988,7 +2988,8 @@ private List applyAdminAccessFilter(List policies) { List listToFilter = entry.getValue(); if (CollectionUtils.isNotEmpty(listToFilter)) { - if (isAdmin || isKeyAdmin) { + boolean isServiceAdminUser=isAdmin || svcStore.isServiceAdminUser(serviceName, userName); + if (isAdmin || isKeyAdmin || isServiceAdminUser) { XXService xService = daoManager.getXXService().findByName(serviceName); Long serviceDefId = xService.getType(); boolean isKmsService = serviceDefId.equals(EmbeddedServiceDefsUtil.instance().getKmsServiceDefId()); @@ -2997,10 +2998,12 @@ private List applyAdminAccessFilter(List policies) { if (!isKmsService) { ret.addAll(listToFilter); } - } else { // isKeyAdmin + } else if (isKeyAdmin) { if (isKmsService) { ret.addAll(listToFilter); } + } else if (isServiceAdminUser) { + ret.addAll(listToFilter); } continue; @@ -3034,16 +3037,11 @@ void ensureAdminAccess(String serviceName, Map res boolean isKeyAdmin = bizUtil.isKeyAdmin(); String userName = bizUtil.getCurrentUserLoginId(); - if(!isAdmin && !isKeyAdmin) { - boolean isAllowed = false; + boolean isSvcAdmin = isAdmin || svcStore.isServiceAdminUser(serviceName, userName); - RangerPolicyEngine policyEngine = getDelegatedAdminPolicyEngine(serviceName); - - if (policyEngine != null) { - Set userGroups = userMgr.getGroupsForUser(userName); - - isAllowed = hasAdminAccess(serviceName, userName, userGroups, resources); - } + if(!isAdmin && !isKeyAdmin && !isSvcAdmin) { + Set userGroups = userMgr.getGroupsForUser(userName); + boolean isAllowed = hasAdminAccess(serviceName, userName, userGroups, resources); if (!isAllowed) { throw restErrorUtil.createRESTException(HttpServletResponse.SC_UNAUTHORIZED, diff --git a/security-admin/src/main/resources/META-INF/jpa_named_queries.xml b/security-admin/src/main/resources/META-INF/jpa_named_queries.xml index 786b4bfdd7..4a7055deea 100644 --- a/security-admin/src/main/resources/META-INF/jpa_named_queries.xml +++ b/security-admin/src/main/resources/META-INF/jpa_named_queries.xml @@ -390,6 +390,11 @@ obj.serviceId = :serviceId and obj.configKey = :configKey + + select obj from XXServiceConfigMap obj, XXService xSvc where + xSvc.name = :name and xSvc.id=obj.serviceId and obj.configKey = :configKey + + select obj from XXService obj where obj.name = :name From b050618aeb094234a3fb535eae3a27e0d2724519 Mon Sep 17 00:00:00 2001 From: Bhavik Patel Date: Fri, 10 Aug 2018 15:19:23 +0530 Subject: [PATCH 133/151] RANGER-2181 : Code Improvement To Follow Best Practices for saving services Signed-off-by: Mehul Parikh (cherry picked from commit 2dfd1ea459ea27461fd2abc108a55ce84e78670a) --- .../plugin/errors/ValidationErrorCode.java | 1 + .../validation/RangerServiceValidator.java | 59 +++-- .../TestRangerServiceValidator.java | 212 ++++++++++++++++++ .../org/apache/ranger/rest/ServiceREST.java | 10 + .../webapp/scripts/models/RangerServiceDef.js | 2 +- security-admin/src/main/webapp/styles/xa.css | 6 + .../webapp/templates/helpers/XAHelpers.js | 2 +- 7 files changed, 276 insertions(+), 16 deletions(-) diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/errors/ValidationErrorCode.java b/agents-common/src/main/java/org/apache/ranger/plugin/errors/ValidationErrorCode.java index d0f015dc45..c2fe4ac778 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/errors/ValidationErrorCode.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/errors/ValidationErrorCode.java @@ -34,6 +34,7 @@ public enum ValidationErrorCode { SERVICE_VALIDATION_ERR_INVALID_SERVICE_ID(1005, "No service found for id [{0}]"), SERVICE_VALIDATION_ERR_INVALID_SERVICE_NAME(1006, "Missing service name"), SERVICE_VALIDATION_ERR_SERVICE_NAME_CONFICT(1007, "Duplicate service name: name=[{0}]"), + SERVICE_VALIDATION_ERR_SPECIAL_CHARACTERS_SERVICE_NAME(3031, "Name should not start with space, it should be less than 256 characters and special characters are not allowed(except _ - and space). : name=[{0}]"), SERVICE_VALIDATION_ERR_ID_NAME_CONFLICT(1008, "Duplicate service name: name=[{0}], id=[{1}]"), SERVICE_VALIDATION_ERR_MISSING_SERVICE_DEF(1009, "Missing service def"), SERVICE_VALIDATION_ERR_INVALID_SERVICE_DEF(1010, "Service def not found: service-def-name=[{0}]"), diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerServiceValidator.java b/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerServiceValidator.java index d3efdc9b55..48b9d28912 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerServiceValidator.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerServiceValidator.java @@ -22,7 +22,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Set; - +import java.util.regex.Pattern; import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -35,9 +35,10 @@ import com.google.common.collect.Sets; public class RangerServiceValidator extends RangerValidator { - private static final Log LOG = LogFactory.getLog(RangerServiceValidator.class); + static final public String VALIDATION_SERVICE_NAME = "^[a-zA-Z0-9_-][a-zA-Z0-9\\s_-]{0,254}"; + static Pattern serviceNameCompiledRegEx; public RangerServiceValidator(ServiceStore store) { super(store); } @@ -151,9 +152,8 @@ boolean isValid(RangerService service, Action action, List listRangerServiceConfigDef = new ArrayList(); + listRangerServiceConfigDef.add(configDef); + + + configDef.setName("myconfig1"); + + Map testMap = new HashMap(); + testMap.put("myconfig1", "myconfig1"); + + rangerService.setConfigs(testMap); + + + RangerServiceDef rangerServiceDef = new RangerServiceDef(); + rangerServiceDef.setConfigs(listRangerServiceConfigDef); + + when(_store.getServiceDefByName("yarn")).thenReturn(rangerServiceDef); + boolean valid = _validator.isValid(rangerService, Action.CREATE, _failures); + Assert.assertEquals(0, _failures.size()); + Assert.assertTrue(valid); + + } + + @Test + public void testIsValidServiceNameUpdationWithOutSpecialCharacters() throws Exception{ + RangerService rangerService = new RangerService(); + rangerService.setId(1L); + rangerService.setName("c1_yarn"); + rangerService.setType("yarn"); + rangerService.setTagService(""); + + RangerServiceConfigDef configDef = new RangerServiceConfigDef(); + configDef.setMandatory(true); + + List listRangerServiceConfigDef = new ArrayList(); + listRangerServiceConfigDef.add(configDef); + + + configDef.setName("myconfig1"); + + Map testMap = new HashMap(); + testMap.put("myconfig1", "myconfig1"); + + rangerService.setConfigs(testMap); + + + RangerServiceDef rangerServiceDef = new RangerServiceDef(); + rangerServiceDef.setConfigs(listRangerServiceConfigDef); + + when(_store.getService(1L)).thenReturn(rangerService); + when(_store.getServiceDefByName("yarn")).thenReturn(rangerServiceDef); + boolean valid = _validator.isValid(rangerService, Action.UPDATE, _failures); + Assert.assertEquals(0, _failures.size()); + Assert.assertTrue(valid); + + } + + @Test + public void testIsValidServiceNameUpdationWithSpecialCharacters() throws Exception{ + RangerService rangerService = new RangerService(); + rangerService.setId(1L); + rangerService.setName("c1_yarn"); + rangerService.setType("yarn"); + rangerService.setTagService(""); + + RangerServiceConfigDef configDef = new RangerServiceConfigDef(); + configDef.setMandatory(true); + + List listRangerServiceConfigDef = new ArrayList(); + listRangerServiceConfigDef.add(configDef); + + + configDef.setName("myconfig1"); + + Map testMap = new HashMap(); + testMap.put("myconfig1", "myconfig1"); + + rangerService.setConfigs(testMap); + + + RangerServiceDef rangerServiceDef = new RangerServiceDef(); + rangerServiceDef.setConfigs(listRangerServiceConfigDef); + + when(_store.getService(1L)).thenReturn(rangerService); + when(_store.getServiceDefByName("yarn")).thenReturn(rangerServiceDef); + boolean valid = _validator.isValid(rangerService, Action.UPDATE, _failures); + ValidationFailureDetails failureMessage = _failures.get(0); + Assert.assertFalse(valid); + Assert.assertEquals("name",failureMessage.getFieldName()); + Assert.assertEquals(serviceNameValidationErrorMessage + ": name=[c1_yarn]",failureMessage._reason); + Assert.assertEquals(3031, failureMessage._errorCode); + + } + + @Test + public void testIsValidServiceNameCreationWithSpecialCharacters() throws Exception{ + RangerService rangerService = new RangerService(); + rangerService.setName(""); + rangerService.setType("yarn"); + rangerService.setTagService(""); + + RangerServiceConfigDef configDef = new RangerServiceConfigDef(); + configDef.setMandatory(true); + + List listRangerServiceConfigDef = new ArrayList(); + listRangerServiceConfigDef.add(configDef); + + + configDef.setName("myconfig1"); + + Map testMap = new HashMap(); + testMap.put("myconfig1", "myconfig1"); + + rangerService.setConfigs(testMap); + + + RangerServiceDef rangerServiceDef = new RangerServiceDef(); + rangerServiceDef.setConfigs(listRangerServiceConfigDef); + + when(_store.getServiceDefByName("yarn")).thenReturn(rangerServiceDef); + boolean valid = _validator.isValid(rangerService, _action, _failures); + ValidationFailureDetails failureMessage = _failures.get(0); + Assert.assertFalse(valid); + Assert.assertEquals("name",failureMessage.getFieldName()); + Assert.assertEquals(serviceNameValidationErrorMessage + ": name=[]",failureMessage._reason); + Assert.assertEquals(3031, failureMessage._errorCode); + + } + + @Test + public void testIsValidServiceNameCreationWithGreater255Characters() throws Exception{ + RangerService rangerService = new RangerService(); + rangerService.setName("c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1"); + rangerService.setType("yarn"); + rangerService.setTagService(""); + + RangerServiceConfigDef configDef = new RangerServiceConfigDef(); + configDef.setMandatory(true); + + List listRangerServiceConfigDef = new ArrayList(); + listRangerServiceConfigDef.add(configDef); + + + configDef.setName("myconfig1"); + + Map testMap = new HashMap(); + testMap.put("myconfig1", "myconfig1"); + + rangerService.setConfigs(testMap); + + + RangerServiceDef rangerServiceDef = new RangerServiceDef(); + rangerServiceDef.setConfigs(listRangerServiceConfigDef); + + when(_store.getServiceDefByName("yarn")).thenReturn(rangerServiceDef); + boolean valid = _validator.isValid(rangerService, _action, _failures); + ValidationFailureDetails failureMessage = _failures.get(0); + Assert.assertFalse(valid); + Assert.assertEquals("name",failureMessage.getFieldName()); + Assert.assertEquals(serviceNameValidationErrorMessage + ": name=[c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1]",failureMessage._reason); + Assert.assertEquals(3031, failureMessage._errorCode); + + } + + @Test + public void testIsValidServiceNameUpdationWithGreater255Characters() throws Exception{ + RangerService rangerService = new RangerService(); + rangerService.setId(1L); + rangerService.setName("c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1"); + rangerService.setType("yarn"); + rangerService.setTagService(""); + + RangerServiceConfigDef configDef = new RangerServiceConfigDef(); + configDef.setMandatory(true); + + List listRangerServiceConfigDef = new ArrayList(); + listRangerServiceConfigDef.add(configDef); + + + configDef.setName("myconfig1"); + + Map testMap = new HashMap(); + testMap.put("myconfig1", "myconfig1"); + + rangerService.setConfigs(testMap); + + + RangerServiceDef rangerServiceDef = new RangerServiceDef(); + rangerServiceDef.setConfigs(listRangerServiceConfigDef); + + when(_store.getService(1L)).thenReturn(rangerService); + when(_store.getServiceDefByName("yarn")).thenReturn(rangerServiceDef); + boolean valid = _validator.isValid(rangerService, Action.UPDATE, _failures); + ValidationFailureDetails failureMessage = _failures.get(0); + Assert.assertFalse(valid); + Assert.assertEquals("name",failureMessage.getFieldName()); + Assert.assertEquals(serviceNameValidationErrorMessage +": name=[c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1]",failureMessage._reason); + Assert.assertEquals(3031, failureMessage._errorCode); + + } + + @Test public void testIsValid_failures() throws Exception { RangerService service = mock(RangerService.class); // passing in a null service to the check itself is an error diff --git a/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java b/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java index 5e5e7dd92b..6685e1a588 100644 --- a/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java +++ b/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java @@ -626,12 +626,17 @@ public RangerService createService(RangerService service) { RangerPerfTracer perf = null; try { + if(RangerPerfTracer.isPerfTraceEnabled(PERF_LOG)) { perf = RangerPerfTracer.getPerfTracer(PERF_LOG, "ServiceREST.createService(serviceName=" + service.getName() + ")"); } RangerServiceValidator validator = validatorFactory.getServiceValidator(svcStore); validator.validate(service, Action.CREATE); + if(!StringUtils.isEmpty(service.getName().trim())){ + service.setName(service.getName().trim()); + } + UserSessionBase session = ContextUtil.getCurrentUserSession(); XXServiceDef xxServiceDef = daoManager.getXXServiceDef().findByName(service.getType()); if(session != null && !session.isSpnegoEnabled()){ @@ -683,12 +688,17 @@ public RangerService updateService(RangerService service, RangerPerfTracer perf = null; try { + if(RangerPerfTracer.isPerfTraceEnabled(PERF_LOG)) { perf = RangerPerfTracer.getPerfTracer(PERF_LOG, "ServiceREST.updateService(serviceName=" + service.getName() + ")"); } RangerServiceValidator validator = validatorFactory.getServiceValidator(svcStore); validator.validate(service, Action.UPDATE); + if(!StringUtils.isEmpty(service.getName().trim())){ + service.setName(service.getName().trim()); + } + bizUtil.hasAdminPermissions("Services"); // TODO: As of now we are allowing SYS_ADMIN to create all the diff --git a/security-admin/src/main/webapp/scripts/models/RangerServiceDef.js b/security-admin/src/main/webapp/scripts/models/RangerServiceDef.js index 73e19a1700..03bd15f357 100644 --- a/security-admin/src/main/webapp/scripts/models/RangerServiceDef.js +++ b/security-admin/src/main/webapp/scripts/models/RangerServiceDef.js @@ -58,7 +58,7 @@ define(function(require){ name : { type : 'Text', title : 'Service Name *', - validators : ['required',{type:'regexp',regexp:/^[a-zA-Z0-9\s_-]{1,512}$/,message :"Name should be less than 512 characters and special characters are not allowed."}], + validators : ['required',{type:'regexp',regexp:/^[a-zA-Z0-9_-][a-zA-Z0-9\s_-]{0,254}/,message :"Name should not start with space, it should be less than 256 characters and special characters are not allowed(except _ - and space)."}], }, description : { type : 'TextArea', diff --git a/security-admin/src/main/webapp/styles/xa.css b/security-admin/src/main/webapp/styles/xa.css index a0e0bf2a7c..f1f7cb12e9 100644 --- a/security-admin/src/main/webapp/styles/xa.css +++ b/security-admin/src/main/webapp/styles/xa.css @@ -2221,3 +2221,9 @@ td.subgrid-custom-cell{ text-overflow: ellipsis; max-width: 95%; } +.serviceNameEllipsis { + max-width: 250px; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} diff --git a/security-admin/src/main/webapp/templates/helpers/XAHelpers.js b/security-admin/src/main/webapp/templates/helpers/XAHelpers.js index f8479e47ff..cdb0602a90 100644 --- a/security-admin/src/main/webapp/templates/helpers/XAHelpers.js +++ b/security-admin/src/main/webapp/templates/helpers/XAHelpers.js @@ -516,7 +516,7 @@
' } tr += '
\ - '+_.escape(serv.attributes.name)+''+serviceOperationDiv+'\ + '+_.escape(serv.attributes.name)+''+serviceOperationDiv+'\
'; }); } From 7deed8eb5abd9cf25e70ba6ece4d7b17c57ca119 Mon Sep 17 00:00:00 2001 From: Pradeep Date: Tue, 9 Oct 2018 11:35:12 +0530 Subject: [PATCH 134/151] RANGER-2169: Create unique index on service and name column of x_policy table --- ...dd-unique-constraint-on-table-x_policy.sql | 35 ++++++++++++++++++ ...dd-unique-constraint-on-table-x_policy.sql | 35 ++++++++++++++++++ ...dd-unique-constraint-on-table-x_policy.sql | 34 ++++++++++++++++++ ...dd-unique-constraint-on-table-x_policy.sql | 36 +++++++++++++++++++ .../current/ranger_core_db_sqlserver.sql | 15 ++++++-- ...dd-unique-constraint-on-table-x_policy.sql | 29 +++++++++++++++ 6 files changed, 181 insertions(+), 3 deletions(-) create mode 100644 security-admin/db/mysql/patches/033-add-unique-constraint-on-table-x_policy.sql create mode 100644 security-admin/db/oracle/patches/033-add-unique-constraint-on-table-x_policy.sql create mode 100644 security-admin/db/postgres/patches/033-add-unique-constraint-on-table-x_policy.sql create mode 100644 security-admin/db/sqlanywhere/patches/033-add-unique-constraint-on-table-x_policy.sql create mode 100644 security-admin/db/sqlserver/patches/033-add-unique-constraint-on-table-x_policy.sql diff --git a/security-admin/db/mysql/patches/033-add-unique-constraint-on-table-x_policy.sql b/security-admin/db/mysql/patches/033-add-unique-constraint-on-table-x_policy.sql new file mode 100644 index 0000000000..8deb285f7d --- /dev/null +++ b/security-admin/db/mysql/patches/033-add-unique-constraint-on-table-x_policy.sql @@ -0,0 +1,35 @@ +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +drop procedure if exists create_unique_constraint_on_name_service; + +delimiter ;; +create procedure create_unique_constraint_on_name_service() begin + /* check tables exist or not */ + if exists (select * from information_schema.columns where table_schema=database() and table_name = 'x_policy' and column_name in('service','name')) then + /* check unique constraint exist on service and name column or not */ + if not exists (select * from information_schema.columns where table_schema=database() and table_name = 'x_policy' and column_name in('name') and column_key in('UNI','MUL')) then + if not exists (select * from information_schema.table_constraints where table_schema=database() and table_name = 'x_policy' and constraint_name='x_policy_UK_name_service') then + UPDATE x_policy set name=concat(name,'-duplicate-',id) where id in (select id from (select id from x_policy where concat(service,name) in (select concat(service,name) from x_policy group by service,name having count(*) >1)) as tmp); + ALTER TABLE x_policy ADD UNIQUE INDEX x_policy_UK_name_service(name(180),service); + end if; + end if; + end if; +end;; + +delimiter ; +call create_unique_constraint_on_name_service(); + +drop procedure if exists create_unique_constraint_on_name_service; diff --git a/security-admin/db/oracle/patches/033-add-unique-constraint-on-table-x_policy.sql b/security-admin/db/oracle/patches/033-add-unique-constraint-on-table-x_policy.sql new file mode 100644 index 0000000000..dc97b3700f --- /dev/null +++ b/security-admin/db/oracle/patches/033-add-unique-constraint-on-table-x_policy.sql @@ -0,0 +1,35 @@ +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +DECLARE + v_count number:=0; + sql_stmt VARCHAR2(1000); + duplicate VARCHAR2(11):='-duplicate-'; +BEGIN + select count(*) into v_count from user_tab_cols where table_name='X_POLICY' and column_name IN('NAME','SERVICE'); + if (v_count = 2) then + v_count:=0; + select count(*) into v_count from user_constraints where table_name='X_POLICY' and constraint_name='X_POLICY_UK_NAME_SERVICE' and constraint_type='U'; + if (v_count = 0) then + v_count:=0; + select count(*) into v_count from user_ind_columns WHERE table_name='X_POLICY' and column_name IN('NAME','SERVICE') and index_name='X_POLICY_UK_NAME_SERVICE'; + if (v_count = 0) THEN + sql_stmt := 'UPDATE x_policy set name=concat(concat(name,:1),id) where id in (select id from (select id from x_policy where concat(service,name) in (select concat(service,name) from x_policy group by service,name having count(*) >1)))'; + EXECUTE IMMEDIATE sql_stmt USING duplicate; + EXECUTE IMMEDIATE 'ALTER TABLE X_POLICY ADD CONSTRAINT x_policy_UK_name_service UNIQUE (NAME,SERVICE)'; + end if; + commit; + end if; + end if; +end;/ diff --git a/security-admin/db/postgres/patches/033-add-unique-constraint-on-table-x_policy.sql b/security-admin/db/postgres/patches/033-add-unique-constraint-on-table-x_policy.sql new file mode 100644 index 0000000000..e3ac945672 --- /dev/null +++ b/security-admin/db/postgres/patches/033-add-unique-constraint-on-table-x_policy.sql @@ -0,0 +1,34 @@ +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +select 'delimiter start'; +CREATE OR REPLACE FUNCTION create_unique_constraint_on_policyname() +RETURNS void AS $$ +DECLARE + v_attnum integer := 0; +BEGIN + select attnum into v_attnum from pg_attribute where attrelid in(select oid from pg_class where relname='x_policy') and attname in('name'); + IF v_attnum > 0 THEN + IF not exists (select * from pg_constraint where conrelid in(select oid from pg_class where relname='x_policy') and conname='x_policy_uk_name_service' and contype='u') THEN + IF not exists (select * from pg_index where indrelid in(select oid from pg_class where relname='x_policy') and indkey[0]=v_attnum) THEN + UPDATE x_policy set name=(name || '-duplicate-' || id) where id in (select id from (select id from x_policy where service || name in (select service || name from x_policy group by service,name having count(*) >1)) as tmp); + ALTER TABLE x_policy ADD CONSTRAINT x_policy_uk_name_service UNIQUE(name,service); + END IF; + END IF; + END IF; + +END; +$$ LANGUAGE plpgsql; +select create_unique_constraint_on_policyname(); +select 'delimiter end'; diff --git a/security-admin/db/sqlanywhere/patches/033-add-unique-constraint-on-table-x_policy.sql b/security-admin/db/sqlanywhere/patches/033-add-unique-constraint-on-table-x_policy.sql new file mode 100644 index 0000000000..ace31d68a8 --- /dev/null +++ b/security-admin/db/sqlanywhere/patches/033-add-unique-constraint-on-table-x_policy.sql @@ -0,0 +1,36 @@ +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +BEGIN +DECLARE tableID INT = 0; +DECLARE columnID INT = 0; +DECLARE guTableID INT = 0; +DECLARE guColumnID INT = 0; + IF EXISTS(select * from SYS.SYSCOLUMNS where tname = 'x_policy' and cname='name') THEN + IF NOT EXISTS(select * from SYS.SYSCONSTRAINT where constraint_name = 'x_policy_UK_name_service') THEN + select table_id into tableID from SYS.SYSTAB where table_name = 'x_policy'; + select column_id into columnID from SYS.SYSTABCOL where table_id=tableID and column_name = 'name'; + IF NOT EXISTS(select * from SYS.SYSIDXCOL where table_id=tableID and column_id=columnID) THEN + UPDATE x_policy set name=(name || '-duplicate-' || id) where id in (select id from (select id from x_policy where service || name in (select service || name from x_policy group by service,name having count(*) >1)) as tmp); + DROP INDEX x_policy_service; + ALTER TABLE dbo.x_policy DROP CONSTRAINT x_policy_FK_service; + ALTER TABLE dbo.x_policy ALTER name varchar(512) NOT NULL, ALTER service bigint NOT NULL ; + ALTER TABLE dbo.x_policy ADD CONSTRAINT x_policy_UK_name_service UNIQUE NONCLUSTERED (name,service); + ALTER TABLE dbo.x_policy ADD CONSTRAINT x_policy_FK_service FOREIGN KEY(service) REFERENCES dbo.x_service (id); + CREATE NONCLUSTERED INDEX x_policy_service ON dbo.x_policy(service ASC); + END IF; + END IF; + END IF; +END +GO diff --git a/security-admin/db/sqlserver/optimized/current/ranger_core_db_sqlserver.sql b/security-admin/db/sqlserver/optimized/current/ranger_core_db_sqlserver.sql index a79c954491..bf73c34eab 100644 --- a/security-admin/db/sqlserver/optimized/current/ranger_core_db_sqlserver.sql +++ b/security-admin/db/sqlserver/optimized/current/ranger_core_db_sqlserver.sql @@ -461,6 +461,10 @@ IF (OBJECT_ID('x_plugin_info_UK') IS NOT NULL) BEGIN ALTER TABLE [dbo].[x_plugin_info] DROP CONSTRAINT x_plugin_info_UK END +IF (OBJECT_ID('x_policy$x_policy_UK_name_service') IS NOT NULL) +BEGIN + ALTER TABLE [dbo].[x_policy] DROP CONSTRAINT x_policy$x_policy_UK_name_service +END IF (OBJECT_ID('vx_trx_log') IS NOT NULL) BEGIN DROP VIEW [dbo].[vx_trx_log] @@ -1136,8 +1140,8 @@ CREATE TABLE [dbo].[x_policy] ( [added_by_id] [bigint] DEFAULT NULL NULL, [upd_by_id] [bigint] DEFAULT NULL NULL, [version] [bigint] DEFAULT NULL NULL, - [service] [bigint] DEFAULT NULL NULL, - [name] [varchar](512) DEFAULT NULL NULL, + [service] [bigint] NOT NULL, + [name] [varchar](512) NOT NULL, [policy_type] [int] DEFAULT 0 NULL, [description] [varchar](1024) DEFAULT NULL NULL, [resource_signature] [varchar](128) DEFAULT NULL NULL, @@ -1146,7 +1150,11 @@ CREATE TABLE [dbo].[x_policy] ( PRIMARY KEY CLUSTERED ( [id] ASC -)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY] +)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY], +CONSTRAINT [x_policy$x_policy_UK_name_service] UNIQUE NONCLUSTERED +( + [name] ASC, [service] ASC +)WITH (PAD_INDEX = OFF,STATISTICS_NORECOMPUTE = OFF,IGNORE_DUP_KEY = OFF,ALLOW_ROW_LOCKS = ON,ALLOW_PAGE_LOCKS = ON) ON [PRIMARY] ) ON [PRIMARY] SET ANSI_NULLS ON SET QUOTED_IDENTIFIER ON @@ -2990,6 +2998,7 @@ INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active) VALUES ('027',CURRENT_TIMESTAMP,'Ranger 0.7.0',CURRENT_TIMESTAMP,'localhost','Y'); INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active) VALUES ('028',CURRENT_TIMESTAMP,'Ranger 0.7.0',CURRENT_TIMESTAMP,'localhost','Y'); INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active) VALUES ('029',CURRENT_TIMESTAMP,'Ranger 0.7.0',CURRENT_TIMESTAMP,'localhost','Y'); +INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active) VALUES ('033',CURRENT_TIMESTAMP,'Ranger 0.7.0',CURRENT_TIMESTAMP,'localhost','Y'); INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active) VALUES ('DB_PATCHES',CURRENT_TIMESTAMP,'Ranger 0.7.0',CURRENT_TIMESTAMP,'localhost','Y'); INSERT INTO x_user_module_perm (user_id,module_id,create_time,update_time,added_by_id,upd_by_id,is_allowed) VALUES (1,3,CURRENT_TIMESTAMP,CURRENT_TIMESTAMP,1,1,1); INSERT INTO x_user_module_perm (user_id,module_id,create_time,update_time,added_by_id,upd_by_id,is_allowed) VALUES (1,1,CURRENT_TIMESTAMP,CURRENT_TIMESTAMP,1,1,1); diff --git a/security-admin/db/sqlserver/patches/033-add-unique-constraint-on-table-x_policy.sql b/security-admin/db/sqlserver/patches/033-add-unique-constraint-on-table-x_policy.sql new file mode 100644 index 0000000000..075fe8676c --- /dev/null +++ b/security-admin/db/sqlserver/patches/033-add-unique-constraint-on-table-x_policy.sql @@ -0,0 +1,29 @@ + + +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +IF EXISTS(select * from INFORMATION_SCHEMA.columns where table_name = 'x_policy' and column_name = 'name') +BEGIN + IF NOT EXISTS(select * from INFORMATION_SCHEMA.CONSTRAINT_COLUMN_USAGE where table_name='x_policy' and column_name='name' and constraint_name = 'x_policy$x_policy_UK_name_service') + BEGIN + IF NOT EXISTS(select * from INFORMATION_SCHEMA.TABLE_CONSTRAINTS where table_name='x_policy' and constraint_name = 'x_policy$x_policy_UK_name_service' and CONSTRAINT_TYPE='UNIQUE') + BEGIN + UPDATE [dbo].[x_policy] set name=concat(name, '-duplicate-',id) where id in (select id from (select id from [dbo].[x_policy] where concat(service,name) in (select concat(service,name) from [dbo].[x_policy] group by service,name having count(*) >1)) as tmp); + ALTER TABLE [dbo].[x_policy] ADD CONSTRAINT [x_policy$x_policy_UK_name_service] UNIQUE ([name],[service]); + END + END +END +GO +exit From 1686d4718727a762b733d703dd044075e6676edf Mon Sep 17 00:00:00 2001 From: Abhay Kulkarni Date: Thu, 11 Oct 2018 10:53:55 -0700 Subject: [PATCH 135/151] RANGER-2242: JiSQL utility is failing Oracle UDF --- jisql/src/main/java/org/apache/util/sql/Jisql.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/jisql/src/main/java/org/apache/util/sql/Jisql.java b/jisql/src/main/java/org/apache/util/sql/Jisql.java index 53a6ca4fe9..7c63aff90c 100644 --- a/jisql/src/main/java/org/apache/util/sql/Jisql.java +++ b/jisql/src/main/java/org/apache/util/sql/Jisql.java @@ -429,6 +429,9 @@ public void doIsql() throws IOException, SQLException { if (trimmedLine.toUpperCase().startsWith("DECLARE")) { commandTerminator="/"; } + if ((trimmedLine.toUpperCase().startsWith("CREATE OR REPLACE PROCEDURE")) || (trimmedLine.toUpperCase().startsWith("CREATE OR REPLACE FUNCTION"))) { + commandTerminator="/"; + } } if(connectString.toLowerCase().startsWith("jdbc:postgresql") && inputFileName!=null){ if (trimmedLine.toLowerCase().startsWith("select 'delimiter start';")) { From 2fe9797ae6dd801547699a7d334cb1fca8571afc Mon Sep 17 00:00:00 2001 From: rmani Date: Thu, 11 Oct 2018 17:51:11 -0700 Subject: [PATCH 136/151] RANGER-2249:Ranger Audit not flushed immediately to hdfs Signed-off-by: rmani --- .../apache/ranger/audit/destination/HDFSAuditDestination.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/destination/HDFSAuditDestination.java b/agents-audit/src/main/java/org/apache/ranger/audit/destination/HDFSAuditDestination.java index 8b17fc510d..5cf7e08fe4 100644 --- a/agents-audit/src/main/java/org/apache/ranger/audit/destination/HDFSAuditDestination.java +++ b/agents-audit/src/main/java/org/apache/ranger/audit/destination/HDFSAuditDestination.java @@ -180,7 +180,7 @@ public PrintWriter run() throws Exception { } finally { logger.info("Flushing HDFS audit. Event Size:" + events.size()); if (out != null) { - out.flush(); + flush(); } } addSuccessCount(events.size()); From 63ed5feb116d789ee827c7577e6d93267860c4ef Mon Sep 17 00:00:00 2001 From: Yuan Gao Date: Fri, 17 Aug 2018 10:56:22 -0700 Subject: [PATCH 137/151] RANGER-2209: Added service definition for ABFS Signed-off-by: Velmurugan Periasamy --- .../plugin/store/EmbeddedServiceDefsUtil.java | 5 + .../service-defs/ranger-servicedef-abfs.json | 123 ++++++++++++++++++ 2 files changed, 128 insertions(+) create mode 100644 agents-common/src/main/resources/service-defs/ranger-servicedef-abfs.json diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/store/EmbeddedServiceDefsUtil.java b/agents-common/src/main/java/org/apache/ranger/plugin/store/EmbeddedServiceDefsUtil.java index de12f8939e..7abcedecfc 100755 --- a/agents-common/src/main/java/org/apache/ranger/plugin/store/EmbeddedServiceDefsUtil.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/store/EmbeddedServiceDefsUtil.java @@ -63,6 +63,7 @@ public class EmbeddedServiceDefsUtil { public static final String EMBEDDED_SERVICEDEF_NIFI_NAME = "nifi"; public static final String EMBEDDED_SERVICEDEF_ATLAS_NAME = "atlas"; public static final String EMBEDDED_SERVICEDEF_WASB_NAME = "wasb"; + public static final String EMBEDDED_SERVICEDEF_ABFS_NAME = "abfs"; public static final String PROPERTY_CREATE_EMBEDDED_SERVICE_DEFS = "ranger.service.store.create.embedded.service-defs"; @@ -93,6 +94,7 @@ public class EmbeddedServiceDefsUtil { private RangerServiceDef nifiServiceDef = null; private RangerServiceDef atlasServiceDef = null; private RangerServiceDef wasbServiceDef = null; + private RangerServiceDef abfsServiceDef = null; private RangerServiceDef tagServiceDef = null; @@ -133,6 +135,7 @@ public void init(ServiceStore store) { tagServiceDef = getOrCreateServiceDef(store, EMBEDDED_SERVICEDEF_TAG_NAME); wasbServiceDef = getOrCreateServiceDef(store, EMBEDDED_SERVICEDEF_WASB_NAME); + abfsServiceDef = getOrCreateServiceDef(store, EMBEDDED_SERVICEDEF_ABFS_NAME); // Ensure that tag service def is updated with access types of all service defs @@ -192,6 +195,8 @@ public long getAtlasServiceDefId() { public long getWasbServiceDefId() { return getId(wasbServiceDef); } + public long getAbfsServiceDefId() { return getId(abfsServiceDef); } + public RangerServiceDef getEmbeddedServiceDef(String defType) throws Exception { RangerServiceDef serviceDef=null; if(StringUtils.isNotEmpty(defType)){ diff --git a/agents-common/src/main/resources/service-defs/ranger-servicedef-abfs.json b/agents-common/src/main/resources/service-defs/ranger-servicedef-abfs.json new file mode 100644 index 0000000000..18454defff --- /dev/null +++ b/agents-common/src/main/resources/service-defs/ranger-servicedef-abfs.json @@ -0,0 +1,123 @@ +{ + "id":103, + "name": "abfs", + "implClass": "", + "label": "Azure Blob File System", + "description": "Ranger plugin for ABFS", + "guid":"", + "options": { "enableDenyAndExceptionsInPolicies": "true" }, + "resources": [ + { + "itemId": 1, + "name": "storageaccount", + "type": "string", + "parent": "", + "level": 10, + "mandatory": true, + "lookupSupported": false, + "excludesSupported": false, + "matcher": "org.apache.ranger.plugin.resourcematcher.RangerDefaultResourceMatcher", + "matcherOptions": {"wildCard":true, "ignoreCase":false}, + "validationRegEx": "", + "validationMessage": "", + "uiHint": "", + "label": "Storage Account", + "description": "Storage Account for the Path" + }, + { + "itemId":2, + "name": "container", + "type": "string", + "parent": "storageaccount", + "level":20, + "mandatory": true, + "lookupSupported": false, + "excludesSupported": false, + "matcher": "org.apache.ranger.plugin.resourcematcher.RangerDefaultResourceMatcher", + "matcherOptions": {"wildCard":true, "ignoreCase":false}, + "validationRegEx":"", + "validationMessage": "", + "uiHint":"", + "label": "Storage Account Container", + "description": "Storage Account Container for the Path" + }, + { + "itemId":3, + "name": "relativepath", + "type": "path", + "parent": "container", + "level":30, + "mandatory": true, + "lookupSupported": true, + "recursiveSupported": true, + "excludesSupported": false, + "matcher": "org.apache.ranger.plugin.resourcematcher.RangerPathResourceMatcher", + "matcherOptions": {"wildCard":true, "ignoreCase":false}, + "validationRegEx":"^[/*]$|^\/.*?[^\/]$", + "validationMessage": "Relative Path must not end with a slash", + "uiHint":"", + "label": "Relative Path", + "description": "Relative Path inside Storage Account Container" + } + ], + "accessTypes": + [ + { + "itemId": 1, + "name": "read", + "label": "Read" + }, + { + "itemId": 2, + "name": "write", + "label": "Write" + } + ], + "configs": + [ + { + "itemId": 1, + "name": "username", + "type": "string", + "subType": "", + "mandatory": false, + "validationRegEx":"", + "validationMessage": "", + "uiHint":"", + "label": "Username" + }, + { + "itemId": 2, + "name": "password", + "type": "string", + "subType": "", + "mandatory": false, + "validationRegEx":"", + "validationMessage": "", + "uiHint":"", + "label": "Password" + }, + { + "itemId":3, + "name": "commonNameForCertificate", + "type": "string", + "subType": "", + "mandatory": false, + "validationRegEx":"", + "validationMessage": "", + "uiHint":"", + "label": "Common Name for Certificate" + } + ], + "contextEnrichers": [], + "policyConditions": + [ + { + "itemId": 1, + "name": "ip-range", + "evaluator": "org.apache.ranger.plugin.conditionevaluator.RangerIpMatcher", + "label": "IP Address Range", + "description": "IP Address Range" + } + ] +} \ No newline at end of file From 8ac86f2ee606ff1cd54836b03e64ad2a00f79fca Mon Sep 17 00:00:00 2001 From: Pradeep Date: Wed, 17 Oct 2018 19:50:26 +0530 Subject: [PATCH 138/151] RANGER-2253: Unable to get dashboard page after login --- security-admin/pom.xml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/security-admin/pom.xml b/security-admin/pom.xml index 8283e31594..c20f277c1b 100644 --- a/security-admin/pom.xml +++ b/security-admin/pom.xml @@ -522,7 +522,8 @@ WEB-INF/lib/spring-*.SEC03.jar, WEB-INF/lib/spring-*.RC3.jar, - WEB-INF/lib/spring-2.*.jar + WEB-INF/lib/spring-2.*.jar, + WEB-INF/lib/asm-5*.jar ${project.build.directory}/${project.build.finalName} From 9890a90b98ae766faedf4a03ab8b6b0163c09bcf Mon Sep 17 00:00:00 2001 From: Abhay Kulkarni Date: Wed, 17 Oct 2018 14:54:33 -0700 Subject: [PATCH 139/151] RANGER-2247: RangerRANGER-2247 Ranger Plugin for HDFS throws StringIndexOutOfBounds exception when policy resource is \ --- .../RangerPathResourceMatcher.java | 4 ++-- .../ranger/plugin/util/StringTokenReplacer.java | 16 +++++++++++++++- .../policyengine/test_policyengine_hdfs.json | 6 ++++++ .../test_resourcematcher_dynamic.json | 2 +- ..._resourcematcher_wildcards_as_delimiters.json | 2 +- 5 files changed, 25 insertions(+), 5 deletions(-) diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/resourcematcher/RangerPathResourceMatcher.java b/agents-common/src/main/java/org/apache/ranger/plugin/resourcematcher/RangerPathResourceMatcher.java index 90c132f35a..480522fa52 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/resourcematcher/RangerPathResourceMatcher.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/resourcematcher/RangerPathResourceMatcher.java @@ -227,8 +227,8 @@ abstract class RecursiveMatcher extends ResourceMatcher { } String getStringToCompare(String policyValue) { - if (policyValue == null) { - return null; + if (StringUtils.isEmpty(policyValue)) { + return policyValue; } return (policyValue.lastIndexOf(levelSeparatorChar) == policyValue.length()-1) ? policyValue.substring(0, policyValue.length()-1) : policyValue; diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/StringTokenReplacer.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/StringTokenReplacer.java index 2ec809ceed..da2b866f57 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/util/StringTokenReplacer.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/StringTokenReplacer.java @@ -50,11 +50,25 @@ public String replaceTokens(String value, Map tokens) { i++; if(i < value.length()) { c = value.charAt(i); - if(token != null) { + if (token != null) { + // if next char is not the escape char or endChar, retain the escapeChar + if (c != escapeChar && c != endChar) { + token.append(escapeChar); + } token.append(c); } else { + // if next char is not the escape char or startChar, retain the escapeChar + if (c != escapeChar && c != startChar) { + ret.append(escapeChar); + } ret.append(c); } + } else { + if (token != null) { + token.append(escapeChar); + } else { + ret.append(escapeChar); + } } continue; } diff --git a/agents-common/src/test/resources/policyengine/test_policyengine_hdfs.json b/agents-common/src/test/resources/policyengine/test_policyengine_hdfs.json index ea167f49c1..3833ba13a6 100644 --- a/agents-common/src/test/resources/policyengine/test_policyengine_hdfs.json +++ b/agents-common/src/test/resources/policyengine/test_policyengine_hdfs.json @@ -60,6 +60,12 @@ "values":["var country_code = ctx.getRequestContextAttribute('LOCATION_TEST_COUNTRY_CODE'); ctx.result = !!country_code;"] }]} ] + }, + {"id":4,"name":"invalid policy with a single backslash","isEnabled":true,"isAuditEnabled":true, + "resources":{"path":{"values":["\\"],"isRecursive":true}}, + "policyItems":[ + {"accesses":[{"type":"read","isAllowed":true}],"users":[],"groups":["public"],"delegateAdmin":false} + ] } ], diff --git a/agents-common/src/test/resources/resourcematcher/test_resourcematcher_dynamic.json b/agents-common/src/test/resources/resourcematcher/test_resourcematcher_dynamic.json index 5237d4773e..0079673017 100644 --- a/agents-common/src/test/resources/resourcematcher/test_resourcematcher_dynamic.json +++ b/agents-common/src/test/resources/resourcematcher/test_resourcematcher_dynamic.json @@ -25,7 +25,7 @@ , { "name":"exact-path","input":"/abc@%xyz@w", "evalContext": {"token:somestuff": "somethingelse"}, "result":false} , - { "name":"exact-path","input":"/abc%xyzw", "evalContext": {"token:somestuff": "somethingelse"}, "result":true} + { "name":"exact-path","input":"/abc%xyz@w", "evalContext": {"token:somestuff": "somethingelse"}, "result":true} , { "name":"exact-path","input":"/abcabcdw", "evalContext": {"token:somestuff": "somethingelse", "xyz":"abcd"}, "result":false} , diff --git a/agents-common/src/test/resources/resourcematcher/test_resourcematcher_wildcards_as_delimiters.json b/agents-common/src/test/resources/resourcematcher/test_resourcematcher_wildcards_as_delimiters.json index c907f414ee..7b53dfa8f5 100644 --- a/agents-common/src/test/resources/resourcematcher/test_resourcematcher_wildcards_as_delimiters.json +++ b/agents-common/src/test/resources/resourcematcher/test_resourcematcher_wildcards_as_delimiters.json @@ -23,7 +23,7 @@ , { "name":"exact-path","input":"/xyzsomethingelsez", "evalContext": {"token:somestuff": "somethingelse"}, "result":true} , - { "name":"exact-path","input":"/abc*xyzw", "evalContext": {"token:somestuff": "somethingelse"}, "result":true} + { "name":"exact-path","input":"/abc*xyz@w", "evalContext": {"token:somestuff": "somethingelse"}, "result":true} ] } ] From c84b98fbae6e089c637848218743e195e1259fa9 Mon Sep 17 00:00:00 2001 From: Abhay Kulkarni Date: Thu, 18 Oct 2018 19:28:20 -0700 Subject: [PATCH 140/151] RANGER-2203, RANGER-2219: Review and update database schema for ranger policies and tag objects to minimize database queries/updates; back-port of RANGER-2186, RANGER-2195 --- .../ranger/authorization/utils/JsonUtils.java | 112 ++ .../plugin/store/AbstractServiceStore.java | 2 +- .../apache/ranger/plugin/store/TagStore.java | 2 + .../scripts/ranger-admin-services.sh | 13 +- .../035-update-schema-for-x-policy.sql | 199 +++ .../patches/036-denormalize-tag-tables.sql | 82 ++ .../035-update-schema-for-x-policy.sql | 164 +++ .../patches/036-denormalize-tag-tables.sql | 54 + .../035-update-schema-for-x-policy.sql | 198 +++ .../patches/036-denormalize-tag-tables.sql | 79 ++ .../035-update-schema-for-x-policy.sql | 179 +++ .../patches/036-denormalize-tag-tables.sql | 71 + .../current/ranger_core_db_sqlserver.sql | 284 +++- .../035-update-schema-for-x-policy.sql | 453 ++++++ .../patches/036-denormalize-tag-tables.sql | 97 ++ security-admin/scripts/db_setup.py | 75 +- .../apache/ranger/biz/PolicyRefUpdater.java | 286 ++++ .../ranger/biz/RangerPolicyRetriever.java | 579 +++----- .../ranger/biz/RangerTagDBRetriever.java | 466 +------ .../org/apache/ranger/biz/ServiceDBStore.java | 1235 ++++++++--------- .../org/apache/ranger/biz/TagDBStore.java | 281 +--- ...ngerTransactionSynchronizationAdapter.java | 154 +- .../apache/ranger/db/RangerDaoManager.java | 26 +- .../ranger/db/RangerDaoManagerBase.java | 48 +- .../apache/ranger/db/XXAccessTypeDefDao.java | 1 - .../ranger/db/XXDataMaskTypeDefDao.java | 1 - .../java/org/apache/ranger/db/XXGroupDao.java | 16 - .../ranger/db/XXPolicyConditionDefDao.java | 28 - .../ranger/db/XXPolicyItemAccessDao.java | 25 - .../ranger/db/XXPolicyItemConditionDao.java | 40 - .../db/XXPolicyItemDataMaskInfoDao.java | 25 - .../ranger/db/XXPolicyItemGroupPermDao.java | 13 - .../db/XXPolicyItemRowFilterInfoDao.java | 13 - .../ranger/db/XXPolicyItemUserPermDao.java | 13 - .../ranger/db/XXPolicyRefAccessTypeDao.java | 100 ++ .../ranger/db/XXPolicyRefConditionDao.java | 111 ++ .../ranger/db/XXPolicyRefDataMaskTypeDao.java | 86 ++ .../apache/ranger/db/XXPolicyRefGroupDao.java | 99 ++ .../ranger/db/XXPolicyRefResourceDao.java | 98 ++ .../apache/ranger/db/XXPolicyRefUserDao.java | 111 ++ .../apache/ranger/db/XXPolicyResourceDao.java | 14 - .../ranger/db/XXPolicyResourceMapDao.java | 13 - .../apache/ranger/db/XXResourceDefDao.java | 1 - .../ranger/db/XXServiceResourceDao.java | 36 +- .../db/XXServiceResourceElementDao.java | 12 - .../db/XXServiceResourceElementValueDao.java | 13 - .../ranger/db/XXServiceVersionInfoDao.java | 14 +- .../apache/ranger/db/XXTagAttributeDao.java | 12 - .../ranger/db/XXTagAttributeDefDao.java | 12 - .../java/org/apache/ranger/db/XXTagDao.java | 13 - .../org/apache/ranger/db/XXTagDefDao.java | 53 +- .../apache/ranger/db/XXTagResourceMapDao.java | 11 - .../java/org/apache/ranger/db/XXUserDao.java | 20 +- .../org/apache/ranger/entity/XXDBBase.java | 6 + .../apache/ranger/entity/XXPolicyBase.java | 14 +- .../ranger/entity/XXPolicyRefAccessType.java | 191 +++ .../ranger/entity/XXPolicyRefCondition.java | 191 +++ .../entity/XXPolicyRefDataMaskType.java | 192 +++ .../ranger/entity/XXPolicyRefGroup.java | 206 +++ .../ranger/entity/XXPolicyRefResource.java | 191 +++ .../apache/ranger/entity/XXPolicyRefUser.java | 191 +++ .../ranger/entity/XXServiceResource.java | 30 + .../java/org/apache/ranger/entity/XXTag.java | 16 +- .../org/apache/ranger/entity/XXTagDef.java | 14 + .../PatchForUpdatingPolicyJson_J10019.java | 1125 +++++++++++++++ .../PatchForUpdatingTagsJson_J10020.java | 788 +++++++++++ .../org/apache/ranger/rest/ServiceREST.java | 14 +- .../ranger/service/RangerAuditFields.java | 12 +- .../service/RangerPolicyServiceBase.java | 21 +- .../service/RangerServiceResourceService.java | 194 ++- .../RangerServiceResourceServiceBase.java | 33 +- .../ranger/service/RangerTagDefService.java | 24 + .../ranger/service/RangerTagService.java | 53 +- .../resources/META-INF/jpa_named_queries.xml | 381 ++--- .../apache/ranger/biz/TestServiceDBStore.java | 371 +++-- .../apache/ranger/rest/TestServiceREST.java | 2 +- .../service/TestRangerServiceDefService.java | 9 - .../service/TestRangerTagDefService.java | 41 +- 78 files changed, 7875 insertions(+), 2578 deletions(-) create mode 100644 agents-common/src/main/java/org/apache/ranger/authorization/utils/JsonUtils.java create mode 100644 security-admin/db/mysql/patches/035-update-schema-for-x-policy.sql create mode 100644 security-admin/db/mysql/patches/036-denormalize-tag-tables.sql create mode 100644 security-admin/db/oracle/patches/035-update-schema-for-x-policy.sql create mode 100644 security-admin/db/oracle/patches/036-denormalize-tag-tables.sql create mode 100644 security-admin/db/postgres/patches/035-update-schema-for-x-policy.sql create mode 100644 security-admin/db/postgres/patches/036-denormalize-tag-tables.sql create mode 100644 security-admin/db/sqlanywhere/patches/035-update-schema-for-x-policy.sql create mode 100644 security-admin/db/sqlanywhere/patches/036-denormalize-tag-tables.sql create mode 100644 security-admin/db/sqlserver/patches/035-update-schema-for-x-policy.sql create mode 100644 security-admin/db/sqlserver/patches/036-denormalize-tag-tables.sql create mode 100644 security-admin/src/main/java/org/apache/ranger/biz/PolicyRefUpdater.java create mode 100644 security-admin/src/main/java/org/apache/ranger/db/XXPolicyRefAccessTypeDao.java create mode 100644 security-admin/src/main/java/org/apache/ranger/db/XXPolicyRefConditionDao.java create mode 100644 security-admin/src/main/java/org/apache/ranger/db/XXPolicyRefDataMaskTypeDao.java create mode 100644 security-admin/src/main/java/org/apache/ranger/db/XXPolicyRefGroupDao.java create mode 100644 security-admin/src/main/java/org/apache/ranger/db/XXPolicyRefResourceDao.java create mode 100644 security-admin/src/main/java/org/apache/ranger/db/XXPolicyRefUserDao.java create mode 100644 security-admin/src/main/java/org/apache/ranger/entity/XXPolicyRefAccessType.java create mode 100644 security-admin/src/main/java/org/apache/ranger/entity/XXPolicyRefCondition.java create mode 100644 security-admin/src/main/java/org/apache/ranger/entity/XXPolicyRefDataMaskType.java create mode 100644 security-admin/src/main/java/org/apache/ranger/entity/XXPolicyRefGroup.java create mode 100644 security-admin/src/main/java/org/apache/ranger/entity/XXPolicyRefResource.java create mode 100644 security-admin/src/main/java/org/apache/ranger/entity/XXPolicyRefUser.java create mode 100644 security-admin/src/main/java/org/apache/ranger/patch/PatchForUpdatingPolicyJson_J10019.java create mode 100644 security-admin/src/main/java/org/apache/ranger/patch/PatchForUpdatingTagsJson_J10020.java diff --git a/agents-common/src/main/java/org/apache/ranger/authorization/utils/JsonUtils.java b/agents-common/src/main/java/org/apache/ranger/authorization/utils/JsonUtils.java new file mode 100644 index 0000000000..98d9c0a6ff --- /dev/null +++ b/agents-common/src/main/java/org/apache/ranger/authorization/utils/JsonUtils.java @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ranger.authorization.utils; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.reflect.TypeToken; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.MapUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +import java.lang.reflect.Type; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class JsonUtils { + private static final Log LOG = LogFactory.getLog(JsonUtils.class); + + private static final HashMap MAP_STRING_STRING = new HashMap<>(); + + private static final Gson gson; + + static { + gson = new GsonBuilder().setDateFormat("yyyyMMdd-HH:mm:ss.SSS-Z") + .create(); + } + + public static String mapToJson(Map map) { + String ret = null; + if (MapUtils.isNotEmpty(map)) { + try { + ret = gson.toJson(map); + } catch (Exception e) { + LOG.error("Invalid input data: ", e); + } + } + return ret; + } + + public static String listToJson(List list) { + String ret = null; + if (CollectionUtils.isNotEmpty(list)) { + try { + ret = gson.toJson(list); + } catch (Exception e) { + LOG.error("Invalid input data: ", e); + } + } + return ret; + } + + public static String objectToJson(Object object) { + String ret = null; + + if(object != null) { + try { + ret = gson.toJson(object); + } catch(Exception excp) { + LOG.warn("objectToJson() failed to convert object to Json", excp); + } + } + + return ret; + } + + public static T jsonToObject(String jsonStr, Class clz) { + T ret = null; + + if(StringUtils.isNotEmpty(jsonStr)) { + try { + ret = gson.fromJson(jsonStr, clz); + } catch(Exception excp) { + LOG.warn("jsonToObject() failed to convert json to object: " + jsonStr, excp); + } + } + + return ret; + } + + public static Map jsonToMapStringString(String jsonStr) { + Map ret = null; + + if(StringUtils.isNotEmpty(jsonStr)) { + try { + ret = gson.fromJson(jsonStr, MAP_STRING_STRING.getClass()); + } catch(Exception excp) { + LOG.warn("jsonToObject() failed to convert json to object: " + jsonStr, excp); + } + } + + return ret; + } + +} diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/store/AbstractServiceStore.java b/agents-common/src/main/java/org/apache/ranger/plugin/store/AbstractServiceStore.java index a75ca59e87..bee7520900 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/store/AbstractServiceStore.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/store/AbstractServiceStore.java @@ -138,7 +138,7 @@ protected void postDelete(RangerBaseModelObject obj) throws Exception { } } - protected final long getNextVersion(Long currentVersion) { + public static long getNextVersion(Long currentVersion) { return currentVersion == null ? 1L : currentVersion + 1; } diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/store/TagStore.java b/agents-common/src/main/java/org/apache/ranger/plugin/store/TagStore.java index 5918b12924..fe4b278179 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/store/TagStore.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/store/TagStore.java @@ -84,6 +84,8 @@ public interface TagStore { RangerServiceResource updateServiceResource(RangerServiceResource resource) throws Exception; + void refreshServiceResource(Long resourceId) throws Exception; + void deleteServiceResource(Long id) throws Exception; void deleteServiceResourceByGuid(String guid) throws Exception; diff --git a/embeddedwebserver/scripts/ranger-admin-services.sh b/embeddedwebserver/scripts/ranger-admin-services.sh index 19dab2a9d4..350826ff09 100755 --- a/embeddedwebserver/scripts/ranger-admin-services.sh +++ b/embeddedwebserver/scripts/ranger-admin-services.sh @@ -28,11 +28,12 @@ action=`echo $action | tr '[:lower:]' '[:upper:]'` realScriptPath=`readlink -f $0` realScriptDir=`dirname $realScriptPath` XAPOLICYMGR_DIR=`(cd $realScriptDir/..; pwd)` +max_memory=1g XAPOLICYMGR_EWS_DIR=${XAPOLICYMGR_DIR}/ews RANGER_JAAS_LIB_DIR="${XAPOLICYMGR_EWS_DIR}/ranger_jaas" RANGER_JAAS_CONF_DIR="${XAPOLICYMGR_EWS_DIR}/webapp/WEB-INF/classes/conf/ranger_jaas" -JAVA_OPTS=" ${JAVA_OPTS} -XX:MaxPermSize=256m -Xmx1024m -Xms1024m " +JAVA_OPTS=" ${JAVA_OPTS} -XX:MetaspaceSize=100m -XX:MaxMetaspaceSize=200m -Xmx${max_memory} -Xms1g -Xloggc:${XAPOLICYMGR_EWS_DIR}/logs/gc-worker.log -verbose:gc -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=1m -XX:+PrintGCDetails -XX:+PrintHeapAtGC -XX:+PrintGCDateStamps" if [[ ${JAVA_OPTS} != *"-Duser.timezone"* ]] ;then export JAVA_OPTS=" ${JAVA_OPTS} -Duser.timezone=UTC" ;fi if [ -f ${XAPOLICYMGR_DIR}/ews/webapp/WEB-INF/classes/conf/java_home.sh ]; then . ${XAPOLICYMGR_DIR}/ews/webapp/WEB-INF/classes/conf/java_home.sh @@ -137,11 +138,11 @@ stop(){ } metric(){ - if [ "$JAVA_HOME" == "" ]; then - echo "[E] JAVA_HOME environment variable not defined, aborting Apache Ranger Admin metric collection" 1>&2; - exit 1; - fi - java ${JAVA_OPTS} -Duser=${USER} -Dhostname=${HOSTNAME} -Dlogdir=${RANGER_ADMIN_LOG_DIR} -cp "${XAPOLICYMGR_EWS_DIR}/webapp/WEB-INF/classes/conf:${XAPOLICYMGR_EWS_DIR}/webapp/WEB-INF/classes/:${XAPOLICYMGR_EWS_DIR}/webapp/WEB-INF/classes/lib/*:${XAPOLICYMGR_EWS_DIR}/webapp/WEB-INF/classes/META-INF:${XAPOLICYMGR_EWS_DIR}/webapp/WEB-INF/lib/*:${XAPOLICYMGR_EWS_DIR}/webapp/META-INF:${XAPOLICYMGR_EWS_DIR}/lib/*:${RANGER_JAAS_LIB_DIR}/*:${RANGER_JAAS_CONF_DIR}:${JAVA_HOME}/lib/*:${RANGER_HADOOP_CONF_DIR}/*:$CLASSPATH" org.apache.ranger.patch.cliutil.MetricUtil ${arg2} ${arg3} 2>/dev/null + if [ "$JAVA_HOME" == "" ]; then + echo "[E] JAVA_HOME environment variable not defined, aborting Apache Ranger Admin metric collection" + exit 1; + fi + java ${JAVA_OPTS} -Dlogdir=${RANGER_ADMIN_LOG_DIR} -cp "${XAPOLICYMGR_EWS_DIR}/webapp/WEB-INF/classes/conf:${XAPOLICYMGR_EWS_DIR}/webapp/WEB-INF/classes/:${XAPOLICYMGR_EWS_DIR}/webapp/WEB-INF/classes/lib/*:${XAPOLICYMGR_EWS_DIR}/webapp/WEB-INF/classes/META-INF:${XAPOLICYMGR_EWS_DIR}/webapp/WEB-INF/lib/*:${XAPOLICYMGR_EWS_DIR}/webapp/META-INF:${XAPOLICYMGR_EWS_DIR}/lib/*:${RANGER_JAAS_LIB_DIR}/*:${RANGER_JAAS_CONF_DIR}:${JAVA_HOME}/lib/*:${RANGER_HADOOP_CONF_DIR}/*:$CLASSPATH" org.apache.ranger.patch.cliutil.MetricUtil ${arg2} ${arg3} 2>/dev/null } if [ "${action}" == "START" ]; then diff --git a/security-admin/db/mysql/patches/035-update-schema-for-x-policy.sql b/security-admin/db/mysql/patches/035-update-schema-for-x-policy.sql new file mode 100644 index 0000000000..05bd850f2d --- /dev/null +++ b/security-admin/db/mysql/patches/035-update-schema-for-x-policy.sql @@ -0,0 +1,199 @@ +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +drop procedure if exists alter_table_x_policy; + +delimiter ;; +create procedure alter_table_x_policy() begin + +if exists (select * from information_schema.columns where table_schema=database() and table_name = 'x_policy') then + if not exists (select * from information_schema.columns where table_schema=database() and table_name = 'x_policy' and column_name = 'policy_text') then + ALTER TABLE `x_policy` ADD `policy_text` MEDIUMTEXT DEFAULT NULL; + end if; + end if; +end;; + +delimiter ; +call alter_table_x_policy(); + +drop procedure if exists alter_table_x_policy; + +DROP PROCEDURE IF EXISTS removeConstraints; +DELIMITER ;; +CREATE PROCEDURE removeConstraints(vTableName varchar(128)) +BEGIN + DECLARE done INT DEFAULT FALSE; + DECLARE cName VARCHAR(64); + DECLARE cur CURSOR FOR + SELECT DISTINCT CONSTRAINT_NAME + FROM INFORMATION_SCHEMA.Key_COLUMN_USAGE + WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME = vTableName + AND REFERENCED_TABLE_NAME IS NOT NULL; + DECLARE CONTINUE HANDLER FOR NOT FOUND SET done = TRUE; + + SET AUTOCOMMIT=0; + SET FOREIGN_KEY_CHECKS=0; + + OPEN cur; + + read_loop: LOOP + FETCH cur INTO cName; + IF done THEN + LEAVE read_loop; + END IF; + SET @sql = CONCAT('ALTER TABLE ',vTableName,' DROP FOREIGN KEY ',cName,';'); + PREPARE stmt FROM @sql; + EXECUTE stmt; + DEALLOCATE PREPARE stmt; + END LOOP; + + CLOSE cur; + + SET FOREIGN_KEY_CHECKS=1; + COMMIT; + SET AUTOCOMMIT=1; +END ;; +DELIMITER ; + +call removeConstraints('x_policy_item'); +call removeConstraints('x_policy_item_access'); +call removeConstraints('x_policy_item_condition'); +call removeConstraints('x_policy_item_datamask'); +call removeConstraints('x_policy_item_group_perm'); +call removeConstraints('x_policy_item_user_perm'); +call removeConstraints('x_policy_item_rowfilter'); +call removeConstraints('x_policy_resource'); +call removeConstraints('x_policy_resource_map'); + +DROP PROCEDURE removeConstraints; + +DROP TABLE IF EXISTS `x_policy_ref_resource`; +CREATE TABLE IF NOT EXISTS `x_policy_ref_resource` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `guid` varchar(1024) DEFAULT NULL, + `create_time` datetime DEFAULT NULL, + `update_time` datetime DEFAULT NULL, + `added_by_id` bigint(20) DEFAULT NULL, + `upd_by_id` bigint(20) DEFAULT NULL, + `policy_id` bigint(20) NOT NULL, + `resource_def_id` bigint(20) NOT NULL, + `resource_name` varchar(4000) DEFAULT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `x_policy_ref_res_UK_polId_resDefId`(`policy_id`, `resource_def_id`), + CONSTRAINT `x_policy_ref_res_FK_added_by_id` FOREIGN KEY (`added_by_id`) REFERENCES `x_portal_user` (`id`), + CONSTRAINT `x_policy_ref_res_FK_upd_by_id` FOREIGN KEY (`upd_by_id`) REFERENCES `x_portal_user` (`id`), + CONSTRAINT `x_policy_ref_res_FK_policy_id` FOREIGN KEY (`policy_id`) REFERENCES `x_policy` (`id`), + CONSTRAINT `x_policy_ref_res_FK_resource_def_id` FOREIGN KEY (`resource_def_id`) REFERENCES `x_resource_def` (`id`) + +) ROW_FORMAT=DYNAMIC; + + +DROP TABLE IF EXISTS `x_policy_ref_access_type`; +CREATE TABLE IF NOT EXISTS `x_policy_ref_access_type` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `guid` varchar(1024) DEFAULT NULL, + `create_time` datetime DEFAULT NULL, + `update_time` datetime DEFAULT NULL, + `added_by_id` bigint(20) DEFAULT NULL, + `upd_by_id` bigint(20) DEFAULT NULL, + `policy_id` bigint(20) NOT NULL, + `access_def_id` bigint(20) NOT NULL, + `access_type_name` varchar(4000) DEFAULT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `x_policy_ref_access_UK_polId_accessDefId`(`policy_id`, `access_def_id`), + CONSTRAINT `x_policy_ref_access_FK_added_by_id` FOREIGN KEY (`added_by_id`) REFERENCES `x_portal_user` (`id`), + CONSTRAINT `x_policy_ref_access_FK_upd_by_id` FOREIGN KEY (`upd_by_id`) REFERENCES `x_portal_user` (`id`), + CONSTRAINT `x_policy_ref_access_FK_policy_id` FOREIGN KEY (`policy_id`) REFERENCES `x_policy` (`id`), + CONSTRAINT `x_policy_ref_access_FK_access_def_id` FOREIGN KEY (`access_def_id`) REFERENCES `x_access_type_def` (`id`) +) ROW_FORMAT=DYNAMIC; + + +DROP TABLE IF EXISTS `x_policy_ref_condition`; +CREATE TABLE IF NOT EXISTS `x_policy_ref_condition` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `guid` varchar(1024) DEFAULT NULL, + `create_time` datetime DEFAULT NULL, + `update_time` datetime DEFAULT NULL, + `added_by_id` bigint(20) DEFAULT NULL, + `upd_by_id` bigint(20) DEFAULT NULL, + `policy_id` bigint(20) NOT NULL, + `condition_def_id` bigint(20) NOT NULL, + `condition_name` varchar(4000) DEFAULT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `x_policy_ref_condition_UK_polId_condDefId`(`policy_id`, `condition_def_id`), + CONSTRAINT `x_policy_ref_condition_FK_added_by_id` FOREIGN KEY (`added_by_id`) REFERENCES `x_portal_user` (`id`), + CONSTRAINT `x_policy_ref_condition_FK_upd_by_id` FOREIGN KEY (`upd_by_id`) REFERENCES `x_portal_user` (`id`), + CONSTRAINT `x_policy_ref_condition_FK_policy_id` FOREIGN KEY (`policy_id`) REFERENCES `x_policy` (`id`), + CONSTRAINT `x_policy_ref_condition_FK_condition_def_id` FOREIGN KEY (`condition_def_id`) REFERENCES `x_policy_condition_def` (`id`) + +) ROW_FORMAT=DYNAMIC; + +DROP TABLE IF EXISTS `x_policy_ref_datamask_type`; +CREATE TABLE IF NOT EXISTS `x_policy_ref_datamask_type` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `guid` varchar(1024) DEFAULT NULL, + `create_time` datetime DEFAULT NULL, + `update_time` datetime DEFAULT NULL, + `added_by_id` bigint(20) DEFAULT NULL, + `upd_by_id` bigint(20) DEFAULT NULL, + `policy_id` bigint(20) NOT NULL, + `datamask_def_id` bigint(20) NOT NULL, + `datamask_type_name` varchar(4000) DEFAULT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `x_policy_ref_datamask_UK_polId_dmaskDefId`(`policy_id`, `datamask_def_id`), + CONSTRAINT `x_policy_ref_datamask_FK_added_by_id` FOREIGN KEY (`added_by_id`) REFERENCES `x_portal_user` (`id`), + CONSTRAINT `x_policy_ref_datamask_FK_upd_by_id` FOREIGN KEY (`upd_by_id`) REFERENCES `x_portal_user` (`id`), + CONSTRAINT `x_policy_ref_datamask_FK_policy_id` FOREIGN KEY (`policy_id`) REFERENCES `x_policy` (`id`), + CONSTRAINT `x_policy_ref_datamask_FK_datamask_def_id` FOREIGN KEY (`datamask_def_id`) REFERENCES `x_datamask_type_def` (`id`) +) ROW_FORMAT=DYNAMIC; + +DROP TABLE IF EXISTS `x_policy_ref_user`; +CREATE TABLE IF NOT EXISTS `x_policy_ref_user` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `guid` varchar(1024) DEFAULT NULL, + `create_time` datetime DEFAULT NULL, + `update_time` datetime DEFAULT NULL, + `added_by_id` bigint(20) DEFAULT NULL, + `upd_by_id` bigint(20) DEFAULT NULL, + `policy_id` bigint(20) NOT NULL, + `user_id` bigint(20) NOT NULL, + `user_name` varchar(4000) DEFAULT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `x_policy_ref_user_UK_polId_userId`(`policy_id`, `user_id`), + CONSTRAINT `x_policy_ref_user_FK_added_by_id` FOREIGN KEY (`added_by_id`) REFERENCES `x_portal_user` (`id`), + CONSTRAINT `x_policy_ref_user_FK_upd_by_id` FOREIGN KEY (`upd_by_id`) REFERENCES `x_portal_user` (`id`), + CONSTRAINT `x_policy_ref_user_FK_policy_id` FOREIGN KEY (`policy_id`) REFERENCES `x_policy` (`id`), + CONSTRAINT `x_policy_ref_user_FK_user_id` FOREIGN KEY (`user_id`) REFERENCES `x_user` (`id`) +) ROW_FORMAT=DYNAMIC; + + +DROP TABLE IF EXISTS `x_policy_ref_group`; +CREATE TABLE IF NOT EXISTS `x_policy_ref_group` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `guid` varchar(1024) DEFAULT NULL, + `create_time` datetime DEFAULT NULL, + `update_time` datetime DEFAULT NULL, + `added_by_id` bigint(20) DEFAULT NULL, + `upd_by_id` bigint(20) DEFAULT NULL, + `policy_id` bigint(20) NOT NULL, + `group_id` bigint(20) NOT NULL, + `group_name` varchar(4000) DEFAULT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `x_policy_ref_group_UK_polId_groupId`(`policy_id`, `group_id`), + CONSTRAINT `x_policy_ref_group_FK_added_by_id` FOREIGN KEY (`added_by_id`) REFERENCES `x_portal_user` (`id`), + CONSTRAINT `x_policy_ref_group_FK_upd_by_id` FOREIGN KEY (`upd_by_id`) REFERENCES `x_portal_user` (`id`), + CONSTRAINT `x_policy_ref_group_FK_policy_id` FOREIGN KEY (`policy_id`) REFERENCES `x_policy` (`id`), + CONSTRAINT `x_policy_ref_group_FK_group_id` FOREIGN KEY (`group_id`) REFERENCES `x_group` (`id`) +) ROW_FORMAT=DYNAMIC; diff --git a/security-admin/db/mysql/patches/036-denormalize-tag-tables.sql b/security-admin/db/mysql/patches/036-denormalize-tag-tables.sql new file mode 100644 index 0000000000..63035bc407 --- /dev/null +++ b/security-admin/db/mysql/patches/036-denormalize-tag-tables.sql @@ -0,0 +1,82 @@ +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +drop procedure if exists denormalize_tag_tables; + +delimiter ;; +create procedure denormalize_tag_tables() begin + +if not exists (select * from information_schema.columns where table_schema=database() and table_name = 'x_tag_def' and column_name='tag_attrs_def_text') then + ALTER TABLE x_tag_def ADD tag_attrs_def_text MEDIUMTEXT NULL DEFAULT NULL; +end if; +if not exists (select * from information_schema.columns where table_schema=database() and table_name = 'x_tag' and column_name='tag_attrs_text') then + ALTER TABLE x_tag ADD tag_attrs_text MEDIUMTEXT NULL DEFAULT NULL; +end if; +if not exists (select * from information_schema.columns where table_schema=database() and table_name = 'x_service_resource' and column_name='service_resource_elements_text') then + ALTER TABLE x_service_resource ADD service_resource_elements_text MEDIUMTEXT NULL DEFAULT NULL; +end if; +if not exists (select * from information_schema.columns where table_schema=database() and table_name = 'x_service_resource' and column_name='tags_text') then + ALTER TABLE x_service_resource ADD tags_text MEDIUMTEXT NULL DEFAULT NULL; +end if; +end;; + +delimiter ; +call denormalize_tag_tables(); + +drop procedure if exists denormalize_tag_tables; + +DROP PROCEDURE IF EXISTS removeConstraints; +DELIMITER ;; +CREATE PROCEDURE removeConstraints(vTableName varchar(128)) +BEGIN + DECLARE done INT DEFAULT FALSE; + DECLARE cName VARCHAR(64); + DECLARE cur CURSOR FOR + SELECT DISTINCT CONSTRAINT_NAME + FROM INFORMATION_SCHEMA.Key_COLUMN_USAGE + WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME = vTableName + AND REFERENCED_TABLE_NAME IS NOT NULL; + DECLARE CONTINUE HANDLER FOR NOT FOUND SET done = TRUE; + + SET AUTOCOMMIT=0; + SET FOREIGN_KEY_CHECKS=0; + + OPEN cur; + + read_loop: LOOP + FETCH cur INTO cName; + IF done THEN + LEAVE read_loop; + END IF; + SET @sql = CONCAT('ALTER TABLE ',vTableName,' DROP FOREIGN KEY ',cName,';'); + PREPARE stmt FROM @sql; + EXECUTE stmt; + DEALLOCATE PREPARE stmt; + END LOOP; + + CLOSE cur; + + SET FOREIGN_KEY_CHECKS=1; + COMMIT; + SET AUTOCOMMIT=1; +END ;; +DELIMITER ; + +call removeConstraints('x_tag_attr_def'); +call removeConstraints('x_tag_attr'); +call removeConstraints('x_service_resource_element'); +call removeConstraints('x_service_resource_element_val'); + +DROP PROCEDURE removeConstraints; diff --git a/security-admin/db/oracle/patches/035-update-schema-for-x-policy.sql b/security-admin/db/oracle/patches/035-update-schema-for-x-policy.sql new file mode 100644 index 0000000000..745f7f080e --- /dev/null +++ b/security-admin/db/oracle/patches/035-update-schema-for-x-policy.sql @@ -0,0 +1,164 @@ +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +/ +CREATE SEQUENCE X_POLICY_REF_RESOURCE_SEQ START WITH 1 INCREMENT BY 1 NOCACHE NOCYCLE; +CREATE SEQUENCE X_POLICY_REF_ACCESS_TYPE_SEQ START WITH 1 INCREMENT BY 1 NOCACHE NOCYCLE; +CREATE SEQUENCE X_POLICY_REF_CONDITION_SEQ START WITH 1 INCREMENT BY 1 NOCACHE NOCYCLE; +CREATE SEQUENCE X_POLICY_REF_DATAMASK_TYPE_SEQ START WITH 1 INCREMENT BY 1 NOCACHE NOCYCLE; +CREATE SEQUENCE X_POLICY_REF_USER_SEQ START WITH 1 INCREMENT BY 1 NOCACHE NOCYCLE; +CREATE SEQUENCE X_POLICY_REF_GROUP_SEQ START WITH 1 INCREMENT BY 1 NOCACHE NOCYCLE; +commit; +CREATE TABLE x_policy_ref_resource ( +id NUMBER(20) NOT NULL, +guid VARCHAR(1024) DEFAULT NULL NULL, +create_time DATE DEFAULT NULL NULL, +update_time DATE DEFAULT NULL NULL, +added_by_id NUMBER(20) DEFAULT NULL NULL, +upd_by_id NUMBER(20) DEFAULT NULL NULL, +policy_id NUMBER(20) NOT NULL, +resource_def_id NUMBER(20) NOT NULL, +resource_name VARCHAR(4000) DEFAULT NULL NULL, +primary key (id), +CONSTRAINT x_p_ref_res_UK_polId_resDefId UNIQUE (policy_id, resource_def_id), +CONSTRAINT x_p_ref_res_FK_policy_id FOREIGN KEY (policy_id) REFERENCES x_policy (id), +CONSTRAINT x_p_ref_res_FK_resource_def_id FOREIGN KEY (resource_def_id) REFERENCES x_resource_def (id), +CONSTRAINT x_p_ref_res_FK_added_by_id FOREIGN KEY (added_by_id) REFERENCES x_portal_user (id), +CONSTRAINT x_p_ref_res_FK_upd_by_id FOREIGN KEY (upd_by_id) REFERENCES x_portal_user (id) +); +commit; +CREATE TABLE x_policy_ref_access_type ( +id NUMBER(20) NOT NULL, +guid VARCHAR(1024) DEFAULT NULL NULL, +create_time DATE DEFAULT NULL NULL, +update_time DATE DEFAULT NULL NULL, +added_by_id NUMBER(20) DEFAULT NULL NULL, +upd_by_id NUMBER(20) DEFAULT NULL NULL, +policy_id NUMBER(20) NOT NULL, +access_def_id NUMBER(20) NOT NULL, +access_type_name VARCHAR(4000) DEFAULT NULL NULL, +primary key (id), +CONSTRAINT x_p_ref_acc_UK_polId_accDefId UNIQUE(policy_id, access_def_id), +CONSTRAINT x_p_ref_acc_FK_policy_id FOREIGN KEY (policy_id) REFERENCES x_policy (id), +CONSTRAINT x_p_ref_acc_FK_acc_def_id FOREIGN KEY (access_def_id) REFERENCES x_access_type_def (id), +CONSTRAINT x_p_ref_acc_FK_added_by_id FOREIGN KEY (added_by_id) REFERENCES x_portal_user (id), +CONSTRAINT x_p_ref_acc_FK_upd_by_id FOREIGN KEY (upd_by_id) REFERENCES x_portal_user (id) +); +commit; +CREATE TABLE x_policy_ref_condition ( +id NUMBER(20) NOT NULL, +guid VARCHAR(1024) DEFAULT NULL NULL, +create_time DATE DEFAULT NULL NULL, +update_time DATE DEFAULT NULL NULL, +added_by_id NUMBER(20) DEFAULT NULL NULL, +upd_by_id NUMBER(20) DEFAULT NULL NULL, +policy_id NUMBER(20) NOT NULL, +condition_def_id NUMBER(20) NOT NULL, +condition_name VARCHAR(4000) DEFAULT NULL NULL, +primary key (id), +CONSTRAINT x_p_ref_cond_UK_polId_cDefId UNIQUE(policy_id, condition_def_id), +CONSTRAINT x_p_ref_cond_FK_policy_id FOREIGN KEY (policy_id) REFERENCES x_policy (id), +CONSTRAINT x_p_ref_cond_FK_cond_def_id FOREIGN KEY (condition_def_id) REFERENCES x_policy_condition_def (id), +CONSTRAINT x_p_ref_cond_FK_added_by_id FOREIGN KEY (added_by_id) REFERENCES x_portal_user (id), +CONSTRAINT x_p_ref_cond_FK_upd_by_id FOREIGN KEY (upd_by_id) REFERENCES x_portal_user (id) +); +commit; +CREATE TABLE x_policy_ref_datamask_type ( +id NUMBER(20) NOT NULL, +guid VARCHAR(1024) DEFAULT NULL NULL, +create_time DATE DEFAULT NULL NULL, +update_time DATE DEFAULT NULL NULL, +added_by_id NUMBER(20) DEFAULT NULL NULL, +upd_by_id NUMBER(20) DEFAULT NULL NULL, +policy_id NUMBER(20) NOT NULL, +datamask_def_id NUMBER(20) NOT NULL, +datamask_type_name VARCHAR(4000) DEFAULT NULL NULL, +primary key (id), +CONSTRAINT x_p_ref_dmsk_UK_polId_dDefId UNIQUE(policy_id, datamask_def_id), +CONSTRAINT x_p_ref_dmsk_FK_policy_id FOREIGN KEY (policy_id) REFERENCES x_policy (id), +CONSTRAINT x_p_ref_dmsk_FK_dmk_def_id FOREIGN KEY (datamask_def_id) REFERENCES x_datamask_type_def (id), +CONSTRAINT x_p_ref_dmsk_FK_added_by_id FOREIGN KEY (added_by_id) REFERENCES x_portal_user (id), +CONSTRAINT x_p_ref_dmsk_FK_upd_by_id FOREIGN KEY (upd_by_id) REFERENCES x_portal_user (id) +); +commit; +CREATE TABLE x_policy_ref_user ( +id NUMBER(20) NOT NULL, +guid VARCHAR(1024) DEFAULT NULL NULL, +create_time DATE DEFAULT NULL NULL, +update_time DATE DEFAULT NULL NULL, +added_by_id NUMBER(20) DEFAULT NULL NULL, +upd_by_id NUMBER(20) DEFAULT NULL NULL, +policy_id NUMBER(20) NOT NULL, +user_id NUMBER(20) NOT NULL, +user_name VARCHAR(4000) DEFAULT NULL NULL, +primary key (id), +CONSTRAINT x_p_ref_usr_UK_polId_userId UNIQUE(policy_id, user_id), +CONSTRAINT x_p_ref_usr_FK_policy_id FOREIGN KEY (policy_id) REFERENCES x_policy (id), +CONSTRAINT x_p_ref_usr_FK_user_id FOREIGN KEY (user_id) REFERENCES x_user (id), +CONSTRAINT x_p_ref_usr_FK_added_by_id FOREIGN KEY (added_by_id) REFERENCES x_portal_user (id), +CONSTRAINT x_p_ref_usr_FK_upd_by_id FOREIGN KEY (upd_by_id) REFERENCES x_portal_user (id) +); +commit; +CREATE TABLE x_policy_ref_group ( +id NUMBER(20) NOT NULL, +guid VARCHAR(1024) DEFAULT NULL NULL, +create_time DATE DEFAULT NULL NULL, +update_time DATE DEFAULT NULL NULL, +added_by_id NUMBER(20) DEFAULT NULL NULL, +upd_by_id NUMBER(20) DEFAULT NULL NULL, +policy_id NUMBER(20) NOT NULL, +group_id NUMBER(20) NOT NULL, +group_name VARCHAR(4000) DEFAULT NULL NULL, +primary key (id), +CONSTRAINT x_p_ref_grp_UK_polId_grpId UNIQUE(policy_id, group_id), +CONSTRAINT x_p_ref_grp_FK_policy_id FOREIGN KEY (policy_id) REFERENCES x_policy (id), +CONSTRAINT x_p_ref_grp_FK_group_id FOREIGN KEY (group_id) REFERENCES x_group (id), +CONSTRAINT x_p_ref_grp_FK_added_by_id FOREIGN KEY (added_by_id) REFERENCES x_portal_user (id), +CONSTRAINT x_p_ref_grp_FK_upd_by_id FOREIGN KEY (upd_by_id) REFERENCES x_portal_user (id) +); +commit; +DECLARE + v_column_exists number := 0; +BEGIN +Select count(*) into v_column_exists from user_tab_cols where column_name = upper('policy_text') and table_name = upper('x_policy'); + if (v_column_exists = 0) then + execute immediate 'ALTER TABLE x_policy ADD policy_text CLOB DEFAULT NULL NULL'; + commit; + end if; +end;/ + +CREATE OR REPLACE PROCEDURE removeConstraints(ObjName IN varchar2) IS +BEGIN +FOR rec IN( +select owner, constraint_name +from all_constraints +where owner = sys_context('userenv','current_schema') +and table_name = ObjName +and constraint_type = 'R') +LOOP +execute immediate 'ALTER TABLE ' || rec.owner || '.' || ObjName || ' DROP CONSTRAINT ' || rec.constraint_name; +END LOOP; +END;/ +/ + +CALL removeConstraints('X_POLICY_ITEM'); +CALL removeConstraints('X_POLICY_ITEM_ACCESS'); +CALL removeConstraints('X_POLICY_ITEM_CONDITION'); +CALL removeConstraints('X_POLICY_ITEM_DATAMASK'); +CALL removeConstraints('X_POLICY_ITEM_GROUP_PERM'); +CALL removeConstraints('X_POLICY_RESOURCE'); +CALL removeConstraints('X_POLICY_RESOURCE_MAP'); +CALL removeConstraints('X_POLICY_ITEM_USER_PERM'); +CALL removeConstraints('X_POLICY_ITEM_ROWFILTER'); + diff --git a/security-admin/db/oracle/patches/036-denormalize-tag-tables.sql b/security-admin/db/oracle/patches/036-denormalize-tag-tables.sql new file mode 100644 index 0000000000..cae29272c4 --- /dev/null +++ b/security-admin/db/oracle/patches/036-denormalize-tag-tables.sql @@ -0,0 +1,54 @@ +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +DECLARE + v_count number:=0; +BEGIN + select count(*) into v_count from user_tab_cols where table_name='X_TAG_DEF' and column_name='TAG_ATTRS_DEF_TEXT'; + if (v_count = 0) then + execute immediate 'ALTER TABLE X_TAG_DEF ADD TAG_ATTRS_DEF_TEXT CLOB DEFAULT NULL NULL'; + end if; + select count(*) into v_count from user_tab_cols where table_name='X_TAG' and column_name='TAG_ATTRS_TEXT'; + if (v_count = 0) then + execute immediate 'ALTER TABLE X_TAG ADD TAG_ATTRS_TEXT CLOB DEFAULT NULL NULL'; + end if; + select count(*) into v_count from user_tab_cols where table_name='X_SERVICE_RESOURCE' and column_name='SERVICE_RESOURCE_ELEMENTS_TEXT'; + if (v_count = 0) then + execute immediate 'ALTER TABLE X_SERVICE_RESOURCE ADD SERVICE_RESOURCE_ELEMENTS_TEXT CLOB DEFAULT NULL NULL'; + end if; + select count(*) into v_count from user_tab_cols where table_name='X_SERVICE_RESOURCE' and column_name='TAGS_TEXT'; + if (v_count = 0) then + execute immediate 'ALTER TABLE X_SERVICE_RESOURCE ADD TAGS_TEXT CLOB DEFAULT NULL NULL'; + end if; + commit; +END;/ + +CREATE OR REPLACE PROCEDURE removeConstraints(ObjName IN varchar2) IS +BEGIN +FOR rec IN( +select owner, constraint_name +from all_constraints +where owner = sys_context('userenv','current_schema') +and table_name = ObjName +and constraint_type = 'R') +LOOP +execute immediate 'ALTER TABLE ' || rec.owner || '.' || ObjName || ' DROP CONSTRAINT ' || rec.constraint_name; +END LOOP; +END;/ +/ + +CALL removeConstraints('X_TAG_ATTR_DEF'); +CALL removeConstraints('X_TAG_ATTR'); +CALL removeConstraints('X_SERVICE_RESOURCE_ELEMENT'); +CALL removeConstraints('X_SERVICE_RESOURCE_ELEMENT_VAL'); diff --git a/security-admin/db/postgres/patches/035-update-schema-for-x-policy.sql b/security-admin/db/postgres/patches/035-update-schema-for-x-policy.sql new file mode 100644 index 0000000000..1414fe300f --- /dev/null +++ b/security-admin/db/postgres/patches/035-update-schema-for-x-policy.sql @@ -0,0 +1,198 @@ +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +DROP TABLE IF EXISTS x_policy_ref_resource CASCADE; +DROP SEQUENCE IF EXISTS x_policy_ref_resource_seq; +CREATE SEQUENCE x_policy_ref_resource_seq; +CREATE TABLE x_policy_ref_resource( +id BIGINT DEFAULT nextval('x_policy_ref_resource_seq'::regclass), +guid VARCHAR(1024) DEFAULT NULL NULL, +create_time TIMESTAMP DEFAULT NULL NULL, +update_time TIMESTAMP DEFAULT NULL NULL, +added_by_id BIGINT DEFAULT NULL NULL, +upd_by_id BIGINT DEFAULT NULL NULL, +policy_id BIGINT NOT NULL, +resource_def_id BIGINT NOT NULL, +resource_name varchar(4000) DEFAULT NULL, +primary key(id), +CONSTRAINT x_p_ref_res_UK_polId_resDefId UNIQUE (policy_id, resource_def_id), +CONSTRAINT x_p_ref_res_FK_policy_id FOREIGN KEY(policy_id) REFERENCES x_policy(id), +CONSTRAINT x_p_ref_res_FK_resource_def_id FOREIGN KEY(resource_def_id) REFERENCES x_resource_def(id), +CONSTRAINT x_p_ref_res_FK_added_by_id FOREIGN KEY(added_by_id) REFERENCES x_portal_user(id), +CONSTRAINT x_p_ref_res_FK_upd_by_id FOREIGN KEY(upd_by_id) REFERENCES x_portal_user(id) +); +commit; +DROP TABLE IF EXISTS x_policy_ref_access_type CASCADE; +DROP SEQUENCE IF EXISTS x_policy_ref_access_type_seq; +CREATE SEQUENCE x_policy_ref_access_type_seq; +CREATE TABLE x_policy_ref_access_type( +id BIGINT DEFAULT nextval('x_policy_ref_access_type_seq'::regclass), +guid VARCHAR(1024) DEFAULT NULL NULL, +create_time TIMESTAMP DEFAULT NULL NULL, +update_time TIMESTAMP DEFAULT NULL NULL, +added_by_id BIGINT DEFAULT NULL NULL, +upd_by_id BIGINT DEFAULT NULL NULL, +policy_id BIGINT NOT NULL, +access_def_id BIGINT NOT NULL, +access_type_name varchar(4000) DEFAULT NULL, +primary key(id), +CONSTRAINT x_p_ref_acc_UK_polId_accDefId UNIQUE(policy_id, access_def_id), +CONSTRAINT x_p_ref_acc_FK_policy_id FOREIGN KEY(policy_id) REFERENCES x_policy(id), +CONSTRAINT x_p_ref_acc_FK_acc_def_id FOREIGN KEY(access_def_id) REFERENCES x_access_type_def(id), +CONSTRAINT x_p_ref_acc_FK_added_by_id FOREIGN KEY(added_by_id) REFERENCES x_portal_user(id), +CONSTRAINT x_p_ref_acc_FK_upd_by_id FOREIGN KEY(upd_by_id) REFERENCES x_portal_user(id) +); +commit; +DROP TABLE IF EXISTS x_policy_ref_condition CASCADE; +DROP SEQUENCE IF EXISTS x_policy_ref_condition_seq; +CREATE SEQUENCE x_policy_ref_condition_seq; +CREATE TABLE x_policy_ref_condition( +id BIGINT DEFAULT nextval('x_policy_ref_condition_seq'::regclass), +guid VARCHAR(1024) DEFAULT NULL NULL, +create_time TIMESTAMP DEFAULT NULL NULL, +update_time TIMESTAMP DEFAULT NULL NULL, +added_by_id BIGINT DEFAULT NULL NULL, +upd_by_id BIGINT DEFAULT NULL NULL, +policy_id BIGINT NOT NULL, +condition_def_id BIGINT NOT NULL, +condition_name varchar(4000) DEFAULT NULL, +primary key(id), +CONSTRAINT x_p_ref_cond_UK_polId_cDefId UNIQUE(policy_id, condition_def_id), +CONSTRAINT x_p_ref_cond_FK_policy_id FOREIGN KEY(policy_id) REFERENCES x_policy(id), +CONSTRAINT x_p_ref_cond_FK_cond_def_id FOREIGN KEY(condition_def_id) REFERENCES x_policy_condition_def(id), +CONSTRAINT x_p_ref_cond_FK_added_by_id FOREIGN KEY(added_by_id) REFERENCES x_portal_user(id), +CONSTRAINT x_p_ref_cond_FK_upd_by_id FOREIGN KEY(upd_by_id) REFERENCES x_portal_user(id) +); +commit; +DROP TABLE IF EXISTS x_policy_ref_datamask_type CASCADE; +DROP SEQUENCE IF EXISTS x_policy_ref_datamask_type_seq; +CREATE SEQUENCE x_policy_ref_datamask_type_seq; +CREATE TABLE x_policy_ref_datamask_type( +id BIGINT DEFAULT nextval('x_policy_ref_datamask_type_seq'::regclass), +guid VARCHAR(1024) DEFAULT NULL NULL, +create_time TIMESTAMP DEFAULT NULL NULL, +update_time TIMESTAMP DEFAULT NULL NULL, +added_by_id BIGINT DEFAULT NULL NULL, +upd_by_id BIGINT DEFAULT NULL NULL, +policy_id BIGINT NOT NULL, +datamask_def_id BIGINT NOT NULL, +datamask_type_name varchar(4000) DEFAULT NULL, +primary key(id), +CONSTRAINT x_p_ref_dmk_UK_polId_dDefId UNIQUE(policy_id, datamask_def_id), +CONSTRAINT x_p_ref_dmk_FK_policy_id FOREIGN KEY(policy_id) REFERENCES x_policy(id), +CONSTRAINT x_p_ref_dmk_FK_dmk_def_id FOREIGN KEY(datamask_def_id) REFERENCES x_datamask_type_def(id), +CONSTRAINT x_p_ref_dmk_FK_added_by_id FOREIGN KEY(added_by_id) REFERENCES x_portal_user(id), +CONSTRAINT x_p_ref_dmk_FK_upd_by_id FOREIGN KEY(upd_by_id) REFERENCES x_portal_user(id) +); +commit; +DROP TABLE IF EXISTS x_policy_ref_user CASCADE; +DROP SEQUENCE IF EXISTS x_policy_ref_user_seq; +CREATE SEQUENCE x_policy_ref_user_seq; +CREATE TABLE x_policy_ref_user( +id BIGINT DEFAULT nextval('x_policy_ref_user_seq'::regclass), +guid VARCHAR(1024) DEFAULT NULL NULL, +create_time TIMESTAMP DEFAULT NULL NULL, +update_time TIMESTAMP DEFAULT NULL NULL, +added_by_id BIGINT DEFAULT NULL NULL, +upd_by_id BIGINT DEFAULT NULL NULL, +policy_id BIGINT NOT NULL, +user_id BIGINT NOT NULL, +user_name varchar(4000) DEFAULT NULL, +primary key(id), +CONSTRAINT x_p_ref_usr_UK_polId_userId UNIQUE(policy_id, user_id), +CONSTRAINT x_p_ref_usr_FK_policy_id FOREIGN KEY(policy_id) REFERENCES x_policy(id), +CONSTRAINT x_p_ref_usr_FK_user_id FOREIGN KEY(user_id) REFERENCES x_user(id), +CONSTRAINT x_p_ref_usr_FK_added_by_id FOREIGN KEY(added_by_id) REFERENCES x_portal_user(id), +CONSTRAINT x_p_ref_usr_FK_upd_by_id FOREIGN KEY(upd_by_id) REFERENCES x_portal_user(id) +); +commit; +DROP TABLE IF EXISTS x_policy_ref_group CASCADE; +DROP SEQUENCE IF EXISTS x_policy_ref_group_seq; +CREATE SEQUENCE x_policy_ref_group_seq; +CREATE TABLE x_policy_ref_group( +id BIGINT DEFAULT nextval('x_policy_ref_group_seq'::regclass), +guid VARCHAR(1024) DEFAULT NULL NULL, +create_time TIMESTAMP DEFAULT NULL NULL, +update_time TIMESTAMP DEFAULT NULL NULL, +added_by_id BIGINT DEFAULT NULL NULL, +upd_by_id BIGINT DEFAULT NULL NULL, +policy_id BIGINT NOT NULL, +group_id BIGINT NOT NULL, +group_name varchar(4000) DEFAULT NULL, +primary key(id), +CONSTRAINT x_p_ref_grp_UK_polId_grpId UNIQUE(policy_id, group_id), +CONSTRAINT x_p_ref_grp_FK_policy_id FOREIGN KEY(policy_id) REFERENCES x_policy(id), +CONSTRAINT x_p_ref_grp_FK_group_id FOREIGN KEY(group_id) REFERENCES x_group(id), +CONSTRAINT x_p_ref_grp_FK_added_by_id FOREIGN KEY(added_by_id) REFERENCES x_portal_user(id), +CONSTRAINT x_p_ref_grp_FK_upd_by_id FOREIGN KEY(upd_by_id) REFERENCES x_portal_user(id) +); +commit; +select 'delimiter start'; +CREATE OR REPLACE FUNCTION add_x_policy_json() +RETURNS void AS $$ +DECLARE + v_column_exists integer := 0; +BEGIN + select count(*) into v_column_exists from pg_attribute where attrelid in(select oid from pg_class where relname='x_policy') and attname='policy_text'; + IF v_column_exists = 0 THEN + ALTER TABLE x_policy ADD COLUMN policy_text TEXT DEFAULT NULL NULL; + END IF; +END; +$$ LANGUAGE plpgsql; +select 'delimiter end'; + +select add_x_policy_json(); +select 'delimiter end'; + +select 'delimiter start'; +CREATE OR REPLACE FUNCTION remove_foreign_key(objName varchar(4000)) +RETURNS void AS $$ +declare + tableName VARCHAR(256); + constraintName VARCHAR(512); + query varchar(4000); + curs CURSOR FOR SELECT table_name,constraint_name from information_schema.key_column_usage where constraint_catalog=current_catalog and table_name=objName and position_in_unique_constraint notnull; +begin + OPEN curs; + loop + FETCH curs INTO tableName,constraintName; + EXIT WHEN NOT FOUND; + query :='ALTER TABLE ' || objName || ' drop constraint ' || constraintName; + execute query; + end loop; + close curs; +END; +$$ LANGUAGE plpgsql; +select 'delimiter end'; + +CREATE OR REPLACE FUNCTION removekeys() +RETURNS void AS +$$ +BEGIN + perform remove_foreign_key('x_policy_item'); + perform remove_foreign_key('x_policy_item_access'); + perform remove_foreign_key('x_policy_item_condition'); + perform remove_foreign_key('x_policy_item_datamask'); + perform remove_foreign_key('x_policy_item_group_perm'); + perform remove_foreign_key('x_policy_resource'); + perform remove_foreign_key('x_policy_resource_map'); + perform remove_foreign_key('x_policy_item_user_perm'); + perform remove_foreign_key('x_policy_item_rowfilter'); + +END; +$$ LANGUAGE plpgsql; +select removekeys(); + +select 'delimiter end'; + diff --git a/security-admin/db/postgres/patches/036-denormalize-tag-tables.sql b/security-admin/db/postgres/patches/036-denormalize-tag-tables.sql new file mode 100644 index 0000000000..e5ed27221c --- /dev/null +++ b/security-admin/db/postgres/patches/036-denormalize-tag-tables.sql @@ -0,0 +1,79 @@ +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +-- function denormalize_tag_tables() +select 'delimiter start'; +CREATE OR REPLACE FUNCTION denormalize_tag_tables() +RETURNS void AS $$ +DECLARE + v_column_exists integer := 0; +BEGIN + select count(*) into v_column_exists from pg_attribute where attrelid in(select oid from pg_class where relname='x_tag_def') and attname='tag_attrs_def_text'; + IF v_column_exists = 0 THEN + ALTER TABLE x_tag_def ADD COLUMN tag_attrs_def_text TEXT DEFAULT NULL NULL; + END IF; + select count(*) into v_column_exists from pg_attribute where attrelid in(select oid from pg_class where relname='x_tag') and attname='tag_attrs_text'; + IF v_column_exists = 0 THEN + ALTER TABLE x_tag ADD COLUMN tag_attrs_text TEXT DEFAULT NULL NULL; + END IF; + select count(*) into v_column_exists from pg_attribute where attrelid in(select oid from pg_class where relname='x_service_resource') and attname='service_resource_elements_text'; + IF v_column_exists = 0 THEN + ALTER TABLE x_service_resource ADD COLUMN service_resource_elements_text TEXT DEFAULT NULL NULL; + END IF; + select count(*) into v_column_exists from pg_attribute where attrelid in(select oid from pg_class where relname='x_service_resource') and attname='tags_text'; + IF v_column_exists = 0 THEN + ALTER TABLE x_service_resource ADD COLUMN tags_text TEXT DEFAULT NULL NULL; + END IF; +END; +$$ LANGUAGE plpgsql; +select 'delimiter end'; + +select denormalize_tag_tables(); +select 'delimiter end'; + +select 'delimiter start'; +CREATE OR REPLACE FUNCTION remove_foreign_key(objName varchar(4000)) +RETURNS void AS $$ +declare + tableName VARCHAR(256); + constraintName VARCHAR(512); + query varchar(4000); + curs CURSOR FOR SELECT table_name,constraint_name from information_schema.key_column_usage where constraint_catalog=current_catalog and table_name=objName and position_in_unique_constraint notnull; +begin + OPEN curs; + loop + FETCH curs INTO tableName,constraintName; + EXIT WHEN NOT FOUND; + query :='ALTER TABLE ' || objName || ' drop constraint ' || constraintName; + execute query; + end loop; + close curs; +END; +$$ LANGUAGE plpgsql; +select 'delimiter end'; + +CREATE OR REPLACE FUNCTION removekeys() +RETURNS void AS +$$ +BEGIN + perform remove_foreign_key('x_tag_attr_def'); + perform remove_foreign_key('x_tag_attr'); + perform remove_foreign_key('x_service_resource_element'); + perform remove_foreign_key('x_service_resource_element_val'); +END; +$$ LANGUAGE plpgsql; +select removekeys(); + +select 'delimiter end'; diff --git a/security-admin/db/sqlanywhere/patches/035-update-schema-for-x-policy.sql b/security-admin/db/sqlanywhere/patches/035-update-schema-for-x-policy.sql new file mode 100644 index 0000000000..5da4538d23 --- /dev/null +++ b/security-admin/db/sqlanywhere/patches/035-update-schema-for-x-policy.sql @@ -0,0 +1,179 @@ + +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +GO +create table dbo.x_policy_ref_resource ( + id bigint IDENTITY NOT NULL, + guid varchar(1024) DEFAULT NULL NULL, + create_time datetime DEFAULT NULL NULL, + update_time datetime DEFAULT NULL NULL, + added_by_id bigint DEFAULT NULL NULL, + upd_by_id bigint DEFAULT NULL NULL, + policy_id bigint NOT NULL, + resource_def_id bigint NOT NULL, + resource_name varchar(4000) DEFAULT NULL NULL, + CONSTRAINT x_policy_ref_res_PK_id PRIMARY KEY CLUSTERED(id), + CONSTRAINT x_p_ref_res_UK_polId_resDefId UNIQUE NONCLUSTERED (policy_id, resource_def_id) +) +GO + +create table dbo.x_policy_ref_access_type ( + id bigint IDENTITY NOT NULL, + guid varchar(1024) DEFAULT NULL NULL, + create_time datetime DEFAULT NULL NULL, + update_time datetime DEFAULT NULL NULL, + added_by_id bigint DEFAULT NULL NULL, + upd_by_id bigint DEFAULT NULL NULL, + policy_id bigint NOT NULL, + access_def_id bigint NOT NULL, + access_type_name varchar(4000) DEFAULT NULL NULL, + CONSTRAINT x_policy_ref_acc_PK_id PRIMARY KEY CLUSTERED(id), + CONSTRAINT x_p_ref_acc_UK_polId_accDefId UNIQUE NONCLUSTERED (policy_id, access_def_id) +) +GO + +create table dbo.x_policy_ref_condition ( + id bigint IDENTITY NOT NULL, + guid varchar(1024) DEFAULT NULL NULL, + create_time datetime DEFAULT NULL NULL, + update_time datetime DEFAULT NULL NULL, + added_by_id bigint DEFAULT NULL NULL, + upd_by_id bigint DEFAULT NULL NULL, + policy_id bigint NOT NULL, + condition_def_id bigint NOT NULL, + condition_name varchar(4000) DEFAULT NULL NULL, + CONSTRAINT x_policy_ref_cond_PK_id PRIMARY KEY CLUSTERED(id), + CONSTRAINT x_p_ref_cond_UK_polId_cDefId UNIQUE NONCLUSTERED (policy_id, condition_def_id) +) +GO + +create table dbo.x_policy_ref_datamask_type ( + id bigint IDENTITY NOT NULL, + guid varchar(1024) DEFAULT NULL NULL, + create_time datetime DEFAULT NULL NULL, + update_time datetime DEFAULT NULL NULL, + added_by_id bigint DEFAULT NULL NULL, + upd_by_id bigint DEFAULT NULL NULL, + policy_id bigint NOT NULL, + datamask_def_id bigint NOT NULL, + datamask_type_name varchar(4000) DEFAULT NULL NULL, + CONSTRAINT x_policy_ref_dmk_PK_id PRIMARY KEY CLUSTERED(id), + CONSTRAINT x_p_ref_dmk_UK_polId_dDefId UNIQUE NONCLUSTERED (policy_id, datamask_def_id) +) +GO + +create table dbo.x_policy_ref_user ( + id bigint IDENTITY NOT NULL, + guid varchar(1024) DEFAULT NULL NULL, + create_time datetime DEFAULT NULL NULL, + update_time datetime DEFAULT NULL NULL, + added_by_id bigint DEFAULT NULL NULL, + upd_by_id bigint DEFAULT NULL NULL, + policy_id bigint NOT NULL, + user_id bigint NOT NULL, + user_name varchar(4000) DEFAULT NULL NULL, + CONSTRAINT x_policy_ref_user_PK_id PRIMARY KEY CLUSTERED(id), + CONSTRAINT x_p_ref_usr_UK_polId_userId UNIQUE NONCLUSTERED (policy_id, user_id) +) +GO + +create table dbo.x_policy_ref_group ( + id bigint IDENTITY NOT NULL, + guid varchar(1024) DEFAULT NULL NULL, + create_time datetime DEFAULT NULL NULL, + update_time datetime DEFAULT NULL NULL, + added_by_id bigint DEFAULT NULL NULL, + upd_by_id bigint DEFAULT NULL NULL, + policy_id bigint NOT NULL, + group_id bigint NOT NULL, + group_name varchar(4000) DEFAULT NULL NULL, + CONSTRAINT x_policy_ref_group_PK_id PRIMARY KEY CLUSTERED(id), + CONSTRAINT x_p_ref_grp_UK_polId_grpId UNIQUE NONCLUSTERED (policy_id, group_id) +) +GO + +IF NOT EXISTS(select * from SYS.SYSCOLUMNS where tname = 'x_policy' and cname='policy_text') THEN + ALTER TABLE dbo.x_policy ADD (policy_text text DEFAULT NULL NULL); +END IF; +GO + +IF EXISTS ( + SELECT 1 + FROM sysobjects + WHERE NAME = 'removeForeignKeyConstraint' + AND TYPE = 'P' +) +BEGIN + drop procedure dbo.removeForeignKeyConstraint +END +GO + +CREATE PROCEDURE dbo.removeForeignKeyConstraint (IN table_name varchar(100)) +AS +BEGIN + DECLARE @stmt VARCHAR(300) + DECLARE cur CURSOR FOR + select 'alter table dbo.' + table_name + ' drop constraint ' + role + from SYS.SYSFOREIGNKEYS + where foreign_creator ='dbo' and foreign_tname = table_name + + OPEN cur WITH HOLD + fetch cur into @stmt + if (@@sqlstatus = 2) + BEGIN + close cur + DEALLOCATE CURSOR cur + END + + WHILE (@@sqlstatus = 0) + BEGIN + + execute(@stmt) + fetch cur into @stmt + END + close cur + DEALLOCATE CURSOR cur + +END +GO + +call dbo.removeForeignKeyConstraint('x_policy_item') +GO + +call dbo.removeForeignKeyConstraint('x_policy_item_access') +GO + +call dbo.removeForeignKeyConstraint('x_policy_item_condition') +GO + +call dbo.removeForeignKeyConstraint('x_policy_item_datamask') +GO + +call dbo.removeForeignKeyConstraint('x_policy_item_group_perm') +GO + +call dbo.removeForeignKeyConstraint('x_policy_item_user_perm') +GO + +call dbo.removeForeignKeyConstraint('x_policy_item_rowfilter') +GO + +call dbo.removeForeignKeyConstraint('x_policy_resource') +GO + +call dbo.removeForeignKeyConstraint('x_policy_resource_map') +GO + +exit diff --git a/security-admin/db/sqlanywhere/patches/036-denormalize-tag-tables.sql b/security-admin/db/sqlanywhere/patches/036-denormalize-tag-tables.sql new file mode 100644 index 0000000000..1fdbfaa539 --- /dev/null +++ b/security-admin/db/sqlanywhere/patches/036-denormalize-tag-tables.sql @@ -0,0 +1,71 @@ +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +IF NOT EXISTS(select * from SYS.SYSCOLUMNS where tname = 'x_tag_def' and cname = 'tag_attrs_def_text') THEN + ALTER TABLE dbo.x_tag_def ADD tag_attrs_def_text text DEFAULT NULL NULL; +END IF; +IF NOT EXISTS(select * from SYS.SYSCOLUMNS where tname = 'x_tag' and cname = 'tag_attrs_text') THEN + ALTER TABLE dbo.x_tag ADD tag_attrs_text text DEFAULT NULL NULL; +END IF; +IF NOT EXISTS(select * from SYS.SYSCOLUMNS where tname = 'x_service_resource' and cname = 'service_resource_elements_text') THEN + ALTER TABLE dbo.x_service_resource ADD service_resource_elements_text text DEFAULT NULL NULL; +END IF; +IF NOT EXISTS(select * from SYS.SYSCOLUMNS where tname = 'x_service_resource' and cname = 'tags_text') THEN + ALTER TABLE dbo.x_service_resource ADD tags_text text DEFAULT NULL NULL; +END IF; +GO + +CREATE PROCEDURE dbo.removeTagForeignKeyConstraint (IN table_name varchar(100)) +AS +BEGIN + DECLARE @stmt VARCHAR(300) + DECLARE cur CURSOR FOR + select 'alter table dbo.' + table_name + ' drop constraint ' + role + from SYS.SYSFOREIGNKEYS + where foreign_creator ='dbo' and foreign_tname = table_name + + OPEN cur WITH HOLD + fetch cur into @stmt + if (@@sqlstatus = 2) + BEGIN + close cur + DEALLOCATE CURSOR cur + END + + WHILE (@@sqlstatus = 0) + BEGIN + + execute(@stmt) + fetch cur into @stmt + END + close cur + DEALLOCATE CURSOR cur + +END +GO + +call dbo.removeTagForeignKeyConstraint('x_tag_attr_def') +GO + +call dbo.removeTagForeignKeyConstraint('x_tag_attr') +GO + +call dbo.removeTagForeignKeyConstraint('x_service_resource_element') +GO + +call dbo.removeTagForeignKeyConstraint('x_service_resource_element_val') +GO + +exit diff --git a/security-admin/db/sqlserver/optimized/current/ranger_core_db_sqlserver.sql b/security-admin/db/sqlserver/optimized/current/ranger_core_db_sqlserver.sql index bf73c34eab..d33dae05d9 100644 --- a/security-admin/db/sqlserver/optimized/current/ranger_core_db_sqlserver.sql +++ b/security-admin/db/sqlserver/optimized/current/ranger_core_db_sqlserver.sql @@ -253,6 +253,78 @@ IF (OBJECT_ID('x_enum_def_FK_defid') IS NOT NULL) BEGIN ALTER TABLE [dbo].[x_enum_def] DROP CONSTRAINT x_enum_def_FK_defid END +IF (OBJECT_ID('x_policy_ref_resource_FK_policy_id') IS NOT NULL) +BEGIN + ALTER TABLE [dbo].[x_policy_ref_resource] DROP CONSTRAINT x_policy_ref_resource_FK_policy_id +END +IF (OBJECT_ID('x_policy_ref_resource_FK_res_def_id') IS NOT NULL) +BEGIN + ALTER TABLE [dbo].[x_policy_ref_resource] DROP CONSTRAINT x_policy_ref_resource_FK_res_def_id +END +IF (OBJECT_ID('x_policy_ref_resource') IS NOT NULL) +BEGIN + DROP TABLE [dbo].[x_policy_ref_resource] +END +IF (OBJECT_ID('x_policy_ref_access_type_FK_policy_id') IS NOT NULL) +BEGIN + ALTER TABLE [dbo].[x_policy_ref_access_type] DROP CONSTRAINT x_policy_ref_access_type_FK_policy_id +END +IF (OBJECT_ID('x_policy_ref_access_type_FK_access_def_id') IS NOT NULL) +BEGIN + ALTER TABLE [dbo].[x_policy_ref_access_type] DROP CONSTRAINT x_policy_ref_access_type_FK_access_def_id +END +IF (OBJECT_ID('x_policy_ref_access_type') IS NOT NULL) +BEGIN + DROP TABLE [dbo].[x_policy_ref_access_type] +END +IF (OBJECT_ID('x_policy_ref_condition_FK_policy_id') IS NOT NULL) +BEGIN + ALTER TABLE [dbo].[x_policy_ref_condition] DROP CONSTRAINT x_policy_ref_condition_FK_policy_id +END +IF (OBJECT_ID('x_policy_ref_condition_FK_condition_def_id') IS NOT NULL) +BEGIN + ALTER TABLE [dbo].[x_policy_ref_condition] DROP CONSTRAINT x_policy_ref_condition_FK_condition_def_id +END +IF (OBJECT_ID('x_policy_ref_condition') IS NOT NULL) +BEGIN + DROP TABLE [dbo].[x_policy_ref_condition] +END +IF (OBJECT_ID('x_policy_ref_datamask_type_FK_policy_id') IS NOT NULL) +BEGIN + ALTER TABLE [dbo].[x_policy_ref_datamask_type] DROP CONSTRAINT x_policy_ref_datamask_type_FK_policy_id +END +IF (OBJECT_ID('x_policy_ref_datamask_type_FK_datamask_def_id') IS NOT NULL) +BEGIN + ALTER TABLE [dbo].[x_policy_ref_datamask_type] DROP CONSTRAINT x_policy_ref_datamask_type_FK_datamask_def_id +END +IF (OBJECT_ID('x_policy_ref_datamask_type') IS NOT NULL) +BEGIN + DROP TABLE [dbo].[x_policy_ref_datamask_type] +END +IF (OBJECT_ID('x_policy_ref_user_FK_policy_id') IS NOT NULL) +BEGIN + ALTER TABLE [dbo].[x_policy_ref_user] DROP CONSTRAINT x_policy_ref_user_FK_policy_id +END +IF (OBJECT_ID('x_policy_ref_user_FK_user_id') IS NOT NULL) +BEGIN + ALTER TABLE [dbo].[x_policy_ref_user] DROP CONSTRAINT x_policy_ref_user_FK_user_id +END +IF (OBJECT_ID('x_policy_ref_user') IS NOT NULL) +BEGIN + DROP TABLE [dbo].[x_policy_ref_user] +END +IF (OBJECT_ID('x_policy_ref_group_FK_policy_id') IS NOT NULL) +BEGIN + ALTER TABLE [dbo].[x_policy_ref_group] DROP CONSTRAINT x_policy_ref_group_FK_policy_id +END +IF (OBJECT_ID('x_policy_ref_group_FK_group_id') IS NOT NULL) +BEGIN + ALTER TABLE [dbo].[x_policy_ref_group] DROP CONSTRAINT x_policy_ref_group_FK_group_id +END +IF (OBJECT_ID('x_policy_ref_group') IS NOT NULL) +BEGIN + DROP TABLE [dbo].[x_policy_ref_group] +END IF (OBJECT_ID('x_enum_element_def_FK_defid') IS NOT NULL) BEGIN ALTER TABLE [dbo].[x_enum_element_def] DROP CONSTRAINT x_enum_element_def_FK_defid @@ -937,7 +1009,7 @@ PRIMARY KEY CLUSTERED CONSTRAINT [[x_group_users$x_group_users_UK_uid_gname] UNIQUE NONCLUSTERED ( [user_id] ASC, - [group_name] ASC + [group_name] ASC )WITH (PAD_INDEX = OFF,STATISTICS_NORECOMPUTE = OFF,IGNORE_DUP_KEY = OFF,ALLOW_ROW_LOCKS = ON,ALLOW_PAGE_LOCKS = ON) ON [PRIMARY] ) ON [PRIMARY] @@ -1147,6 +1219,7 @@ CREATE TABLE [dbo].[x_policy] ( [resource_signature] [varchar](128) DEFAULT NULL NULL, [is_enabled] [tinyint] DEFAULT 0 NOT NULL, [is_audit_enabled] [tinyint] DEFAULT 0 NOT NULL, + [policy_text] [nvarchar](max) DEFAULT NULL NULL, PRIMARY KEY CLUSTERED ( [id] ASC @@ -1587,6 +1660,7 @@ CREATE TABLE [dbo].[x_tag_def]( [name] [varchar](255) NOT NULL, [source] [varchar](128) DEFAULT NULL NULL, [is_enabled] [tinyint] DEFAULT 0 NOT NULL, + [tag_attrs_def_text] [nvarchar](max) DEFAULT NULL NULL, PRIMARY KEY CLUSTERED ( [id] ASC @@ -1613,6 +1687,7 @@ CREATE TABLE [dbo].[x_tag]( [version] [bigint] DEFAULT NULL NULL, [type] [bigint] NOT NULL, [owned_by] [smallint] DEFAULT 0 NOT NULL, + [tag_attrs_text] [nvarchar](max) DEFAULT NULL NULL, PRIMARY KEY CLUSTERED ( [id] ASC @@ -1636,6 +1711,8 @@ CREATE TABLE [dbo].[x_service_resource]( [service_id] [bigint] NOT NULL, [resource_signature] [varchar](128) DEFAULT NULL NULL, [is_enabled] [tinyint] DEFAULT 1 NOT NULL, + [service_resource_elements_text] [nvarchar](max) DEFAULT NULL NULL, + [tags_text] [nvarchar](max) DEFAULT NULL NULL, PRIMARY KEY CLUSTERED ( [id] ASC @@ -1829,6 +1906,138 @@ CONSTRAINT [x_plugin_info$x_plugin_info_UK] UNIQUE NONCLUSTERED SET ANSI_NULLS ON SET QUOTED_IDENTIFIER ON SET ANSI_PADDING ON +CREATE TABLE [dbo].[x_policy_ref_resource] ( + [id] [bigint] IDENTITY (1, 1) NOT NULL, + [guid] [varchar](1024) DEFAULT NULL NULL, + [create_time] [datetime2] DEFAULT NULL NULL, + [update_time] [datetime2] DEFAULT NULL NULL, + [added_by_id] [bigint] DEFAULT NULL NULL, + [upd_by_id] [bigint] DEFAULT NULL NULL, + [policy_id] [bigint] NOT NULL, + [resource_def_id] [bigint] NOT NULL, + [resource_name] [varchar](4000) DEFAULT NULL NULL, + PRIMARY KEY CLUSTERED + ( + [id] ASC + ) WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY], + CONSTRAINT [x_policy_ref_resource$x_policy_ref_resource_UK] UNIQUE NONCLUSTERED + ( + [policy_id] ASC, [resource_def_id] ASC + )WITH (PAD_INDEX = OFF,STATISTICS_NORECOMPUTE = OFF,IGNORE_DUP_KEY = OFF,ALLOW_ROW_LOCKS = ON,ALLOW_PAGE_LOCKS = ON) ON [PRIMARY] +) ON [PRIMARY] +SET ANSI_NULLS ON +SET QUOTED_IDENTIFIER ON +SET ANSI_PADDING ON +CREATE TABLE [dbo].[x_policy_ref_access_type] ( + [id] [bigint] IDENTITY (1, 1) NOT NULL, + [guid] [varchar](1024) DEFAULT NULL NULL, + [create_time] [datetime2] DEFAULT NULL NULL, + [update_time] [datetime2] DEFAULT NULL NULL, + [added_by_id] [bigint] DEFAULT NULL NULL, + [upd_by_id] [bigint] DEFAULT NULL NULL, + [policy_id] [bigint] NOT NULL, + [access_def_id] [bigint] NOT NULL, + [access_type_name] [varchar](4000) DEFAULT NULL NULL, + PRIMARY KEY CLUSTERED + ( + [id] ASC + ) WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY], + CONSTRAINT [x_policy_ref_access_type$x_policy_ref_access_type_UK] UNIQUE NONCLUSTERED +( + [policy_id] ASC, [access_def_id] ASC +)WITH (PAD_INDEX = OFF,STATISTICS_NORECOMPUTE = OFF,IGNORE_DUP_KEY = OFF,ALLOW_ROW_LOCKS = ON,ALLOW_PAGE_LOCKS = ON) ON [PRIMARY] +) ON [PRIMARY] +SET ANSI_NULLS ON +SET QUOTED_IDENTIFIER ON +SET ANSI_PADDING ON +CREATE TABLE [dbo].[x_policy_ref_condition] ( + [id] [bigint] IDENTITY (1, 1) NOT NULL, + [guid] [varchar](1024) DEFAULT NULL NULL, + [create_time] [datetime2] DEFAULT NULL NULL, + [update_time] [datetime2] DEFAULT NULL NULL, + [added_by_id] [bigint] DEFAULT NULL NULL, + [upd_by_id] [bigint] DEFAULT NULL NULL, + [policy_id] [bigint] NOT NULL, + [condition_def_id] [bigint] NOT NULL, + [condition_name] [varchar](4000) DEFAULT NULL NULL, + PRIMARY KEY CLUSTERED + ( + [id] ASC + ) WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY], + CONSTRAINT [x_policy_ref_condition$x_policy_ref_condition_UK] UNIQUE NONCLUSTERED +( + [policy_id] ASC, [condition_def_id] ASC +)WITH (PAD_INDEX = OFF,STATISTICS_NORECOMPUTE = OFF,IGNORE_DUP_KEY = OFF,ALLOW_ROW_LOCKS = ON,ALLOW_PAGE_LOCKS = ON) ON [PRIMARY] +) ON [PRIMARY] +SET ANSI_NULLS ON +SET QUOTED_IDENTIFIER ON +SET ANSI_PADDING ON +CREATE TABLE [dbo].[x_policy_ref_datamask_type] ( + [id] [bigint] IDENTITY (1, 1) NOT NULL, + [guid] [varchar](1024) DEFAULT NULL NULL, + [create_time] [datetime2] DEFAULT NULL NULL, + [update_time] [datetime2] DEFAULT NULL NULL, + [added_by_id] [bigint] DEFAULT NULL NULL, + [upd_by_id] [bigint] DEFAULT NULL NULL, + [policy_id] [bigint] NOT NULL, + [datamask_def_id] [bigint] NOT NULL, + [datamask_type_name] [varchar](4000) DEFAULT NULL NULL, + PRIMARY KEY CLUSTERED + ( + [id] ASC + ) WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY], + CONSTRAINT [x_policy_ref_datamask_type$x_policy_ref_datamask_type_UK] UNIQUE NONCLUSTERED +( + [policy_id] ASC, [datamask_def_id] ASC +)WITH (PAD_INDEX = OFF,STATISTICS_NORECOMPUTE = OFF,IGNORE_DUP_KEY = OFF,ALLOW_ROW_LOCKS = ON,ALLOW_PAGE_LOCKS = ON) ON [PRIMARY] +) ON [PRIMARY] +SET ANSI_NULLS ON +SET QUOTED_IDENTIFIER ON +SET ANSI_PADDING ON +CREATE TABLE [dbo].[x_policy_ref_user] ( + [id] [bigint] IDENTITY (1, 1) NOT NULL, + [guid] [varchar](1024) DEFAULT NULL NULL, + [create_time] [datetime2] DEFAULT NULL NULL, + [update_time] [datetime2] DEFAULT NULL NULL, + [added_by_id] [bigint] DEFAULT NULL NULL, + [upd_by_id] [bigint] DEFAULT NULL NULL, + [policy_id] [bigint] NOT NULL, + [user_id] [bigint] NOT NULL, + [user_name] [varchar](4000) DEFAULT NULL NULL, + PRIMARY KEY CLUSTERED + ( + [id] ASC + ) WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY], + CONSTRAINT [x_policy_ref_user$x_policy_ref_user_UK] UNIQUE NONCLUSTERED +( + [policy_id] ASC, [user_id] ASC +)WITH (PAD_INDEX = OFF,STATISTICS_NORECOMPUTE = OFF,IGNORE_DUP_KEY = OFF,ALLOW_ROW_LOCKS = ON,ALLOW_PAGE_LOCKS = ON) ON [PRIMARY] +) ON [PRIMARY] +SET ANSI_NULLS ON +SET QUOTED_IDENTIFIER ON +SET ANSI_PADDING ON +CREATE TABLE [dbo].[x_policy_ref_group] ( + [id] [bigint] IDENTITY (1, 1) NOT NULL, + [guid] [varchar](1024) DEFAULT NULL NULL, + [create_time] [datetime2] DEFAULT NULL NULL, + [update_time] [datetime2] DEFAULT NULL NULL, + [added_by_id] [bigint] DEFAULT NULL NULL, + [upd_by_id] [bigint] DEFAULT NULL NULL, + [policy_id] [bigint] NOT NULL, + [group_id] [bigint] NOT NULL, + [group_name] [varchar](4000) DEFAULT NULL NULL, + PRIMARY KEY CLUSTERED + ( + [id] ASC + ) WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY], + CONSTRAINT [x_policy_ref_group$x_policy_ref_group_UK] UNIQUE NONCLUSTERED +( + [policy_id] ASC, [group_id] ASC +)WITH (PAD_INDEX = OFF,STATISTICS_NORECOMPUTE = OFF,IGNORE_DUP_KEY = OFF,ALLOW_ROW_LOCKS = ON,ALLOW_PAGE_LOCKS = ON) ON [PRIMARY] +) ON [PRIMARY] +SET ANSI_NULLS ON +SET QUOTED_IDENTIFIER ON +SET ANSI_PADDING ON ALTER TABLE [dbo].[x_asset] WITH CHECK ADD CONSTRAINT [x_asset_FK_added_by_id] FOREIGN KEY([added_by_id]) REFERENCES [dbo].[x_portal_user] ([id]) @@ -2191,6 +2400,73 @@ ALTER TABLE [dbo].[x_policy_item_rowfilter] WITH CHECK ADD CONSTRAINT [x_policy_ ALTER TABLE [dbo].[x_policy_item_rowfilter] WITH CHECK ADD CONSTRAINT [x_policy_item_rowfilter_FK_added_by_id] FOREIGN KEY([added_by_id]) REFERENCES [dbo].[x_portal_user] ([id]) ALTER TABLE [dbo].[x_policy_item_rowfilter] WITH CHECK ADD CONSTRAINT [x_policy_item_rowfilter_FK_upd_by_id] FOREIGN KEY([upd_by_id]) REFERENCES [dbo].[x_portal_user] ([id]) ALTER TABLE [dbo].[x_service_version_info] WITH CHECK ADD CONSTRAINT [x_service_version_info_service_id] FOREIGN KEY([service_id]) REFERENCES [dbo].[x_service] ([id]) +ALTER TABLE [dbo].[x_policy_ref_resource] WITH CHECK ADD CONSTRAINT [x_policy_ref_resource_FK_policy_id] FOREIGN KEY ([policy_id]) +REFERENCES [dbo].[x_policy] ([id]) +ALTER TABLE [dbo].[x_policy_ref_resource] CHECK CONSTRAINT [x_policy_ref_resource_FK_policy_id] +ALTER TABLE [dbo].[x_policy_ref_resource] WITH CHECK ADD CONSTRAINT [x_policy_ref_resource_FK_resource_def_id] FOREIGN KEY ([resource_def_id]) +REFERENCES [dbo].[x_resource_def] ([id]) +ALTER TABLE [dbo].[x_policy_ref_resource] CHECK CONSTRAINT [x_policy_ref_resource_FK_resource_def_id] +ALTER TABLE [dbo].[x_policy_ref_resource] WITH CHECK ADD CONSTRAINT [x_policy_ref_resource_FK_added_by] FOREIGN KEY ([added_by_id]) +REFERENCES [dbo].[x_portal_user] ([id]) +ALTER TABLE [dbo].[x_policy_ref_resource] CHECK CONSTRAINT [x_policy_ref_resource_FK_added_by] +ALTER TABLE [dbo].[x_policy_ref_resource] WITH CHECK ADD CONSTRAINT [x_policy_ref_resource_FK_upd_by] FOREIGN KEY ([upd_by_id]) +REFERENCES [dbo].[x_portal_user] ([id]) +ALTER TABLE [dbo].[x_policy_ref_access_type] WITH CHECK ADD CONSTRAINT [x_policy_ref_access_type_FK_policy_id] FOREIGN KEY ([policy_id]) +REFERENCES [dbo].[x_policy] ([id]) +ALTER TABLE [dbo].[x_policy_ref_access_type] CHECK CONSTRAINT [x_policy_ref_access_type_FK_policy_id] +ALTER TABLE [dbo].[x_policy_ref_access_type] WITH CHECK ADD CONSTRAINT [x_policy_ref_access_type_FK_access_def_id] FOREIGN KEY ([access_def_id]) +REFERENCES [dbo].[x_access_type_def] ([id]) +ALTER TABLE [dbo].[x_policy_ref_access_type] CHECK CONSTRAINT [x_policy_ref_access_type_FK_access_def_id] +ALTER TABLE [dbo].[x_policy_ref_access_type] WITH CHECK ADD CONSTRAINT [x_policy_ref_access_type_FK_added_by] FOREIGN KEY ([added_by_id]) +REFERENCES [dbo].[x_portal_user] ([id]) +ALTER TABLE [dbo].[x_policy_ref_access_type] CHECK CONSTRAINT [x_policy_ref_access_type_FK_added_by] +ALTER TABLE [dbo].[x_policy_ref_access_type] WITH CHECK ADD CONSTRAINT [x_policy_ref_access_type_FK_upd_by] FOREIGN KEY ([upd_by_id]) +REFERENCES [dbo].[x_portal_user] ([id]) +ALTER TABLE [dbo].[x_policy_ref_condition] WITH CHECK ADD CONSTRAINT [x_policy_ref_condition_FK_policy_id] FOREIGN KEY ([policy_id]) +REFERENCES [dbo].[x_policy] ([id]) +ALTER TABLE [dbo].[x_policy_ref_condition] CHECK CONSTRAINT [x_policy_ref_condition_FK_policy_id] +ALTER TABLE [dbo].[x_policy_ref_condition] WITH CHECK ADD CONSTRAINT [x_policy_ref_condition_FK_condition_def_id] FOREIGN KEY ([condition_def_id]) +REFERENCES [dbo].[x_policy_condition_def] ([id]) +ALTER TABLE [dbo].[x_policy_ref_condition] CHECK CONSTRAINT [x_policy_ref_condition_FK_condition_def_id] +ALTER TABLE [dbo].[x_policy_ref_condition] WITH CHECK ADD CONSTRAINT [x_policy_ref_condition_FK_added_by] FOREIGN KEY ([added_by_id]) +REFERENCES [dbo].[x_portal_user] ([id]) +ALTER TABLE [dbo].[x_policy_ref_condition] CHECK CONSTRAINT [x_policy_ref_condition_FK_added_by] +ALTER TABLE [dbo].[x_policy_ref_condition] WITH CHECK ADD CONSTRAINT [x_policy_ref_condition_FK_upd_by] FOREIGN KEY ([upd_by_id]) +REFERENCES [dbo].[x_portal_user] ([id]) +ALTER TABLE [dbo].[x_policy_ref_datamask_type] WITH CHECK ADD CONSTRAINT [x_policy_ref_datamask_type_FK_policy_id] FOREIGN KEY ([policy_id]) +REFERENCES [dbo].[x_policy] ([id]) +ALTER TABLE [dbo].[x_policy_ref_datamask_type] CHECK CONSTRAINT [x_policy_ref_datamask_type_FK_policy_id] +ALTER TABLE [dbo].[x_policy_ref_datamask_type] WITH CHECK ADD CONSTRAINT [x_policy_ref_datamask_type_FK_datamask_def_id] FOREIGN KEY ([datamask_def_id]) +REFERENCES [dbo].[x_datamask_type_def] ([id]) +ALTER TABLE [dbo].[x_policy_ref_datamask_type] CHECK CONSTRAINT [x_policy_ref_datamask_type_FK_datamask_def_id] +ALTER TABLE [dbo].[x_policy_ref_datamask_type] WITH CHECK ADD CONSTRAINT [x_policy_ref_datamask_type_FK_added_by] FOREIGN KEY ([added_by_id]) +REFERENCES [dbo].[x_portal_user] ([id]) +ALTER TABLE [dbo].[x_policy_ref_datamask_type] CHECK CONSTRAINT [x_policy_ref_datamask_type_FK_added_by] +ALTER TABLE [dbo].[x_policy_ref_datamask_type] WITH CHECK ADD CONSTRAINT [x_policy_ref_datamask_type_FK_upd_by] FOREIGN KEY ([upd_by_id]) +REFERENCES [dbo].[x_portal_user] ([id]) +ALTER TABLE [dbo].[x_policy_ref_user] WITH CHECK ADD CONSTRAINT [x_policy_ref_user_FK_policy_id] FOREIGN KEY ([policy_id]) +REFERENCES [dbo].[x_policy] ([id]) +ALTER TABLE [dbo].[x_policy_ref_user] CHECK CONSTRAINT [x_policy_ref_user_FK_policy_id] +ALTER TABLE [dbo].[x_policy_ref_user] WITH CHECK ADD CONSTRAINT [x_policy_ref_user_FK_user_id] FOREIGN KEY ([user_id]) +REFERENCES [dbo].[x_user] ([id]) +ALTER TABLE [dbo].[x_policy_ref_user] CHECK CONSTRAINT [x_policy_ref_user_FK_user_id] +ALTER TABLE [dbo].[x_policy_ref_user] WITH CHECK ADD CONSTRAINT [x_policy_ref_user_FK_added_by] FOREIGN KEY ([added_by_id]) +REFERENCES [dbo].[x_portal_user] ([id]) +ALTER TABLE [dbo].[x_policy_ref_user] CHECK CONSTRAINT [x_policy_ref_user_FK_added_by] +ALTER TABLE [dbo].[x_policy_ref_user] WITH CHECK ADD CONSTRAINT [x_policy_ref_user_FK_upd_by] FOREIGN KEY ([upd_by_id]) +REFERENCES [dbo].[x_portal_user] ([id]) +ALTER TABLE [dbo].[x_policy_ref_user] CHECK CONSTRAINT [x_policy_ref_user_FK_upd_by] +ALTER TABLE [dbo].[x_policy_ref_group] WITH CHECK ADD CONSTRAINT [x_policy_ref_group_FK_policy_id] FOREIGN KEY ([policy_id]) +REFERENCES [dbo].[x_policy] ([id]) +ALTER TABLE [dbo].[x_policy_ref_group] CHECK CONSTRAINT [x_policy_ref_group_FK_policy_id] +ALTER TABLE [dbo].[x_policy_ref_group] WITH CHECK ADD CONSTRAINT [x_policy_ref_group_FK_group_id] FOREIGN KEY ([group_id]) +REFERENCES [dbo].[x_group] ([id]) +ALTER TABLE [dbo].[x_policy_ref_group] CHECK CONSTRAINT [x_policy_ref_group_FK_group_id] +ALTER TABLE [dbo].[x_policy_ref_group] WITH CHECK ADD CONSTRAINT [x_policy_ref_group_FK_added_by] FOREIGN KEY ([added_by_id]) +REFERENCES [dbo].[x_portal_user] ([id]) +ALTER TABLE [dbo].[x_policy_ref_group] CHECK CONSTRAINT [x_policy_ref_group_FK_added_by] +ALTER TABLE [dbo].[x_policy_ref_group] WITH CHECK ADD CONSTRAINT [x_policy_ref_group_FK_upd_by] FOREIGN KEY ([upd_by_id]) +REFERENCES [dbo].[x_portal_user] ([id]) CREATE NONCLUSTERED INDEX [x_asset_cr_time] ON [x_asset] ( [create_time] ASC @@ -2999,6 +3275,8 @@ INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active) VALUES ('028',CURRENT_TIMESTAMP,'Ranger 0.7.0',CURRENT_TIMESTAMP,'localhost','Y'); INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active) VALUES ('029',CURRENT_TIMESTAMP,'Ranger 0.7.0',CURRENT_TIMESTAMP,'localhost','Y'); INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active) VALUES ('033',CURRENT_TIMESTAMP,'Ranger 0.7.0',CURRENT_TIMESTAMP,'localhost','Y'); +INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active) VALUES ('035',CURRENT_TIMESTAMP,'Ranger 0.7.0',CURRENT_TIMESTAMP,'localhost','Y'); +INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active) VALUES ('040',CURRENT_TIMESTAMP,'Ranger 0.7.0',CURRENT_TIMESTAMP,'localhost','Y'); INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active) VALUES ('DB_PATCHES',CURRENT_TIMESTAMP,'Ranger 0.7.0',CURRENT_TIMESTAMP,'localhost','Y'); INSERT INTO x_user_module_perm (user_id,module_id,create_time,update_time,added_by_id,upd_by_id,is_allowed) VALUES (1,3,CURRENT_TIMESTAMP,CURRENT_TIMESTAMP,1,1,1); INSERT INTO x_user_module_perm (user_id,module_id,create_time,update_time,added_by_id,upd_by_id,is_allowed) VALUES (1,1,CURRENT_TIMESTAMP,CURRENT_TIMESTAMP,1,1,1); @@ -3026,7 +3304,11 @@ INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active) VALUES ('J10006',CURRENT_TIMESTAMP,'Ranger 0.7.0',CURRENT_TIMESTAMP,'localhost','Y'); INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active) VALUES ('J10007',CURRENT_TIMESTAMP,'Ranger 0.7.0',CURRENT_TIMESTAMP,'localhost','Y'); INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active) VALUES ('J10008',CURRENT_TIMESTAMP,'Ranger 0.7.0',CURRENT_TIMESTAMP,'localhost','Y'); +INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active) VALUES ('J10009',CURRENT_TIMESTAMP,'Ranger 0.7.0',CURRENT_TIMESTAMP,'localhost','Y'); +INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active) VALUES ('J10010',CURRENT_TIMESTAMP,'Ranger 0.7.0',CURRENT_TIMESTAMP,'localhost','Y'); INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active) VALUES ('J10011',CURRENT_TIMESTAMP,'Ranger 0.7.0',CURRENT_TIMESTAMP,'localhost','Y'); +INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active) VALUES ('J10017',CURRENT_TIMESTAMP,'Ranger 0.7.0',CURRENT_TIMESTAMP,'localhost','Y'); +INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active) VALUES ('J10020',CURRENT_TIMESTAMP,'Ranger 0.7.0',CURRENT_TIMESTAMP,'localhost','Y'); INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active) VALUES ('JAVA_PATCHES',CURRENT_TIMESTAMP,'Ranger 0.7.0',CURRENT_TIMESTAMP,'localhost','Y'); GO CREATE VIEW [dbo].[vx_trx_log] AS diff --git a/security-admin/db/sqlserver/patches/035-update-schema-for-x-policy.sql b/security-admin/db/sqlserver/patches/035-update-schema-for-x-policy.sql new file mode 100644 index 0000000000..ebf44ac836 --- /dev/null +++ b/security-admin/db/sqlserver/patches/035-update-schema-for-x-policy.sql @@ -0,0 +1,453 @@ +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +GO +IF (OBJECT_ID('x_policy_ref_resource_FK_policy_id') IS NOT NULL) +BEGIN + ALTER TABLE [dbo].[x_policy_ref_resource] DROP CONSTRAINT x_policy_ref_resource_FK_policy_id +END +GO +IF (OBJECT_ID('x_policy_ref_resource_FK_resource_def_id') IS NOT NULL) +BEGIN + ALTER TABLE [dbo].[x_policy_ref_resource] DROP CONSTRAINT x_policy_ref_resource_FK_resource_def_id +END +GO +IF (OBJECT_ID('x_policy_ref_resource_UK') IS NOT NULL) +BEGIN + ALTER TABLE [dbo].[x_policy_ref_resource] DROP CONSTRAINT x_policy_ref_resource_UK +END +GO +IF (OBJECT_ID('x_policy_ref_resource') IS NOT NULL) +BEGIN + DROP TABLE [dbo].[x_policy_ref_resource] +END +GO +IF (OBJECT_ID('x_policy_ref_access_type_FK_policy_id') IS NOT NULL) +BEGIN + ALTER TABLE [dbo].[x_policy_ref_access_type] DROP CONSTRAINT x_policy_ref_access_type_FK_policy_id +END +GO +IF (OBJECT_ID('x_policy_ref_access_type_FK_access_def_id') IS NOT NULL) +BEGIN + ALTER TABLE [dbo].[x_policy_ref_access_type] DROP CONSTRAINT x_policy_ref_access_type_FK_access_def_id +END +GO +IF (OBJECT_ID('x_policy_ref_access_type_UK') IS NOT NULL) +BEGIN + ALTER TABLE [dbo].[x_policy_ref_access_type] DROP CONSTRAINT x_policy_ref_access_type_UK +END +GO +IF (OBJECT_ID('x_policy_ref_access_type') IS NOT NULL) +BEGIN + DROP TABLE [dbo].[x_policy_ref_access_type] +END +GO +IF (OBJECT_ID('x_policy_ref_condition_FK_policy_id') IS NOT NULL) +BEGIN + ALTER TABLE [dbo].[x_policy_ref_condition] DROP CONSTRAINT x_policy_ref_condition_FK_policy_id +END +GO +IF (OBJECT_ID('x_policy_ref_condition_FK_condition_def_id') IS NOT NULL) +BEGIN + ALTER TABLE [dbo].[x_policy_ref_condition] DROP CONSTRAINT x_policy_ref_condition_FK_condition_def_id +END +GO +IF (OBJECT_ID('x_policy_ref_condition_UK') IS NOT NULL) +BEGIN + ALTER TABLE [dbo].[x_policy_ref_condition] DROP CONSTRAINT x_policy_ref_condition_UK +END +GO +IF (OBJECT_ID('x_policy_ref_condition') IS NOT NULL) +BEGIN + DROP TABLE [dbo].[x_policy_ref_condition] +END +GO +IF (OBJECT_ID('x_policy_ref_datamask_type_FK_policy_id') IS NOT NULL) +BEGIN + ALTER TABLE [dbo].[x_policy_ref_datamask_type] DROP CONSTRAINT x_policy_ref_datamask_type_FK_policy_id +END +GO +IF (OBJECT_ID('x_policy_ref_datamask_type_FK_datamask_def_id') IS NOT NULL) +BEGIN + ALTER TABLE [dbo].[x_policy_ref_datamask_type] DROP CONSTRAINT x_policy_ref_datamask_type_FK_datamask_def_id +END +GO +IF (OBJECT_ID('x_policy_ref_datamask_type_UK') IS NOT NULL) +BEGIN + ALTER TABLE [dbo].[x_policy_ref_datamask_type] DROP CONSTRAINT x_policy_ref_datamask_type_UK +END +GO +IF (OBJECT_ID('x_policy_ref_datamask_type') IS NOT NULL) +BEGIN + DROP TABLE [dbo].[x_policy_ref_datamask_type] +END +GO +IF (OBJECT_ID('x_policy_ref_user_FK_policy_id') IS NOT NULL) +BEGIN + ALTER TABLE [dbo].[x_policy_ref_user] DROP CONSTRAINT x_policy_ref_user_FK_policy_id +END +GO +IF (OBJECT_ID('x_policy_ref_user_FK_user_id') IS NOT NULL) +BEGIN + ALTER TABLE [dbo].[x_policy_ref_user] DROP CONSTRAINT x_policy_ref_user_FK_user_id +END +GO +IF (OBJECT_ID('x_policy_ref_user_UK') IS NOT NULL) +BEGIN + ALTER TABLE [dbo].[x_policy_ref_user] DROP CONSTRAINT x_policy_ref_user_UK +END +GO +IF (OBJECT_ID('x_policy_ref_user') IS NOT NULL) +BEGIN + DROP TABLE [dbo].[x_policy_ref_user] +END +GO +IF (OBJECT_ID('x_policy_ref_group_FK_policy_id') IS NOT NULL) +BEGIN + ALTER TABLE [dbo].[x_policy_ref_group] DROP CONSTRAINT x_policy_ref_group_FK_policy_id +END +GO +IF (OBJECT_ID('x_policy_ref_group_FK_group_id') IS NOT NULL) +BEGIN + ALTER TABLE [dbo].[x_policy_ref_group] DROP CONSTRAINT x_policy_ref_group_FK_group_id +END +GO +IF (OBJECT_ID('x_policy_ref_group_UK') IS NOT NULL) +BEGIN + ALTER TABLE [dbo].[x_policy_ref_group] DROP CONSTRAINT x_policy_ref_group_UK +END +GO +IF (OBJECT_ID('x_policy_ref_group') IS NOT NULL) +BEGIN + DROP TABLE [dbo].[x_policy_ref_group] +END +GO +SET ANSI_NULLS ON +GO +SET QUOTED_IDENTIFIER ON +GO +SET ANSI_PADDING ON +GO +CREATE TABLE [dbo].[x_policy_ref_resource] ( + [id] [bigint] IDENTITY (1, 1) NOT NULL, + [guid] [varchar](1024) DEFAULT NULL NULL, + [create_time] [datetime2] DEFAULT NULL NULL, + [update_time] [datetime2] DEFAULT NULL NULL, + [added_by_id] [bigint] DEFAULT NULL NULL, + [upd_by_id] [bigint] DEFAULT NULL NULL, + [policy_id] [bigint] NOT NULL, + [resource_def_id] [bigint] NOT NULL, + [resource_name] [varchar](4000) DEFAULT NULL NULL, + PRIMARY KEY CLUSTERED + ( + [id] ASC + ) WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY], + CONSTRAINT [x_policy_ref_resource$x_policy_ref_resource_UK] UNIQUE NONCLUSTERED + ( + [policy_id] ASC, [resource_def_id] ASC + )WITH (PAD_INDEX = OFF,STATISTICS_NORECOMPUTE = OFF,IGNORE_DUP_KEY = OFF,ALLOW_ROW_LOCKS = ON,ALLOW_PAGE_LOCKS = ON) ON [PRIMARY] +) ON [PRIMARY] +GO +SET ANSI_NULLS ON +GO +SET QUOTED_IDENTIFIER ON +GO +SET ANSI_PADDING ON +GO +CREATE TABLE [dbo].[x_policy_ref_access_type] ( + [id] [bigint] IDENTITY (1, 1) NOT NULL, + [guid] [varchar](1024) DEFAULT NULL NULL, + [create_time] [datetime2] DEFAULT NULL NULL, + [update_time] [datetime2] DEFAULT NULL NULL, + [added_by_id] [bigint] DEFAULT NULL NULL, + [upd_by_id] [bigint] DEFAULT NULL NULL, + [policy_id] [bigint] NOT NULL, + [access_def_id] [bigint] NOT NULL, + [access_type_name] [varchar](4000) DEFAULT NULL NULL, + PRIMARY KEY CLUSTERED + ( + [id] ASC + ) WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY], + CONSTRAINT [x_policy_ref_access_type$x_policy_ref_access_type_UK] UNIQUE NONCLUSTERED +( + [policy_id] ASC, [access_def_id] ASC +)WITH (PAD_INDEX = OFF,STATISTICS_NORECOMPUTE = OFF,IGNORE_DUP_KEY = OFF,ALLOW_ROW_LOCKS = ON,ALLOW_PAGE_LOCKS = ON) ON [PRIMARY] +) ON [PRIMARY] +GO +SET ANSI_NULLS ON +GO +SET QUOTED_IDENTIFIER ON +GO +SET ANSI_PADDING ON +GO +CREATE TABLE [dbo].[x_policy_ref_condition] ( + [id] [bigint] IDENTITY (1, 1) NOT NULL, + [guid] [varchar](1024) DEFAULT NULL NULL, + [create_time] [datetime2] DEFAULT NULL NULL, + [update_time] [datetime2] DEFAULT NULL NULL, + [added_by_id] [bigint] DEFAULT NULL NULL, + [upd_by_id] [bigint] DEFAULT NULL NULL, + [policy_id] [bigint] NOT NULL, + [condition_def_id] [bigint] NOT NULL, + [condition_name] [varchar](4000) DEFAULT NULL NULL, + PRIMARY KEY CLUSTERED + ( + [id] ASC + ) WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY], + CONSTRAINT [x_policy_ref_condition$x_policy_ref_condition_UK] UNIQUE NONCLUSTERED +( + [policy_id] ASC, [condition_def_id] ASC +)WITH (PAD_INDEX = OFF,STATISTICS_NORECOMPUTE = OFF,IGNORE_DUP_KEY = OFF,ALLOW_ROW_LOCKS = ON,ALLOW_PAGE_LOCKS = ON) ON [PRIMARY] +) ON [PRIMARY] +GO +SET ANSI_NULLS ON +GO +SET QUOTED_IDENTIFIER ON +GO +SET ANSI_PADDING ON +GO +CREATE TABLE [dbo].[x_policy_ref_datamask_type] ( + [id] [bigint] IDENTITY (1, 1) NOT NULL, + [guid] [varchar](1024) DEFAULT NULL NULL, + [create_time] [datetime2] DEFAULT NULL NULL, + [update_time] [datetime2] DEFAULT NULL NULL, + [added_by_id] [bigint] DEFAULT NULL NULL, + [upd_by_id] [bigint] DEFAULT NULL NULL, + [policy_id] [bigint] NOT NULL, + [datamask_def_id] [bigint] NOT NULL, + [datamask_type_name] [varchar](4000) DEFAULT NULL NULL, + PRIMARY KEY CLUSTERED + ( + [id] ASC + ) WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY], + CONSTRAINT [x_policy_ref_datamask_type$x_policy_ref_datamask_type_UK] UNIQUE NONCLUSTERED +( + [policy_id] ASC, [datamask_def_id] ASC +)WITH (PAD_INDEX = OFF,STATISTICS_NORECOMPUTE = OFF,IGNORE_DUP_KEY = OFF,ALLOW_ROW_LOCKS = ON,ALLOW_PAGE_LOCKS = ON) ON [PRIMARY] +) ON [PRIMARY] +GO +SET ANSI_NULLS ON +GO +SET QUOTED_IDENTIFIER ON +GO +SET ANSI_PADDING ON +GO +CREATE TABLE [dbo].[x_policy_ref_user] ( + [id] [bigint] IDENTITY (1, 1) NOT NULL, + [guid] [varchar](1024) DEFAULT NULL NULL, + [create_time] [datetime2] DEFAULT NULL NULL, + [update_time] [datetime2] DEFAULT NULL NULL, + [added_by_id] [bigint] DEFAULT NULL NULL, + [upd_by_id] [bigint] DEFAULT NULL NULL, + [policy_id] [bigint] NOT NULL, + [user_id] [bigint] NOT NULL, + [user_name] [varchar](4000) DEFAULT NULL NULL, + PRIMARY KEY CLUSTERED + ( + [id] ASC + ) WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY], + CONSTRAINT [x_policy_ref_user$x_policy_ref_user_UK] UNIQUE NONCLUSTERED +( + [policy_id] ASC, [user_id] ASC +)WITH (PAD_INDEX = OFF,STATISTICS_NORECOMPUTE = OFF,IGNORE_DUP_KEY = OFF,ALLOW_ROW_LOCKS = ON,ALLOW_PAGE_LOCKS = ON) ON [PRIMARY] +) ON [PRIMARY] +GO +SET ANSI_NULLS ON +GO +SET QUOTED_IDENTIFIER ON +GO +SET ANSI_PADDING ON +GO +CREATE TABLE [dbo].[x_policy_ref_group] ( + [id] [bigint] IDENTITY (1, 1) NOT NULL, + [guid] [varchar](1024) DEFAULT NULL NULL, + [create_time] [datetime2] DEFAULT NULL NULL, + [update_time] [datetime2] DEFAULT NULL NULL, + [added_by_id] [bigint] DEFAULT NULL NULL, + [upd_by_id] [bigint] DEFAULT NULL NULL, + [policy_id] [bigint] NOT NULL, + [group_id] [bigint] NOT NULL, + [group_name] [varchar](4000) DEFAULT NULL NULL, + PRIMARY KEY CLUSTERED + ( + [id] ASC + ) WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY], + CONSTRAINT [x_policy_ref_group$x_policy_ref_group_UK] UNIQUE NONCLUSTERED +( + [policy_id] ASC, [group_id] ASC +)WITH (PAD_INDEX = OFF,STATISTICS_NORECOMPUTE = OFF,IGNORE_DUP_KEY = OFF,ALLOW_ROW_LOCKS = ON,ALLOW_PAGE_LOCKS = ON) ON [PRIMARY] +) ON [PRIMARY] +GO +ALTER TABLE [dbo].[x_policy_ref_resource] WITH CHECK ADD CONSTRAINT [x_policy_ref_resource_FK_policy_id] FOREIGN KEY ([policy_id]) +REFERENCES [dbo].[x_policy] ([id]) +ALTER TABLE [dbo].[x_policy_ref_resource] CHECK CONSTRAINT [x_policy_ref_resource_FK_policy_id] +ALTER TABLE [dbo].[x_policy_ref_resource] WITH CHECK ADD CONSTRAINT [x_policy_ref_resource_FK_resource_def_id] FOREIGN KEY ([resource_def_id]) +REFERENCES [dbo].[x_resource_def] ([id]) +ALTER TABLE [dbo].[x_policy_ref_resource] CHECK CONSTRAINT [x_policy_ref_resource_FK_resource_def_id] +ALTER TABLE [dbo].[x_policy_ref_resource] WITH CHECK ADD CONSTRAINT [x_policy_ref_resource_FK_added_by] FOREIGN KEY ([added_by_id]) +REFERENCES [dbo].[x_portal_user] ([id]) +ALTER TABLE [dbo].[x_policy_ref_resource] CHECK CONSTRAINT [x_policy_ref_resource_FK_added_by] +ALTER TABLE [dbo].[x_policy_ref_resource] WITH CHECK ADD CONSTRAINT [x_policy_ref_resource_FK_upd_by] FOREIGN KEY ([upd_by_id]) +REFERENCES [dbo].[x_portal_user] ([id]) +ALTER TABLE [dbo].[x_policy_ref_access_type] WITH CHECK ADD CONSTRAINT [x_policy_ref_access_type_FK_policy_id] FOREIGN KEY ([policy_id]) +REFERENCES [dbo].[x_policy] ([id]) +ALTER TABLE [dbo].[x_policy_ref_access_type] CHECK CONSTRAINT [x_policy_ref_access_type_FK_policy_id] +ALTER TABLE [dbo].[x_policy_ref_access_type] WITH CHECK ADD CONSTRAINT [x_policy_ref_access_type_FK_access_def_id] FOREIGN KEY ([access_def_id]) +REFERENCES [dbo].[x_access_type_def] ([id]) +ALTER TABLE [dbo].[x_policy_ref_access_type] CHECK CONSTRAINT [x_policy_ref_access_type_FK_access_def_id] +ALTER TABLE [dbo].[x_policy_ref_access_type] WITH CHECK ADD CONSTRAINT [x_policy_ref_access_type_FK_added_by] FOREIGN KEY ([added_by_id]) +REFERENCES [dbo].[x_portal_user] ([id]) +ALTER TABLE [dbo].[x_policy_ref_access_type] CHECK CONSTRAINT [x_policy_ref_access_type_FK_added_by] +ALTER TABLE [dbo].[x_policy_ref_access_type] WITH CHECK ADD CONSTRAINT [x_policy_ref_access_type_FK_upd_by] FOREIGN KEY ([upd_by_id]) +REFERENCES [dbo].[x_portal_user] ([id]) +ALTER TABLE [dbo].[x_policy_ref_condition] WITH CHECK ADD CONSTRAINT [x_policy_ref_condition_FK_policy_id] FOREIGN KEY ([policy_id]) +REFERENCES [dbo].[x_policy] ([id]) +ALTER TABLE [dbo].[x_policy_ref_condition] CHECK CONSTRAINT [x_policy_ref_condition_FK_policy_id] +ALTER TABLE [dbo].[x_policy_ref_condition] WITH CHECK ADD CONSTRAINT [x_policy_ref_condition_FK_condition_def_id] FOREIGN KEY ([condition_def_id]) +REFERENCES [dbo].[x_policy_condition_def] ([id]) +ALTER TABLE [dbo].[x_policy_ref_condition] CHECK CONSTRAINT [x_policy_ref_condition_FK_condition_def_id] +ALTER TABLE [dbo].[x_policy_ref_condition] WITH CHECK ADD CONSTRAINT [x_policy_ref_condition_FK_added_by] FOREIGN KEY ([added_by_id]) +REFERENCES [dbo].[x_portal_user] ([id]) +ALTER TABLE [dbo].[x_policy_ref_condition] CHECK CONSTRAINT [x_policy_ref_condition_FK_added_by] +ALTER TABLE [dbo].[x_policy_ref_condition] WITH CHECK ADD CONSTRAINT [x_policy_ref_condition_FK_upd_by] FOREIGN KEY ([upd_by_id]) +REFERENCES [dbo].[x_portal_user] ([id]) +ALTER TABLE [dbo].[x_policy_ref_datamask_type] WITH CHECK ADD CONSTRAINT [x_policy_ref_datamask_type_FK_policy_id] FOREIGN KEY ([policy_id]) +REFERENCES [dbo].[x_policy] ([id]) +ALTER TABLE [dbo].[x_policy_ref_datamask_type] CHECK CONSTRAINT [x_policy_ref_datamask_type_FK_policy_id] +ALTER TABLE [dbo].[x_policy_ref_datamask_type] WITH CHECK ADD CONSTRAINT [x_policy_ref_datamask_type_FK_datamask_def_id] FOREIGN KEY ([datamask_def_id]) +REFERENCES [dbo].[x_datamask_type_def] ([id]) +ALTER TABLE [dbo].[x_policy_ref_datamask_type] CHECK CONSTRAINT [x_policy_ref_datamask_type_FK_datamask_def_id] +ALTER TABLE [dbo].[x_policy_ref_datamask_type] WITH CHECK ADD CONSTRAINT [x_policy_ref_datamask_type_FK_added_by] FOREIGN KEY ([added_by_id]) +REFERENCES [dbo].[x_portal_user] ([id]) +ALTER TABLE [dbo].[x_policy_ref_datamask_type] CHECK CONSTRAINT [x_policy_ref_datamask_type_FK_added_by] +ALTER TABLE [dbo].[x_policy_ref_datamask_type] WITH CHECK ADD CONSTRAINT [x_policy_ref_datamask_type_FK_upd_by] FOREIGN KEY ([upd_by_id]) +REFERENCES [dbo].[x_portal_user] ([id]) + + +ALTER TABLE [dbo].[x_policy_ref_user] WITH CHECK ADD CONSTRAINT [x_policy_ref_user_FK_policy_id] FOREIGN KEY ([policy_id]) +REFERENCES [dbo].[x_policy] ([id]) +ALTER TABLE [dbo].[x_policy_ref_user] CHECK CONSTRAINT [x_policy_ref_user_FK_policy_id] + +ALTER TABLE [dbo].[x_policy_ref_user] WITH CHECK ADD CONSTRAINT [x_policy_ref_user_FK_user_id] FOREIGN KEY ([user_id]) +REFERENCES [dbo].[x_user] ([id]) +ALTER TABLE [dbo].[x_policy_ref_user] CHECK CONSTRAINT [x_policy_ref_user_FK_user_id] + +ALTER TABLE [dbo].[x_policy_ref_user] WITH CHECK ADD CONSTRAINT [x_policy_ref_user_FK_added_by] FOREIGN KEY ([added_by_id]) +REFERENCES [dbo].[x_portal_user] ([id]) +ALTER TABLE [dbo].[x_policy_ref_user] CHECK CONSTRAINT [x_policy_ref_user_FK_added_by] + +ALTER TABLE [dbo].[x_policy_ref_user] WITH CHECK ADD CONSTRAINT [x_policy_ref_user_FK_upd_by] FOREIGN KEY ([upd_by_id]) +REFERENCES [dbo].[x_portal_user] ([id]) +ALTER TABLE [dbo].[x_policy_ref_user] CHECK CONSTRAINT [x_policy_ref_user_FK_upd_by] + +ALTER TABLE [dbo].[x_policy_ref_group] WITH CHECK ADD CONSTRAINT [x_policy_ref_group_FK_policy_id] FOREIGN KEY ([policy_id]) +REFERENCES [dbo].[x_policy] ([id]) +ALTER TABLE [dbo].[x_policy_ref_group] CHECK CONSTRAINT [x_policy_ref_group_FK_policy_id] +ALTER TABLE [dbo].[x_policy_ref_group] WITH CHECK ADD CONSTRAINT [x_policy_ref_group_FK_group_id] FOREIGN KEY ([group_id]) +REFERENCES [dbo].[x_group] ([id]) +ALTER TABLE [dbo].[x_policy_ref_group] CHECK CONSTRAINT [x_policy_ref_group_FK_group_id] +ALTER TABLE [dbo].[x_policy_ref_group] WITH CHECK ADD CONSTRAINT [x_policy_ref_group_FK_added_by] FOREIGN KEY ([added_by_id]) +REFERENCES [dbo].[x_portal_user] ([id]) +ALTER TABLE [dbo].[x_policy_ref_group] CHECK CONSTRAINT [x_policy_ref_group_FK_added_by] +ALTER TABLE [dbo].[x_policy_ref_group] WITH CHECK ADD CONSTRAINT [x_policy_ref_group_FK_upd_by] FOREIGN KEY ([upd_by_id]) +REFERENCES [dbo].[x_portal_user] ([id]) +GO +IF NOT EXISTS (SELECT + * + FROM INFORMATION_SCHEMA.columns + WHERE table_name = 'x_policy' + AND column_name = 'policy_text') +BEGIN + ALTER TABLE [dbo].[x_policy] ADD [policy_text] [nvarchar](max) DEFAULT NULL NULL; +END +GO + +SET ANSI_NULLS ON +GO +SET QUOTED_IDENTIFIER ON +GO +SET ANSI_PADDING ON +GO +IF EXISTS ( + SELECT type_desc, type + FROM sys.procedures WITH(NOLOCK) + WHERE NAME = 'removeConstraints' + AND type = 'P' + ) +BEGIN + PRINT 'Proc exist with name dbo.removeConstraints' + DROP PROCEDURE dbo.removeConstraints + PRINT 'Proc dropped dbo.removeConstraints' +END +GO +CREATE PROCEDURE dbo.removeConstraints + -- Add the parameters for the stored procedure here + @tablename nvarchar(100) +AS +BEGIN + + DECLARE @stmt VARCHAR(300); + + -- Cursor to generate ALTER TABLE DROP CONSTRAINT statements + DECLARE cur CURSOR FOR + SELECT 'ALTER TABLE ' + OBJECT_SCHEMA_NAME(parent_object_id) + '.' + OBJECT_NAME(parent_object_id) + + ' DROP CONSTRAINT ' + name + FROM sys.foreign_keys + WHERE OBJECT_SCHEMA_NAME(referenced_object_id) = 'dbo' AND + OBJECT_NAME(referenced_object_id) = @tablename; + + OPEN cur; + FETCH cur INTO @stmt; + + -- Drop each found foreign key constraint + WHILE @@FETCH_STATUS = 0 + BEGIN + EXEC (@stmt); + FETCH cur INTO @stmt; + END + + CLOSE cur; + DEALLOCATE cur; + +END +GO + +EXEC dbo.removeConstraints 'x_policy_item' +GO + +EXEC dbo.removeConstraints 'x_policy_item_access' +GO + +EXEC dbo.removeConstraints 'x_policy_item_condition' +GO + +EXEC dbo.removeConstraints 'x_policy_item_datamask' +GO + +EXEC dbo.removeConstraints 'x_policy_item_group_perm' +GO + +EXEC dbo.removeConstraints 'x_policy_item_user_perm' +GO + +EXEC dbo.removeConstraints 'x_policy_item_rowfilter' +GO + +EXEC dbo.removeConstraints 'x_policy_resource' +GO + +EXEC dbo.removeConstraints 'x_policy_resource_map' +GO + +EXIT diff --git a/security-admin/db/sqlserver/patches/036-denormalize-tag-tables.sql b/security-admin/db/sqlserver/patches/036-denormalize-tag-tables.sql new file mode 100644 index 0000000000..9bfae30f21 --- /dev/null +++ b/security-admin/db/sqlserver/patches/036-denormalize-tag-tables.sql @@ -0,0 +1,97 @@ +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +GO +IF NOT EXISTS(select * from INFORMATION_SCHEMA.columns where table_name = 'x_tag_def' and column_name = 'tag_attrs_def_text') +BEGIN + ALTER TABLE [dbo].[x_tag_def] ADD [tag_attrs_def_text] [nvarchar](max) DEFAULT NULL NULL; +END +IF NOT EXISTS(select * from INFORMATION_SCHEMA.columns where table_name = 'x_tag' and column_name = 'tag_attrs_text') +BEGIN + ALTER TABLE [dbo].[x_tag] ADD [tag_attrs_text] [nvarchar](max) DEFAULT NULL NULL; +END +IF NOT EXISTS(select * from INFORMATION_SCHEMA.columns where table_name = 'x_service_resource' and column_name = 'service_resource_elements_text') +BEGIN + ALTER TABLE [dbo].[x_service_resource] ADD [service_resource_elements_text] [nvarchar](max) DEFAULT NULL NULL; +END +IF NOT EXISTS(select * from INFORMATION_SCHEMA.columns where table_name = 'x_service_resource' and column_name = 'tags_text') +BEGIN + ALTER TABLE [dbo].[x_service_resource] ADD [tags_text] [nvarchar](max) DEFAULT NULL NULL; +END +GO + +SET ANSI_NULLS ON +GO +SET QUOTED_IDENTIFIER ON +GO +SET ANSI_PADDING ON +GO +IF EXISTS ( + SELECT type_desc, type + FROM sys.procedures WITH(NOLOCK) + WHERE NAME = 'removeConstraints' + AND type = 'P' + ) +BEGIN + PRINT 'Proc exist with name dbo.removeConstraints' + DROP PROCEDURE dbo.removeConstraints + PRINT 'Proc dropped dbo.removeConstraints' +END +GO +CREATE PROCEDURE dbo.removeConstraints + -- Add the parameters for the stored procedure here + @tablename nvarchar(100) +AS +BEGIN + + DECLARE @stmt VARCHAR(300); + + -- Cursor to generate ALTER TABLE DROP CONSTRAINT statements + DECLARE cur CURSOR FOR + SELECT 'ALTER TABLE ' + OBJECT_SCHEMA_NAME(parent_object_id) + '.' + OBJECT_NAME(parent_object_id) + + ' DROP CONSTRAINT ' + name + FROM sys.foreign_keys + WHERE OBJECT_SCHEMA_NAME(referenced_object_id) = 'dbo' AND + OBJECT_NAME(referenced_object_id) = @tablename; + + OPEN cur; + FETCH cur INTO @stmt; + + -- Drop each found foreign key constraint + WHILE @@FETCH_STATUS = 0 + BEGIN + EXEC (@stmt); + FETCH cur INTO @stmt; + END + + CLOSE cur; + DEALLOCATE cur; + +END +GO + +EXEC dbo.removeConstraints 'x_tag_attr_def' +GO + +EXEC dbo.removeConstraints 'x_tag_attr' +GO + +EXEC dbo.removeConstraints 'x_service_resource_element' +GO + +EXEC dbo.removeConstraints 'x_service_resource_element_val' +GO + +EXIT diff --git a/security-admin/scripts/db_setup.py b/security-admin/scripts/db_setup.py index d4f37ed6fb..83463cfaf3 100644 --- a/security-admin/scripts/db_setup.py +++ b/security-admin/scripts/db_setup.py @@ -35,8 +35,9 @@ ranger_version='' jisql_debug=True retryPatchAfterSeconds=120 - +java_patch_regex="^Patch.*?J\d{5}.class$" is_unix = os_name == "LINUX" or os_name == "DARWIN" +max_memory='1g' if is_unix: RANGER_ADMIN_HOME = os.getenv("RANGER_ADMIN_HOME") @@ -183,14 +184,14 @@ def import_core_db_schema(self, db_name, db_user, db_password, file_name,first_t class MysqlConf(BaseDB): # Constructor - def __init__(self, host,SQL_CONNECTOR_JAR,JAVA_BIN,db_ssl_enabled,db_ssl_required,db_ssl_verifyServerCertificate,javax_net_ssl_keyStore,javax_net_ssl_keyStorePassword,javax_net_ssl_trustStore,javax_net_ssl_trustStorePassword,db_ssl_auth_type): + def __init__(self, host,SQL_CONNECTOR_JAR,JAVA_BIN,db_ssl_enabled,db_ssl_required,db_ssl_verifyServerCertificate,javax_net_ssl_keyStore,javax_net_ssl_keyStorePassword,javax_net_ssl_trustStore,javax_net_ssl_trustStorePassword,db_ssl_auth_type): self.host = host self.SQL_CONNECTOR_JAR = SQL_CONNECTOR_JAR self.JAVA_BIN = JAVA_BIN self.db_ssl_enabled=db_ssl_enabled.lower() self.db_ssl_required=db_ssl_required.lower() self.db_ssl_verifyServerCertificate=db_ssl_verifyServerCertificate.lower() - self.db_ssl_auth_type=db_ssl_auth_type.lower() + self.db_ssl_auth_type=db_ssl_auth_type.lower() self.javax_net_ssl_keyStore=javax_net_ssl_keyStore self.javax_net_ssl_keyStorePassword=javax_net_ssl_keyStorePassword self.javax_net_ssl_trustStore=javax_net_ssl_trustStore @@ -203,10 +204,10 @@ def get_jisql_cmd(self, user, password ,db_name): if self.db_ssl_enabled == 'true': db_ssl_param="?useSSL=%s&requireSSL=%s&verifyServerCertificate=%s" %(self.db_ssl_enabled,self.db_ssl_required,self.db_ssl_verifyServerCertificate) if self.db_ssl_verifyServerCertificate == 'true': - if self.db_ssl_auth_type == '1-way': - db_ssl_cert_param=" -Djavax.net.ssl.trustStore=%s -Djavax.net.ssl.trustStorePassword=%s " %(self.javax_net_ssl_trustStore,self.javax_net_ssl_trustStorePassword) - else: - db_ssl_cert_param=" -Djavax.net.ssl.keyStore=%s -Djavax.net.ssl.keyStorePassword=%s -Djavax.net.ssl.trustStore=%s -Djavax.net.ssl.trustStorePassword=%s " %(self.javax_net_ssl_keyStore,self.javax_net_ssl_keyStorePassword,self.javax_net_ssl_trustStore,self.javax_net_ssl_trustStorePassword) + if self.db_ssl_auth_type == '1-way': + db_ssl_cert_param=" -Djavax.net.ssl.trustStore=%s -Djavax.net.ssl.trustStorePassword=%s " %(self.javax_net_ssl_trustStore,self.javax_net_ssl_trustStorePassword) + else: + db_ssl_cert_param=" -Djavax.net.ssl.keyStore=%s -Djavax.net.ssl.keyStorePassword=%s -Djavax.net.ssl.trustStore=%s -Djavax.net.ssl.trustStorePassword=%s " %(self.javax_net_ssl_keyStore,self.javax_net_ssl_keyStorePassword,self.javax_net_ssl_trustStore,self.javax_net_ssl_trustStorePassword) self.JAVA_BIN = self.JAVA_BIN.strip("'") if is_unix: jisql_cmd = "%s %s -cp %s:%s/jisql/lib/* org.apache.util.sql.Jisql -driver mysqlconj -cstring jdbc:mysql://%s/%s%s -u '%s' -p '%s' -noheader -trim -c \;" %(self.JAVA_BIN,db_ssl_cert_param,self.SQL_CONNECTOR_JAR,path,self.host,db_name,db_ssl_param,user,password) @@ -498,7 +499,7 @@ def execute_java_patches(self, xa_db_host, db_user, db_password, db_name): files = os.listdir(javaFiles) if files: for filename in files: - f = re.match("^Patch.*?.class$",filename) + f = re.match(java_patch_regex,filename) if f: className = re.match("(Patch.*?)_.*.class",filename) className = className.group(1) @@ -555,7 +556,7 @@ def execute_java_patches(self, xa_db_host, db_user, db_password, db_name): path = os.path.join("%s","WEB-INF","classes","conf:%s","WEB-INF","classes","lib","*:%s","WEB-INF",":%s","META-INF",":%s","WEB-INF","lib","*:%s","WEB-INF","classes",":%s","WEB-INF","classes","META-INF:%s" )%(app_home ,app_home ,app_home, app_home, app_home, app_home ,app_home ,self.SQL_CONNECTOR_JAR) elif os_name == "WINDOWS": path = os.path.join("%s","WEB-INF","classes","conf;%s","WEB-INF","classes","lib","*;%s","WEB-INF",";%s","META-INF",";%s","WEB-INF","lib","*;%s","WEB-INF","classes",";%s","WEB-INF","classes","META-INF;%s" )%(app_home ,app_home ,app_home, app_home, app_home, app_home ,app_home ,self.SQL_CONNECTOR_JAR) - get_java_cmd = "%s -Dlogdir=%s -Dlog4j.configuration=db_patch.log4j.xml -cp %s org.apache.ranger.patch.%s"%(self.JAVA_BIN,ranger_log,path,className) + get_java_cmd = "%s -XX:MetaspaceSize=100m -XX:MaxMetaspaceSize=200m -Xmx%s -Xms1g -Dlogdir=%s -Dlog4j.configuration=db_patch.log4j.xml -cp %s org.apache.ranger.patch.%s"%(self.JAVA_BIN,max_memory,ranger_log,path,className) if is_unix: ret = subprocess.call(shlex.split(get_java_cmd)) elif os_name == "WINDOWS": @@ -1188,7 +1189,7 @@ def execute_java_patches(self, xa_db_host, db_user, db_password, db_name): files = os.listdir(javaFiles) if files: for filename in files: - f = re.match("^Patch.*?.class$",filename) + f = re.match(java_patch_regex,filename) if f: className = re.match("(Patch.*?)_.*.class",filename) className = className.group(1) @@ -1270,7 +1271,7 @@ def execute_java_patches(self, xa_db_host, db_user, db_password, db_name): path = os.path.join("%s","WEB-INF","classes","conf:%s","WEB-INF","classes","lib","*:%s","WEB-INF",":%s","META-INF",":%s","WEB-INF","lib","*:%s","WEB-INF","classes",":%s","WEB-INF","classes","META-INF:%s" )%(app_home ,app_home ,app_home, app_home, app_home, app_home ,app_home ,self.SQL_CONNECTOR_JAR) elif os_name == "WINDOWS": path = os.path.join("%s","WEB-INF","classes","conf;%s","WEB-INF","classes","lib","*;%s","WEB-INF",";%s","META-INF",";%s","WEB-INF","lib","*;%s","WEB-INF","classes",";%s","WEB-INF","classes","META-INF;%s" )%(app_home ,app_home ,app_home, app_home, app_home, app_home ,app_home ,self.SQL_CONNECTOR_JAR) - get_java_cmd = "%s -Djava.security.egd=file:///dev/urandom -Dlogdir=%s -Dlog4j.configuration=db_patch.log4j.xml -cp %s org.apache.ranger.patch.%s"%(self.JAVA_BIN,ranger_log,path,className) + get_java_cmd = "%s -XX:MetaspaceSize=100m -XX:MaxMetaspaceSize=200m -Xmx%s -Xms1g -Djava.security.egd=file:///dev/urandom -Dlogdir=%s -Dlog4j.configuration=db_patch.log4j.xml -cp %s org.apache.ranger.patch.%s"%(self.JAVA_BIN,max_memory,ranger_log,path,className) if is_unix: ret = subprocess.call(shlex.split(get_java_cmd)) elif os_name == "WINDOWS": @@ -1359,7 +1360,7 @@ def change_admin_default_password(self, xa_db_host, db_user, db_password, db_nam path = os.path.join("%s","WEB-INF","classes","conf:%s","WEB-INF","classes","lib","*:%s","WEB-INF",":%s","META-INF",":%s","WEB-INF","lib","*:%s","WEB-INF","classes",":%s","WEB-INF","classes","META-INF:%s" )%(app_home ,app_home ,app_home, app_home, app_home, app_home ,app_home ,self.SQL_CONNECTOR_JAR) elif os_name == "WINDOWS": path = os.path.join("%s","WEB-INF","classes","conf;%s","WEB-INF","classes","lib","*;%s","WEB-INF",";%s","META-INF",";%s","WEB-INF","lib","*;%s","WEB-INF","classes",";%s","WEB-INF","classes","META-INF;%s" )%(app_home ,app_home ,app_home, app_home, app_home, app_home ,app_home ,self.SQL_CONNECTOR_JAR) - get_java_cmd = "%s -Dlogdir=%s -Dlog4j.configuration=db_patch.log4j.xml -cp %s org.apache.ranger.patch.cliutil.%s %s %s %s -default"%(self.JAVA_BIN,ranger_log,path,className,userName,oldPassword,newPassword) + get_java_cmd = "%s -XX:MetaspaceSize=100m -XX:MaxMetaspaceSize=200m -Xmx%s -Xms1g -Dlogdir=%s -Dlog4j.configuration=db_patch.log4j.xml -cp %s org.apache.ranger.patch.cliutil.%s %s %s %s -default"%(self.JAVA_BIN,max_memory,ranger_log,path,className,userName,oldPassword,newPassword) if is_unix: status = subprocess.call(shlex.split(get_java_cmd)) elif os_name == "WINDOWS": @@ -1883,7 +1884,7 @@ def execute_java_patches(self, xa_db_host, db_user, db_password, db_name): files = os.listdir(javaFiles) if files: for filename in files: - f = re.match("^Patch.*?.class$",filename) + f = re.match(java_patch_regex,filename) if f: className = re.match("(Patch.*?)_.*.class",filename) className = className.group(1) @@ -1940,7 +1941,7 @@ def execute_java_patches(self, xa_db_host, db_user, db_password, db_name): path = os.path.join("%s","WEB-INF","classes","conf:%s","WEB-INF","classes","lib","*:%s","WEB-INF",":%s","META-INF",":%s","WEB-INF","lib","*:%s","WEB-INF","classes",":%s","WEB-INF","classes","META-INF:%s" )%(app_home ,app_home ,app_home, app_home, app_home, app_home ,app_home ,self.SQL_CONNECTOR_JAR) elif os_name == "WINDOWS": path = os.path.join("%s","WEB-INF","classes","conf;%s","WEB-INF","classes","lib","*;%s","WEB-INF",";%s","META-INF",";%s","WEB-INF","lib","*;%s","WEB-INF","classes",";%s","WEB-INF","classes","META-INF;%s" )%(app_home ,app_home ,app_home, app_home, app_home, app_home ,app_home ,self.SQL_CONNECTOR_JAR) - get_java_cmd = "%s -Dlogdir=%s -Dlog4j.configuration=db_patch.log4j.xml -cp %s org.apache.ranger.patch.%s"%(self.JAVA_BIN,ranger_log,path,className) + get_java_cmd = "%s -XX:MetaspaceSize=100m -XX:MaxMetaspaceSize=200m -Xmx%s -Xms1g -Dlogdir=%s -Dlog4j.configuration=db_patch.log4j.xml -cp %s org.apache.ranger.patch.%s"%(self.JAVA_BIN,max_memory,ranger_log,path,className) if is_unix: ret = subprocess.call(shlex.split(get_java_cmd)) elif os_name == "WINDOWS": @@ -2029,7 +2030,7 @@ def change_admin_default_password(self, xa_db_host, db_user, db_password, db_nam path = os.path.join("%s","WEB-INF","classes","conf:%s","WEB-INF","classes","lib","*:%s","WEB-INF",":%s","META-INF",":%s","WEB-INF","lib","*:%s","WEB-INF","classes",":%s","WEB-INF","classes","META-INF:%s" )%(app_home ,app_home ,app_home, app_home, app_home, app_home ,app_home ,self.SQL_CONNECTOR_JAR) elif os_name == "WINDOWS": path = os.path.join("%s","WEB-INF","classes","conf;%s","WEB-INF","classes","lib","*;%s","WEB-INF",";%s","META-INF",";%s","WEB-INF","lib","*;%s","WEB-INF","classes",";%s","WEB-INF","classes","META-INF;%s" )%(app_home ,app_home ,app_home, app_home, app_home, app_home ,app_home ,self.SQL_CONNECTOR_JAR) - get_java_cmd = "%s -Dlogdir=%s -Dlog4j.configuration=db_patch.log4j.xml -cp %s org.apache.ranger.patch.cliutil.%s %s %s %s -default"%(self.JAVA_BIN,ranger_log,path,className,userName,oldPassword,newPassword) + get_java_cmd = "%s -XX:MetaspaceSize=100m -XX:MaxMetaspaceSize=200m -Xmx%s -Xms1g -Dlogdir=%s -Dlog4j.configuration=db_patch.log4j.xml -cp %s org.apache.ranger.patch.cliutil.%s %s %s %s -default"%(self.JAVA_BIN,max_memory,ranger_log,path,className,userName,oldPassword,newPassword) if is_unix: status = subprocess.call(shlex.split(get_java_cmd)) elif os_name == "WINDOWS": @@ -2515,7 +2516,7 @@ def execute_java_patches(self, xa_db_host, db_user, db_password, db_name): files = os.listdir(javaFiles) if files: for filename in files: - f = re.match("^Patch.*?.class$",filename) + f = re.match(java_patch_regex,filename) if f: className = re.match("(Patch.*?)_.*.class",filename) className = className.group(1) @@ -2572,7 +2573,7 @@ def execute_java_patches(self, xa_db_host, db_user, db_password, db_name): path = os.path.join("%s","WEB-INF","classes","conf:%s","WEB-INF","classes","lib","*:%s","WEB-INF",":%s","META-INF",":%s","WEB-INF","lib","*:%s","WEB-INF","classes",":%s","WEB-INF","classes","META-INF:%s" )%(app_home ,app_home ,app_home, app_home, app_home, app_home ,app_home ,self.SQL_CONNECTOR_JAR) elif os_name == "WINDOWS": path = os.path.join("%s","WEB-INF","classes","conf;%s","WEB-INF","classes","lib","*;%s","WEB-INF",";%s","META-INF",";%s","WEB-INF","lib","*;%s","WEB-INF","classes",";%s","WEB-INF","classes","META-INF;%s" )%(app_home ,app_home ,app_home, app_home, app_home, app_home ,app_home ,self.SQL_CONNECTOR_JAR) - get_java_cmd = "%s -Dlogdir=%s -Dlog4j.configuration=db_patch.log4j.xml -cp %s org.apache.ranger.patch.%s"%(self.JAVA_BIN,ranger_log,path,className) + get_java_cmd = "%s -XX:MetaspaceSize=100m -XX:MaxMetaspaceSize=200m -Xmx%s -Xms1g -Dlogdir=%s -Dlog4j.configuration=db_patch.log4j.xml -cp %s org.apache.ranger.patch.%s"%(self.JAVA_BIN,max_memory,ranger_log,path,className) if is_unix: ret = subprocess.call(shlex.split(get_java_cmd)) elif os_name == "WINDOWS": @@ -2661,7 +2662,7 @@ def change_admin_default_password(self, xa_db_host, db_user, db_password, db_nam path = os.path.join("%s","WEB-INF","classes","conf:%s","WEB-INF","classes","lib","*:%s","WEB-INF",":%s","META-INF",":%s","WEB-INF","lib","*:%s","WEB-INF","classes",":%s","WEB-INF","classes","META-INF:%s" )%(app_home ,app_home ,app_home, app_home, app_home, app_home ,app_home ,self.SQL_CONNECTOR_JAR) elif os_name == "WINDOWS": path = os.path.join("%s","WEB-INF","classes","conf;%s","WEB-INF","classes","lib","*;%s","WEB-INF",";%s","META-INF",";%s","WEB-INF","lib","*;%s","WEB-INF","classes",";%s","WEB-INF","classes","META-INF;%s" )%(app_home ,app_home ,app_home, app_home, app_home, app_home ,app_home ,self.SQL_CONNECTOR_JAR) - get_java_cmd = "%s -Dlogdir=%s -Dlog4j.configuration=db_patch.log4j.xml -cp %s org.apache.ranger.patch.cliutil.%s %s %s %s -default"%(self.JAVA_BIN,ranger_log,path,className,userName,oldPassword,newPassword) + get_java_cmd = "%s -XX:MetaspaceSize=100m -XX:MaxMetaspaceSize=200m -Xmx%s -Xms1g -Dlogdir=%s -Dlog4j.configuration=db_patch.log4j.xml -cp %s org.apache.ranger.patch.cliutil.%s %s %s %s -default"%(self.JAVA_BIN,max_memory,ranger_log,path,className,userName,oldPassword,newPassword) if is_unix: status = subprocess.call(shlex.split(get_java_cmd)) elif os_name == "WINDOWS": @@ -3139,7 +3140,7 @@ def execute_java_patches(self, xa_db_host, db_user, db_password, db_name): files = os.listdir(javaFiles) if files: for filename in files: - f = re.match("^Patch.*?.class$",filename) + f = re.match(java_patch_regex,filename) if f: className = re.match("(Patch.*?)_.*.class",filename) className = className.group(1) @@ -3196,7 +3197,7 @@ def execute_java_patches(self, xa_db_host, db_user, db_password, db_name): path = os.path.join("%s","WEB-INF","classes","conf:%s","WEB-INF","classes","lib","*:%s","WEB-INF",":%s","META-INF",":%s","WEB-INF","lib","*:%s","WEB-INF","classes",":%s","WEB-INF","classes","META-INF:%s" )%(app_home ,app_home ,app_home, app_home, app_home, app_home ,app_home ,self.SQL_CONNECTOR_JAR) elif os_name == "WINDOWS": path = os.path.join("%s","WEB-INF","classes","conf;%s","WEB-INF","classes","lib","*;%s","WEB-INF",";%s","META-INF",";%s","WEB-INF","lib","*;%s","WEB-INF","classes",";%s","WEB-INF","classes","META-INF;%s" )%(app_home ,app_home ,app_home, app_home, app_home, app_home ,app_home ,self.SQL_CONNECTOR_JAR) - get_java_cmd = "%s -Dlogdir=%s -Dlog4j.configuration=db_patch.log4j.xml -cp %s org.apache.ranger.patch.%s"%(self.JAVA_BIN,ranger_log,path,className) + get_java_cmd = "%s -XX:MetaspaceSize=100m -XX:MaxMetaspaceSize=200m -Xmx%s -Xms1g -Dlogdir=%s -Dlog4j.configuration=db_patch.log4j.xml -cp %s org.apache.ranger.patch.%s"%(self.JAVA_BIN,max_memory,ranger_log,path,className) if is_unix: ret = subprocess.call(shlex.split(get_java_cmd)) elif os_name == "WINDOWS": @@ -3306,7 +3307,7 @@ def change_admin_default_password(self, xa_db_host, db_user, db_password, db_nam path = os.path.join("%s","WEB-INF","classes","conf:%s","WEB-INF","classes","lib","*:%s","WEB-INF",":%s","META-INF",":%s","WEB-INF","lib","*:%s","WEB-INF","classes",":%s","WEB-INF","classes","META-INF:%s" )%(app_home ,app_home ,app_home, app_home, app_home, app_home ,app_home ,self.SQL_CONNECTOR_JAR) elif os_name == "WINDOWS": path = os.path.join("%s","WEB-INF","classes","conf;%s","WEB-INF","classes","lib","*;%s","WEB-INF",";%s","META-INF",";%s","WEB-INF","lib","*;%s","WEB-INF","classes",";%s","WEB-INF","classes","META-INF;%s" )%(app_home ,app_home ,app_home, app_home, app_home, app_home ,app_home ,self.SQL_CONNECTOR_JAR) - get_java_cmd = "%s -Dlogdir=%s -Dlog4j.configuration=db_patch.log4j.xml -cp %s org.apache.ranger.patch.cliutil.%s %s %s %s -default"%(self.JAVA_BIN,ranger_log,path,className,userName,oldPassword,newPassword) + get_java_cmd = "%s -XX:MetaspaceSize=100m -XX:MaxMetaspaceSize=200m -Xmx%s -Xms1g -Dlogdir=%s -Dlog4j.configuration=db_patch.log4j.xml -cp %s org.apache.ranger.patch.cliutil.%s %s %s %s -default"%(self.JAVA_BIN,max_memory,ranger_log,path,className,userName,oldPassword,newPassword) if is_unix: status = subprocess.call(shlex.split(get_java_cmd)) elif os_name == "WINDOWS": @@ -3608,7 +3609,7 @@ def main(argv): db_ssl_enabled='false' db_ssl_required='false' db_ssl_verifyServerCertificate='false' - db_ssl_auth_type='2-way' + db_ssl_auth_type='2-way' javax_net_ssl_keyStore='' javax_net_ssl_keyStorePassword='' javax_net_ssl_trustStore='' @@ -3622,8 +3623,8 @@ def main(argv): db_ssl_required=globalDict['db_ssl_required'].lower() if 'db_ssl_verifyServerCertificate' in globalDict: db_ssl_verifyServerCertificate=globalDict['db_ssl_verifyServerCertificate'].lower() - if 'db_ssl_auth_type' in globalDict: - db_ssl_auth_type=globalDict['db_ssl_auth_type'].lower() + if 'db_ssl_auth_type' in globalDict: + db_ssl_auth_type=globalDict['db_ssl_auth_type'].lower() if db_ssl_verifyServerCertificate == 'true': if 'javax_net_ssl_trustStore' in globalDict: javax_net_ssl_trustStore=globalDict['javax_net_ssl_trustStore'] @@ -3635,20 +3636,20 @@ def main(argv): if javax_net_ssl_trustStorePassword is None or javax_net_ssl_trustStorePassword =="": log("[E] Invalid ssl truststore password!","error") sys.exit(1) - if db_ssl_auth_type == '2-way': - if 'javax_net_ssl_keyStore' in globalDict: - javax_net_ssl_keyStore=globalDict['javax_net_ssl_keyStore'] - if 'javax_net_ssl_keyStorePassword' in globalDict: - javax_net_ssl_keyStorePassword=globalDict['javax_net_ssl_keyStorePassword'] - if not os.path.exists(javax_net_ssl_keyStore): - log("[E] Invalid file Name! Unable to find keystore file:"+javax_net_ssl_keyStore,"error") - sys.exit(1) - if javax_net_ssl_keyStorePassword is None or javax_net_ssl_keyStorePassword =="": - log("[E] Invalid ssl keystore password!","error") - sys.exit(1) + if db_ssl_auth_type == '2-way': + if 'javax_net_ssl_keyStore' in globalDict: + javax_net_ssl_keyStore=globalDict['javax_net_ssl_keyStore'] + if 'javax_net_ssl_keyStorePassword' in globalDict: + javax_net_ssl_keyStorePassword=globalDict['javax_net_ssl_keyStorePassword'] + if not os.path.exists(javax_net_ssl_keyStore): + log("[E] Invalid file Name! Unable to find keystore file:"+javax_net_ssl_keyStore,"error") + sys.exit(1) + if javax_net_ssl_keyStorePassword is None or javax_net_ssl_keyStorePassword =="": + log("[E] Invalid ssl keystore password!","error") + sys.exit(1) MYSQL_CONNECTOR_JAR=globalDict['SQL_CONNECTOR_JAR'] - xa_sqlObj = MysqlConf(xa_db_host, MYSQL_CONNECTOR_JAR, JAVA_BIN,db_ssl_enabled,db_ssl_required,db_ssl_verifyServerCertificate,javax_net_ssl_keyStore,javax_net_ssl_keyStorePassword,javax_net_ssl_trustStore,javax_net_ssl_trustStorePassword,db_ssl_auth_type) + xa_sqlObj = MysqlConf(xa_db_host, MYSQL_CONNECTOR_JAR, JAVA_BIN,db_ssl_enabled,db_ssl_required,db_ssl_verifyServerCertificate,javax_net_ssl_keyStore,javax_net_ssl_keyStorePassword,javax_net_ssl_trustStore,javax_net_ssl_trustStorePassword,db_ssl_auth_type) xa_db_version_file = os.path.join(RANGER_ADMIN_HOME , mysql_dbversion_catalog) xa_db_core_file = os.path.join(RANGER_ADMIN_HOME , mysql_core_file) xa_patch_file = os.path.join(RANGER_ADMIN_HOME ,mysql_patches) @@ -3708,7 +3709,7 @@ def main(argv): if AUDIT_DB_FLAVOR == "MYSQL": MYSQL_CONNECTOR_JAR=globalDict['SQL_CONNECTOR_JAR'] - audit_sqlObj = MysqlConf(audit_db_host,MYSQL_CONNECTOR_JAR,JAVA_BIN,db_ssl_enabled,db_ssl_required,db_ssl_verifyServerCertificate,javax_net_ssl_keyStore,javax_net_ssl_keyStorePassword,javax_net_ssl_trustStore,javax_net_ssl_trustStorePassword,db_ssl_auth_type) + audit_sqlObj = MysqlConf(audit_db_host,MYSQL_CONNECTOR_JAR,JAVA_BIN,db_ssl_enabled,db_ssl_required,db_ssl_verifyServerCertificate,javax_net_ssl_keyStore,javax_net_ssl_keyStorePassword,javax_net_ssl_trustStore,javax_net_ssl_trustStorePassword,db_ssl_auth_type) audit_db_file = os.path.join(RANGER_ADMIN_HOME ,mysql_audit_file) elif AUDIT_DB_FLAVOR == "ORACLE": diff --git a/security-admin/src/main/java/org/apache/ranger/biz/PolicyRefUpdater.java b/security-admin/src/main/java/org/apache/ranger/biz/PolicyRefUpdater.java new file mode 100644 index 0000000000..25b48bb508 --- /dev/null +++ b/security-admin/src/main/java/org/apache/ranger/biz/PolicyRefUpdater.java @@ -0,0 +1,286 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ranger.biz; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.ranger.db.RangerDaoManager; +import org.apache.ranger.db.XXPolicyRefAccessTypeDao; +import org.apache.ranger.db.XXPolicyRefConditionDao; +import org.apache.ranger.db.XXPolicyRefDataMaskTypeDao; +import org.apache.ranger.db.XXPolicyRefGroupDao; +import org.apache.ranger.db.XXPolicyRefResourceDao; +import org.apache.ranger.db.XXPolicyRefUserDao; +import org.apache.ranger.entity.XXAccessTypeDef; +import org.apache.ranger.entity.XXDataMaskTypeDef; +import org.apache.ranger.entity.XXGroup; +import org.apache.ranger.entity.XXPolicy; +import org.apache.ranger.entity.XXPolicyConditionDef; +import org.apache.ranger.entity.XXPolicyRefAccessType; +import org.apache.ranger.entity.XXPolicyRefCondition; +import org.apache.ranger.entity.XXPolicyRefDataMaskType; +import org.apache.ranger.entity.XXPolicyRefGroup; +import org.apache.ranger.entity.XXPolicyRefResource; +import org.apache.ranger.entity.XXPolicyRefUser; +import org.apache.ranger.entity.XXResourceDef; +import org.apache.ranger.entity.XXServiceDef; +import org.apache.ranger.entity.XXUser; +import org.apache.ranger.plugin.model.RangerPolicy; +import org.apache.ranger.plugin.model.RangerPolicy.RangerDataMaskPolicyItem; +import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyItem; +import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyItemAccess; +import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyItemCondition; +import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyItemDataMaskInfo; +import org.apache.ranger.service.RangerAuditFields; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +@Component +public class PolicyRefUpdater { + + @Autowired + RangerDaoManager daoMgr; + + @Autowired + RangerAuditFields rangerAuditFields; + + public void createNewPolMappingForRefTable(RangerPolicy policy, XXPolicy xPolicy, XXServiceDef xServiceDef) throws Exception { + if(policy == null) { + return; + } + + cleanupRefTables(policy); + + final Set resourceNames = policy.getResources().keySet(); + final Set groupNames = new HashSet<>(); + final Set userNames = new HashSet<>(); + final Set accessTypes = new HashSet<>(); + final Set conditionTypes = new HashSet<>(); + final Set dataMaskTypes = new HashSet<>(); + + for (List policyItems : getAllPolicyItems(policy)) { + if (CollectionUtils.isEmpty(policyItems)) { + continue; + } + + for (RangerPolicyItem policyItem : policyItems) { + groupNames.addAll(policyItem.getGroups()); + userNames.addAll(policyItem.getUsers()); + + if (CollectionUtils.isNotEmpty(policyItem.getAccesses())) { + for (RangerPolicyItemAccess access : policyItem.getAccesses()) { + accessTypes.add(access.getType()); + } + } + + if (CollectionUtils.isNotEmpty(policyItem.getConditions())) { + for (RangerPolicyItemCondition condition : policyItem.getConditions()) { + conditionTypes.add(condition.getType()); + } + } + + if (policyItem instanceof RangerDataMaskPolicyItem) { + RangerPolicyItemDataMaskInfo dataMaskInfo = ((RangerDataMaskPolicyItem) policyItem).getDataMaskInfo(); + + dataMaskTypes.add(dataMaskInfo.getDataMaskType()); + } + } + } + + for (String resource : resourceNames) { + XXResourceDef xResDef = daoMgr.getXXResourceDef().findByNameAndPolicyId(resource, policy.getId()); + + if (xResDef == null) { + throw new Exception(resource + ": is not a valid resource-type. policy='"+ policy.getName() + "' service='"+ policy.getService() + "'"); + } + + XXPolicyRefResource xPolRes = rangerAuditFields.populateAuditFields(new XXPolicyRefResource(), xPolicy); + + xPolRes.setPolicyId(policy.getId()); + xPolRes.setResourceDefId(xResDef.getId()); + xPolRes.setResourceName(resource); + + daoMgr.getXXPolicyRefResource().create(xPolRes); + } + + for (String group : groupNames) { + if (StringUtils.isBlank(group)) { + continue; + } + + XXGroup xGroup = daoMgr.getXXGroup().findByGroupName(group); + + if (xGroup == null) { + throw new Exception(group + ": group does not exist. policy='"+ policy.getName() + "' service='"+ policy.getService() + "' group='" + group + "'"); + } + + XXPolicyRefGroup xPolGroup = rangerAuditFields.populateAuditFields(new XXPolicyRefGroup(), xPolicy); + + xPolGroup.setPolicyId(policy.getId()); + xPolGroup.setGroupId(xGroup.getId()); + xPolGroup.setGroupName(group); + + daoMgr.getXXPolicyRefGroup().create(xPolGroup); + } + + for (String user : userNames) { + if (StringUtils.isBlank(user)) { + continue; + } + + XXUser xUser = daoMgr.getXXUser().findByUserName(user); + + if (xUser == null) { + throw new Exception(user + ": user does not exist. policy='"+ policy.getName() + "' service='"+ policy.getService() + "' user='" + user +"'"); + } + + XXPolicyRefUser xPolUser = rangerAuditFields.populateAuditFields(new XXPolicyRefUser(), xPolicy); + + xPolUser.setPolicyId(policy.getId()); + xPolUser.setUserId(xUser.getId()); + xPolUser.setUserName(user); + + daoMgr.getXXPolicyRefUser().create(xPolUser); + } + + for (String accessType : accessTypes) { + XXAccessTypeDef xAccTypeDef = daoMgr.getXXAccessTypeDef().findByNameAndServiceId(accessType, xPolicy.getService()); + + if (xAccTypeDef == null) { + throw new Exception(accessType + ": is not a valid access-type. policy='" + policy.getName() + "' service='" + policy.getService() + "'"); + } + + XXPolicyRefAccessType xPolAccess = rangerAuditFields.populateAuditFields(new XXPolicyRefAccessType(), xPolicy); + + xPolAccess.setPolicyId(policy.getId()); + xPolAccess.setAccessDefId(xAccTypeDef.getId()); + xPolAccess.setAccessTypeName(accessType); + + daoMgr.getXXPolicyRefAccessType().create(xPolAccess); + } + + for (String condition : conditionTypes) { + XXPolicyConditionDef xPolCondDef = daoMgr.getXXPolicyConditionDef().findByServiceDefIdAndName(xServiceDef.getId(), condition); + + if (xPolCondDef == null) { + throw new Exception(condition + ": is not a valid condition-type. policy='"+ xPolicy.getName() + "' service='"+ xPolicy.getService() + "'"); + } + + XXPolicyRefCondition xPolCond = rangerAuditFields.populateAuditFields(new XXPolicyRefCondition(), xPolicy); + + xPolCond.setPolicyId(policy.getId()); + xPolCond.setConditionDefId(xPolCondDef.getId()); + xPolCond.setConditionName(condition); + + daoMgr.getXXPolicyRefCondition().create(xPolCond); + } + + for (String dataMaskType : dataMaskTypes ) { + XXDataMaskTypeDef dataMaskDef = daoMgr.getXXDataMaskTypeDef().findByNameAndServiceId(dataMaskType, xPolicy.getService()); + + if (dataMaskDef == null) { + throw new Exception(dataMaskType + ": is not a valid datamask-type. policy='" + policy.getName() + "' service='" + policy.getService() + "'"); + } + + XXPolicyRefDataMaskType xxDataMaskInfo = new XXPolicyRefDataMaskType(); + + xxDataMaskInfo.setPolicyId(policy.getId()); + xxDataMaskInfo.setDataMaskDefId(dataMaskDef.getId()); + xxDataMaskInfo.setDataMaskTypeName(dataMaskType); + + daoMgr.getXXPolicyRefDataMaskType().create(xxDataMaskInfo); + } + } + + public Boolean cleanupRefTables(RangerPolicy policy) { + final Long policyId = policy == null ? null : policy.getId(); + + if (policyId == null) { + return false; + } + + XXPolicyRefResourceDao xPolResDao = daoMgr.getXXPolicyRefResource(); + XXPolicyRefGroupDao xPolGroupDao = daoMgr.getXXPolicyRefGroup(); + XXPolicyRefUserDao xPolUserDao = daoMgr.getXXPolicyRefUser(); + XXPolicyRefAccessTypeDao xPolAccessDao = daoMgr.getXXPolicyRefAccessType(); + XXPolicyRefConditionDao xPolCondDao = daoMgr.getXXPolicyRefCondition(); + XXPolicyRefDataMaskTypeDao xPolDataMaskDao = daoMgr.getXXPolicyRefDataMaskType(); + + for (XXPolicyRefResource resource : xPolResDao.findByPolicyId(policyId)) { + xPolResDao.remove(resource); + } + + for(XXPolicyRefGroup group : xPolGroupDao.findByPolicyId(policyId)) { + xPolGroupDao.remove(group); + } + + for(XXPolicyRefUser user : xPolUserDao.findByPolicyId(policyId)) { + xPolUserDao.remove(user); + } + + for(XXPolicyRefAccessType access : xPolAccessDao.findByPolicyId(policyId)) { + xPolAccessDao.remove(access); + } + + for(XXPolicyRefCondition condVal : xPolCondDao.findByPolicyId(policyId)) { + xPolCondDao.remove(condVal); + } + + for(XXPolicyRefDataMaskType dataMask : xPolDataMaskDao.findByPolicyId(policyId)) { + xPolDataMaskDao.remove(dataMask); + } + + return true; + } + + static List> getAllPolicyItems(RangerPolicy policy) { + List> ret = new ArrayList<>(); + + if (CollectionUtils.isNotEmpty(policy.getPolicyItems())) { + ret.add(policy.getPolicyItems()); + } + + if (CollectionUtils.isNotEmpty(policy.getDenyPolicyItems())) { + ret.add(policy.getDenyPolicyItems()); + } + + if (CollectionUtils.isNotEmpty(policy.getAllowExceptions())) { + ret.add(policy.getAllowExceptions()); + } + + if (CollectionUtils.isNotEmpty(policy.getDenyExceptions())) { + ret.add(policy.getDenyExceptions()); + } + + if (CollectionUtils.isNotEmpty(policy.getDataMaskPolicyItems())) { + ret.add(policy.getDataMaskPolicyItems()); + } + + if (CollectionUtils.isNotEmpty(policy.getRowFilterPolicyItems())) { + ret.add(policy.getRowFilterPolicyItems()); + } + + return ret; + } +} diff --git a/security-admin/src/main/java/org/apache/ranger/biz/RangerPolicyRetriever.java b/security-admin/src/main/java/org/apache/ranger/biz/RangerPolicyRetriever.java index 354ab1cba9..b0734e4f30 100644 --- a/security-admin/src/main/java/org/apache/ranger/biz/RangerPolicyRetriever.java +++ b/security-admin/src/main/java/org/apache/ranger/biz/RangerPolicyRetriever.java @@ -26,28 +26,28 @@ import java.util.Map; import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.collections.MapUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.ranger.authorization.utils.JsonUtils; import org.apache.ranger.authorization.utils.StringUtil; import org.apache.ranger.db.RangerDaoManager; -import org.apache.ranger.entity.*; +import org.apache.ranger.entity.XXPolicy; +import org.apache.ranger.entity.XXPortalUser; +import org.apache.ranger.entity.XXService; import org.apache.ranger.plugin.model.RangerPolicy; import org.apache.ranger.plugin.model.RangerPolicy.RangerDataMaskPolicyItem; import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyItem; import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyItemAccess; import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyItemCondition; -import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyItemDataMaskInfo; -import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyItemRowFilterInfo; import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyResource; -import org.apache.ranger.plugin.model.RangerPolicy.RangerRowFilterPolicyItem; -import org.apache.ranger.plugin.policyevaluator.RangerPolicyItemEvaluator; import org.apache.ranger.plugin.util.RangerPerfTracer; import org.springframework.transaction.PlatformTransactionManager; import org.springframework.transaction.TransactionStatus; import org.springframework.transaction.support.TransactionCallback; import org.springframework.transaction.support.TransactionTemplate; + public class RangerPolicyRetriever { static final Log LOG = LogFactory.getLog(RangerPolicyRetriever.class); static final Log PERF_LOG = RangerPerfTracer.getPerfLogger("db.RangerPolicyRetriever"); @@ -299,33 +299,13 @@ private XXPolicy getXXPolicy(Long policyId) { } class LookupCache { - final Map userNames = new HashMap(); - final Map userScreenNames = new HashMap(); - final Map groupNames = new HashMap(); - final Map accessTypes = new HashMap(); - final Map conditions = new HashMap(); - final Map resourceDefs = new HashMap(); - final Map dataMasks = new HashMap(); - - String getUserName(Long userId) { - String ret = null; - - if(userId != null) { - ret = userNames.get(userId); - - if(ret == null) { - XXUser user = daoMgr.getXXUser().getById(userId); - - if(user != null) { - ret = user.getName(); // Name is `loginId` - - userNames.put(userId, ret); - } - } - } - - return ret; - } + final Map userScreenNames = new HashMap(); + final Map> groupMappingsPerPolicy = new HashMap<>(); + final Map> userMappingsPerPolicy = new HashMap<>(); + final Map> accessMappingsPerPolicy = new HashMap<>(); + final Map> resourceMappingsPerPolicy = new HashMap<>(); + final Map> dataMaskMappingsPerPolicy = new HashMap<>(); + final Map> conditionMappingsPerPolicy = new HashMap<>(); String getUserScreenName(Long userId) { String ret = null; @@ -361,257 +341,135 @@ String getUserScreenName(Long userId) { return ret; } - String getGroupName(Long groupId) { - String ret = null; - - if(groupId != null) { - ret = groupNames.get(groupId); + void setNameMapping(Map> nameMappingContainer, List nameMappings) { + nameMappingContainer.clear(); - if(ret == null) { - XXGroup group = daoMgr.getXXGroup().getById(groupId); + for (PolicyTextNameMap nameMapping : nameMappings) { + Map policyNameMap = nameMappingContainer.get(nameMapping.policyId); - if(group != null) { - ret = group.getName(); + if (policyNameMap == null) { + policyNameMap = new HashMap<>(); - groupNames.put(groupId, ret); - } + nameMappingContainer.put(nameMapping.policyId, policyNameMap); } - } - - return ret; - } - String getAccessType(Long accessTypeId) { - String ret = null; - - if(accessTypeId != null) { - ret = accessTypes.get(accessTypeId); - - if(ret == null) { - XXAccessTypeDef xAccessType = daoMgr.getXXAccessTypeDef().getById(accessTypeId); - - if(xAccessType != null) { - ret = xAccessType.getName(); - - accessTypes.put(accessTypeId, ret); - } - } + policyNameMap.put(nameMapping.oldName, nameMapping.currentName); } - - return ret; } - String getConditionType(Long conditionDefId) { - String ret = null; - - if(conditionDefId != null) { - ret = conditions.get(conditionDefId); + String getMappedName(Map> nameMappingContainer, Long policyId, String nameToMap) { + Map policyNameMap = nameMappingContainer.get(policyId); - if(ret == null) { - XXPolicyConditionDef xPolicyConditionDef = daoMgr.getXXPolicyConditionDef().getById(conditionDefId); - - if(xPolicyConditionDef != null) { - ret = xPolicyConditionDef.getName(); - - conditions.put(conditionDefId, ret); - } - } - } - - return ret; + return policyNameMap != null ? policyNameMap.get(nameToMap) : null; } - String getResourceName(Long resourceDefId) { - String ret = null; - - if(resourceDefId != null) { - ret = resourceDefs.get(resourceDefId); - - if(ret == null) { - XXResourceDef xResourceDef = daoMgr.getXXResourceDef().getById(resourceDefId); - - if(xResourceDef != null) { - ret = xResourceDef.getName(); + void setGroupNameMapping(List groupNameMapping) { + setNameMapping(groupMappingsPerPolicy, groupNameMapping); + } - resourceDefs.put(resourceDefId, ret); - } - } - } + void setUserNameMapping(List userNameMapping) { + setNameMapping(userMappingsPerPolicy, userNameMapping); + } - return ret; + void setAccessNameMapping(List accessNameMapping) { + setNameMapping(accessMappingsPerPolicy, accessNameMapping); } - String getDataMaskName(Long dataMaskDefId) { - String ret = null; + public void setResourceNameMapping(List resourceNameMapping) { + setNameMapping(resourceMappingsPerPolicy, resourceNameMapping); + } - if(dataMaskDefId != null) { - ret = dataMasks.get(dataMaskDefId); + public void setDataMaskNameMapping(List dataMaskMapping) { + setNameMapping(dataMaskMappingsPerPolicy, dataMaskMapping); + } - if(ret == null) { - XXDataMaskTypeDef xDataMaskDef = daoMgr.getXXDataMaskTypeDef().getById(dataMaskDefId); + public void setConditionNameMapping(List conditionNameMapping) { + setNameMapping(conditionMappingsPerPolicy, conditionNameMapping); + } - if(xDataMaskDef != null) { - ret = xDataMaskDef.getName(); + } - dataMasks.put(dataMaskDefId, ret); - } - } - } + public static class PolicyTextNameMap { + final Long policyId; + final String oldName; + final String currentName; - return ret; + public PolicyTextNameMap(Long policyId, String oldName, String currentName) { + this.policyId = policyId; + this.oldName = oldName; + this.currentName = currentName; } } - static List asList(XXPolicy policy) { - List ret = new ArrayList(); + static List asList(XXPolicy policy) { + List ret = new ArrayList<>(); - if(policy != null) { - ret.add(policy); - } + if (policy != null) { + ret.add(policy); + } - return ret; - } + return ret; + } class RetrieverContext { - final XXService service; - final ListIterator iterPolicy; - final ListIterator iterResources; - final ListIterator iterResourceMaps; - final ListIterator iterPolicyItems; - final ListIterator iterUserPerms; - final ListIterator iterGroupPerms; - final ListIterator iterAccesses; - final ListIterator iterConditions; - final ListIterator iterDataMaskInfos; - final ListIterator iterRowFilterInfos; + final XXService service; + final ListIterator iterPolicy; RetrieverContext(XXService xService) { - Long serviceId = xService == null ? null : xService.getId(); - - List xPolicies = daoMgr.getXXPolicy().findByServiceId(serviceId); - List xResources = daoMgr.getXXPolicyResource().findByServiceId(serviceId); - List xResourceMaps = daoMgr.getXXPolicyResourceMap().findByServiceId(serviceId); - List xPolicyItems = daoMgr.getXXPolicyItem().findByServiceId(serviceId); - List xUserPerms = daoMgr.getXXPolicyItemUserPerm().findByServiceId(serviceId); - List xGroupPerms = daoMgr.getXXPolicyItemGroupPerm().findByServiceId(serviceId); - List xAccesses = daoMgr.getXXPolicyItemAccess().findByServiceId(serviceId); - List xConditions = daoMgr.getXXPolicyItemCondition().findByServiceId(serviceId); - List xDataMaskInfos = daoMgr.getXXPolicyItemDataMaskInfo().findByServiceId(serviceId); - List xRowFilterInfos = daoMgr.getXXPolicyItemRowFilterInfo().findByServiceId(serviceId); - - this.service = xService; - this.iterPolicy = xPolicies.listIterator(); - this.iterResources = xResources.listIterator(); - this.iterResourceMaps = xResourceMaps.listIterator(); - this.iterPolicyItems = xPolicyItems.listIterator(); - this.iterUserPerms = xUserPerms.listIterator(); - this.iterGroupPerms = xGroupPerms.listIterator(); - this.iterAccesses = xAccesses.listIterator(); - this.iterConditions = xConditions.listIterator(); - this.iterDataMaskInfos = xDataMaskInfos.listIterator(); - this.iterRowFilterInfos = xRowFilterInfos.listIterator(); - } - - RetrieverContext(XXPolicy xPolicy) { - this(xPolicy, getXXService(xPolicy.getService())); + if (xService != null) { + Long serviceId = xService.getId(); + + lookupCache.setGroupNameMapping(daoMgr.getXXPolicyRefGroup().findUpdatedGroupNamesByService(serviceId)); + lookupCache.setUserNameMapping(daoMgr.getXXPolicyRefUser().findUpdatedUserNamesByService(serviceId)); + lookupCache.setAccessNameMapping(daoMgr.getXXPolicyRefAccessType().findUpdatedAccessNamesByService(serviceId)); + lookupCache.setResourceNameMapping(daoMgr.getXXPolicyRefResource().findUpdatedResourceNamesByService(serviceId)); + lookupCache.setDataMaskNameMapping(daoMgr.getXXPolicyRefDataMaskType().findUpdatedDataMaskNamesByService(serviceId)); + lookupCache.setConditionNameMapping(daoMgr.getXXPolicyRefCondition().findUpdatedConditionNamesByService(serviceId)); + + this.service = xService; + this.iterPolicy = daoMgr.getXXPolicy().findByServiceId(serviceId).listIterator(); + } else { + this.service = null; + this.iterPolicy = null; + } } RetrieverContext(XXPolicy xPolicy, XXService xService) { - Long policyId = xPolicy == null ? null : xPolicy.getId(); - - List xPolicies = asList(xPolicy); - List xResources = daoMgr.getXXPolicyResource().findByPolicyId(policyId); - List xResourceMaps = daoMgr.getXXPolicyResourceMap().findByPolicyId(policyId); - List xPolicyItems = daoMgr.getXXPolicyItem().findByPolicyId(policyId); - List xUserPerms = daoMgr.getXXPolicyItemUserPerm().findByPolicyId(policyId); - List xGroupPerms = daoMgr.getXXPolicyItemGroupPerm().findByPolicyId(policyId); - List xAccesses = daoMgr.getXXPolicyItemAccess().findByPolicyId(policyId); - List xConditions = daoMgr.getXXPolicyItemCondition().findByPolicyId(policyId); - List xDataMaskInfos = daoMgr.getXXPolicyItemDataMaskInfo().findByPolicyId(policyId); - List xRowFilterInfos = daoMgr.getXXPolicyItemRowFilterInfo().findByPolicyId(policyId); - - this.service = xService; - this.iterPolicy = xPolicies.listIterator(); - this.iterResources = xResources.listIterator(); - this.iterResourceMaps = xResourceMaps.listIterator(); - this.iterPolicyItems = xPolicyItems.listIterator(); - this.iterUserPerms = xUserPerms.listIterator(); - this.iterGroupPerms = xGroupPerms.listIterator(); - this.iterAccesses = xAccesses.listIterator(); - this.iterConditions = xConditions.listIterator(); - this.iterDataMaskInfos = xDataMaskInfos.listIterator(); - this.iterRowFilterInfos = xRowFilterInfos.listIterator(); + Long policyId = xPolicy.getId(); + + lookupCache.setGroupNameMapping(daoMgr.getXXPolicyRefGroup().findUpdatedGroupNamesByPolicy(policyId)); + lookupCache.setUserNameMapping(daoMgr.getXXPolicyRefUser().findUpdatedUserNamesByPolicy(policyId)); + lookupCache.setAccessNameMapping(daoMgr.getXXPolicyRefAccessType().findUpdatedAccessNamesByPolicy(policyId)); + lookupCache.setResourceNameMapping(daoMgr.getXXPolicyRefResource().findUpdatedResourceNamesByPolicy(policyId)); + lookupCache.setDataMaskNameMapping(daoMgr.getXXPolicyRefDataMaskType().findUpdatedDataMaskNamesByPolicy(policyId)); + lookupCache.setConditionNameMapping(daoMgr.getXXPolicyRefCondition().findUpdatedConditionNamesByPolicy(policyId)); + + this.service = xService; + this.iterPolicy = asList(xPolicy).listIterator(); } RangerPolicy getNextPolicy() { RangerPolicy ret = null; - if(iterPolicy.hasNext()) { + if (service != null && iterPolicy != null && iterPolicy.hasNext()) { XXPolicy xPolicy = iterPolicy.next(); - if(xPolicy != null) { - ret = new RangerPolicy(); - - ret.setId(xPolicy.getId()); - ret.setGuid(xPolicy.getGuid()); - ret.setIsEnabled(xPolicy.getIsEnabled()); - ret.setCreatedBy(lookupCache.getUserScreenName(xPolicy.getAddedByUserId())); - ret.setUpdatedBy(lookupCache.getUserScreenName(xPolicy.getUpdatedByUserId())); - ret.setCreateTime(xPolicy.getCreateTime()); - ret.setUpdateTime(xPolicy.getUpdateTime()); - ret.setVersion(xPolicy.getVersion()); - ret.setService(service == null ? null : service.getName()); - ret.setName(StringUtils.trim(xPolicy.getName())); - ret.setPolicyType(xPolicy.getPolicyType() == null ? RangerPolicy.POLICY_TYPE_ACCESS : xPolicy.getPolicyType()); - ret.setDescription(xPolicy.getDescription()); - ret.setResourceSignature(xPolicy.getResourceSignature()); - ret.setIsAuditEnabled(xPolicy.getIsAuditEnabled()); - - getResource(ret); - getPolicyItems(ret); - } - } - - return ret; - } - - List getAllPolicies() { - List ret = new ArrayList(); - - while(iterPolicy.hasNext()) { - RangerPolicy policy = getNextPolicy(); - - if(policy != null) { - ret.add(policy); - } - } - - if(! hasProcessedAll()) { - LOG.warn("getAllPolicies(): perhaps one or more policies got updated during retrieval. Falling back to secondary method"); - - ret = getAllPoliciesBySecondary(); - } - - return ret; - } - - List getAllPoliciesBySecondary() { - List ret = null; - - if(service != null) { - List xPolicies = daoMgr.getXXPolicy().findByServiceId(service.getId()); - - if(CollectionUtils.isNotEmpty(xPolicies)) { - ret = new ArrayList(xPolicies.size()); - - for(XXPolicy xPolicy : xPolicies) { - RetrieverContext ctx = new RetrieverContext(xPolicy, service); - - RangerPolicy policy = ctx.getNextPolicy(); - - if(policy != null) { - ret.add(policy); - } + if (xPolicy != null) { + String policyText = xPolicy.getPolicyText(); + + ret = JsonUtils.jsonToObject(policyText, RangerPolicy.class); + + if (ret != null) { + ret.setId(xPolicy.getId()); + ret.setGuid(xPolicy.getGuid()); + ret.setCreatedBy(lookupCache.getUserScreenName(xPolicy.getAddedByUserId())); + ret.setUpdatedBy(lookupCache.getUserScreenName(xPolicy.getUpdatedByUserId())); + ret.setCreateTime(xPolicy.getCreateTime()); + ret.setUpdateTime(xPolicy.getUpdateTime()); + ret.setVersion(xPolicy.getVersion()); + ret.setPolicyType(xPolicy.getPolicyType() == null ? RangerPolicy.POLICY_TYPE_ACCESS : xPolicy.getPolicyType()); + ret.setService(service.getName()); + updatePolicyReferenceFields(ret); } } } @@ -619,206 +477,121 @@ List getAllPoliciesBySecondary() { return ret; } - private boolean hasProcessedAll() { - boolean moreToProcess = iterPolicy.hasNext() - || iterResources.hasNext() - || iterResourceMaps.hasNext() - || iterPolicyItems.hasNext() - || iterUserPerms.hasNext() - || iterGroupPerms.hasNext() - || iterAccesses.hasNext() - || iterConditions.hasNext() - || iterDataMaskInfos.hasNext() - || iterRowFilterInfos.hasNext(); - - return !moreToProcess; - } + void updatePolicyReferenceFields(final RangerPolicy policy) { + final Long policyId = policy.getId(); - private void getResource(RangerPolicy policy) { - while(iterResources.hasNext()) { - XXPolicyResource xResource = iterResources.next(); + Map policyResourceNameMap = lookupCache.resourceMappingsPerPolicy.get(policyId); - if(xResource.getPolicyid().equals(policy.getId())) { - RangerPolicyResource resource = new RangerPolicyResource(); + if (MapUtils.isNotEmpty(policyResourceNameMap) && CollectionUtils.containsAny(policyResourceNameMap.keySet(), policy.getResources().keySet())) { + Map updatedResources = new HashMap<>(); - resource.setIsExcludes(xResource.getIsexcludes()); - resource.setIsRecursive(xResource.getIsrecursive()); + for (Map.Entry entry : policy.getResources().entrySet()) { + String resourceName = entry.getKey(); + RangerPolicyResource policyResource = entry.getValue(); + String updatedName = policyResourceNameMap.get(resourceName); - while(iterResourceMaps.hasNext()) { - XXPolicyResourceMap xResourceMap = iterResourceMaps.next(); - - if(xResourceMap.getResourceid().equals(xResource.getId())) { - resource.getValues().add(xResourceMap.getValue()); - } else { - if(iterResourceMaps.hasPrevious()) { - iterResourceMaps.previous(); - } - break; - } + if (updatedName == null) { + updatedName = resourceName; } - policy.getResources().put(lookupCache.getResourceName(xResource.getResdefid()), resource); - } else if(xResource.getPolicyid().compareTo(policy.getId()) > 0) { - if(iterResources.hasPrevious()) { - iterResources.previous(); - } - break; + updatedResources.put(updatedName, policyResource); } + + policy.setResources(updatedResources); } - } - private void getPolicyItems(RangerPolicy policy) { - while(iterPolicyItems.hasNext()) { - XXPolicyItem xPolicyItem = iterPolicyItems.next(); + for (List policyItems : PolicyRefUpdater.getAllPolicyItems(policy)) { + if (CollectionUtils.isEmpty(policyItems)) { + continue; + } - if(xPolicyItem.getPolicyid().equals(policy.getId())) { - final RangerPolicyItem policyItem; - final RangerDataMaskPolicyItem dataMaskPolicyItem; - final RangerRowFilterPolicyItem rowFilterPolicyItem; + for (RangerPolicyItem policyItem : policyItems) { + if (lookupCache.groupMappingsPerPolicy.containsKey(policyId)) { + List updatedGroups = getUpdatedNames(lookupCache.groupMappingsPerPolicy, policyId, policyItem.getGroups()); - if(xPolicyItem.getItemType() == RangerPolicyItemEvaluator.POLICY_ITEM_TYPE_DATAMASK) { - dataMaskPolicyItem = new RangerDataMaskPolicyItem(); - rowFilterPolicyItem = null; - policyItem = dataMaskPolicyItem; - } else if(xPolicyItem.getItemType() == RangerPolicyItemEvaluator.POLICY_ITEM_TYPE_ROWFILTER) { - dataMaskPolicyItem = null; - rowFilterPolicyItem = new RangerRowFilterPolicyItem(); - policyItem = rowFilterPolicyItem; - } else { - dataMaskPolicyItem = null; - rowFilterPolicyItem = null; - policyItem = new RangerPolicyItem(); + if (updatedGroups != null) { + policyItem.setGroups(updatedGroups); + } } + if (lookupCache.userMappingsPerPolicy.containsKey(policyId)) { + List updatedUsers = getUpdatedNames(lookupCache.userMappingsPerPolicy, policyId, policyItem.getUsers()); - while(iterAccesses.hasNext()) { - XXPolicyItemAccess xAccess = iterAccesses.next(); - - if(xAccess.getPolicyitemid().equals(xPolicyItem.getId())) { - policyItem.getAccesses().add(new RangerPolicyItemAccess(lookupCache.getAccessType(xAccess.getType()), xAccess.getIsallowed())); - } else { - if(iterAccesses.hasPrevious()) { - iterAccesses.previous(); - } - break; + if (updatedUsers != null) { + policyItem.setUsers(updatedUsers); } } - while(iterUserPerms.hasNext()) { - XXPolicyItemUserPerm xUserPerm = iterUserPerms.next(); + if (lookupCache.accessMappingsPerPolicy.containsKey(policyId)) { + for (RangerPolicyItemAccess itemAccess : policyItem.getAccesses()) { + String updatedName = lookupCache.getMappedName(lookupCache.accessMappingsPerPolicy, policyId, itemAccess.getType()); - if(xUserPerm.getPolicyitemid().equals(xPolicyItem.getId())) { - String userName = lookupCache.getUserName(xUserPerm.getUserid()); - if (userName != null) { - policyItem.getUsers().add(userName); + if (updatedName != null) { + itemAccess.setType(updatedName); } - } else { - if(iterUserPerms.hasPrevious()) { - iterUserPerms.previous(); - } - break; } } - while(iterGroupPerms.hasNext()) { - XXPolicyItemGroupPerm xGroupPerm = iterGroupPerms.next(); + if (lookupCache.conditionMappingsPerPolicy.containsKey(policyId)) { + for (RangerPolicyItemCondition condition : policyItem.getConditions()) { + String updatedName = lookupCache.getMappedName(lookupCache.conditionMappingsPerPolicy, policyId, condition.getType()); - if(xGroupPerm.getPolicyitemid().equals(xPolicyItem.getId())) { - String groupName = lookupCache.getGroupName(xGroupPerm.getGroupid()); - if (groupName != null) { - policyItem.getGroups().add(groupName); - } - } else { - if(iterGroupPerms.hasPrevious()) { - iterGroupPerms.previous(); + if (updatedName != null) { + condition.setType(updatedName); } - break; } } - RangerPolicyItemCondition condition = null; - Long prevConditionType = null; - while(iterConditions.hasNext()) { - XXPolicyItemCondition xCondition = iterConditions.next(); + if (policyItem instanceof RangerDataMaskPolicyItem && lookupCache.dataMaskMappingsPerPolicy.containsKey(policyId)) { + RangerDataMaskPolicyItem dataMaskItem = (RangerDataMaskPolicyItem) policyItem; + String updatedName = lookupCache.getMappedName(lookupCache.dataMaskMappingsPerPolicy, policyId, dataMaskItem.getDataMaskInfo().getDataMaskType()); - if(xCondition.getPolicyitemid().equals(xPolicyItem.getId())) { - if(! xCondition.getType().equals(prevConditionType)) { - condition = new RangerPolicyItemCondition(); - condition.setType(lookupCache.getConditionType(xCondition.getType())); - condition.getValues().add(xCondition.getValue()); - - policyItem.getConditions().add(condition); - - prevConditionType = xCondition.getType(); - } else { - condition.getValues().add(xCondition.getValue()); - } - } else { - if(iterConditions.hasPrevious()) { - iterConditions.previous(); - } - break; + if (updatedName != null) { + dataMaskItem.getDataMaskInfo().setDataMaskType(updatedName); } } + } + } + } - policyItem.setDelegateAdmin(xPolicyItem.getDelegateAdmin()); + List getUpdatedNames(final Map> nameMappingContainer, final Long policyId, final List namesToMap) { + List ret = null; + Map policyNameMap = nameMappingContainer.get(policyId); - if(dataMaskPolicyItem != null) { - while (iterDataMaskInfos.hasNext()) { - XXPolicyItemDataMaskInfo xDataMaskInfo = iterDataMaskInfos.next(); + if (MapUtils.isNotEmpty(policyNameMap) && CollectionUtils.containsAny(policyNameMap.keySet(), namesToMap)) { + ret = new ArrayList<>(); - if (xDataMaskInfo.getPolicyItemId().equals(xPolicyItem.getId())) { - dataMaskPolicyItem.setDataMaskInfo(new RangerPolicyItemDataMaskInfo(lookupCache.getDataMaskName(xDataMaskInfo.getType()), xDataMaskInfo.getConditionExpr(), xDataMaskInfo.getValueExpr())); - } else { - if (iterDataMaskInfos.hasPrevious()) { - iterDataMaskInfos.previous(); - } - break; - } - } + for (String nameToMap : namesToMap) { + String mappedName = policyNameMap.get(nameToMap); + + if (mappedName != null) { + ret.add(mappedName); + } else { + ret.add(nameToMap); } + } - if(rowFilterPolicyItem != null) { - while (iterRowFilterInfos.hasNext()) { - XXPolicyItemRowFilterInfo xRowFilterInfo = iterRowFilterInfos.next(); + } - if (xRowFilterInfo.getPolicyItemId().equals(xPolicyItem.getId())) { - rowFilterPolicyItem.setRowFilterInfo(new RangerPolicyItemRowFilterInfo(xRowFilterInfo.getFilterExpr())); - } else { - if (iterRowFilterInfos.hasPrevious()) { - iterRowFilterInfos.previous(); - } - break; - } - } - } + return ret; + } + List getAllPolicies() { + List ret = new ArrayList<>(); - int itemType = xPolicyItem.getItemType() == null ? RangerPolicyItemEvaluator.POLICY_ITEM_TYPE_ALLOW : xPolicyItem.getItemType(); - - if(itemType == RangerPolicyItemEvaluator.POLICY_ITEM_TYPE_ALLOW) { - policy.getPolicyItems().add(policyItem); - } else if(itemType == RangerPolicyItemEvaluator.POLICY_ITEM_TYPE_DENY) { - policy.getDenyPolicyItems().add(policyItem); - } else if(itemType == RangerPolicyItemEvaluator.POLICY_ITEM_TYPE_ALLOW_EXCEPTIONS) { - policy.getAllowExceptions().add(policyItem); - } else if(itemType == RangerPolicyItemEvaluator.POLICY_ITEM_TYPE_DENY_EXCEPTIONS) { - policy.getDenyExceptions().add(policyItem); - } else if(itemType == RangerPolicyItemEvaluator.POLICY_ITEM_TYPE_DATAMASK) { - policy.getDataMaskPolicyItems().add(dataMaskPolicyItem); - } else if(itemType == RangerPolicyItemEvaluator.POLICY_ITEM_TYPE_ROWFILTER) { - policy.getRowFilterPolicyItems().add(rowFilterPolicyItem); - } else { // unknown itemType - LOG.warn("RangerPolicyRetriever.getPolicy(policyId=" + policy.getId() + "): ignoring unknown policyItemType " + itemType); - } - } else if(xPolicyItem.getPolicyid().compareTo(policy.getId()) > 0) { - if(iterPolicyItems.hasPrevious()) { - iterPolicyItems.previous(); + if (iterPolicy != null) { + while (iterPolicy.hasNext()) { + RangerPolicy policy = getNextPolicy(); + + if (policy != null) { + ret.add(policy); } - break; } } + + return ret; } } + } diff --git a/security-admin/src/main/java/org/apache/ranger/biz/RangerTagDBRetriever.java b/security-admin/src/main/java/org/apache/ranger/biz/RangerTagDBRetriever.java index 53683ec04f..09494fc4be 100644 --- a/security-admin/src/main/java/org/apache/ranger/biz/RangerTagDBRetriever.java +++ b/security-admin/src/main/java/org/apache/ranger/biz/RangerTagDBRetriever.java @@ -19,61 +19,63 @@ package org.apache.ranger.biz; +import java.lang.reflect.Type; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.ListIterator; import java.util.Map; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.reflect.TypeToken; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.ranger.authorization.hadoop.config.RangerConfiguration; import org.apache.ranger.authorization.utils.StringUtil; import org.apache.ranger.db.RangerDaoManager; import org.apache.ranger.entity.*; import org.apache.ranger.plugin.model.*; -import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyResource; import org.apache.ranger.plugin.util.RangerPerfTracer; +import org.apache.ranger.service.RangerServiceResourceService; import org.springframework.transaction.PlatformTransactionManager; import org.springframework.transaction.TransactionStatus; import org.springframework.transaction.support.TransactionCallback; import org.springframework.transaction.support.TransactionTemplate; - public class RangerTagDBRetriever { static final Log LOG = LogFactory.getLog(RangerTagDBRetriever.class); static final Log PERF_LOG = RangerPerfTracer.getPerfLogger("db.RangerTagDBRetriever"); public static final String OPTION_RANGER_FILTER_TAGS_FOR_SERVICE_PLUGIN = "ranger.filter.tags.for.service.plugin"; + public static final Type subsumedDataType = new TypeToken>() {}.getType(); + + public static final Gson gsonBuilder = new GsonBuilder().setDateFormat("yyyyMMdd-HH:mm:ss.SSS-Z") + .create(); + private final RangerDaoManager daoMgr; - private final XXService xService; private final LookupCache lookupCache; - private final PlatformTransactionManager txManager; - private final TransactionTemplate txTemplate; - private List serviceResources; private Map tagDefs; - private Map tags; - private List tagResourceMaps; - private boolean filterForServicePlugin; + RangerTagDBRetriever(final RangerDaoManager daoMgr, final PlatformTransactionManager txManager, final XXService xService) { - public RangerTagDBRetriever(final RangerDaoManager daoMgr, final PlatformTransactionManager txManager, final XXService xService) { this.daoMgr = daoMgr; - this.txManager = txManager; - if (this.txManager != null) { - this.txTemplate = new TransactionTemplate(this.txManager); - this.txTemplate.setReadOnly(true); + + final TransactionTemplate txTemplate; + + if (txManager != null) { + txTemplate = new TransactionTemplate(txManager); + txTemplate.setReadOnly(true); } else { - this.txTemplate = null; + txTemplate = null; } - this.xService = xService; this.lookupCache = new LookupCache(); - if (this.daoMgr != null && this.xService != null) { + if (this.daoMgr != null && xService != null) { RangerPerfTracer perf = null; @@ -81,13 +83,11 @@ public RangerTagDBRetriever(final RangerDaoManager daoMgr, final PlatformTransac perf = RangerPerfTracer.getPerfTracer(PERF_LOG, "RangerTagDBReceiver.getTags(serviceName=" + xService.getName()); } - filterForServicePlugin = RangerConfiguration.getInstance().getBoolean(OPTION_RANGER_FILTER_TAGS_FOR_SERVICE_PLUGIN, false); - - if (this.txTemplate == null) { + if (txTemplate == null) { if (LOG.isDebugEnabled()) { LOG.debug("Load Tags in the same thread and using an existing transaction"); } - if (initializeTagCache(xService) == false) { + if (!initializeTagCache(xService)) { LOG.error("Failed to get tags for service:[" + xService.getName() + "] in the same thread and using an existing transaction"); } } else { @@ -110,20 +110,48 @@ public RangerTagDBRetriever(final RangerDaoManager daoMgr, final PlatformTransac } } - public List getTagResourceMaps() { - return tagResourceMaps; - } - - public List getServiceResources() { + List getServiceResources() { return serviceResources; } - public Map getTagDefs() { + Map getTagDefs() { return tagDefs; } - public Map getTags() { - return tags; + Map getTags() { + + Map ret = new HashMap<>(); + + if (CollectionUtils.isNotEmpty(serviceResources)) { + for (RangerServiceResource serviceResource : serviceResources) { + List tags = lookupCache.serviceResourceToTags.get(serviceResource.getId()); + if (CollectionUtils.isNotEmpty(tags)) { + for (RangerTag tag : tags) { + ret.put(tag.getId(), tag); + } + } + } + } + + return ret; + } + + Map> getResourceToTagIds() { + Map> ret = new HashMap<>(); + + if (CollectionUtils.isNotEmpty(serviceResources)) { + for (RangerServiceResource serviceResource : serviceResources) { + List tags = lookupCache.serviceResourceToTags.get(serviceResource.getId()); + if (CollectionUtils.isNotEmpty(tags)) { + List tagIds = new ArrayList<>(); + ret.put(serviceResource.getId(), tagIds); + for (RangerTag tag : tags) { + tagIds.add(tag.getId()); + } + } + } + } + return ret; } private boolean initializeTagCache(XXService xService) { @@ -131,69 +159,23 @@ private boolean initializeTagCache(XXService xService) { try { TagRetrieverServiceResourceContext serviceResourceContext = new TagRetrieverServiceResourceContext(xService); TagRetrieverTagDefContext tagDefContext = new TagRetrieverTagDefContext(xService); - TagRetrieverTagContext tagContext = new TagRetrieverTagContext(xService); serviceResources = serviceResourceContext.getAllServiceResources(); tagDefs = tagDefContext.getAllTagDefs(); - tags = tagContext.getAllTags(); - - tagResourceMaps = getAllTagResourceMaps(); ret = true; } catch (Exception ex) { - LOG.error("Failed to get tags for service:[" + xService.getName() + "]"); + LOG.error("Failed to get tags for service:[" + xService.getName() + "]", ex); serviceResources = null; tagDefs = null; - tags = null; - tagResourceMaps = null; ret = false; } return ret; } - private List getAllTagResourceMaps() { - - List xTagResourceMaps = filterForServicePlugin ? daoMgr.getXXTagResourceMap().findForServicePlugin(xService.getId()) : daoMgr.getXXTagResourceMap().findByServiceId(xService.getId()); - - ListIterator iterTagResourceMap = xTagResourceMaps.listIterator(); - - List ret = new ArrayList(); - - while (iterTagResourceMap.hasNext()) { - - XXTagResourceMap xTagResourceMap = iterTagResourceMap.next(); - - if (xTagResourceMap != null) { - - RangerTagResourceMap tagResourceMap = new RangerTagResourceMap(); - - tagResourceMap.setId(xTagResourceMap.getId()); - tagResourceMap.setGuid(xTagResourceMap.getGuid()); - tagResourceMap.setCreatedBy(lookupCache.getUserScreenName(xTagResourceMap.getAddedByUserId())); - tagResourceMap.setUpdatedBy(lookupCache.getUserScreenName(xTagResourceMap.getUpdatedByUserId())); - tagResourceMap.setCreateTime(xTagResourceMap.getCreateTime()); - tagResourceMap.setUpdateTime(xTagResourceMap.getUpdateTime()); - tagResourceMap.setResourceId(xTagResourceMap.getResourceId()); - tagResourceMap.setTagId(xTagResourceMap.getTagId()); - - ret.add(tagResourceMap); - } - } - return ret; - } - - static List asList(T obj) { - List ret = new ArrayList(); - - if (obj != null) { - ret.add(obj); - } - - return ret; - } private class LookupCache { - final Map userScreenNames = new HashMap(); - final Map resourceDefs = new HashMap(); + final Map userScreenNames = new HashMap<>(); + final Map> serviceResourceToTags = new HashMap<>(); String getUserScreenName(Long userId) { String ret = null; @@ -229,25 +211,6 @@ String getUserScreenName(Long userId) { return ret; } - String getResourceName(Long resourceDefId) { - String ret = null; - - if (resourceDefId != null) { - ret = resourceDefs.get(resourceDefId); - - if (ret == null) { - XXResourceDef xResourceDef = daoMgr.getXXResourceDef().getById(resourceDefId); - - if (xResourceDef != null) { - ret = xResourceDef.getName(); - - resourceDefs.put(resourceDefId, ret); - } - } - } - - return ret; - } } private class TagLoaderThread extends Thread { @@ -287,39 +250,19 @@ private class TagRetrieverServiceResourceContext { final XXService service; final ListIterator iterServiceResource; - final ListIterator iterServiceResourceElement; - final ListIterator iterServiceResourceElementValue; TagRetrieverServiceResourceContext(XXService xService) { Long serviceId = xService == null ? null : xService.getId(); - - List xServiceResources = filterForServicePlugin ? daoMgr.getXXServiceResource().findForServicePlugin(serviceId) : daoMgr.getXXServiceResource().findTaggedResourcesInServiceId(serviceId); - List xServiceResourceElements = filterForServicePlugin ? daoMgr.getXXServiceResourceElement().findForServicePlugin(serviceId) : daoMgr.getXXServiceResourceElement().findTaggedResourcesInServiceId(serviceId); - List xServiceResourceElementValues = filterForServicePlugin ? daoMgr.getXXServiceResourceElementValue().findForServicePlugin(serviceId) : daoMgr.getXXServiceResourceElementValue().findTaggedResourcesInServiceId(serviceId); - this.service = xService; - this.iterServiceResource = xServiceResources.listIterator(); - this.iterServiceResourceElement = xServiceResourceElements.listIterator(); - this.iterServiceResourceElementValue = xServiceResourceElementValues.listIterator(); - - } - TagRetrieverServiceResourceContext(XXServiceResource xServiceResource, XXService xService) { - Long resourceId = xServiceResource == null ? null : xServiceResource.getId(); + List xServiceResources = daoMgr.getXXServiceResource().findTaggedResourcesInServiceId(serviceId); - List xServiceResources = asList(xServiceResource); - List xServiceResourceElements = daoMgr.getXXServiceResourceElement().findByResourceId(resourceId); - List xServiceResourceElementValues = daoMgr.getXXServiceResourceElementValue().findByResourceId(resourceId); - - this.service = xService; this.iterServiceResource = xServiceResources.listIterator(); - this.iterServiceResourceElement = xServiceResourceElements.listIterator(); - this.iterServiceResourceElementValue = xServiceResourceElementValues.listIterator(); } List getAllServiceResources() { - List ret = new ArrayList(); + List ret = new ArrayList<>(); while (iterServiceResource.hasNext()) { RangerServiceResource serviceResource = getNextServiceResource(); @@ -329,12 +272,6 @@ List getAllServiceResources() { } } - if (!hasProcessedAll()) { - LOG.warn("getAllServiceResources(): perhaps one or more serviceResources got updated during retrieval. Using fallback ... "); - - ret = getServiceResourcesBySecondary(); - } - return ret; } @@ -344,7 +281,7 @@ RangerServiceResource getNextServiceResource() { if (iterServiceResource.hasNext()) { XXServiceResource xServiceResource = iterServiceResource.next(); - if (xServiceResource != null) { + if (xServiceResource != null && StringUtils.isNotEmpty(xServiceResource.getTags())) { ret = new RangerServiceResource(); ret.setId(xServiceResource.getId()); @@ -357,108 +294,35 @@ RangerServiceResource getNextServiceResource() { ret.setVersion(xServiceResource.getVersion()); ret.setResourceSignature(xServiceResource.getResourceSignature()); - getServiceResourceElements(ret); - } - } - - return ret; - } - - void getServiceResourceElements(RangerServiceResource serviceResource) { - while (iterServiceResourceElement.hasNext()) { - XXServiceResourceElement xServiceResourceElement = iterServiceResourceElement.next(); - - if (xServiceResourceElement.getResourceId().equals(serviceResource.getId())) { - RangerPolicyResource resource = new RangerPolicyResource(); - - resource.setIsExcludes(xServiceResourceElement.getIsExcludes()); - resource.setIsRecursive(xServiceResourceElement.getIsRecursive()); - - while (iterServiceResourceElementValue.hasNext()) { - XXServiceResourceElementValue xServiceResourceElementValue = iterServiceResourceElementValue.next(); + Map serviceResourceElements = gsonBuilder.fromJson(xServiceResource.getServiceResourceElements(), RangerServiceResourceService.subsumedDataType); + ret.setResourceElements(serviceResourceElements); - if (xServiceResourceElementValue.getResElementId().equals(xServiceResourceElement.getId())) { - resource.getValues().add(xServiceResourceElementValue.getValue()); - } else { - if (iterServiceResourceElementValue.hasPrevious()) { - iterServiceResourceElementValue.previous(); - } - break; - } - } - - serviceResource.getResourceElements().put(lookupCache.getResourceName(xServiceResourceElement.getResDefId()), resource); - } else if (xServiceResourceElement.getResourceId().compareTo(serviceResource.getId()) > 0) { - if (iterServiceResourceElement.hasPrevious()) { - iterServiceResourceElement.previous(); - } - break; + List tags = gsonBuilder.fromJson(xServiceResource.getTags(), RangerServiceResourceService.duplicatedDataType); + lookupCache.serviceResourceToTags.put(xServiceResource.getId(), tags); } } - } - boolean hasProcessedAll() { - boolean moreToProcess = iterServiceResource.hasNext() - || iterServiceResourceElement.hasNext() - || iterServiceResourceElementValue.hasNext(); - return !moreToProcess; - } - - List getServiceResourcesBySecondary() { - List ret = null; - - if (service != null) { - List xServiceResources = filterForServicePlugin ? daoMgr.getXXServiceResource().findForServicePlugin(service.getId()) : daoMgr.getXXServiceResource().findTaggedResourcesInServiceId(service.getId()); - - if (CollectionUtils.isNotEmpty(xServiceResources)) { - ret = new ArrayList(xServiceResources.size()); - - for (XXServiceResource xServiceResource : xServiceResources) { - TagRetrieverServiceResourceContext ctx = new TagRetrieverServiceResourceContext(xServiceResource, service); - - RangerServiceResource serviceResource = ctx.getNextServiceResource(); - - if (serviceResource != null) { - ret.add(serviceResource); - } - } - } - } return ret; } + } private class TagRetrieverTagDefContext { final XXService service; final ListIterator iterTagDef; - final ListIterator iterTagAttributeDef; - TagRetrieverTagDefContext(XXService xService) { Long serviceId = xService == null ? null : xService.getId(); - List xTagDefs = filterForServicePlugin ? daoMgr.getXXTagDef().findForServicePlugin(serviceId) : daoMgr.getXXTagDef().findByServiceId(serviceId); - List xTagAttributeDefs = filterForServicePlugin ? daoMgr.getXXTagAttributeDef().findForServicePlugin(serviceId) : daoMgr.getXXTagAttributeDef().findByServiceId(serviceId); - - this.service = xService; - this.iterTagDef = xTagDefs.listIterator(); - this.iterTagAttributeDef = xTagAttributeDefs.listIterator(); - } - - TagRetrieverTagDefContext(XXTagDef xTagDef, XXService xService) { - Long tagDefId = xTagDef == null ? null : xTagDef.getId(); - - List xTagDefs = asList(xTagDef); - List xTagAttributeDefs = daoMgr.getXXTagAttributeDef().findByTagDefId(tagDefId); + List xTagDefs = daoMgr.getXXTagDef().findByServiceId(serviceId); this.service = xService; this.iterTagDef = xTagDefs.listIterator(); - this.iterTagAttributeDef = xTagAttributeDefs.listIterator(); } Map getAllTagDefs() { - Map ret = new HashMap(); + Map ret = new HashMap<>(); while (iterTagDef.hasNext()) { RangerTagDef tagDef = getNextTagDef(); @@ -468,13 +332,6 @@ Map getAllTagDefs() { } } - if (!hasProcessedAllTagDefs()) { - LOG.warn("getAllTagDefs(): perhaps one or more tag-definitions got updated during retrieval. Using fallback ... "); - - ret = getTagDefsBySecondary(); - - } - return ret; } @@ -497,192 +354,15 @@ RangerTagDef getNextTagDef() { ret.setVersion(xTagDef.getVersion()); ret.setName(xTagDef.getName()); ret.setSource(xTagDef.getSource()); - - getTagAttributeDefs(ret); + List attributeDefs = gsonBuilder.fromJson(xTagDef.getTagAttrDefs(), RangerTagDBRetriever.subsumedDataType); + ret.setAttributeDefs(attributeDefs); } } return ret; } - void getTagAttributeDefs(RangerTagDef tagDef) { - while (iterTagAttributeDef.hasNext()) { - XXTagAttributeDef xTagAttributeDef = iterTagAttributeDef.next(); - - if (xTagAttributeDef.getTagDefId().equals(tagDef.getId())) { - RangerTagDef.RangerTagAttributeDef tagAttributeDef = new RangerTagDef.RangerTagAttributeDef(); - - tagAttributeDef.setName(xTagAttributeDef.getName()); - tagAttributeDef.setType(xTagAttributeDef.getType()); - - tagDef.getAttributeDefs().add(tagAttributeDef); - } else if (xTagAttributeDef.getTagDefId().compareTo(tagDef.getId()) > 0) { - if (iterTagAttributeDef.hasPrevious()) { - iterTagAttributeDef.previous(); - } - break; - } - } - } - - boolean hasProcessedAllTagDefs() { - boolean moreToProcess = iterTagAttributeDef.hasNext(); - return !moreToProcess; - } - - Map getTagDefsBySecondary() { - Map ret = null; - - if (service != null) { - List xTagDefs = daoMgr.getXXTagDef().findByServiceId(service.getId()); - - if (CollectionUtils.isNotEmpty(xTagDefs)) { - ret = new HashMap(xTagDefs.size()); - - for (XXTagDef xTagDef : xTagDefs) { - TagRetrieverTagDefContext ctx = new TagRetrieverTagDefContext(xTagDef, service); - - RangerTagDef tagDef = ctx.getNextTagDef(); - - if (tagDef != null) { - ret.put(tagDef.getId(), tagDef); - } - } - } - } - return ret; - } } - private class TagRetrieverTagContext { - - final XXService service; - final ListIterator iterTag; - final ListIterator iterTagAttribute; - - TagRetrieverTagContext(XXService xService) { - Long serviceId = xService == null ? null : xService.getId(); - - List xTags = filterForServicePlugin ? daoMgr.getXXTag().findForServicePlugin(serviceId) : daoMgr.getXXTag().findByServiceId(serviceId); - List xTagAttributes = filterForServicePlugin ? daoMgr.getXXTagAttribute().findForServicePlugin(serviceId) : daoMgr.getXXTagAttribute().findByServiceId(serviceId); - - this.service = xService; - this.iterTag = xTags.listIterator(); - this.iterTagAttribute = xTagAttributes.listIterator(); - - } - - TagRetrieverTagContext(XXTag xTag, XXService xService) { - Long tagId = xTag == null ? null : xTag.getId(); - - List xTags = asList(xTag); - List xTagAttributes = daoMgr.getXXTagAttribute().findByTagId(tagId); - - this.service = xService; - this.iterTag = xTags.listIterator(); - this.iterTagAttribute = xTagAttributes.listIterator(); - } - - - Map getAllTags() { - Map ret = new HashMap(); - - while (iterTag.hasNext()) { - RangerTag tag = getNextTag(); - - if (tag != null) { - ret.put(tag.getId(), tag); - } - } - - if (!hasProcessedAllTags()) { - LOG.warn("getAllTags(): perhaps one or more tags got updated during retrieval. Using fallback ... "); - - ret = getTagsBySecondary(); - } - - return ret; - } - - RangerTag getNextTag() { - RangerTag ret = null; - - if (iterTag.hasNext()) { - XXTag xTag = iterTag.next(); - - if (xTag != null) { - ret = new RangerTag(); - - ret.setId(xTag.getId()); - ret.setGuid(xTag.getGuid()); - ret.setOwner(xTag.getOwner()); - ret.setCreatedBy(lookupCache.getUserScreenName(xTag.getAddedByUserId())); - ret.setUpdatedBy(lookupCache.getUserScreenName(xTag.getUpdatedByUserId())); - ret.setCreateTime(xTag.getCreateTime()); - ret.setUpdateTime(xTag.getUpdateTime()); - ret.setVersion(xTag.getVersion()); - - Map tagDefs = getTagDefs(); - if (tagDefs != null) { - RangerTagDef tagDef = tagDefs.get(xTag.getType()); - if (tagDef != null) { - ret.setType(tagDef.getName()); - } - } - - getTagAttributes(ret); - } - } - - return ret; - } - - void getTagAttributes(RangerTag tag) { - while (iterTagAttribute.hasNext()) { - XXTagAttribute xTagAttribute = iterTagAttribute.next(); - - if (xTagAttribute.getTagId().equals(tag.getId())) { - String attributeName = xTagAttribute.getName(); - String attributeValue = xTagAttribute.getValue(); - - - tag.getAttributes().put(attributeName, attributeValue); - } else if (xTagAttribute.getTagId().compareTo(tag.getId()) > 0) { - if (iterTagAttribute.hasPrevious()) { - iterTagAttribute.previous(); - } - break; - } - } - } - - boolean hasProcessedAllTags() { - boolean moreToProcess = iterTagAttribute.hasNext(); - return !moreToProcess; - } - - Map getTagsBySecondary() { - Map ret = null; - - if (service != null) { - List xTags = daoMgr.getXXTag().findByServiceId(service.getId()); - - if (CollectionUtils.isNotEmpty(xTags)) { - ret = new HashMap(xTags.size()); - - for (XXTag xTag : xTags) { - TagRetrieverTagContext ctx = new TagRetrieverTagContext(xTag, service); - - RangerTag tag = ctx.getNextTag(); - - if (tag != null) { - ret.put(tag.getId(), tag); - } - } - } - } - return ret; - } - } } diff --git a/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java b/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java index ceee8cef29..a3e02612b1 100644 --- a/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java +++ b/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java @@ -19,6 +19,12 @@ package org.apache.ranger.biz; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.UnknownHostException; +import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -27,18 +33,11 @@ import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashMap; -import java.util.LinkedHashSet; import java.util.List; import java.util.Map; -import java.util.Set; import java.util.Map.Entry; +import java.util.Set; import java.util.StringTokenizer; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.net.UnknownHostException; -import java.text.SimpleDateFormat; import javax.annotation.PostConstruct; import javax.servlet.ServletOutputStream; @@ -50,22 +49,30 @@ import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.security.SecureClientLogin; +import org.apache.hadoop.security.authentication.util.KerberosName; +import org.apache.poi.hssf.usermodel.HSSFWorkbook; +import org.apache.poi.ss.usermodel.Cell; +import org.apache.poi.ss.usermodel.CellStyle; +import org.apache.poi.ss.usermodel.Font; +import org.apache.poi.ss.usermodel.Row; +import org.apache.poi.ss.usermodel.Sheet; +import org.apache.poi.ss.usermodel.Workbook; import org.apache.ranger.audit.provider.MiscUtil; import org.apache.ranger.authorization.hadoop.config.RangerConfiguration; import org.apache.ranger.common.AppConstants; import org.apache.ranger.common.ContextUtil; import org.apache.ranger.common.MessageEnums; -import org.apache.ranger.common.RangerCommonEnums; +import org.apache.ranger.common.db.RangerTransactionSynchronizationAdapter; import org.apache.ranger.plugin.policyengine.RangerPolicyEngine; import org.apache.ranger.plugin.policyresourcematcher.RangerDefaultPolicyResourceMatcher; import org.apache.ranger.plugin.policyresourcematcher.RangerPolicyResourceMatcher; -import org.apache.ranger.plugin.resourcematcher.RangerAbstractResourceMatcher; -import org.apache.ranger.plugin.service.RangerBaseService; import org.apache.ranger.plugin.store.ServiceStore; import org.apache.ranger.plugin.util.PasswordUtils; import org.apache.ranger.common.JSONUtil; import org.apache.ranger.common.PropertiesUtil; import org.apache.ranger.common.RESTErrorUtil; +import org.apache.ranger.common.RangerConstants; import org.apache.ranger.common.RangerFactory; import org.apache.ranger.common.RangerServicePoliciesCache; import org.apache.ranger.common.RangerVersionInfo; @@ -79,15 +86,6 @@ import org.apache.ranger.db.XXEnumDefDao; import org.apache.ranger.db.XXEnumElementDefDao; import org.apache.ranger.db.XXPolicyConditionDefDao; -import org.apache.ranger.db.XXPolicyItemAccessDao; -import org.apache.ranger.db.XXPolicyItemConditionDao; -import org.apache.ranger.db.XXPolicyItemDao; -import org.apache.ranger.db.XXPolicyItemDataMaskInfoDao; -import org.apache.ranger.db.XXPolicyItemGroupPermDao; -import org.apache.ranger.db.XXPolicyItemRowFilterInfoDao; -import org.apache.ranger.db.XXPolicyItemUserPermDao; -import org.apache.ranger.db.XXPolicyResourceDao; -import org.apache.ranger.db.XXPolicyResourceMapDao; import org.apache.ranger.db.XXResourceDefDao; import org.apache.ranger.db.XXServiceConfigDefDao; import org.apache.ranger.db.XXServiceConfigMapDao; @@ -101,18 +99,11 @@ import org.apache.ranger.entity.XXDataMaskTypeDef; import org.apache.ranger.entity.XXEnumDef; import org.apache.ranger.entity.XXEnumElementDef; -import org.apache.ranger.entity.XXGroup; import org.apache.ranger.entity.XXPolicy; import org.apache.ranger.entity.XXPolicyConditionDef; -import org.apache.ranger.entity.XXPolicyItem; -import org.apache.ranger.entity.XXPolicyItemAccess; -import org.apache.ranger.entity.XXPolicyItemCondition; -import org.apache.ranger.entity.XXPolicyItemDataMaskInfo; -import org.apache.ranger.entity.XXPolicyItemGroupPerm; -import org.apache.ranger.entity.XXPolicyItemRowFilterInfo; -import org.apache.ranger.entity.XXPolicyItemUserPerm; -import org.apache.ranger.entity.XXPolicyResource; -import org.apache.ranger.entity.XXPolicyResourceMap; +import org.apache.ranger.entity.XXPolicyRefAccessType; +import org.apache.ranger.entity.XXPolicyRefCondition; +import org.apache.ranger.entity.XXPolicyRefResource; import org.apache.ranger.entity.XXResourceDef; import org.apache.ranger.entity.XXService; import org.apache.ranger.entity.XXServiceConfigDef; @@ -123,13 +114,11 @@ import org.apache.ranger.entity.XXUser; import org.apache.ranger.plugin.model.RangerPolicy; import org.apache.ranger.plugin.model.RangerPolicy.RangerDataMaskPolicyItem; -import org.apache.ranger.plugin.model.RangerPolicy.RangerRowFilterPolicyItem; import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyItem; import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyItemAccess; import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyItemCondition; -import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyItemDataMaskInfo; -import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyItemRowFilterInfo; import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyResource; +import org.apache.ranger.plugin.model.RangerPolicy.RangerRowFilterPolicyItem; import org.apache.ranger.plugin.model.RangerPolicyResourceSignature; import org.apache.ranger.plugin.model.RangerService; import org.apache.ranger.plugin.model.RangerServiceDef; @@ -144,7 +133,6 @@ import org.apache.ranger.plugin.model.RangerServiceDef.RangerRowFilterDef; import org.apache.ranger.plugin.model.RangerServiceDef.RangerServiceConfigDef; import org.apache.ranger.plugin.model.validation.RangerServiceDefHelper; -import org.apache.ranger.plugin.policyevaluator.RangerPolicyItemEvaluator; import org.apache.ranger.plugin.store.AbstractServiceStore; import org.apache.ranger.plugin.store.EmbeddedServiceDefsUtil; import org.apache.ranger.plugin.store.PList; @@ -161,15 +149,14 @@ import org.apache.ranger.service.RangerServiceDefWithAssignedIdService; import org.apache.ranger.service.RangerServiceService; import org.apache.ranger.service.RangerServiceWithAssignedIdService; -import org.apache.ranger.service.XGroupService; import org.apache.ranger.service.XUserService; import org.apache.ranger.view.RangerExportPolicyList; import org.apache.ranger.view.RangerPolicyList; import org.apache.ranger.view.RangerServiceDefList; import org.apache.ranger.view.RangerServiceList; -import org.apache.ranger.view.VXGroup; import org.apache.ranger.view.VXString; import org.apache.ranger.view.VXUser; +import org.codehaus.jettison.json.JSONException; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Component; @@ -177,14 +164,6 @@ import org.springframework.transaction.TransactionStatus; import org.springframework.transaction.support.TransactionCallback; import org.springframework.transaction.support.TransactionTemplate; -import org.apache.poi.hssf.usermodel.HSSFWorkbook; -import org.apache.poi.ss.usermodel.Cell; -import org.apache.poi.ss.usermodel.CellStyle; -import org.apache.poi.ss.usermodel.Font; -import org.apache.poi.ss.usermodel.Row; -import org.apache.poi.ss.usermodel.Sheet; -import org.apache.poi.ss.usermodel.Workbook; -import org.codehaus.jettison.json.JSONException; import com.google.gson.Gson; @@ -192,26 +171,38 @@ public class ServiceDBStore extends AbstractServiceStore { private static final Log LOG = LogFactory.getLog(ServiceDBStore.class); + public static final String RANGER_TAG_EXPIRY_CONDITION_NAME = "accessed-after-expiry"; + + private static final String ADMIN_USER_PRINCIPAL = "ranger.admin.kerberos.principal"; + private static final String ADMIN_USER_KEYTAB = "ranger.admin.kerberos.keytab"; + private static final String LOOKUP_PRINCIPAL = "ranger.lookup.kerberos.principal"; + private static final String LOOKUP_KEYTAB = "ranger.lookup.kerberos.keytab"; + private static final String RANGER_AUTH_TYPE = "hadoop.security.authentication"; + private static final String AMBARI_SERVICE_CHECK_USER = "ambari.service.check.user"; + + private static final String KERBEROS_TYPE = "kerberos"; + private static final String POLICY_ALLOW_EXCLUDE = "Policy Allow:Exclude"; - //private static final String POLICY_ALLOW_INCLUDE = "Policy Allow:Include"; - private static final String POLICY_DENY_EXCLUDE = "Policy Deny:Exclude"; - private static final String POLICY_DENY_INCLUDE = "Policy Deny:Include"; - - private static String LOCAL_HOSTNAME = "unknown"; - private static final String HOSTNAME = "Host name"; - private static final String USER_NAME = "Exported by"; + private static final String POLICY_ALLOW_INCLUDE = "Policy Allow:Include"; + private static final String POLICY_DENY_EXCLUDE = "Policy Deny:Exclude"; + private static final String POLICY_DENY_INCLUDE = "Policy Deny:Include"; + + private static String LOCAL_HOSTNAME = "unknown"; + private static final String HOSTNAME = "Host name"; + private static final String USER_NAME = "Exported by"; private static final String RANGER_VERSION = "Ranger apache version"; - private static final String TIMESTAMP = "Export time"; + private static final String TIMESTAMP = "Export time"; - private static final String AMBARI_SERVICE_CHECK_USER = "ambari.service.check.user"; - private static final String SERVICE_ADMIN_USERS = "service.admin.users"; - - public static final String CRYPT_ALGO = PropertiesUtil.getProperty("ranger.password.encryption.algorithm", PasswordUtils.DEFAULT_CRYPT_ALGO); - public static final String ENCRYPT_KEY = PropertiesUtil.getProperty("ranger.password.encryption.key", PasswordUtils.DEFAULT_ENCRYPT_KEY); - public static final String SALT = PropertiesUtil.getProperty("ranger.password.salt", PasswordUtils.DEFAULT_SALT); - public static final Integer ITERATION_COUNT = PropertiesUtil.getIntProperty("ranger.password.iteration.count", PasswordUtils.DEFAULT_ITERATION_COUNT); + private static final String AUDITTOHDFS_KMS_PATH = "/ranger/audit/kms"; + private static final String AUDITTOHDFS_POLICY_NAME = "kms-audit-path"; + private static final String SERVICE_ADMIN_USERS = "service.admin.users"; - static { + public static final String CRYPT_ALGO = PropertiesUtil.getProperty("ranger.password.encryption.algorithm", PasswordUtils.DEFAULT_CRYPT_ALGO); + public static final String ENCRYPT_KEY = PropertiesUtil.getProperty("ranger.password.encryption.key", PasswordUtils.DEFAULT_ENCRYPT_KEY); + public static final String SALT = PropertiesUtil.getProperty("ranger.password.salt", PasswordUtils.DEFAULT_SALT); + public static final Integer ITERATION_COUNT = PropertiesUtil.getIntProperty("ranger.password.iteration.count", PasswordUtils.DEFAULT_ITERATION_COUNT); + + static { try { LOCAL_HOSTNAME = java.net.InetAddress.getLocalHost().getCanonicalHostName(); } catch (UnknownHostException e) { @@ -247,7 +238,7 @@ public class ServiceDBStore extends AbstractServiceStore { XUserMgr xUserMgr; @Autowired - XGroupService xGroupService; + PolicyRefUpdater policyRefUpdater; @Autowired RangerDataHistService dataHistService; @@ -277,7 +268,14 @@ public class ServiceDBStore extends AbstractServiceStore { @Autowired ServiceMgr serviceMgr; + @Autowired + AssetMgr assetMgr; + + @Autowired + RangerTransactionSynchronizationAdapter transactionSynchronizationAdapter; + private static volatile boolean legacyServiceDefsInitDone = false; + private Boolean populateExistingBaseFields = false; public static final String HIDDEN_PASSWORD_STR = "*****"; @@ -288,7 +286,6 @@ public class ServiceDBStore extends AbstractServiceStore { private ServicePredicateUtil predicateUtil = null; - @Override public void init() throws Exception { if (LOG.isDebugEnabled()) { @@ -735,8 +732,8 @@ private void updateChildObjectsOfServiceDef(XXServiceDef createdSvcDef, List policyResList = daoMgr.getXXPolicyResource().findByResDefId(xRes.getId()); - if (!stringUtil.isEmpty(policyResList)) { + List xxPolicyRefResource = daoMgr.getXXPolicyRefResource().findByResourceDefID(xRes.getId()); + if (!stringUtil.isEmpty(xxPolicyRefResource)) { throw restErrorUtil.createRESTException("Policy/Policies are referring to this resource: " + xRes.getName() + ". Please remove such references from policy before updating service-def.", MessageEnums.DATA_NOT_UPDATABLE); @@ -822,8 +819,8 @@ private void updateChildObjectsOfServiceDef(XXServiceDef createdSvcDef, List polItemAccessList = daoMgr.getXXPolicyItemAccess().findByType(xAccess.getId()); - if(!stringUtil.isEmpty(polItemAccessList)) { + List policyRefAccessTypeList = daoMgr.getXXPolicyRefAccessType().findByAccessTypeDefId(xAccess.getId()); + if(!stringUtil.isEmpty(policyRefAccessTypeList)) { throw restErrorUtil.createRESTException("Policy/Policies are referring to this access-type: " + xAccess.getName() + ". Please remove such references from policy before updating service-def.", MessageEnums.DATA_NOT_UPDATABLE); @@ -865,15 +862,14 @@ private void updateChildObjectsOfServiceDef(XXServiceDef createdSvcDef, List policyItemCondList = daoMgr.getXXPolicyItemCondition() - .findByPolicyConditionDefId(xCondition.getId()); - if(!stringUtil.isEmpty(policyItemCondList)) { + List xxPolicyRefConditions = daoMgr.getXXPolicyRefCondition().findByConditionDefId(xCondition.getId()); + if(!stringUtil.isEmpty(xxPolicyRefConditions)) { throw restErrorUtil.createRESTException("Policy/Policies are referring to this policy-condition: " + xCondition.getName() + ". Please remove such references from policy before updating service-def.", MessageEnums.DATA_NOT_UPDATABLE); } - for(XXPolicyItemCondition policyItemCond : policyItemCondList) { - daoMgr.getXXPolicyItemCondition().remove(policyItemCond); + for(XXPolicyRefCondition xxPolicyRefCondition : xxPolicyRefConditions) { + daoMgr.getXXPolicyRefCondition().remove(xxPolicyRefCondition); } xxPolCondDao.remove(xCondition); } @@ -1206,9 +1202,9 @@ public void deleteServiceDef(Long serviceDefId, Boolean forceDelete) throws Exce List policyCondList = policyCondDao.findByServiceDefId(serviceDefId); for (XXPolicyConditionDef policyCond : policyCondList) { - List policyItemCondList = daoMgr.getXXPolicyItemCondition().findByPolicyConditionDefId(policyCond.getId()); - for (XXPolicyItemCondition policyItemCond : policyItemCondList) { - daoMgr.getXXPolicyItemCondition().remove(policyItemCond); + List xxPolicyRefConditions = daoMgr.getXXPolicyRefCondition().findByConditionDefId(policyCond.getId()); + for (XXPolicyRefCondition XXPolicyRefCondition : xxPolicyRefConditions) { + daoMgr.getXXPolicyRefCondition().remove(XXPolicyRefCondition); } policyCondDao.remove(policyCond); } @@ -1258,37 +1254,25 @@ public void deleteXXAccessTypeDef(XXAccessTypeDef xAccess) { daoMgr.getXXAccessTypeDefGrants().remove(atdGrant); } - List policyItemAccessList = daoMgr.getXXPolicyItemAccess().findByType(xAccess.getId()); - for (XXPolicyItemAccess policyItemAccess : policyItemAccessList) { - daoMgr.getXXPolicyItemAccess().remove(policyItemAccess); + List policyRefAccessTypeList = daoMgr.getXXPolicyRefAccessType().findByAccessTypeDefId(xAccess.getId()); + for (XXPolicyRefAccessType xxPolicyRefAccessType : policyRefAccessTypeList) { + daoMgr.getXXPolicyRefAccessType().remove(xxPolicyRefAccessType); } daoMgr.getXXAccessTypeDef().remove(xAccess); } public void deleteXXResourceDef(XXResourceDef xRes) { - List xChildObjs = daoMgr.getXXResourceDef().findByParentResId(xRes.getId()); for(XXResourceDef childRes : xChildObjs) { deleteXXResourceDef(childRes); } - - List xxResources = daoMgr.getXXPolicyResource().findByResDefId(xRes.getId()); - for (XXPolicyResource xPolRes : xxResources) { - deleteXXPolicyResource(xPolRes); + List xxPolicyRefResources = daoMgr.getXXPolicyRefResource().findByResourceDefID(xRes.getId()); + for (XXPolicyRefResource xPolRefRes : xxPolicyRefResources) { + daoMgr.getXXPolicyRefResource().remove(xPolRefRes); } - daoMgr.getXXResourceDef().remove(xRes); } - public void deleteXXPolicyResource(XXPolicyResource xPolRes) { - List polResMapList = daoMgr.getXXPolicyResourceMap().findByPolicyResId(xPolRes.getId()); - XXPolicyResourceMapDao polResMapDao = daoMgr.getXXPolicyResourceMap(); - for (XXPolicyResourceMap xxPolResMap : polResMapList) { - polResMapDao.remove(xxPolResMap); - } - daoMgr.getXXPolicyResource().remove(xPolRes); - } - @Override public RangerServiceDef getServiceDef(Long id) throws Exception { if (LOG.isDebugEnabled()) { @@ -1439,10 +1423,7 @@ public RangerService createService(RangerService service) throws Exception { xConfMap.setServiceId(xCreatedService.getId()); xConfMap.setConfigkey(configKey); xConfMap.setConfigvalue(configValue); - xConfMapDao.create(xConfMap); - } - if (LOG.isDebugEnabled()) { - LOG.debug("vXUser:[" + vXUser + "]"); + xConfMap = xConfMapDao.create(xConfMap); } RangerService createdService = svcService.getPopulatedViewObject(xCreatedService); @@ -1457,7 +1438,7 @@ public RangerService createService(RangerService service) throws Exception { bizUtil.createTrxLog(trxLogList); if (createDefaultPolicy) { - createDefaultPolicies(createdService); + createDefaultPolicies(xCreatedService, vXUser); } return createdService; @@ -1599,25 +1580,21 @@ public RangerService updateService(RangerService service, Map op + userName + "] please use existing user", MessageEnums.OPER_NO_PERMISSION); } vXUser = xUserMgr.createServiceConfigUser(userName); + if (LOG.isDebugEnabled()) { + LOG.debug("Service config user created:[" + vXUser + "]"); + } } } - + if (StringUtils.equalsIgnoreCase(configKey, CONFIG_KEY_PASSWORD)) { if (StringUtils.equalsIgnoreCase(configValue, HIDDEN_PASSWORD_STR)) { - String[] crypt_algo_array = null; - if (configValue.contains(",")) { - crypt_algo_array = configValue.split(","); - } - if (oldPassword != null && oldPassword.contains(",")) { - String encryptKey = null; - String salt = null; - int iterationCount = 0; - crypt_algo_array = oldPassword.split(","); - String OLD_CRYPT_ALGO = crypt_algo_array[0]; - encryptKey = crypt_algo_array[1]; - salt = crypt_algo_array[2]; - iterationCount = Integer.parseInt(crypt_algo_array[3]); - + if (oldPassword != null && oldPassword.contains(",")) { + String[] crypt_algo_array = oldPassword.split(","); + String OLD_CRYPT_ALGO = crypt_algo_array[0]; + String encryptKey = crypt_algo_array[1]; + String salt = crypt_algo_array[2]; + int iterationCount = Integer.parseInt(crypt_algo_array[3]); + if (!OLD_CRYPT_ALGO.equalsIgnoreCase(CRYPT_ALGO)) { String decryptedPwd = PasswordUtils.decryptPassword(oldPassword); String paddingString = CRYPT_ALGO + "," + encryptKey + "," + salt + "," + iterationCount; @@ -1642,15 +1619,13 @@ public RangerService updateService(RangerService service, Map op } } } + XXServiceConfigMap xConfMap = new XXServiceConfigMap(); xConfMap = (XXServiceConfigMap) rangerAuditFields.populateAuditFields(xConfMap, xUpdService); xConfMap.setServiceId(service.getId()); xConfMap.setConfigkey(configKey); xConfMap.setConfigvalue(configValue); - xConfMapDao.create(xConfMap); - } - if (LOG.isDebugEnabled()) { - LOG.debug("vXUser:[" + vXUser + "]"); + xConfMap = xConfMapDao.create(xConfMap); } RangerService updService = svcService.getPopulatedViewObject(xUpdService); dataHistService.createObjectDataHistory(updService, RangerDataHistService.ACTION_UPDATE); @@ -1672,11 +1647,11 @@ public void deleteService(Long id) throws Exception { } List policies = daoMgr.getXXPolicy().findByServiceId(service.getId()); - RangerPolicy rangerPolicy =null; + //RangerPolicy rangerPolicy =null; for(XXPolicy policy : policies) { LOG.info("Deleting Policy, policyName: " + policy.getName()); - rangerPolicy = getPolicy(policy.getId()); - deletePolicy(rangerPolicy); + //rangerPolicy = getPolicy(policy.getId()); + deletePolicy(policy.getId()); } XXServiceConfigMapDao configDao = daoMgr.getXXServiceConfigMap(); @@ -1834,14 +1809,6 @@ public RangerPolicy createPolicy(RangerPolicy policy) throws Exception { throw new Exception("policy already exists: ServiceName=" + policy.getService() + "; PolicyName=" + policy.getName() + ". ID=" + existing.getId()); } - Map resources = policy.getResources(); - List policyItems = policy.getPolicyItems(); - List denyPolicyItems = policy.getDenyPolicyItems(); - List allowExceptions = policy.getAllowExceptions(); - List denyExceptions = policy.getDenyExceptions(); - List dataMaskItems = policy.getDataMaskPolicyItems(); - List rowFilterItems = policy.getRowFilterPolicyItems(); - policy.setVersion(Long.valueOf(1)); updatePolicySignature(policy); @@ -1859,14 +1826,8 @@ public RangerPolicy createPolicy(RangerPolicy policy) throws Exception { } XXPolicy xCreatedPolicy = daoMgr.getXXPolicy().getById(policy.getId()); + policyRefUpdater.createNewPolMappingForRefTable(policy, xCreatedPolicy, xServiceDef); - createNewResourcesForPolicy(policy, xCreatedPolicy, resources); - createNewPolicyItemsForPolicy(policy, xCreatedPolicy, policyItems, xServiceDef, RangerPolicyItemEvaluator.POLICY_ITEM_TYPE_ALLOW); - createNewPolicyItemsForPolicy(policy, xCreatedPolicy, denyPolicyItems, xServiceDef, RangerPolicyItemEvaluator.POLICY_ITEM_TYPE_DENY); - createNewPolicyItemsForPolicy(policy, xCreatedPolicy, allowExceptions, xServiceDef, RangerPolicyItemEvaluator.POLICY_ITEM_TYPE_ALLOW_EXCEPTIONS); - createNewPolicyItemsForPolicy(policy, xCreatedPolicy, denyExceptions, xServiceDef, RangerPolicyItemEvaluator.POLICY_ITEM_TYPE_DENY_EXCEPTIONS); - createNewDataMaskPolicyItemsForPolicy(policy, xCreatedPolicy, dataMaskItems, xServiceDef, RangerPolicyItemEvaluator.POLICY_ITEM_TYPE_DATAMASK); - createNewRowFilterPolicyItemsForPolicy(policy, xCreatedPolicy, rowFilterItems, xServiceDef, RangerPolicyItemEvaluator.POLICY_ITEM_TYPE_ROWFILTER); handlePolicyUpdate(service, true); RangerPolicy createdPolicy = policyService.getPopulatedViewObject(xCreatedPolicy); dataHistService.createObjectDataHistory(createdPolicy, RangerDataHistService.ACTION_CREATE); @@ -1877,29 +1838,6 @@ public RangerPolicy createPolicy(RangerPolicy policy) throws Exception { return createdPolicy; } - private boolean validatePolicyItem(List policyItems) { - boolean isPolicyItemValid=true; - for (RangerPolicyItem policyItem : policyItems) { - if (policyItem != null) { - if (CollectionUtils.isEmpty(policyItem.getUsers()) - || (policyItem.getUsers() != null) && policyItem.getUsers().contains(null) - || (policyItem.getUsers().contains(""))) { - if (CollectionUtils.isEmpty(policyItem.getGroups()) - || (policyItem.getGroups() != null) && policyItem.getGroups().contains(null) - || (policyItem.getGroups().contains(""))) { - - isPolicyItemValid = false; - } - } - if (CollectionUtils.isEmpty(policyItem.getAccesses()) - || (policyItem.getAccesses() != null) && policyItem.getAccesses().contains(null)) { - isPolicyItemValid = false; - } - } - } - return isPolicyItemValid; - } - @Override public RangerPolicy updatePolicy(RangerPolicy policy) throws Exception { if(LOG.isDebugEnabled()) { @@ -1937,14 +1875,7 @@ public RangerPolicy updatePolicy(RangerPolicy policy) throws Exception { throw new Exception("another policy already exists with name '" + policy.getName() + "'. ID=" + newNamePolicy.getId()); } } - Map newResources = policy.getResources(); - List policyItems = policy.getPolicyItems(); - List denyPolicyItems = policy.getDenyPolicyItems(); - List allowExceptions = policy.getAllowExceptions(); - List denyExceptions = policy.getDenyExceptions(); - List dataMaskPolicyItems = policy.getDataMaskPolicyItems(); - List rowFilterItems = policy.getRowFilterPolicyItems(); - + policy.setCreateTime(xxExisting.getCreateTime()); policy.setGuid(xxExisting.getGuid()); policy.setVersion(xxExisting.getVersion()); @@ -1958,26 +1889,19 @@ public RangerPolicy updatePolicy(RangerPolicy policy) throws Exception { isTagVersionUpdateNeeded = existing.getIsEnabled() ? !policy.getIsEnabled() : policy.getIsEnabled(); isTagVersionUpdateNeeded = isTagVersionUpdateNeeded || !StringUtils.equals(existing.getResourceSignature(), policy.getResourceSignature()); } + policy = policyService.update(policy); XXPolicy newUpdPolicy = daoMgr.getXXPolicy().getById(policy.getId()); - deleteExistingPolicyResources(policy); - deleteExistingPolicyItems(policy); - - createNewResourcesForPolicy(policy, newUpdPolicy, newResources); - createNewPolicyItemsForPolicy(policy, newUpdPolicy, policyItems, xServiceDef, RangerPolicyItemEvaluator.POLICY_ITEM_TYPE_ALLOW); - createNewPolicyItemsForPolicy(policy, newUpdPolicy, denyPolicyItems, xServiceDef, RangerPolicyItemEvaluator.POLICY_ITEM_TYPE_DENY); - createNewPolicyItemsForPolicy(policy, newUpdPolicy, allowExceptions, xServiceDef, RangerPolicyItemEvaluator.POLICY_ITEM_TYPE_ALLOW_EXCEPTIONS); - createNewPolicyItemsForPolicy(policy, newUpdPolicy, denyExceptions, xServiceDef, RangerPolicyItemEvaluator.POLICY_ITEM_TYPE_DENY_EXCEPTIONS); - createNewDataMaskPolicyItemsForPolicy(policy, newUpdPolicy, dataMaskPolicyItems, xServiceDef, RangerPolicyItemEvaluator.POLICY_ITEM_TYPE_DATAMASK); - createNewRowFilterPolicyItemsForPolicy(policy, newUpdPolicy, rowFilterItems, xServiceDef, RangerPolicyItemEvaluator.POLICY_ITEM_TYPE_ROWFILTER); - - handlePolicyUpdate(service, isTagVersionUpdateNeeded); + policyRefUpdater.cleanupRefTables(policy); + policyRefUpdater.createNewPolMappingForRefTable(policy, newUpdPolicy, xServiceDef); + + handlePolicyUpdate(service, isTagVersionUpdateNeeded); RangerPolicy updPolicy = policyService.getPopulatedViewObject(newUpdPolicy); dataHistService.createObjectDataHistory(updPolicy, RangerDataHistService.ACTION_UPDATE); bizUtil.createTrxLog(trxLogList); - + return updPolicy; } @@ -2011,10 +1935,7 @@ public void deletePolicy(Long policyId) throws Exception { policy.setVersion(version); List trxLogList = policyService.getTransactionLog(policy, RangerPolicyService.OPERATION_DELETE_CONTEXT); - - deleteExistingPolicyItems(policy); - deleteExistingPolicyResources(policy); - + policyRefUpdater.cleanupRefTables(policy); policyService.delete(policy); handlePolicyUpdate(service, true); @@ -2024,7 +1945,7 @@ public void deletePolicy(Long policyId) throws Exception { LOG.info("Policy Deleted Successfully. PolicyName : " + policyName); } - +/* public void deletePolicy(RangerPolicy policy) throws Exception { if(policy == null) { return; @@ -2045,13 +1966,11 @@ public void deletePolicy(RangerPolicy policy) throws Exception { } policy.setVersion(version); List trxLogList = policyService.getTransactionLog(policy, RangerPolicyService.OPERATION_DELETE_CONTEXT); - deleteExistingPolicyItemsNative(policy); - deleteExistingPolicyResourcesNative(policy); daoMgr.getXXPolicy().deletePolicyIDReference("id",policy.getId()); handlePolicyUpdate(service, true); dataHistService.createObjectDataHistory(policy, RangerDataHistService.ACTION_DELETE); bizUtil.createTrxLog(trxLogList); - } + }*/ @Override public RangerPolicy getPolicy(Long id) throws Exception { @@ -2087,7 +2006,6 @@ public Long getPolicyId(final Long serviceId, final String policyName) { return ret; } - public void getPoliciesInExcel(List policies, HttpServletResponse response) throws Exception { if (LOG.isDebugEnabled()) { LOG.debug("==> ServiceDBStore.getPoliciesInExcel()"); @@ -2570,134 +2488,428 @@ public ServicePolicies getServicePolicies(String serviceName) throws Exception { return ret; } - void createDefaultPolicies(RangerService createdService) throws Exception { + void createDefaultPolicies(XXService createdService, VXUser vXUser) throws Exception { + RangerServiceDef serviceDef = getServiceDef(createdService.getType()); + + if (serviceDef.getName().equals(EmbeddedServiceDefsUtil.EMBEDDED_SERVICEDEF_TAG_NAME)) { + createDefaultTagPolicy(createdService); + } else { + // we need to create one policy for each resource hierarchy + RangerServiceDefHelper serviceDefHelper = new RangerServiceDefHelper(serviceDef); + for (List aHierarchy : serviceDefHelper.getResourceHierarchies(RangerPolicy.POLICY_TYPE_ACCESS)) { + RangerPolicy policy = new RangerPolicy(); + createDefaultPolicy(policy, createdService, vXUser, aHierarchy); + policy = createPolicy(policy); + RangerPolicy policyAudit = new RangerPolicy(); + createPolicyForKeyAdmin(policyAudit, serviceDef, aHierarchy, createdService); + } + } + } + + private void createPolicyForKeyAdmin(RangerPolicy policyAudit, RangerServiceDef serviceDef, List aHierarchy, XXService createdService) { + if (serviceDef.getName().equals(EmbeddedServiceDefsUtil.EMBEDDED_SERVICEDEF_HDFS_NAME)) { + try { + // we need to create one policy for keyadmin user for audit to HDFS + RangerPolicy policy = getPolicyForKMSAudit(aHierarchy , createdService.getName(), serviceDef); + if (policy != null) { + createPolicy(policy); + } + } catch (Exception e) { + LOG.error("Error creating policy for keyadmin for audit to HDFS : " + serviceDef.getName(), e); + } + } + } + + private RangerPolicy getPolicyForKMSAudit(List resourceHierarchy, String serviceName, RangerServiceDef serviceDef) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> ServiceDBStore.getPolicyForKMSAudit()"); + } - RangerBaseService svc = serviceMgr.getRangerServiceByService(createdService, this); + RangerPolicy policy = new RangerPolicy(); - if (svc != null) { + policy.setIsEnabled(true); + policy.setVersion(1L); + policy.setName(AUDITTOHDFS_POLICY_NAME); + policy.setService(serviceName); + policy.setDescription("Policy for " + AUDITTOHDFS_POLICY_NAME); + policy.setIsAuditEnabled(true); + policy.setResources(createKMSAuditResource(resourceHierarchy)); - List serviceCheckUsers = getServiceCheckUsers(createdService); + List policyItems = new ArrayList(); + //Create policy item for keyadmin + RangerPolicy.RangerPolicyItem policyItem = new RangerPolicy.RangerPolicyItem(); + List userKeyAdmin = new ArrayList(); + userKeyAdmin.add("keyadmin"); + policyItem.setUsers(userKeyAdmin); + policyItem.setAccesses(getAndAllowAllAccesses(serviceDef)); + policyItem.setDelegateAdmin(false); - List allAccesses = svc.getAndAllowAllAccesses(); + policyItems.add(policyItem); + policy.setPolicyItems(policyItems); - List defaultPolicies = svc.getDefaultRangerPolicies(); + if (LOG.isDebugEnabled()) { + LOG.debug("<== ServiceDBStore.getPolicyForKMSAudit()" + policy); + } - if (CollectionUtils.isNotEmpty(defaultPolicies)) { + return policy; + } - createDefaultPolicyUsersAndGroups(defaultPolicies); + public List getAndAllowAllAccesses(RangerServiceDef serviceDef) { + List ret = new ArrayList(); + + for (RangerServiceDef.RangerAccessTypeDef accessTypeDef : serviceDef.getAccessTypes()) { + RangerPolicy.RangerPolicyItemAccess access = new RangerPolicy.RangerPolicyItemAccess(); + access.setType(accessTypeDef.getName()); + access.setIsAllowed(true); + ret.add(access); + } + return ret; + } - for (RangerPolicy defaultPolicy : defaultPolicies) { - List policyItems = defaultPolicy.getPolicyItems(); - if (CollectionUtils.isNotEmpty(serviceCheckUsers) - && StringUtils.equalsIgnoreCase(defaultPolicy.getService(), createdService.getName())) { + private Map createKMSAuditResource( + List resourceHierarchy) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> ServiceDBStore.createKMSAuditResource()"); + } + Map resourceMap = new HashMap<>(); - RangerPolicy.RangerPolicyItem policyItem = new RangerPolicy.RangerPolicyItem(); + for (RangerServiceDef.RangerResourceDef resourceDef : resourceHierarchy) { + RangerPolicy.RangerPolicyResource polRes = new RangerPolicy.RangerPolicyResource(); - policyItem.setUsers(serviceCheckUsers); - policyItem.setAccesses(allAccesses); - policyItem.setDelegateAdmin(true); + polRes.setIsExcludes(false); + polRes.setIsRecursive(resourceDef.getRecursiveSupported()); + polRes.setValue(AUDITTOHDFS_KMS_PATH); - defaultPolicy.getPolicyItems().add(policyItem); - } - boolean isPolicyItemValid=validatePolicyItem(policyItems); - if (isPolicyItemValid) { - createPolicy(defaultPolicy); - } else { - LOG.warn("Default policy won't be created,since policyItems not valid-either users/groups not present or access not present in policy."); - } + resourceMap.put(resourceDef.getName(), polRes); + } - } - } + if (LOG.isDebugEnabled()) { + LOG.debug("<== ServiceDBStore.createKMSAuditResource():" + + resourceMap); } + return resourceMap; } - void createDefaultPolicyUsersAndGroups(List defaultPolicies) { - Set defaultPolicyUsers = new HashSet(); - Set defaultPolicyGroups = new HashSet(); + private void createDefaultTagPolicy(XXService createdService) throws Exception { + if (LOG.isDebugEnabled()) { + LOG.debug("==> ServiceDBStore.createDefaultTagPolicy() "); + } - for (RangerPolicy defaultPolicy : defaultPolicies) { + String tagResourceDefName = null; + boolean isConditionDefFound = false; - for (RangerPolicyItem defaultPolicyItem : defaultPolicy.getPolicyItems()) { - defaultPolicyUsers.addAll(defaultPolicyItem.getUsers()); - defaultPolicyGroups.addAll(defaultPolicyItem.getGroups()); - } - for (RangerPolicyItem defaultPolicyItem : defaultPolicy.getAllowExceptions()) { - defaultPolicyUsers.addAll(defaultPolicyItem.getUsers()); - defaultPolicyGroups.addAll(defaultPolicyItem.getGroups()); - } - for (RangerPolicyItem defaultPolicyItem : defaultPolicy.getDenyPolicyItems()) { - defaultPolicyUsers.addAll(defaultPolicyItem.getUsers()); - defaultPolicyGroups.addAll(defaultPolicyItem.getGroups()); + RangerServiceDef tagServiceDef = getServiceDef(createdService.getType()); + List tagResourceDef = tagServiceDef.getResources(); + if (tagResourceDef != null && tagResourceDef.size() > 0) { + // Assumption : First (and perhaps the only) resourceDef is the name of the tag resource + RangerResourceDef theTagResourceDef = tagResourceDef.get(0); + tagResourceDefName = theTagResourceDef.getName(); + } else { + LOG.error("ServiceDBStore.createService() - Cannot create default TAG policy: Cannot get tagResourceDef Name."); + } + + List policyConditionDefs = tagServiceDef.getPolicyConditions(); + + if (CollectionUtils.isNotEmpty(policyConditionDefs)) { + for (RangerPolicyConditionDef conditionDef : policyConditionDefs) { + if (conditionDef.getName().equals(RANGER_TAG_EXPIRY_CONDITION_NAME)) { + isConditionDefFound = true; + break; + } } - for (RangerPolicyItem defaultPolicyItem : defaultPolicy.getDenyExceptions()) { - defaultPolicyUsers.addAll(defaultPolicyItem.getUsers()); - defaultPolicyGroups.addAll(defaultPolicyItem.getGroups()); + } + if (!isConditionDefFound) { + LOG.error("ServiceDBStore.createService() - Cannot create default TAG policy: Cannot get tagPolicyConditionDef with name=" + RANGER_TAG_EXPIRY_CONDITION_NAME); + } + + if (tagResourceDefName != null && isConditionDefFound) { + + String tagType = "EXPIRES_ON"; + + String policyName = tagType; + + RangerPolicy policy = new RangerPolicy(); + + policy.setIsEnabled(true); + policy.setVersion(1L); + policy.setName(StringUtils.trim(policyName)); + policy.setService(createdService.getName()); + policy.setDescription("Policy for data with " + tagType + " tag"); + policy.setIsAuditEnabled(true); + + Map resourceMap = new HashMap(); + + RangerPolicyResource polRes = new RangerPolicyResource(); + polRes.setIsExcludes(false); + polRes.setIsRecursive(false); + polRes.setValue(tagType); + resourceMap.put(tagResourceDefName, polRes); + + policy.setResources(resourceMap); + + List policyItems = new ArrayList(); + + RangerPolicyItem policyItem = new RangerPolicyItem(); + + List groups = new ArrayList(); + groups.add(RangerConstants.GROUP_PUBLIC); + policyItem.setGroups(groups); + + List accessTypeDefs = daoMgr.getXXAccessTypeDef().findByServiceDefId(createdService.getType()); + List accesses = new ArrayList(); + for (XXAccessTypeDef accessTypeDef : accessTypeDefs) { + RangerPolicyItemAccess access = new RangerPolicyItemAccess(); + access.setType(accessTypeDef.getName()); + access.setIsAllowed(true); + accesses.add(access); } - for (RangerPolicyItem defaultPolicyItem : defaultPolicy.getDataMaskPolicyItems()) { - defaultPolicyUsers.addAll(defaultPolicyItem.getUsers()); - defaultPolicyGroups.addAll(defaultPolicyItem.getGroups()); + policyItem.setAccesses(accesses); + + List policyItemConditions = new ArrayList(); + List values = new ArrayList(); + values.add("yes"); + RangerPolicyItemCondition policyItemCondition = new RangerPolicyItemCondition(RANGER_TAG_EXPIRY_CONDITION_NAME, values); + policyItemConditions.add(policyItemCondition); + + policyItem.setConditions(policyItemConditions); + policyItem.setDelegateAdmin(Boolean.FALSE); + + policyItems.add(policyItem); + + policy.setDenyPolicyItems(policyItems); + + policy = createPolicy(policy); + } else { + LOG.error("ServiceDBStore.createService() - Cannot create default TAG policy, tagResourceDefName=" + tagResourceDefName + + ", tagPolicyConditionName=" + RANGER_TAG_EXPIRY_CONDITION_NAME); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== ServiceDBStore.createDefaultTagPolicy()"); + } + } + + private String buildPolicyName(List resourceHierarchy) { + String ret = "all"; + if (CollectionUtils.isNotEmpty(resourceHierarchy)) { + int resourceDefCount = 0; + for (RangerResourceDef resourceDef : resourceHierarchy) { + if (resourceDefCount > 0) { + ret += ", "; + } else { + ret += " - "; + } + ret += resourceDef.getName(); + resourceDefCount++; } - for (RangerPolicyItem defaultPolicyItem : defaultPolicy.getRowFilterPolicyItems()) { - defaultPolicyUsers.addAll(defaultPolicyItem.getUsers()); - defaultPolicyGroups.addAll(defaultPolicyItem.getGroups()); + } + return ret; + } + + void createDefaultPolicy(RangerPolicy policy, XXService createdService, VXUser vXUser, List resourceHierarchy) throws Exception { + + String policyName=buildPolicyName(resourceHierarchy); + + policy.setIsEnabled(true); + policy.setVersion(1L); + policy.setName(StringUtils.trim(policyName)); + policy.setService(createdService.getName()); + policy.setDescription("Policy for " + policyName); + policy.setIsAuditEnabled(true); + policy.setPolicyType(policy.getPolicyType() == null ? RangerPolicy.POLICY_TYPE_ACCESS : policy.getPolicyType()); + + policy.setResources(createDefaultPolicyResource(resourceHierarchy)); + + if (vXUser != null) { + List policyItems = new ArrayList(); + List accessTypeDefs = daoMgr.getXXAccessTypeDef().findByServiceDefId(createdService.getType()); + //Create Default policy item for the service user + RangerPolicyItem policyItem = createDefaultPolicyItem(createdService, vXUser, accessTypeDefs); + policyItems.add(policyItem); + // For KMS add default policies for HDFS & HIVE users. + XXServiceDef xServiceDef = daoMgr.getXXServiceDef().getById(createdService.getType()); + if (EmbeddedServiceDefsUtil.KMS_IMPL_CLASS_NAME.equals(xServiceDef.getImplclassname())) { + List hdfsAccessTypeDefs = new ArrayList(); + List hiveAccessTypeDefs = new ArrayList(); + for(XXAccessTypeDef accessTypeDef : accessTypeDefs) { + if (accessTypeDef.getName().equalsIgnoreCase(ACCESS_TYPE_GET_METADATA)) { + hdfsAccessTypeDefs.add(accessTypeDef); + hiveAccessTypeDefs.add(accessTypeDef); + } else if (accessTypeDef.getName().equalsIgnoreCase(ACCESS_TYPE_GENERATE_EEK)) { + hdfsAccessTypeDefs.add(accessTypeDef); + } else if (accessTypeDef.getName().equalsIgnoreCase(ACCESS_TYPE_DECRYPT_EEK)) { + hiveAccessTypeDefs.add(accessTypeDef); + } + } + + String hdfsUser = PropertiesUtil.getProperty("ranger.kms.service.user.hdfs", "hdfs"); + if (hdfsUser != null && !hdfsUser.isEmpty()) { + XXUser xxUser = daoMgr.getXXUser().findByUserName(hdfsUser); + if (xxUser != null) { + vXUser = xUserService.populateViewBean(xxUser); + } else { + vXUser = xUserMgr.createServiceConfigUser(hdfsUser); + } + if (vXUser != null) { + LOG.info("Creating default KMS policy item for " + hdfsUser); + policyItem = createDefaultPolicyItem(createdService, vXUser, hdfsAccessTypeDefs); + policyItems.add(policyItem); + } + } + + String hiveUser = PropertiesUtil.getProperty("ranger.kms.service.user.hive", "hive"); + if (hiveUser != null && !hiveUser.isEmpty()) { + XXUser xxUser = daoMgr.getXXUser().findByUserName(hiveUser); + if (xxUser != null) { + vXUser = xUserService.populateViewBean(xxUser); + } else { + vXUser = xUserMgr.createServiceConfigUser(hiveUser); + } + if (vXUser != null) { + LOG.info("Creating default KMS policy item for " + hiveUser); + policyItem = createDefaultPolicyItem(createdService, vXUser, hiveAccessTypeDefs); + policyItems.add(policyItem); + } + } } + policy.setPolicyItems(policyItems); } - for (String policyUser : defaultPolicyUsers) { - if (LOG.isDebugEnabled()) { - LOG.debug("Checking policyUser:[" + policyUser + "] for existence"); + } + + private RangerPolicyItem createDefaultPolicyItem(XXService createdService, VXUser vXUser, List accessTypeDefs) throws Exception { + String adminPrincipal = PropertiesUtil.getProperty(ADMIN_USER_PRINCIPAL); + String adminKeytab = PropertiesUtil.getProperty(ADMIN_USER_KEYTAB); + String authType = PropertiesUtil.getProperty(RANGER_AUTH_TYPE,"simple"); + String lookupPrincipal = PropertiesUtil.getProperty(LOOKUP_PRINCIPAL); + String lookupKeytab = PropertiesUtil.getProperty(LOOKUP_KEYTAB); + + RangerPolicyItem policyItem = new RangerPolicyItem(); + + List users = new ArrayList(); + users.add(vXUser.getName()); + VXUser vXLookupUser = getLookupUser(authType, lookupPrincipal, lookupKeytab); + + XXService xService = daoMgr.getXXService().findByName(createdService.getName()); + XXServiceDef xServiceDef = daoMgr.getXXServiceDef().getById(xService.getType()); + if (StringUtils.equals(xServiceDef.getImplclassname(), EmbeddedServiceDefsUtil.KMS_IMPL_CLASS_NAME)){ + VXUser vXAdminUser = getLookupUser(authType, adminPrincipal, adminKeytab); + if(vXAdminUser != null){ + users.add(vXAdminUser.getName()); + } + }else if(vXLookupUser != null){ + users.add(vXLookupUser.getName()); + }else{ + // do nothing + } + + if (StringUtils.equals(xServiceDef.getImplclassname(), EmbeddedServiceDefsUtil.ATLAS_IMPL_CLASS_NAME)){ + VXUser vXUserAdmin = chkAdminUserExists("admin"); + if(vXUserAdmin != null){ + users.add(vXUserAdmin.getName()); } - if (StringUtils.isNotBlank(policyUser) && !StringUtils.equals(policyUser, RangerPolicyEngine.USER_CURRENT) - && !StringUtils.equals(policyUser, RangerPolicyEngine.RESOURCE_OWNER)) { - XXUser xxUser = daoMgr.getXXUser().findByUserName(policyUser); - if (xxUser == null) { - UserSessionBase usb = ContextUtil.getCurrentUserSession(); - if (usb != null && !usb.isKeyAdmin() && !usb.isUserAdmin() && !usb.isSpnegoEnabled()) { - throw restErrorUtil.createRESTException("User does not exist with given username: [" - + policyUser + "] please use existing user", MessageEnums.OPER_NO_PERMISSION); + } + + RangerService rangerService = getServiceByName(createdService.getName()); + if (rangerService != null){ + Map map = rangerService.getConfigs(); + if (map != null && map.containsKey(AMBARI_SERVICE_CHECK_USER)){ + String userNames = map.get(AMBARI_SERVICE_CHECK_USER); + String[] userList = userNames.split(","); + if(userList != null){ + for (String userName : userList) { + if(!StringUtils.isEmpty(userName)){ + XXUser xxUser = daoMgr.getXXUser().findByUserName(userName); + if (xxUser != null) { + vXUser = xUserService.populateViewBean(xxUser); + } else { + vXUser = xUserMgr.createServiceConfigUser(userName); + LOG.info("Creating Ambari Service Check User : "+vXUser.getName()); + } + if(vXUser != null){ + users.add(vXUser.getName()); + } + } } - xUserMgr.createServiceConfigUser(policyUser); } } } - for (String policyGroup : defaultPolicyGroups) { - if (LOG.isDebugEnabled()) { - LOG.debug("Checking policyGroup:[" + policyGroup + "] for existence"); + policyItem.setUsers(users); + + List accesses = new ArrayList(); + for(XXAccessTypeDef accessTypeDef : accessTypeDefs) { + RangerPolicyItemAccess access = new RangerPolicyItemAccess(); + access.setType(accessTypeDef.getName()); + access.setIsAllowed(true); + accesses.add(access); + } + policyItem.setAccesses(accesses); + + policyItem.setDelegateAdmin(true); + return policyItem; + } + + private VXUser chkAdminUserExists(String adminUser) { + VXUser vXUser = null; + if(!StringUtils.isEmpty(adminUser)){ + XXUser xxUser = daoMgr.getXXUser().findByUserName(adminUser); + if (xxUser != null) { + vXUser = xUserService.populateViewBean(xxUser); } - if (StringUtils.isNotBlank(policyGroup)) { - XXGroup xxGroup = daoMgr.getXXGroup().findByGroupName(policyGroup); - if (xxGroup == null) { - UserSessionBase usb = ContextUtil.getCurrentUserSession(); - if (usb != null && !usb.isKeyAdmin() && !usb.isUserAdmin() && !usb.isSpnegoEnabled()) { - throw restErrorUtil.createRESTException("Group does not exist with given groupname: [" - + policyGroup + "] please use existing group", MessageEnums.OPER_NO_PERMISSION); + } + return vXUser; + } + + private VXUser getLookupUser(String authType, String lookupPrincipal, String lookupKeytab) { + VXUser vXUser = null; + if(!StringUtils.isEmpty(authType) && authType.equalsIgnoreCase(KERBEROS_TYPE)){ + if(SecureClientLogin.isKerberosCredentialExists(lookupPrincipal, lookupKeytab)){ + KerberosName krbName = new KerberosName(lookupPrincipal); + String lookupUser=null; + try { + lookupUser = krbName.getShortName(); + } catch (IOException e) { + throw restErrorUtil.createRESTException("Please provide proper value of lookup user principal : "+ lookupPrincipal, MessageEnums.INVALID_INPUT_DATA); + } + + if(LOG.isDebugEnabled()){ + LOG.debug("Checking for Lookup User : "+lookupUser); + } + if(!StringUtils.isEmpty(lookupUser)){ + XXUser xxUser = daoMgr.getXXUser().findByUserName(lookupUser); + if (xxUser != null) { + vXUser = xUserService.populateViewBean(xxUser); + } else { + vXUser = xUserMgr.createServiceConfigUser(lookupUser); + LOG.info("Creating Lookup User : "+vXUser.getName()); } - VXGroup vXGroup = new VXGroup(); - vXGroup.setName(policyGroup); - vXGroup.setDescription(policyGroup); - vXGroup.setGroupSource(RangerCommonEnums.GROUP_INTERNAL); - vXGroup.setIsVisible(RangerCommonEnums.IS_VISIBLE); - xGroupService.createResource(vXGroup); } } } + return vXUser; } - List getServiceCheckUsers(RangerService createdService) { - List ret = new ArrayList(); - Map serviceConfig = createdService.getConfigs(); + Map createDefaultPolicyResource(List resourceHierarchy) throws Exception { + Map resourceMap = new HashMap<>(); - if (serviceConfig.containsKey(AMBARI_SERVICE_CHECK_USER)) { - String userNames = serviceConfig.get(AMBARI_SERVICE_CHECK_USER); - String[] userList = userNames.split(","); - for (String userName : userList) { - if (!StringUtils.isEmpty(userName)) { - ret.add(userName); - } + for (RangerResourceDef resourceDef : resourceHierarchy) { + RangerPolicyResource polRes = new RangerPolicyResource(); + polRes.setIsExcludes(false); + polRes.setIsRecursive(false); + + String value = "*"; + if("path".equalsIgnoreCase(resourceDef.getName())) { + value = "/*"; } - } - return ret; + if(resourceDef.getRecursiveSupported()) { + polRes.setIsRecursive(Boolean.TRUE); + } + + polRes.setValue(value); + resourceMap.put(resourceDef.getName(), polRes); + } + return resourceMap; } private Map validateRequiredConfigParams(RangerService service, Map configs) { @@ -2732,381 +2944,77 @@ private void handlePolicyUpdate(RangerService service, boolean isTagVersionUpdat updatePolicyVersion(service, isTagVersionUpdateNeeded); } + public enum VERSION_TYPE { POLICY_VERSION, TAG_VERSION, POLICY_AND_TAG_VERSION } + private void updatePolicyVersion(RangerService service, boolean isTagVersionUpdateNeeded) throws Exception { if(service == null || service.getId() == null) { return; } - boolean filterForServicePlugin = RangerConfiguration.getInstance().getBoolean(RangerTagDBRetriever.OPTION_RANGER_FILTER_TAGS_FOR_SERVICE_PLUGIN, false); - XXServiceDao serviceDao = daoMgr.getXXService(); - XXService serviceDbObj = serviceDao.getById(service.getId()); + final XXService serviceDbObj = serviceDao.getById(service.getId()); if(serviceDbObj == null) { LOG.warn("updatePolicyVersion(serviceId=" + service.getId() + "): service not found"); return; } - XXServiceVersionInfoDao serviceVersionInfoDao = daoMgr.getXXServiceVersionInfo(); - - XXServiceVersionInfo serviceVersionInfoDbObj = serviceVersionInfoDao.findByServiceId(service.getId()); + final RangerDaoManager daoManager = daoMgr; + final Long serviceId = serviceDbObj.getId(); + final VERSION_TYPE versionType = VERSION_TYPE.POLICY_VERSION; - if(serviceVersionInfoDbObj != null) { - serviceVersionInfoDbObj.setPolicyVersion(getNextVersion(serviceVersionInfoDbObj.getPolicyVersion())); - serviceVersionInfoDbObj.setPolicyUpdateTime(new Date()); - - serviceVersionInfoDao.update(serviceVersionInfoDbObj); - - } else { - LOG.warn("updatePolicyVersion(service=" + serviceDbObj.getName() + "): serviceVersionInfo not found, creating it.."); - - serviceVersionInfoDbObj = new XXServiceVersionInfo(); - serviceVersionInfoDbObj.setServiceId(serviceDbObj.getId()); - serviceVersionInfoDbObj.setPolicyVersion(getNextVersion(serviceDbObj.getPolicyVersion())); - serviceVersionInfoDbObj.setPolicyUpdateTime(new Date()); - serviceVersionInfoDbObj.setTagVersion(serviceDbObj.getTagVersion()); - serviceVersionInfoDbObj.setTagUpdateTime(serviceDbObj.getTagUpdateTime()); - - serviceVersionInfoDao.create(serviceVersionInfoDbObj); - } + Runnable serviceVersionUpdater = new ServiceVersionUpdater(daoManager, serviceId, versionType); + transactionSynchronizationAdapter.executeOnTransactionCommit(serviceVersionUpdater); // if this is a tag service, update all services that refer to this tag service // so that next policy-download from plugins will get updated tag policies boolean isTagService = serviceDbObj.getType() == EmbeddedServiceDefsUtil.instance().getTagServiceDefId(); if(isTagService) { - List referringServices = serviceDao.findByTagServiceId(serviceDbObj.getId()); + List referringServices = serviceDao.findByTagServiceId(serviceId); if(CollectionUtils.isNotEmpty(referringServices)) { for(XXService referringService : referringServices) { - serviceVersionInfoDbObj = serviceVersionInfoDao.findByServiceId(referringService.getId()); - if (serviceVersionInfoDbObj != null) { - - serviceVersionInfoDbObj.setPolicyVersion(getNextVersion(serviceVersionInfoDbObj.getPolicyVersion())); - serviceVersionInfoDbObj.setPolicyUpdateTime(new Date()); + final Long referringServiceId = referringService.getId(); + final VERSION_TYPE tagServiceversionType = VERSION_TYPE.POLICY_VERSION; - if (filterForServicePlugin && isTagVersionUpdateNeeded) { - serviceVersionInfoDbObj.setTagVersion(getNextVersion(serviceVersionInfoDbObj.getTagVersion())); - serviceVersionInfoDbObj.setTagUpdateTime(new Date()); - } - serviceVersionInfoDao.update(serviceVersionInfoDbObj); - } else { - LOG.warn("updatePolicyVersion(service=" + referringService.getName() + "): serviceVersionInfo not found, creating it.."); - serviceVersionInfoDbObj = new XXServiceVersionInfo(); - serviceVersionInfoDbObj.setServiceId(referringService.getId()); - serviceVersionInfoDbObj.setPolicyVersion(getNextVersion(referringService.getPolicyVersion())); - serviceVersionInfoDbObj.setPolicyUpdateTime(new Date()); - if (filterForServicePlugin && isTagVersionUpdateNeeded) { - serviceVersionInfoDbObj.setTagVersion(getNextVersion(referringService.getTagVersion())); - serviceVersionInfoDbObj.setTagUpdateTime(new Date()); - } else { - serviceVersionInfoDbObj.setTagVersion(referringService.getTagVersion()); - serviceVersionInfoDbObj.setTagUpdateTime(referringService.getTagUpdateTime()); - } - serviceVersionInfoDao.create(serviceVersionInfoDbObj); - } + Runnable tagServiceVersionUpdater = new ServiceVersionUpdater(daoManager, referringServiceId, tagServiceversionType); + transactionSynchronizationAdapter.executeOnTransactionCommit(tagServiceVersionUpdater); } } } } - private XXPolicyItem createNewPolicyItemForPolicy(RangerPolicy policy, XXPolicy xPolicy, RangerPolicyItem policyItem, XXServiceDef xServiceDef, int itemOrder, int policyItemType) throws Exception { - XXPolicyItem xPolicyItem = new XXPolicyItem(); - - xPolicyItem = (XXPolicyItem) rangerAuditFields.populateAuditFields(xPolicyItem, xPolicy); - - xPolicyItem.setDelegateAdmin(policyItem.getDelegateAdmin()); - xPolicyItem.setItemType(policyItemType); - xPolicyItem.setIsEnabled(Boolean.TRUE); - xPolicyItem.setComments(null); - xPolicyItem.setPolicyId(policy.getId()); - xPolicyItem.setOrder(itemOrder); - xPolicyItem = daoMgr.getXXPolicyItem().create(xPolicyItem); - - List accesses = policyItem.getAccesses(); - for (int i = 0; i < accesses.size(); i++) { - RangerPolicyItemAccess access = accesses.get(i); - - XXAccessTypeDef xAccTypeDef = daoMgr.getXXAccessTypeDef() - .findByNameAndServiceId(access.getType(), - xPolicy.getService()); - if (xAccTypeDef == null) { - throw new Exception(access.getType() + ": is not a valid access-type. policy='" + policy.getName() + "' service='" + policy.getService() + "'"); - } - - XXPolicyItemAccess xPolItemAcc = new XXPolicyItemAccess(); - - xPolItemAcc = (XXPolicyItemAccess) rangerAuditFields.populateAuditFields(xPolItemAcc, xPolicyItem); - xPolItemAcc.setIsAllowed(access.getIsAllowed()); - xPolItemAcc.setType(xAccTypeDef.getId()); - xPolItemAcc.setPolicyitemid(xPolicyItem.getId()); - xPolItemAcc.setOrder(i); - - daoMgr.getXXPolicyItemAccess().create(xPolItemAcc); - } - - List users = policyItem.getUsers(); - for(int i = 0; i < users.size(); i++) { - String user = users.get(i); - if (StringUtils.isBlank(user)) { - continue; - } - XXUser xUser = daoMgr.getXXUser().findByUserName(user); - if(xUser == null) { - throw new Exception(user + ": user does not exist. policy='"+ policy.getName() + "' service='"+ policy.getService() + "' user='" + user +"'"); - } - XXPolicyItemUserPerm xUserPerm = new XXPolicyItemUserPerm(); - xUserPerm = (XXPolicyItemUserPerm) rangerAuditFields.populateAuditFields(xUserPerm, xPolicyItem); - xUserPerm.setUserId(xUser.getId()); - xUserPerm.setPolicyItemId(xPolicyItem.getId()); - xUserPerm.setOrder(i); - xUserPerm = daoMgr.getXXPolicyItemUserPerm().create(xUserPerm); - } - - List groups = policyItem.getGroups(); - for(int i = 0; i < groups.size(); i++) { - String group = groups.get(i); - if (StringUtils.isBlank(group)) { - continue; - } - XXGroup xGrp = daoMgr.getXXGroup().findByGroupName(group); - if(xGrp == null) { - throw new Exception(group + ": group does not exist. policy='"+ policy.getName() + "' service='"+ policy.getService() + "' group='" + group + "'"); - } - XXPolicyItemGroupPerm xGrpPerm = new XXPolicyItemGroupPerm(); - xGrpPerm = (XXPolicyItemGroupPerm) rangerAuditFields.populateAuditFields(xGrpPerm, xPolicyItem); - xGrpPerm.setGroupId(xGrp.getId()); - xGrpPerm.setPolicyItemId(xPolicyItem.getId()); - xGrpPerm.setOrder(i); - xGrpPerm = daoMgr.getXXPolicyItemGroupPerm().create(xGrpPerm); - } - - List conditions = policyItem.getConditions(); - for(RangerPolicyItemCondition condition : conditions) { - XXPolicyConditionDef xPolCond = daoMgr - .getXXPolicyConditionDef().findByServiceDefIdAndName( - xServiceDef.getId(), condition.getType()); - - if(xPolCond == null) { - throw new Exception(condition.getType() + ": is not a valid condition-type. policy='"+ xPolicy.getName() + "' service='"+ xPolicy.getService() + "'"); - } - - for(int i = 0; i < condition.getValues().size(); i++) { - String value = condition.getValues().get(i); - XXPolicyItemCondition xPolItemCond = new XXPolicyItemCondition(); - xPolItemCond = (XXPolicyItemCondition) rangerAuditFields.populateAuditFields(xPolItemCond, xPolicyItem); - xPolItemCond.setPolicyItemId(xPolicyItem.getId()); - xPolItemCond.setType(xPolCond.getId()); - xPolItemCond.setValue(value); - xPolItemCond.setOrder(i); - - daoMgr.getXXPolicyItemCondition().create(xPolItemCond); - } - } - - return xPolicyItem; - } - - private void createNewPolicyItemsForPolicy(RangerPolicy policy, XXPolicy xPolicy, List policyItems, XXServiceDef xServiceDef, int policyItemType) throws Exception { - if(CollectionUtils.isNotEmpty(policyItems)) { - for (int itemOrder = 0; itemOrder < policyItems.size(); itemOrder++) { - RangerPolicyItem policyItem = policyItems.get(itemOrder); - createNewPolicyItemForPolicy(policy, xPolicy, policyItem, xServiceDef, itemOrder, policyItemType); - } - } - } - - private void createNewDataMaskPolicyItemsForPolicy(RangerPolicy policy, XXPolicy xPolicy, List policyItems, XXServiceDef xServiceDef, int policyItemType) throws Exception { - if(CollectionUtils.isNotEmpty(policyItems)) { - for (int itemOrder = 0; itemOrder < policyItems.size(); itemOrder++) { - RangerDataMaskPolicyItem policyItem = policyItems.get(itemOrder); - - XXPolicyItem xPolicyItem = createNewPolicyItemForPolicy(policy, xPolicy, policyItem, xServiceDef, itemOrder, policyItemType); - - RangerPolicyItemDataMaskInfo dataMaskInfo = policyItem.getDataMaskInfo(); - - if(dataMaskInfo != null) { - XXDataMaskTypeDef dataMaskDef = daoMgr.getXXDataMaskTypeDef().findByNameAndServiceId(dataMaskInfo.getDataMaskType(), xPolicy.getService()); - - if(dataMaskDef == null) { - throw new Exception(dataMaskInfo.getDataMaskType() + ": is not a valid datamask-type. policy='" + policy.getName() + "' service='" + policy.getService() + "'"); - } - - XXPolicyItemDataMaskInfo xxDataMaskInfo = new XXPolicyItemDataMaskInfo(); - - xxDataMaskInfo.setPolicyItemId(xPolicyItem.getId()); - xxDataMaskInfo.setType(dataMaskDef.getId()); - xxDataMaskInfo.setConditionExpr(dataMaskInfo.getConditionExpr()); - xxDataMaskInfo.setValueExpr(dataMaskInfo.getValueExpr()); - - daoMgr.getXXPolicyItemDataMaskInfo().create(xxDataMaskInfo); - } - } - } - } - - private void createNewRowFilterPolicyItemsForPolicy(RangerPolicy policy, XXPolicy xPolicy, List policyItems, XXServiceDef xServiceDef, int policyItemType) throws Exception { - if(CollectionUtils.isNotEmpty(policyItems)) { - for (int itemOrder = 0; itemOrder < policyItems.size(); itemOrder++) { - RangerRowFilterPolicyItem policyItem = policyItems.get(itemOrder); - - XXPolicyItem xPolicyItem = createNewPolicyItemForPolicy(policy, xPolicy, policyItem, xServiceDef, itemOrder, policyItemType); - - RangerPolicyItemRowFilterInfo dataMaskInfo = policyItem.getRowFilterInfo(); - - if(dataMaskInfo != null) { - XXPolicyItemRowFilterInfo xxRowFilterInfo = new XXPolicyItemRowFilterInfo(); - - xxRowFilterInfo.setPolicyItemId(xPolicyItem.getId()); - xxRowFilterInfo.setFilterExpr(dataMaskInfo.getFilterExpr()); - - xxRowFilterInfo = daoMgr.getXXPolicyItemRowFilterInfo().create(xxRowFilterInfo); - } - } - } - } + public static void persistVersionChange(RangerDaoManager daoMgr, Long id, VERSION_TYPE versionType) { + XXServiceVersionInfoDao serviceVersionInfoDao = daoMgr.getXXServiceVersionInfo(); - private void createNewResourcesForPolicy(RangerPolicy policy, XXPolicy xPolicy, Map resources) throws Exception { - - for (Entry resource : resources.entrySet()) { - RangerPolicyResource policyRes = resource.getValue(); - - XXResourceDef xResDef = daoMgr.getXXResourceDef() - .findByNameAndPolicyId(resource.getKey(), policy.getId()); - if (xResDef == null) { - throw new Exception(resource.getKey() + ": is not a valid resource-type. policy='"+ policy.getName() + "' service='"+ policy.getService() + "'"); - } - - XXPolicyResource xPolRes = new XXPolicyResource(); - xPolRes = (XXPolicyResource) rangerAuditFields.populateAuditFields(xPolRes, xPolicy); - - xPolRes.setIsExcludes(policyRes.getIsExcludes()); - xPolRes.setIsRecursive(policyRes.getIsRecursive()); - xPolRes.setPolicyId(policy.getId()); - xPolRes.setResDefId(xResDef.getId()); - xPolRes = daoMgr.getXXPolicyResource().create(xPolRes); - - List values = policyRes.getValues(); - if (CollectionUtils.isNotEmpty(values)) { - Set uniqueValues = new LinkedHashSet(values); - int i = 0; - if (CollectionUtils.isNotEmpty(uniqueValues)) { - for (String uniqValue : uniqueValues) { - if (!StringUtils.isEmpty(uniqValue)) { - XXPolicyResourceMap xPolResMap = new XXPolicyResourceMap(); - xPolResMap = (XXPolicyResourceMap) rangerAuditFields.populateAuditFields(xPolResMap,xPolRes); - xPolResMap.setResourceId(xPolRes.getId()); - xPolResMap.setValue(uniqValue); - xPolResMap.setOrder(i); - xPolResMap = daoMgr.getXXPolicyResourceMap().create(xPolResMap); - i++; - } - } - } - } - } - } + XXServiceVersionInfo serviceVersionInfoDbObj = serviceVersionInfoDao.findByServiceId(id); - private Boolean deleteExistingPolicyItems(RangerPolicy policy) { - if(policy == null) { - return false; - } - - XXPolicyItemDao policyItemDao = daoMgr.getXXPolicyItem(); - List policyItems = policyItemDao.findByPolicyId(policy.getId()); - for(XXPolicyItem policyItem : policyItems) { - Long polItemId = policyItem.getId(); - - XXPolicyItemConditionDao polCondDao = daoMgr.getXXPolicyItemCondition(); - List conditions = polCondDao.findByPolicyItemId(polItemId); - for(XXPolicyItemCondition condition : conditions) { - polCondDao.remove(condition); - } - - XXPolicyItemGroupPermDao grpPermDao = daoMgr.getXXPolicyItemGroupPerm(); - List groups = grpPermDao.findByPolicyItemId(polItemId); - for(XXPolicyItemGroupPerm group : groups) { - grpPermDao.remove(group); - } - - XXPolicyItemUserPermDao userPermDao = daoMgr.getXXPolicyItemUserPerm(); - List users = userPermDao.findByPolicyItemId(polItemId); - for(XXPolicyItemUserPerm user : users) { - userPermDao.remove(user); + if(serviceVersionInfoDbObj != null) { + if (versionType == VERSION_TYPE.POLICY_VERSION || versionType == VERSION_TYPE.POLICY_AND_TAG_VERSION) { + serviceVersionInfoDbObj.setPolicyVersion(getNextVersion(serviceVersionInfoDbObj.getPolicyVersion())); + serviceVersionInfoDbObj.setPolicyUpdateTime(new Date()); } - - XXPolicyItemAccessDao polItemAccDao = daoMgr.getXXPolicyItemAccess(); - List accesses = polItemAccDao.findByPolicyItemId(polItemId); - for(XXPolicyItemAccess access : accesses) { - polItemAccDao.remove(access); + if (versionType == VERSION_TYPE.TAG_VERSION || versionType == VERSION_TYPE.POLICY_AND_TAG_VERSION) { + serviceVersionInfoDbObj.setTagVersion(getNextVersion(serviceVersionInfoDbObj.getTagVersion())); + serviceVersionInfoDbObj.setTagUpdateTime(new Date()); } - XXPolicyItemDataMaskInfoDao polItemDataMaskInfoDao = daoMgr.getXXPolicyItemDataMaskInfo(); - List dataMaskInfos = polItemDataMaskInfoDao.findByPolicyItemId(polItemId); - for(XXPolicyItemDataMaskInfo dataMaskInfo : dataMaskInfos) { - polItemDataMaskInfoDao.remove(dataMaskInfo); - } - - XXPolicyItemRowFilterInfoDao polItemRowFilterInfoDao = daoMgr.getXXPolicyItemRowFilterInfo(); - List rowFilterInfos = polItemRowFilterInfoDao.findByPolicyItemId(polItemId); - for(XXPolicyItemRowFilterInfo rowFilterInfo : rowFilterInfos) { - polItemRowFilterInfoDao.remove(rowFilterInfo); - } + serviceVersionInfoDao.update(serviceVersionInfoDbObj); - policyItemDao.remove(policyItem); - } - return true; - } + } else { + XXService service = daoMgr.getXXService().getById(id); + if (service != null) { + serviceVersionInfoDbObj = new XXServiceVersionInfo(); + serviceVersionInfoDbObj.setServiceId(service.getId()); + serviceVersionInfoDbObj.setPolicyVersion(1L); + serviceVersionInfoDbObj.setPolicyUpdateTime(new Date()); + serviceVersionInfoDbObj.setTagVersion(1L); + serviceVersionInfoDbObj.setTagUpdateTime(new Date()); - private Boolean deleteExistingPolicyResources(RangerPolicy policy) { - if(policy == null) { - return false; - } - - List resources = daoMgr.getXXPolicyResource().findByPolicyId(policy.getId()); - - XXPolicyResourceDao resDao = daoMgr.getXXPolicyResource(); - for(XXPolicyResource resource : resources) { - List resMapList = daoMgr.getXXPolicyResourceMap().findByPolicyResId(resource.getId()); - - XXPolicyResourceMapDao resMapDao = daoMgr.getXXPolicyResourceMap(); - for(XXPolicyResourceMap resMap : resMapList) { - resMapDao.remove(resMap); + serviceVersionInfoDao.create(serviceVersionInfoDbObj); } - resDao.remove(resource); } - return true; - } - - private Boolean deleteExistingPolicyItemsNative(RangerPolicy policy) { - if(policy == null) { - return false; - } - XXPolicyItemDao policyItemDao = daoMgr.getXXPolicyItem(); - List policyItems = policyItemDao.findByPolicyId(policy.getId()); - for(XXPolicyItem policyItem : policyItems) { - Long polItemId = policyItem.getId(); - daoMgr.getXXPolicyItemRowFilterInfo().deletePolicyIDReference("policy_item_id", polItemId); - daoMgr.getXXPolicyItemDataMaskInfo().deletePolicyIDReference("policy_item_id", polItemId); - daoMgr.getXXPolicyItemGroupPerm().deletePolicyIDReference("policy_item_id", polItemId); - daoMgr.getXXPolicyItemUserPerm().deletePolicyIDReference("policy_item_id", polItemId); - daoMgr.getXXPolicyItemCondition().deletePolicyIDReference("policy_item_id", polItemId); - daoMgr.getXXPolicyItemAccess().deletePolicyIDReference("policy_item_id", polItemId); - } - daoMgr.getXXPolicyItem().deletePolicyIDReference("policy_id", policy.getId()); - return true; - } - - private Boolean deleteExistingPolicyResourcesNative(RangerPolicy policy) { - if(policy == null) { - return false; - } - List resources = daoMgr.getXXPolicyResource().findByPolicyId(policy.getId()); - for(XXPolicyResource resource : resources) { - daoMgr.getXXPolicyResourceMap().deletePolicyIDReference("resource_id", resource.getId()); - daoMgr.getXXPolicyResource().deletePolicyIDReference("id", resource.getId()); - } - return true; } @Override @@ -3177,55 +3085,34 @@ protected void updateServicesForServiceDefUpdate(RangerServiceDef serviceDef) th return; } + final RangerDaoManager daoManager = daoMgr; + boolean isTagServiceDef = StringUtils.equals(serviceDef.getName(), EmbeddedServiceDefsUtil.EMBEDDED_SERVICEDEF_TAG_NAME); XXServiceDao serviceDao = daoMgr.getXXService(); - XXServiceVersionInfoDao serviceVersionInfoDao = daoMgr.getXXServiceVersionInfo(); List services = serviceDao.findByServiceDefId(serviceDef.getId()); if(CollectionUtils.isNotEmpty(services)) { for(XXService service : services) { - XXServiceVersionInfo serviceVersionInfo = serviceVersionInfoDao.findByServiceId(service.getId()); - if (serviceVersionInfo != null) { - serviceVersionInfo.setPolicyVersion(getNextVersion(serviceVersionInfo.getPolicyVersion())); - serviceVersionInfo.setPolicyUpdateTime(serviceDef.getUpdateTime()); - serviceVersionInfoDao.update(serviceVersionInfo); - } else { - LOG.warn("updateServicesForServiceDefUpdate(service=" + service.getName() + "): serviceVersionInfo not found, creating it.."); - serviceVersionInfo = new XXServiceVersionInfo(); - serviceVersionInfo.setServiceId(service.getId()); - serviceVersionInfo.setPolicyVersion(getNextVersion(service.getPolicyVersion())); - serviceVersionInfo.setTagVersion(service.getTagVersion()); - serviceVersionInfo.setPolicyUpdateTime(new Date()); - serviceVersionInfo.setTagUpdateTime(service.getTagUpdateTime()); + final Long serviceId = service.getId(); + final VERSION_TYPE versionType = VERSION_TYPE.POLICY_VERSION; - serviceVersionInfoDao.create(serviceVersionInfo); - } + Runnable serviceVersionUpdater = new ServiceVersionUpdater(daoManager, serviceId, versionType); + transactionSynchronizationAdapter.executeOnTransactionCommit(serviceVersionUpdater); if(isTagServiceDef) { - List referrringServices = serviceDao.findByTagServiceId(service.getId()); + List referringServices = serviceDao.findByTagServiceId(service.getId()); - if(CollectionUtils.isNotEmpty(referrringServices)) { - for(XXService referringService : referrringServices) { - serviceVersionInfo = serviceVersionInfoDao.findByServiceId(referringService.getId()); - if (serviceVersionInfo != null) { - serviceVersionInfo.setPolicyVersion(getNextVersion(serviceVersionInfo.getPolicyVersion())); - serviceVersionInfo.setPolicyUpdateTime(serviceDef.getUpdateTime()); + if(CollectionUtils.isNotEmpty(referringServices)) { + for(XXService referringService : referringServices) { - serviceVersionInfoDao.update(serviceVersionInfo); - } else { - LOG.warn("updateServicesForServiceDefUpdate(service=" + referringService.getName() + "): serviceVersionInfo not found, creating it.."); - serviceVersionInfo = new XXServiceVersionInfo(); - serviceVersionInfo.setServiceId(referringService.getId()); - serviceVersionInfo.setPolicyVersion(getNextVersion(referringService.getPolicyVersion())); - serviceVersionInfo.setTagVersion(referringService.getTagVersion()); - serviceVersionInfo.setPolicyUpdateTime(new Date()); - serviceVersionInfo.setTagUpdateTime(referringService.getTagUpdateTime()); - - serviceVersionInfoDao.create(serviceVersionInfo); - } + final Long referringServiceId = referringService.getId(); + final VERSION_TYPE tagServiceVersionType = VERSION_TYPE.POLICY_VERSION; + + Runnable tagServiceVersionUpdater = new ServiceVersionUpdater(daoManager, referringServiceId, tagServiceVersionType); + transactionSynchronizationAdapter.executeOnTransactionCommit(tagServiceVersionUpdater); } } } @@ -3711,10 +3598,6 @@ public Map createPolicyMap( private void writeBookForPolicyItems(RangerPolicy policy, RangerPolicyItem policyItem, RangerDataMaskPolicyItem dataMaskPolicyItem, RangerRowFilterPolicyItem rowFilterPolicyItem, Row row, String policyConditonType) { - if (LOG.isDebugEnabled()) { - // To avoid PMD violation - LOG.debug("policyConditonType:[" + policyConditonType + "]"); - } List groups = new ArrayList(); List users = new ArrayList(); String groupNames = ""; @@ -3995,18 +3878,34 @@ private void createGenericUsers() { xUserService.createXUserWithOutLogin(genericUser); } - public boolean isServiceAdminUser(String serviceName, String userName) { - boolean ret=false; - XXServiceConfigMap cfgSvcAdminUsers = daoMgr.getXXServiceConfigMap().findByServiceNameAndConfigKey(serviceName, SERVICE_ADMIN_USERS); - String svcAdminUsers = cfgSvcAdminUsers != null ? cfgSvcAdminUsers.getConfigvalue() : null; - if (svcAdminUsers != null) { - for (String svcAdminUser : svcAdminUsers.split(",")) { - if (userName.equals(svcAdminUser)) { - ret=true; - break; - } - } - } - return ret; - } + public boolean isServiceAdminUser(String serviceName, String userName) { + boolean ret=false; + XXServiceConfigMap cfgSvcAdminUsers = daoMgr.getXXServiceConfigMap().findByServiceNameAndConfigKey(serviceName, SERVICE_ADMIN_USERS); + String svcAdminUsers = cfgSvcAdminUsers != null ? cfgSvcAdminUsers.getConfigvalue() : null; + if (svcAdminUsers != null) { + for (String svcAdminUser : svcAdminUsers.split(",")) { + if (userName.equals(svcAdminUser)) { + ret=true; + break; + } + } + } + return ret; + } + + public static class ServiceVersionUpdater implements Runnable { + final Long serviceId; + final RangerDaoManager daoManager; + final VERSION_TYPE versionType; + + public ServiceVersionUpdater(RangerDaoManager daoManager, Long serviceId, VERSION_TYPE versionType ) { + this.serviceId = serviceId; + this.daoManager = daoManager; + this.versionType = versionType; + } + @Override + public void run() { + ServiceDBStore.persistVersionChange(this.daoManager, this.serviceId, this.versionType); + } + } } diff --git a/security-admin/src/main/java/org/apache/ranger/biz/TagDBStore.java b/security-admin/src/main/java/org/apache/ranger/biz/TagDBStore.java index 3234be6a29..793ddf4cf0 100644 --- a/security-admin/src/main/java/org/apache/ranger/biz/TagDBStore.java +++ b/security-admin/src/main/java/org/apache/ranger/biz/TagDBStore.java @@ -20,15 +20,14 @@ package org.apache.ranger.biz; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; import java.util.Map; import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.collections.MapUtils; import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.ranger.authorization.utils.JsonUtils; import org.apache.ranger.common.GUIDUtil; import org.apache.ranger.common.MessageEnums; import org.apache.ranger.common.RESTErrorUtil; @@ -36,20 +35,12 @@ import org.apache.ranger.common.RangerServiceTagsCache; import org.apache.ranger.db.RangerDaoManager; import org.apache.ranger.entity.XXDBBase; -import org.apache.ranger.entity.XXResourceDef; import org.apache.ranger.entity.XXService; -import org.apache.ranger.entity.XXServiceDef; import org.apache.ranger.entity.XXServiceResource; import org.apache.ranger.entity.XXServiceVersionInfo; import org.apache.ranger.entity.XXTag; -import org.apache.ranger.entity.XXTagAttribute; -import org.apache.ranger.entity.XXTagAttributeDef; -import org.apache.ranger.entity.XXServiceResourceElement; -import org.apache.ranger.entity.XXServiceResourceElementValue; import org.apache.ranger.entity.XXTagResourceMap; import org.apache.ranger.plugin.model.*; -import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyResource; -import org.apache.ranger.plugin.model.RangerTagDef.RangerTagAttributeDef; import org.apache.ranger.plugin.store.AbstractTagStore; import org.apache.ranger.plugin.store.PList; import org.apache.ranger.plugin.store.RangerServiceResourceSignature; @@ -117,8 +108,6 @@ public RangerTagDef createTagDef(RangerTagDef tagDef) throws Exception { RangerTagDef ret = rangerTagDefService.create(tagDef); - createTagAttributeDefs(ret.getId(), tagDef.getAttributeDefs()); - ret = rangerTagDefService.read(ret.getId()); if (LOG.isDebugEnabled()) { @@ -138,6 +127,8 @@ public RangerTagDef updateTagDef(RangerTagDef tagDef) throws Exception { if (existing == null) { throw errorUtil.createRESTException("failed to update tag-def [" + tagDef.getName() + "], Reason: No TagDef found with id: [" + tagDef.getId() + "]", MessageEnums.DATA_NOT_UPDATABLE); + } else if (!existing.getName().equals(tagDef.getName())) { + throw errorUtil.createRESTException("Cannot change tag-def name; existing-name:[" + existing.getName() + "], new-name:[" + tagDef.getName() + "]", MessageEnums.DATA_NOT_UPDATABLE); } tagDef.setCreatedBy(existing.getCreatedBy()); @@ -147,10 +138,6 @@ public RangerTagDef updateTagDef(RangerTagDef tagDef) throws Exception { RangerTagDef ret = rangerTagDefService.update(tagDef); - // TODO: delete attributes might fail; so instead of delete+create, following should be updated to deal with only attributes that changed - deleteTagAttributeDefs(ret.getId()); - createTagAttributeDefs(ret.getId(), tagDef.getAttributeDefs()); - ret = rangerTagDefService.read(ret.getId()); if (LOG.isDebugEnabled()) { @@ -174,7 +161,6 @@ public void deleteTagDefByName(String name) throws Exception { LOG.debug("Deleting tag-def [name=" + name + "; id=" + tagDef.getId() + "]"); } - deleteTagAttributeDefs(tagDef.getId()); rangerTagDefService.delete(tagDef); } } @@ -194,7 +180,6 @@ public void deleteTagDef(Long id) throws Exception { RangerTagDef tagDef = rangerTagDefService.read(id); if(tagDef != null) { - deleteTagAttributeDefs(tagDef.getId()); rangerTagDefService.delete(tagDef); } } @@ -307,8 +292,6 @@ public RangerTag createTag(RangerTag tag) throws Exception { RangerTag ret = rangerTagService.create(tag); - createTagAttributes(ret.getId(), tag.getAttributes()); - ret = rangerTagService.read(ret.getId()); if (LOG.isDebugEnabled()) { @@ -337,9 +320,6 @@ public RangerTag updateTag(RangerTag tag) throws Exception { RangerTag ret = rangerTagService.update(tag); - deleteTagAttributes(existing.getId()); - createTagAttributes(existing.getId(), tag.getAttributes()); - ret = rangerTagService.read(ret.getId()); if (LOG.isDebugEnabled()) { @@ -357,8 +337,6 @@ public void deleteTag(Long id) throws Exception { RangerTag tag = rangerTagService.read(id); - deleteTagAttributes(id); - rangerTagService.delete(tag); if (LOG.isDebugEnabled()) { @@ -498,8 +476,6 @@ public RangerServiceResource createServiceResource(RangerServiceResource resourc RangerServiceResource ret = rangerServiceResourceService.create(resource); - createResourceForServiceResource(ret.getId(), resource); - ret = rangerServiceResourceService.read(ret.getId()); if (LOG.isDebugEnabled()) { @@ -527,9 +503,6 @@ public RangerServiceResource updateServiceResource(RangerServiceResource resourc resource.setResourceSignature(serializer.getSignature()); } - boolean serviceResourceElementUpdateNeeded = - !StringUtils.equals(existing.getResourceSignature(), resource.getResourceSignature()); - resource.setCreatedBy(existing.getCreatedBy()); resource.setCreateTime(existing.getCreateTime()); resource.setGuid(existing.getGuid()); @@ -537,11 +510,6 @@ public RangerServiceResource updateServiceResource(RangerServiceResource resourc rangerServiceResourceService.update(resource); - if (serviceResourceElementUpdateNeeded) { - deleteResourceForServiceResource(existing.getId()); - createResourceForServiceResource(existing.getId(), resource); - } - RangerServiceResource ret = rangerServiceResourceService.read(existing.getId()); if (LOG.isDebugEnabled()) { @@ -551,6 +519,24 @@ public RangerServiceResource updateServiceResource(RangerServiceResource resourc return ret; } + + @Override + public void refreshServiceResource(Long resourceId) throws Exception { + XXServiceResource serviceResourceEntity = daoManager.getXXServiceResource().getById(resourceId); + String tagsText = null; + + List tagResourceMaps = getTagResourceMapsForResourceId(resourceId); + if (tagResourceMaps != null) { + List associatedTags = new ArrayList<>(); + for (RangerTagResourceMap element : tagResourceMaps) { + associatedTags.add(getTag(element.getTagId())); + } + tagsText = JsonUtils.listToJson(associatedTags); + } + serviceResourceEntity.setTags(tagsText); + daoManager.getXXServiceResource().update(serviceResourceEntity); + } + @Override public void deleteServiceResource(Long id) throws Exception { if (LOG.isDebugEnabled()) { @@ -560,7 +546,6 @@ public void deleteServiceResource(Long id) throws Exception { RangerServiceResource resource = getServiceResource(id); if(resource != null) { - deleteResourceForServiceResource(resource.getId()); rangerServiceResourceService.delete(resource); } @@ -578,7 +563,6 @@ public void deleteServiceResourceByGuid(String guid) throws Exception { RangerServiceResource resource = getServiceResourceByGuid(guid); if(resource != null) { - deleteResourceForServiceResource(resource.getId()); rangerServiceResourceService.delete(resource); } @@ -719,6 +703,9 @@ public RangerTagResourceMap createTagResourceMap(RangerTagResourceMap tagResourc RangerTagResourceMap ret = rangerTagResourceMapService.create(tagResourceMap); + // We also need to update tags stored with the resource + refreshServiceResource(tagResourceMap.getResourceId()); + if (LOG.isDebugEnabled()) { LOG.debug("<== TagDBStore.createTagResourceMap(" + tagResourceMap + "): " + ret); } @@ -741,6 +728,8 @@ public void deleteTagResourceMap(Long id) throws Exception { if (tag.getOwner() == RangerTag.OWNER_SERVICERESOURCE) { deleteTag(tagId); } + // We also need to update tags stored with the resource + refreshServiceResource(tagResourceMap.getResourceId()); if (LOG.isDebugEnabled()) { LOG.debug("<== TagDBStore.deleteTagResourceMap(" + id + ")"); @@ -996,31 +985,7 @@ public ServiceTags getServiceTags(String serviceName) throws Exception { Map tagDefMap = tagDBRetriever.getTagDefs(); Map tagMap = tagDBRetriever.getTags(); List resources = tagDBRetriever.getServiceResources(); - List tagResourceMaps = tagDBRetriever.getTagResourceMaps(); - - Map> resourceToTagIds = new HashMap>(); - - if (CollectionUtils.isNotEmpty(tagResourceMaps)) { - Long resourceId = null; - List tagIds = null; - - for (RangerTagResourceMap tagResourceMap : tagResourceMaps) { - if (!tagResourceMap.getResourceId().equals(resourceId)) { - if (resourceId != null) { - resourceToTagIds.put(resourceId, tagIds); - } - - resourceId = tagResourceMap.getResourceId(); - tagIds = new ArrayList(); - } - - tagIds.add(tagResourceMap.getTagId()); - } - - if (resourceId != null) { - resourceToTagIds.put(resourceId, tagIds); - } - } + Map> resourceToTagIds = tagDBRetriever.getResourceToTagIds(); ret = new ServiceTags(); @@ -1039,159 +1004,6 @@ public ServiceTags getServiceTags(String serviceName) throws Exception { } - private List createTagAttributeDefs(Long tagDefId, List tagAttrDefList) { - if (LOG.isDebugEnabled()) { - LOG.debug("==> TagDBStore.createTagAttributeDefs(" + tagDefId + ", attributeDefCount=" + (tagAttrDefList == null ? 0 : tagAttrDefList.size()) + ")"); - } - - if (tagDefId == null) { - throw errorUtil.createRESTException("TagDBStore.createTagAttributeDefs(): Error creating tag-attr def. tagDefId can not be null.", MessageEnums.ERROR_CREATING_OBJECT); - } - - List ret = new ArrayList(); - - if (CollectionUtils.isNotEmpty(tagAttrDefList)) { - for (RangerTagDef.RangerTagAttributeDef attrDef : tagAttrDefList) { - XXTagAttributeDef xAttrDef = new XXTagAttributeDef(); - - xAttrDef.setTagDefId(tagDefId); - xAttrDef.setName(attrDef.getName()); - xAttrDef.setType(attrDef.getType()); - xAttrDef = (XXTagAttributeDef) rangerAuditFields.populateAuditFieldsForCreate(xAttrDef); - - xAttrDef = daoManager.getXXTagAttributeDef().create(xAttrDef); - - ret.add(xAttrDef); - } - } - - if (LOG.isDebugEnabled()) { - LOG.debug("<== TagDBStore.createTagAttributeDefs(" + tagDefId + ", attributeDefCount=" + (tagAttrDefList == null ? 0 : tagAttrDefList.size()) + "): retCount=" + ret.size()); - } - - return ret; - } - - private void deleteTagAttributeDefs(Long tagDefId) { - if (LOG.isDebugEnabled()) { - LOG.debug("==> TagDBStore.deleteTagAttributeDefs(" + tagDefId + ")"); - } - - if (tagDefId != null) { - List tagAttrDefList = daoManager.getXXTagAttributeDef().findByTagDefId(tagDefId); - - if (CollectionUtils.isNotEmpty(tagAttrDefList)) { - for (XXTagAttributeDef xAttrDef : tagAttrDefList) { - if (LOG.isDebugEnabled()) { - LOG.debug("Deleting tag-attribute def [name=" + xAttrDef.getName() + "; id=" + xAttrDef.getId() + "]"); - } - daoManager.getXXTagAttributeDef().remove(xAttrDef); - } - } - } - - if (LOG.isDebugEnabled()) { - LOG.debug("<== TagDBStore.deleteTagAttributeDefs(" + tagDefId + ")"); - } - } - - private List createTagAttributes(Long tagId, Map attributes) { - List ret = new ArrayList(); - - if(MapUtils.isNotEmpty(attributes)) { - for (Map.Entry attr : attributes.entrySet()) { - XXTagAttribute xTagAttr = new XXTagAttribute(); - - xTagAttr.setTagId(tagId); - xTagAttr.setName(attr.getKey()); - xTagAttr.setValue(attr.getValue()); - xTagAttr = (XXTagAttribute) rangerAuditFields.populateAuditFieldsForCreate(xTagAttr); - - xTagAttr = daoManager.getXXTagAttribute().create(xTagAttr); - - ret.add(xTagAttr); - } - } - - return ret; - } - - private void deleteTagAttributes(Long tagId) { - List tagAttrList = daoManager.getXXTagAttribute().findByTagId(tagId); - for (XXTagAttribute tagAttr : tagAttrList) { - daoManager.getXXTagAttribute().remove(tagAttr); - } - } - - private void deleteResourceForServiceResource(Long resourceId) { - List resElements = daoManager.getXXServiceResourceElement().findByResourceId(resourceId); - - if(CollectionUtils.isNotEmpty(resElements)) { - for(XXServiceResourceElement resElement : resElements) { - List elementValues = daoManager.getXXServiceResourceElementValue().findByResValueId(resElement.getId()); - - if(CollectionUtils.isNotEmpty(elementValues)) { - for(XXServiceResourceElementValue elementValue : elementValues) { - daoManager.getXXServiceResourceElementValue().remove(elementValue.getId()); - } - } - - daoManager.getXXServiceResourceElement().remove(resElement.getId()); - } - } - } - - private void createResourceForServiceResource(Long resourceId, RangerServiceResource serviceResource) { - String serviceName = serviceResource.getServiceName(); - - XXService xService = daoManager.getXXService().findByName(serviceName); - - if (xService == null) { - throw errorUtil.createRESTException("No Service found with name: " + serviceName, MessageEnums.ERROR_CREATING_OBJECT); - } - - XXServiceDef xServiceDef = daoManager.getXXServiceDef().getById(xService.getType()); - - if (xServiceDef == null) { - throw errorUtil.createRESTException("No Service-Def found with ID: " + xService.getType(), MessageEnums.ERROR_CREATING_OBJECT); - } - - Map resElements = serviceResource.getResourceElements(); - - for (Map.Entry resElement : resElements.entrySet()) { - XXResourceDef xResDef = daoManager.getXXResourceDef().findByNameAndServiceDefId(resElement.getKey(), xServiceDef.getId()); - - if (xResDef == null) { - LOG.error("TagDBStore.createResource: ResourceType is not valid [" + resElement.getKey() + "]"); - throw errorUtil.createRESTException("Resource Type is not valid [" + resElement.getKey() + "]", MessageEnums.DATA_NOT_FOUND); - } - - RangerPolicyResource policyRes = resElement.getValue(); - - XXServiceResourceElement resourceElement = new XXServiceResourceElement(); - resourceElement.setIsExcludes(policyRes.getIsExcludes()); - resourceElement.setIsRecursive(policyRes.getIsRecursive()); - resourceElement.setResDefId(xResDef.getId()); - resourceElement.setResourceId(resourceId); - - resourceElement = (XXServiceResourceElement) rangerAuditFields.populateAuditFieldsForCreate(resourceElement); - - resourceElement = daoManager.getXXServiceResourceElement().create(resourceElement); - - int sortOrder = 1; - for (String resVal : policyRes.getValues()) { - XXServiceResourceElementValue resourceElementValue = new XXServiceResourceElementValue(); - resourceElementValue.setResElementId(resourceElement.getId()); - resourceElementValue.setValue(resVal); - resourceElementValue.setSortOrder(sortOrder); - resourceElementValue = (XXServiceResourceElementValue) rangerAuditFields.populateAuditFieldsForCreate(resourceElementValue); - - resourceElementValue = daoManager.getXXServiceResourceElementValue().create(resourceElementValue); - sortOrder++; - } - } - } - @Override public void deleteAllTagObjectsForService(String serviceName) throws Exception { @@ -1204,8 +1016,6 @@ public void deleteAllTagObjectsForService(String serviceName) throws Exception { if (service != null) { Long serviceId = service.getId(); - List xxTagAttributes = daoManager.getXXTagAttribute().findByServiceIdAndOwner(serviceId, RangerTag.OWNER_SERVICERESOURCE); - List xxTags = daoManager.getXXTag().findByServiceIdAndOwner(serviceId, RangerTag.OWNER_SERVICERESOURCE); List xxTagResourceMaps = daoManager.getXXTagResourceMap().findByServiceId(serviceId); @@ -1221,17 +1031,6 @@ public void deleteAllTagObjectsForService(String serviceName) throws Exception { } } - if (CollectionUtils.isNotEmpty(xxTagAttributes)) { - for (XXTagAttribute xxTagAttribute : xxTagAttributes) { - try { - daoManager.getXXTagAttribute().remove(xxTagAttribute); - } catch (Exception e) { - LOG.error("Error deleting RangerTagAttribute with id=" + xxTagAttribute.getId(), e); - throw e; - } - } - } - if (CollectionUtils.isNotEmpty(xxTags)) { for (XXTag xxTag : xxTags) { try { @@ -1243,32 +1042,6 @@ public void deleteAllTagObjectsForService(String serviceName) throws Exception { } } - List xxServiceResourceElementValues = daoManager.getXXServiceResourceElementValue().findByServiceId(serviceId); - - if (CollectionUtils.isNotEmpty(xxServiceResourceElementValues)) { - for (XXServiceResourceElementValue xxServiceResourceElementValue : xxServiceResourceElementValues) { - try { - daoManager.getXXServiceResourceElementValue().remove(xxServiceResourceElementValue); - } catch (Exception e) { - LOG.error("Error deleting ServiceResourceElementValue with id=" + xxServiceResourceElementValue.getId(), e); - throw e; - } - } - } - - List xxServiceResourceElements = daoManager.getXXServiceResourceElement().findByServiceId(serviceId); - - if (CollectionUtils.isNotEmpty(xxServiceResourceElements)) { - for (XXServiceResourceElement xxServiceResourceElement : xxServiceResourceElements) { - try { - daoManager.getXXServiceResourceElement().remove(xxServiceResourceElement); - } catch (Exception e) { - LOG.error("Error deleting ServiceResourceElement with id=" + xxServiceResourceElement.getId(), e); - throw e; - } - } - } - List xxServiceResources = daoManager.getXXServiceResource().findByServiceId(serviceId); if (CollectionUtils.isNotEmpty(xxServiceResources)) { diff --git a/security-admin/src/main/java/org/apache/ranger/common/db/RangerTransactionSynchronizationAdapter.java b/security-admin/src/main/java/org/apache/ranger/common/db/RangerTransactionSynchronizationAdapter.java index 2a62fb4087..536ca29e7e 100644 --- a/security-admin/src/main/java/org/apache/ranger/common/db/RangerTransactionSynchronizationAdapter.java +++ b/security-admin/src/main/java/org/apache/ranger/common/db/RangerTransactionSynchronizationAdapter.java @@ -22,6 +22,7 @@ import java.util.ArrayList; import java.util.List; +import org.apache.commons.collections.CollectionUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -46,6 +47,7 @@ public class RangerTransactionSynchronizationAdapter extends TransactionSynchron private static final Log LOG = LogFactory.getLog(RangerTransactionSynchronizationAdapter.class); private static final ThreadLocal> RUNNABLES = new ThreadLocal>(); + private static final ThreadLocal> RUNNABLES_AFTER_COMMIT = new ThreadLocal>(); public void executeOnTransactionCompletion(Runnable runnable) { if (LOG.isDebugEnabled()) { @@ -64,7 +66,7 @@ public void executeOnTransactionCompletion(Runnable runnable) { TransactionSynchronizationAdapter */ - if (!TransactionSynchronizationManager.isSynchronizationActive()) { + if (!registerSynchronization()) { LOG.info("Transaction synchronization is NOT ACTIVE. Executing right now runnable {" + runnable + "}"); runnable.run(); return; @@ -73,9 +75,36 @@ public void executeOnTransactionCompletion(Runnable runnable) { if (threadRunnables == null) { threadRunnables = new ArrayList(); RUNNABLES.set(threadRunnables); - // Register a new transaction synchronization for the current thread. - // TransactionSynchronizationManage will call afterCompletion() when current transaction completes. - TransactionSynchronizationManager.registerSynchronization(this); + } + threadRunnables.add(runnable); + } + + public void executeOnTransactionCommit(Runnable runnable) { + if (LOG.isDebugEnabled()) { + LOG.debug("Submitting new runnable {" + runnable + "} to run after transaction is committed"); + } + + /* + From TransactionSynchronizationManager documentation: + TransactionSynchronizationManager is a central helper that manages resources and transaction synchronizations per thread. + Resource management code should only register synchronizations when this manager is active, + which can be checked via isSynchronizationActive(); it should perform immediate resource cleanup else. + If transaction synchronization isn't active, there is either no current transaction, + or the transaction manager doesn't support transaction synchronization. + + Note: Synchronization is an Interface for transaction synchronization callbacks which is implemented by + TransactionSynchronizationAdapter + */ + + if (!registerSynchronization()) { + LOG.info("Transaction synchronization is NOT ACTIVE. Executing right now runnable {" + runnable + "}"); + runnable.run(); + return; + } + List threadRunnables = RUNNABLES_AFTER_COMMIT.get(); + if (threadRunnables == null) { + threadRunnables = new ArrayList(); + RUNNABLES_AFTER_COMMIT.set(threadRunnables); } threadRunnables.add(runnable); } @@ -83,48 +112,93 @@ public void executeOnTransactionCompletion(Runnable runnable) { @Override public void afterCompletion(int status) { if (LOG.isDebugEnabled()) { - LOG.debug("Transaction completed with status {" + (status == STATUS_COMMITTED ? "COMMITTED" : "ROLLED_BACK") + "}"); + LOG.debug("==> RangerTransactionSynchronizationAdapter.afterCompletion(status=" + (status == STATUS_COMMITTED ? "COMMITTED" : "ROLLED_BACK") + ")"); } - /* Thread runnables are expected to be executed only when the status is STATUS_ROLLED_BACK. Currently, executeOnTransactionCompletion() - * is called only for those changes that are going to be rolled-back by TransactionSynchronizationManager - such - * as when the operation returns HttpServletResponse.SC_NOT_MODIFIED status. - */ - //if (status == STATUS_ROLLED_BACK) { - final List threadRunnables = RUNNABLES.get(); - if (LOG.isDebugEnabled()) { - LOG.debug("Transaction completed, executing {" + threadRunnables.size() + "} runnables"); + + List allRunnables = null; + + if (status == STATUS_COMMITTED) { + final List postCommitRunnables = RUNNABLES_AFTER_COMMIT.get(); + if (CollectionUtils.isNotEmpty(postCommitRunnables)) { + allRunnables = postCommitRunnables; } - if (threadRunnables != null) { - try { - //Create new transaction - TransactionTemplate txTemplate = new TransactionTemplate(txManager); - txTemplate.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRES_NEW); - - txTemplate.execute(new TransactionCallback() { - public Object doInTransaction(TransactionStatus status) { - for (Runnable runnable : threadRunnables) { - if (LOG.isDebugEnabled()) { - LOG.debug("Executing runnable {" + runnable + "}"); - } - try { - runnable.run(); - } catch (RuntimeException e) { - LOG.error("Failed to execute runnable " + runnable, e); - break; - } - } + } - return null; - } - }); - } catch (Exception e) { - LOG.error("Failed to commit TransactionService transaction", e); - LOG.error("Ignoring..."); - } + final List postCompletionRunnables = RUNNABLES.get(); + + if (CollectionUtils.isNotEmpty(postCompletionRunnables)) { + if (allRunnables == null) { + allRunnables = postCompletionRunnables; + } else { + allRunnables.addAll(postCompletionRunnables); } + } - //} + runRunnables(allRunnables); + + RUNNABLES_AFTER_COMMIT.remove(); RUNNABLES.remove(); + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerTransactionSynchronizationAdapter.afterCompletion(status=" + (status == STATUS_COMMITTED ? "COMMITTED" : "ROLLED_BACK") + ")"); + } } + private boolean registerSynchronization() { + final boolean ret = TransactionSynchronizationManager.isSynchronizationActive(); + if (ret) { + List threadRunnablesOnCompletion = RUNNABLES.get(); + List threadRunnablesOnCommit = RUNNABLES_AFTER_COMMIT.get(); + if (threadRunnablesOnCompletion == null && threadRunnablesOnCommit == null) { + TransactionSynchronizationManager.registerSynchronization(this); + } + } + return ret; + } + + private void runRunnables(final List runnables) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerTransactionSynchronizationAdapter.runRunnables()"); + } + + if (runnables != null) { + if (LOG.isDebugEnabled()) { + LOG.debug("Executing {" + runnables.size() + "} runnables"); + } + try { + //Create new transaction + TransactionTemplate txTemplate = new TransactionTemplate(txManager); + txTemplate.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRES_NEW); + + txTemplate.execute(new TransactionCallback() { + public Object doInTransaction(TransactionStatus status) { + for (Runnable runnable : runnables) { + if (LOG.isDebugEnabled()) { + LOG.debug("Executing runnable {" + runnable + "}"); + } + try { + runnable.run(); + } catch (RuntimeException e) { + LOG.error("Failed to execute runnable " + runnable, e); + break; + } + } + + return null; + } + }); + } catch (Exception e) { + LOG.error("Failed to commit TransactionService transaction", e); + LOG.error("Ignoring..."); + } + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("No runnables to execute"); + } + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerTransactionSynchronizationAdapter.runRunnables()"); + } + } } \ No newline at end of file diff --git a/security-admin/src/main/java/org/apache/ranger/db/RangerDaoManager.java b/security-admin/src/main/java/org/apache/ranger/db/RangerDaoManager.java index e3b878b3ec..80cb4be158 100644 --- a/security-admin/src/main/java/org/apache/ranger/db/RangerDaoManager.java +++ b/security-admin/src/main/java/org/apache/ranger/db/RangerDaoManager.java @@ -28,6 +28,7 @@ import org.apache.ranger.common.RangerConstants; import org.apache.ranger.common.StringUtil; import org.apache.ranger.common.db.BaseDao; +import org.apache.ranger.common.db.RangerTransactionSynchronizationAdapter; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; @@ -44,6 +45,9 @@ public class RangerDaoManager extends RangerDaoManagerBase { @Autowired StringUtil stringUtil; + @Autowired + RangerTransactionSynchronizationAdapter transactionSynchronizationAdapter; + @Override public EntityManager getEntityManager() { return em; @@ -68,15 +72,19 @@ public StringUtil getStringUtil() { return stringUtil; } - /* - * (non-Javadoc) - */ - @Override - public BaseDao getDaoForClassType(int classType) { - if (classType == RangerConstants.CLASS_TYPE_NONE) { - return null; - } - return super.getDaoForClassType(classType); + /** + * (non-Javadoc) + */ + @Override + public BaseDao getDaoForClassType(int classType) { + if (classType == RangerConstants.CLASS_TYPE_NONE) { + return null; + } + return super.getDaoForClassType(classType); + } + + public RangerTransactionSynchronizationAdapter getRangerTransactionSynchronizationAdapter() { + return transactionSynchronizationAdapter; } } diff --git a/security-admin/src/main/java/org/apache/ranger/db/RangerDaoManagerBase.java b/security-admin/src/main/java/org/apache/ranger/db/RangerDaoManagerBase.java index d718441ee9..1886aae529 100644 --- a/security-admin/src/main/java/org/apache/ranger/db/RangerDaoManagerBase.java +++ b/security-admin/src/main/java/org/apache/ranger/db/RangerDaoManagerBase.java @@ -24,19 +24,13 @@ */ import javax.persistence.EntityManager; - import org.apache.log4j.Logger; import org.apache.ranger.common.AppConstants; -import org.apache.ranger.common.RESTErrorUtil; import org.apache.ranger.common.db.BaseDao; -import org.springframework.beans.factory.annotation.Autowired; - public abstract class RangerDaoManagerBase { private static final Logger logger = Logger.getLogger(RangerDaoManagerBase.class); - @Autowired - protected RESTErrorUtil restErrorUtil; abstract public EntityManager getEntityManager(); public RangerDaoManagerBase() { @@ -366,6 +360,24 @@ public BaseDao getDaoForClassName(String className) { if (className.equals("XXPluginInfo")) { return getXXPluginInfo(); } + if (className.equals("XXPolicyRefCondition")) { + return getXXPolicyRefCondition(); + } + if (className.equals("XXPolicyRefGroup")) { + return getXXPolicyRefGroup(); + } + if (className.equals("XXPolicyRefDataMaskType")) { + return getXXPolicyRefDataMaskType(); + } + if (className.equals("XXPolicyRefResource")) { + return getXXPolicyRefResource(); + } + if (className.equals("XXPolicyRefUser")) { + return getXXPolicyRefUser(); + } + if (className.equals("XXPolicyRefAccessType")) { + return getXXPolicyRefAccessType(); + } logger.error("No DaoManager found for className=" + className, new Throwable()); return null; } @@ -590,5 +602,29 @@ public XXServiceVersionInfoDao getXXServiceVersionInfo() { public XXPluginInfoDao getXXPluginInfo() { return new XXPluginInfoDao(this); } + + public XXPolicyRefConditionDao getXXPolicyRefCondition() { + return new XXPolicyRefConditionDao(this); + } + + public XXPolicyRefGroupDao getXXPolicyRefGroup() { + return new XXPolicyRefGroupDao(this); + } + + public XXPolicyRefDataMaskTypeDao getXXPolicyRefDataMaskType() { + return new XXPolicyRefDataMaskTypeDao(this); + } + + public XXPolicyRefResourceDao getXXPolicyRefResource() { + return new XXPolicyRefResourceDao(this); + } + + public XXPolicyRefUserDao getXXPolicyRefUser() { + return new XXPolicyRefUserDao(this); + } + + public XXPolicyRefAccessTypeDao getXXPolicyRefAccessType() { + return new XXPolicyRefAccessTypeDao(this); + } } diff --git a/security-admin/src/main/java/org/apache/ranger/db/XXAccessTypeDefDao.java b/security-admin/src/main/java/org/apache/ranger/db/XXAccessTypeDefDao.java index 8f3a506871..8db657f698 100644 --- a/security-admin/src/main/java/org/apache/ranger/db/XXAccessTypeDefDao.java +++ b/security-admin/src/main/java/org/apache/ranger/db/XXAccessTypeDefDao.java @@ -58,5 +58,4 @@ public XXAccessTypeDef findByNameAndServiceId(String name, Long serviceId) { return null; } } - } diff --git a/security-admin/src/main/java/org/apache/ranger/db/XXDataMaskTypeDefDao.java b/security-admin/src/main/java/org/apache/ranger/db/XXDataMaskTypeDefDao.java index f6e1aff0b4..7e43db497a 100644 --- a/security-admin/src/main/java/org/apache/ranger/db/XXDataMaskTypeDefDao.java +++ b/security-admin/src/main/java/org/apache/ranger/db/XXDataMaskTypeDefDao.java @@ -58,5 +58,4 @@ public XXDataMaskTypeDef findByNameAndServiceId(String name, Long serviceId) { return null; } } - } diff --git a/security-admin/src/main/java/org/apache/ranger/db/XXGroupDao.java b/security-admin/src/main/java/org/apache/ranger/db/XXGroupDao.java index 19e2e11f36..acc8700f73 100644 --- a/security-admin/src/main/java/org/apache/ranger/db/XXGroupDao.java +++ b/security-admin/src/main/java/org/apache/ranger/db/XXGroupDao.java @@ -23,8 +23,6 @@ import java.util.ArrayList; import java.util.List; -import javax.persistence.NoResultException; - import org.apache.ranger.common.db.BaseDao; import org.apache.ranger.entity.XXGroup; @@ -66,19 +64,5 @@ public XXGroup findByGroupName(String groupName) { } return null; } - - @SuppressWarnings("unchecked") - public List findByPolicyItemId(Long polItemId) { - if (polItemId == null) { - return null; - } - try { - return getEntityManager() - .createNamedQuery("XXGroup.findByPolicyItemId") - .setParameter("polItemId", polItemId).getResultList(); - } catch (NoResultException e) { - return null; - } - } } diff --git a/security-admin/src/main/java/org/apache/ranger/db/XXPolicyConditionDefDao.java b/security-admin/src/main/java/org/apache/ranger/db/XXPolicyConditionDefDao.java index 21afcac891..e2048aed18 100644 --- a/security-admin/src/main/java/org/apache/ranger/db/XXPolicyConditionDefDao.java +++ b/security-admin/src/main/java/org/apache/ranger/db/XXPolicyConditionDefDao.java @@ -59,33 +59,5 @@ public XXPolicyConditionDef findByServiceDefIdAndName(Long serviceDefId, String return null; } } - - public List findByPolicyItemId(Long polItemId) { - if(polItemId == null) { - return new ArrayList(); - } - try { - return getEntityManager() - .createNamedQuery("XXPolicyConditionDef.findByPolicyItemId", tClass) - .setParameter("polItemId", polItemId).getResultList(); - } catch (NoResultException e) { - return new ArrayList(); - } - } - - public XXPolicyConditionDef findByPolicyItemIdAndName(Long polItemId, String name) { - if(polItemId == null || name == null) { - return null; - } - try { - return getEntityManager() - .createNamedQuery("XXPolicyConditionDef.findByPolicyItemIdAndName", tClass) - .setParameter("polItemId", polItemId) - .setParameter("name", name).getSingleResult(); - } catch (NoResultException e) { - return null; - } - } - } diff --git a/security-admin/src/main/java/org/apache/ranger/db/XXPolicyItemAccessDao.java b/security-admin/src/main/java/org/apache/ranger/db/XXPolicyItemAccessDao.java index de37e10cde..aeafc8ac1f 100644 --- a/security-admin/src/main/java/org/apache/ranger/db/XXPolicyItemAccessDao.java +++ b/security-admin/src/main/java/org/apache/ranger/db/XXPolicyItemAccessDao.java @@ -30,19 +30,6 @@ public class XXPolicyItemAccessDao extends BaseDao { public XXPolicyItemAccessDao(RangerDaoManagerBase daoManager) { super(daoManager); } - - public List findByPolicyItemId(Long polItemId) { - if(polItemId == null) { - return new ArrayList(); - } - try { - return getEntityManager() - .createNamedQuery("XXPolicyItemAccess.findByPolicyItemId", tClass) - .setParameter("polItemId", polItemId).getResultList(); - } catch (NoResultException e) { - return new ArrayList(); - } - } public List findByPolicyId(Long policyId) { if(policyId == null) { @@ -70,16 +57,4 @@ public List findByServiceId(Long serviceId) { } } - public List findByType(Long type) { - if (type == null) { - return new ArrayList(); - } - try { - return getEntityManager().createNamedQuery("XXPolicyItemAccess.findByType", tClass) - .setParameter("type", type).getResultList(); - } catch (NoResultException e) { - return new ArrayList(); - } - } - } diff --git a/security-admin/src/main/java/org/apache/ranger/db/XXPolicyItemConditionDao.java b/security-admin/src/main/java/org/apache/ranger/db/XXPolicyItemConditionDao.java index 11596ef4d1..4b974a7738 100644 --- a/security-admin/src/main/java/org/apache/ranger/db/XXPolicyItemConditionDao.java +++ b/security-admin/src/main/java/org/apache/ranger/db/XXPolicyItemConditionDao.java @@ -30,19 +30,6 @@ public class XXPolicyItemConditionDao extends BaseDao { public XXPolicyItemConditionDao(RangerDaoManagerBase daoManager) { super(daoManager); } - - public List findByPolicyItemId(Long polItemId) { - if(polItemId == null) { - return new ArrayList(); - } - try { - return getEntityManager() - .createNamedQuery("XXPolicyItemCondition.findByPolicyItemId", tClass) - .setParameter("polItemId", polItemId).getResultList(); - } catch (NoResultException e) { - return new ArrayList(); - } - } public List findByPolicyId(Long policyId) { if(policyId == null) { @@ -70,31 +57,4 @@ public List findByServiceId(Long serviceId) { } } - public List findByPolicyItemAndDefId(Long polItemId, - Long polCondDefId) { - if(polItemId == null || polCondDefId == null) { - return new ArrayList(); - } - try { - return getEntityManager() - .createNamedQuery("XXPolicyItemCondition.findByPolicyItemAndDefId", tClass) - .setParameter("polItemId", polItemId) - .setParameter("polCondDefId", polCondDefId).getResultList(); - } catch (NoResultException e) { - return new ArrayList(); - } - } - - public List findByPolicyConditionDefId(Long polCondDefId) { - if (polCondDefId == null) { - return new ArrayList(); - } - try { - return getEntityManager().createNamedQuery("XXPolicyItemCondition.findByPolicyConditionDefId", tClass) - .setParameter("polCondDefId", polCondDefId).getResultList(); - } catch (NoResultException e) { - return new ArrayList(); - } - } - } diff --git a/security-admin/src/main/java/org/apache/ranger/db/XXPolicyItemDataMaskInfoDao.java b/security-admin/src/main/java/org/apache/ranger/db/XXPolicyItemDataMaskInfoDao.java index a8418c6261..7deda623a1 100644 --- a/security-admin/src/main/java/org/apache/ranger/db/XXPolicyItemDataMaskInfoDao.java +++ b/security-admin/src/main/java/org/apache/ranger/db/XXPolicyItemDataMaskInfoDao.java @@ -30,19 +30,6 @@ public class XXPolicyItemDataMaskInfoDao extends BaseDao findByPolicyItemId(Long polItemId) { - if(polItemId == null) { - return new ArrayList(); - } - try { - return getEntityManager() - .createNamedQuery("XXPolicyItemDataMaskInfo.findByPolicyItemId", tClass) - .setParameter("polItemId", polItemId).getResultList(); - } catch (NoResultException e) { - return new ArrayList(); - } - } public List findByPolicyId(Long policyId) { if(policyId == null) { @@ -70,16 +57,4 @@ public List findByServiceId(Long serviceId) { } } - public List findByType(Long type) { - if (type == null) { - return new ArrayList(); - } - try { - return getEntityManager().createNamedQuery("XXPolicyItemDataMaskInfo.findByType", tClass) - .setParameter("type", type).getResultList(); - } catch (NoResultException e) { - return new ArrayList(); - } - } - } diff --git a/security-admin/src/main/java/org/apache/ranger/db/XXPolicyItemGroupPermDao.java b/security-admin/src/main/java/org/apache/ranger/db/XXPolicyItemGroupPermDao.java index 8c05699b25..aa4a3497ed 100644 --- a/security-admin/src/main/java/org/apache/ranger/db/XXPolicyItemGroupPermDao.java +++ b/security-admin/src/main/java/org/apache/ranger/db/XXPolicyItemGroupPermDao.java @@ -31,19 +31,6 @@ public XXPolicyItemGroupPermDao(RangerDaoManagerBase daoManager) { super(daoManager); } - public List findByPolicyItemId(Long polItemId) { - if(polItemId == null) { - return new ArrayList(); - } - try { - return getEntityManager() - .createNamedQuery("XXPolicyItemGroupPerm.findByPolicyItemId", tClass) - .setParameter("polItemId", polItemId).getResultList(); - } catch (NoResultException e) { - return new ArrayList(); - } - } - public List findByPolicyId(Long policyId) { if(policyId == null) { return new ArrayList(); diff --git a/security-admin/src/main/java/org/apache/ranger/db/XXPolicyItemRowFilterInfoDao.java b/security-admin/src/main/java/org/apache/ranger/db/XXPolicyItemRowFilterInfoDao.java index 4618e7dc76..ff889e7a33 100644 --- a/security-admin/src/main/java/org/apache/ranger/db/XXPolicyItemRowFilterInfoDao.java +++ b/security-admin/src/main/java/org/apache/ranger/db/XXPolicyItemRowFilterInfoDao.java @@ -29,19 +29,6 @@ public class XXPolicyItemRowFilterInfoDao extends BaseDao findByPolicyItemId(Long polItemId) { - if(polItemId == null) { - return new ArrayList(); - } - try { - return getEntityManager() - .createNamedQuery("XXPolicyItemRowFilterInfo.findByPolicyItemId", tClass) - .setParameter("polItemId", polItemId).getResultList(); - } catch (NoResultException e) { - return new ArrayList(); - } - } public List findByPolicyId(Long policyId) { if(policyId == null) { diff --git a/security-admin/src/main/java/org/apache/ranger/db/XXPolicyItemUserPermDao.java b/security-admin/src/main/java/org/apache/ranger/db/XXPolicyItemUserPermDao.java index 40a0da13b2..66f156630c 100644 --- a/security-admin/src/main/java/org/apache/ranger/db/XXPolicyItemUserPermDao.java +++ b/security-admin/src/main/java/org/apache/ranger/db/XXPolicyItemUserPermDao.java @@ -31,19 +31,6 @@ public XXPolicyItemUserPermDao(RangerDaoManagerBase daoManager) { super(daoManager); } - public List findByPolicyItemId(Long polItemId) { - if(polItemId == null) { - return new ArrayList(); - } - try { - return getEntityManager() - .createNamedQuery("XXPolicyItemUserPerm.findByPolicyItemId", tClass) - .setParameter("polItemId", polItemId).getResultList(); - } catch (NoResultException e) { - return new ArrayList(); - } - } - public List findByPolicyId(Long policyId) { if(policyId == null) { return new ArrayList(); diff --git a/security-admin/src/main/java/org/apache/ranger/db/XXPolicyRefAccessTypeDao.java b/security-admin/src/main/java/org/apache/ranger/db/XXPolicyRefAccessTypeDao.java new file mode 100644 index 0000000000..1ef01bb28c --- /dev/null +++ b/security-admin/src/main/java/org/apache/ranger/db/XXPolicyRefAccessTypeDao.java @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.db; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import javax.persistence.NoResultException; + +import org.apache.ranger.biz.RangerPolicyRetriever; +import org.apache.ranger.common.db.BaseDao; +import org.apache.ranger.entity.XXPolicyRefAccessType; +import org.springframework.stereotype.Service; + +@Service +public class XXPolicyRefAccessTypeDao extends BaseDao { + + public XXPolicyRefAccessTypeDao(RangerDaoManagerBase daoManager) { + super(daoManager); + } + + public List findByPolicyId(Long polId) { + if(polId == null) { + return Collections.EMPTY_LIST; + } + try { + return getEntityManager() + .createNamedQuery("XXPolicyRefAccessType.findByPolicyId", tClass) + .setParameter("policyId", polId).getResultList(); + } catch (NoResultException e) { + return Collections.EMPTY_LIST; + } + } + + public List findByAccessTypeDefId(Long accessTypeDefId) { + if (accessTypeDefId == null) { + return Collections.EMPTY_LIST; + } + try { + return getEntityManager().createNamedQuery("XXPolicyRefAccessType.findByAccessTypeDefId", tClass) + .setParameter("accessDefId", accessTypeDefId) + .getResultList(); + } catch (NoResultException e) { + return Collections.EMPTY_LIST; + } + } + + @SuppressWarnings("unchecked") + public List findUpdatedAccessNamesByPolicy(Long policyId) { + List ret = new ArrayList<>(); + if (policyId != null) { + List rows = (List) getEntityManager() + .createNamedQuery("XXPolicyRefAccessType.findUpdatedAccessNamesByPolicy") + .setParameter("policy", policyId) + .getResultList(); + if (rows != null) { + for (Object[] row : rows) { + ret.add(new RangerPolicyRetriever.PolicyTextNameMap((Long)row[0], (String)row[1], (String)row[2])); + } + } + } + return ret; + } + + @SuppressWarnings("unchecked") + public List findUpdatedAccessNamesByService(Long serviceId) { + List ret = new ArrayList<>(); + if (serviceId != null) { + List rows = (List) getEntityManager() + .createNamedQuery("XXPolicyRefAccessType.findUpdatedAccessNamesByService") + .setParameter("service", serviceId) + .getResultList(); + if (rows != null) { + for (Object[] row : rows) { + ret.add(new RangerPolicyRetriever.PolicyTextNameMap((Long)row[0], (String)row[1], (String)row[2])); + } + } + } + return ret; + } + +} diff --git a/security-admin/src/main/java/org/apache/ranger/db/XXPolicyRefConditionDao.java b/security-admin/src/main/java/org/apache/ranger/db/XXPolicyRefConditionDao.java new file mode 100644 index 0000000000..2c04ab4834 --- /dev/null +++ b/security-admin/src/main/java/org/apache/ranger/db/XXPolicyRefConditionDao.java @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.db; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import javax.persistence.NoResultException; + +import org.apache.ranger.biz.RangerPolicyRetriever; +import org.apache.ranger.common.db.BaseDao; +import org.apache.ranger.entity.XXPolicyRefCondition; +import org.springframework.stereotype.Service; + +@Service +public class XXPolicyRefConditionDao extends BaseDao { + + public XXPolicyRefConditionDao(RangerDaoManagerBase daoManager) { + super(daoManager); + } + + public List findByPolicyId(Long polId) { + if(polId == null) { + return Collections.EMPTY_LIST; + } + try { + return getEntityManager() + .createNamedQuery("XXPolicyRefCondition.findByPolicyId", tClass) + .setParameter("policyId", polId).getResultList(); + } catch (NoResultException e) { + return Collections.EMPTY_LIST; + } + } + public List findByConditionName(String conditionName) { + if (conditionName == null) { + return Collections.EMPTY_LIST; + } + try { + return getEntityManager().createNamedQuery("XXPolicyRefCondition.findByConditionName", tClass) + .setParameter("conditionName", conditionName).getResultList(); + } catch (NoResultException e) { + return Collections.EMPTY_LIST; + } + } + + public List findByConditionDefId(Long conditionDefId) { + if (conditionDefId == null) { + return Collections.EMPTY_LIST; + } + try { + return getEntityManager().createNamedQuery("XXPolicyRefCondition.findByConditionDefId", tClass) + .setParameter("conditionDefId", conditionDefId) + .getResultList(); + } catch (NoResultException e) { + return Collections.EMPTY_LIST; + } + } + + @SuppressWarnings("unchecked") + public List findUpdatedConditionNamesByPolicy(Long policyId) { + List ret = new ArrayList<>(); + if (policyId != null) { + List rows = (List) getEntityManager() + .createNamedQuery("XXPolicyRefCondition.findUpdatedConditionNamesByPolicy") + .setParameter("policy", policyId) + .getResultList(); + if (rows != null) { + for (Object[] row : rows) { + ret.add(new RangerPolicyRetriever.PolicyTextNameMap((Long)row[0], (String)row[1], (String)row[2])); + } + } + } + return ret; + } + + @SuppressWarnings("unchecked") + public List findUpdatedConditionNamesByService(Long serviceId) { + List ret = new ArrayList<>(); + if (serviceId != null) { + List rows = (List) getEntityManager() + .createNamedQuery("XXPolicyRefCondition.findUpdatedConditionNamesByService") + .setParameter("service", serviceId) + .getResultList(); + if (rows != null) { + for (Object[] row : rows) { + ret.add(new RangerPolicyRetriever.PolicyTextNameMap((Long)row[0], (String)row[1], (String)row[2])); + } + } + } + return ret; + } + +} diff --git a/security-admin/src/main/java/org/apache/ranger/db/XXPolicyRefDataMaskTypeDao.java b/security-admin/src/main/java/org/apache/ranger/db/XXPolicyRefDataMaskTypeDao.java new file mode 100644 index 0000000000..258e3b0bab --- /dev/null +++ b/security-admin/src/main/java/org/apache/ranger/db/XXPolicyRefDataMaskTypeDao.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.db; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import javax.persistence.NoResultException; + +import org.apache.ranger.biz.RangerPolicyRetriever; +import org.apache.ranger.common.db.BaseDao; +import org.apache.ranger.entity.XXPolicyRefDataMaskType; +import org.springframework.stereotype.Service; + +@Service +public class XXPolicyRefDataMaskTypeDao extends BaseDao{ + + public XXPolicyRefDataMaskTypeDao(RangerDaoManagerBase daoManager) { + super(daoManager); + } + + public List findByPolicyId(Long policyId) { + if(policyId == null) { + return Collections.EMPTY_LIST; + } + try { + return getEntityManager() + .createNamedQuery("XXPolicyRefDataMaskType.findByPolicyId", tClass) + .setParameter("policyId", policyId).getResultList(); + } catch (NoResultException e) { + return Collections.EMPTY_LIST; + } + } + + @SuppressWarnings("unchecked") + public List findUpdatedDataMaskNamesByPolicy(Long policyId) { + List ret = new ArrayList<>(); + if (policyId != null) { + List rows = (List) getEntityManager() + .createNamedQuery("XXPolicyRefDataMaskType.findUpdatedDataMaskNamesByPolicy") + .setParameter("policy", policyId) + .getResultList(); + if (rows != null) { + for (Object[] row : rows) { + ret.add(new RangerPolicyRetriever.PolicyTextNameMap((Long)row[0], (String)row[1], (String)row[2])); + } + } + } + return ret; + } + + @SuppressWarnings("unchecked") + public List findUpdatedDataMaskNamesByService(Long serviceId) { + List ret = new ArrayList<>(); + if (serviceId != null) { + List rows = (List) getEntityManager() + .createNamedQuery("XXPolicyRefDataMaskType.findUpdatedDataMaskNamesByService") + .setParameter("service", serviceId) + .getResultList(); + if (rows != null) { + for (Object[] row : rows) { + ret.add(new RangerPolicyRetriever.PolicyTextNameMap((Long)row[0], (String)row[1], (String)row[2])); + } + } + } + return ret; + } +} diff --git a/security-admin/src/main/java/org/apache/ranger/db/XXPolicyRefGroupDao.java b/security-admin/src/main/java/org/apache/ranger/db/XXPolicyRefGroupDao.java new file mode 100644 index 0000000000..08829d4f78 --- /dev/null +++ b/security-admin/src/main/java/org/apache/ranger/db/XXPolicyRefGroupDao.java @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.db; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import javax.persistence.NoResultException; + +import org.apache.ranger.biz.RangerPolicyRetriever; +import org.apache.ranger.common.db.BaseDao; +import org.apache.ranger.entity.XXPolicyRefGroup; +import org.springframework.stereotype.Service; + +@Service +public class XXPolicyRefGroupDao extends BaseDao{ + + + public XXPolicyRefGroupDao(RangerDaoManagerBase daoManager) { + super(daoManager); + } + + public List findByPolicyId(Long policyId) { + if(policyId == null) { + return Collections.EMPTY_LIST; + } + try { + return getEntityManager() + .createNamedQuery("XXPolicyRefGroup.findByPolicyId", tClass) + .setParameter("policyId", policyId).getResultList(); + } catch (NoResultException e) { + return Collections.EMPTY_LIST; + } + } + public List findByGroupName(String groupName) { + if (groupName == null) { + return Collections.EMPTY_LIST; + } + try { + return getEntityManager().createNamedQuery("XXPolicyRefGroup.findByGroupName", tClass) + .setParameter("groupName", groupName).getResultList(); + } catch (NoResultException e) { + return Collections.EMPTY_LIST; + } + } + + @SuppressWarnings("unchecked") + public List findUpdatedGroupNamesByPolicy(Long policyId) { + List ret = new ArrayList<>(); + if (policyId != null) { + List rows = (List) getEntityManager() + .createNamedQuery("XXPolicyRefGroup.findUpdatedGroupNamesByPolicy") + .setParameter("policy", policyId) + .getResultList(); + if (rows != null) { + for (Object[] row : rows) { + ret.add(new RangerPolicyRetriever.PolicyTextNameMap((Long)row[0], (String)row[1], (String)row[2])); + } + } + } + return ret; + } + + @SuppressWarnings("unchecked") + public List findUpdatedGroupNamesByService(Long serviceId) { + List ret = new ArrayList<>(); + if (serviceId != null) { + List rows = (List) getEntityManager() + .createNamedQuery("XXPolicyRefGroup.findUpdatedGroupNamesByService") + .setParameter("service", serviceId) + .getResultList(); + if (rows != null) { + for (Object[] row : rows) { + ret.add(new RangerPolicyRetriever.PolicyTextNameMap((Long)row[0], (String)row[1], (String)row[2])); + } + } + } + return ret; + } + +} diff --git a/security-admin/src/main/java/org/apache/ranger/db/XXPolicyRefResourceDao.java b/security-admin/src/main/java/org/apache/ranger/db/XXPolicyRefResourceDao.java new file mode 100644 index 0000000000..e259ee8646 --- /dev/null +++ b/security-admin/src/main/java/org/apache/ranger/db/XXPolicyRefResourceDao.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.db; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import javax.persistence.NoResultException; + +import org.apache.ranger.biz.RangerPolicyRetriever; +import org.apache.ranger.common.db.BaseDao; +import org.apache.ranger.entity.XXPolicyRefResource; +import org.springframework.stereotype.Service; + +@Service +public class XXPolicyRefResourceDao extends BaseDao{ + + public XXPolicyRefResourceDao(RangerDaoManagerBase daoManager) { + super(daoManager); + } + + public List findByPolicyId(Long policyId) { + if(policyId == null) { + return Collections.EMPTY_LIST; + } + try { + return getEntityManager() + .createNamedQuery("XXPolicyRefResource.findByPolicyId", tClass) + .setParameter("policyId", policyId).getResultList(); + } catch (NoResultException e) { + return Collections.EMPTY_LIST; + } + } + + public List findByResourceDefID(Long resourceDefId) { + if (resourceDefId == null) { + return Collections.EMPTY_LIST; + } + try { + return getEntityManager().createNamedQuery("XXPolicyRefResource.findByResourceDefId", tClass) + .setParameter("resourceDefId", resourceDefId).getResultList(); + } catch (NoResultException e) { + return Collections.EMPTY_LIST; + } + } + + @SuppressWarnings("unchecked") + public List findUpdatedResourceNamesByPolicy(Long policyId) { + List ret = new ArrayList<>(); + if (policyId != null) { + List rows = (List) getEntityManager() + .createNamedQuery("XXPolicyRefResource.findUpdatedResourceNamesByPolicy") + .setParameter("policy", policyId) + .getResultList(); + if (rows != null) { + for (Object[] row : rows) { + ret.add(new RangerPolicyRetriever.PolicyTextNameMap((Long)row[0], (String)row[1], (String)row[2])); + } + } + } + return ret; + } + + @SuppressWarnings("unchecked") + public List findUpdatedResourceNamesByService(Long serviceId) { + List ret = new ArrayList<>(); + if (serviceId != null) { + List rows = (List) getEntityManager() + .createNamedQuery("XXPolicyRefResource.findUpdatedResourceNamesByService") + .setParameter("service", serviceId) + .getResultList(); + if (rows != null) { + for (Object[] row : rows) { + ret.add(new RangerPolicyRetriever.PolicyTextNameMap((Long)row[0], (String)row[1], (String)row[2])); + } + } + } + return ret; + } +} diff --git a/security-admin/src/main/java/org/apache/ranger/db/XXPolicyRefUserDao.java b/security-admin/src/main/java/org/apache/ranger/db/XXPolicyRefUserDao.java new file mode 100644 index 0000000000..f7b6131c6a --- /dev/null +++ b/security-admin/src/main/java/org/apache/ranger/db/XXPolicyRefUserDao.java @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.db; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import javax.persistence.NoResultException; + +import org.apache.ranger.biz.RangerPolicyRetriever; +import org.apache.ranger.common.db.BaseDao; +import org.apache.ranger.entity.XXPolicyRefUser; +import org.springframework.stereotype.Service; + +@Service +public class XXPolicyRefUserDao extends BaseDao{ + + + public XXPolicyRefUserDao(RangerDaoManagerBase daoManager) { + super(daoManager); + } + + public List findByPolicyId(Long policyId) { + if(policyId == null) { + return Collections.EMPTY_LIST; + } + try { + return getEntityManager() + .createNamedQuery("XXPolicyRefUser.findByPolicyId", tClass) + .setParameter("policyId", policyId).getResultList(); + } catch (NoResultException e) { + return Collections.EMPTY_LIST; + } + } + public List findByUserName(String userName) { + if (userName == null) { + return Collections.EMPTY_LIST; + } + try { + return getEntityManager().createNamedQuery("XXPolicyRefUser.findByUserName", tClass) + .setParameter("userName", userName).getResultList(); + } catch (NoResultException e) { + return Collections.EMPTY_LIST; + } + } + + public List findByUserId(String userID) { + if (userID == null) { + return Collections.EMPTY_LIST; + } + try { + return getEntityManager().createNamedQuery("XXPolicyRefUser.findByUserId", tClass) + .setParameter("userID", userID).getResultList(); + } catch (NoResultException e) { + return Collections.EMPTY_LIST; + } + } + + @SuppressWarnings("unchecked") + public List findUpdatedUserNamesByPolicy(Long policyId) { + List ret = new ArrayList<>(); + if (policyId != null) { + List rows = (List) getEntityManager() + .createNamedQuery("XXPolicyRefUser.findUpdatedUserNamesByPolicy") + .setParameter("policy", policyId) + .getResultList(); + if (rows != null) { + for (Object[] row : rows) { + ret.add(new RangerPolicyRetriever.PolicyTextNameMap((Long)row[0], (String)row[1], (String)row[2])); + } + } + } + return ret; + } + + @SuppressWarnings("unchecked") + public List findUpdatedUserNamesByService(Long serviceId) { + List ret = new ArrayList<>(); + if (serviceId != null) { + List rows = (List) getEntityManager() + .createNamedQuery("XXPolicyRefUser.findUpdatedUserNamesByService") + .setParameter("service", serviceId) + .getResultList(); + if (rows != null) { + for (Object[] row : rows) { + ret.add(new RangerPolicyRetriever.PolicyTextNameMap((Long)row[0], (String)row[1], (String)row[2])); + } + } + } + return ret; + } + +} diff --git a/security-admin/src/main/java/org/apache/ranger/db/XXPolicyResourceDao.java b/security-admin/src/main/java/org/apache/ranger/db/XXPolicyResourceDao.java index 4b04b9664d..c34e44e94f 100644 --- a/security-admin/src/main/java/org/apache/ranger/db/XXPolicyResourceDao.java +++ b/security-admin/src/main/java/org/apache/ranger/db/XXPolicyResourceDao.java @@ -30,20 +30,6 @@ public class XXPolicyResourceDao extends BaseDao { public XXPolicyResourceDao(RangerDaoManagerBase daoManager) { super(daoManager); } - - public XXPolicyResource findByResDefIdAndPolicyId(Long resDefId, Long polId) { - if(resDefId == null || polId == null) { - return null; - } - try { - return getEntityManager() - .createNamedQuery("XXPolicyResource.findByResDefIdAndPolicyId", tClass) - .setParameter("resDefId", resDefId).setParameter("polId", polId) - .getSingleResult(); - } catch (NoResultException e) { - return null; - } - } public List findByPolicyId(Long policyId) { if(policyId == null) { diff --git a/security-admin/src/main/java/org/apache/ranger/db/XXPolicyResourceMapDao.java b/security-admin/src/main/java/org/apache/ranger/db/XXPolicyResourceMapDao.java index 70657370af..edda109f1f 100644 --- a/security-admin/src/main/java/org/apache/ranger/db/XXPolicyResourceMapDao.java +++ b/security-admin/src/main/java/org/apache/ranger/db/XXPolicyResourceMapDao.java @@ -30,19 +30,6 @@ public class XXPolicyResourceMapDao extends BaseDao { public XXPolicyResourceMapDao(RangerDaoManagerBase daoManager) { super(daoManager); } - - public List findByPolicyResId(Long polResId) { - if(polResId == null) { - return new ArrayList(); - } - try { - return getEntityManager() - .createNamedQuery("XXPolicyResourceMap.findByPolicyResId", tClass) - .setParameter("polResId", polResId).getResultList(); - } catch (NoResultException e) { - return new ArrayList(); - } - } public List findByPolicyId(Long policyId) { if(policyId == null) { diff --git a/security-admin/src/main/java/org/apache/ranger/db/XXResourceDefDao.java b/security-admin/src/main/java/org/apache/ranger/db/XXResourceDefDao.java index aee56e833e..646e61d897 100644 --- a/security-admin/src/main/java/org/apache/ranger/db/XXResourceDefDao.java +++ b/security-admin/src/main/java/org/apache/ranger/db/XXResourceDefDao.java @@ -97,5 +97,4 @@ public List findByParentResId(Long parentId) { return new ArrayList(); } } - } diff --git a/security-admin/src/main/java/org/apache/ranger/db/XXServiceResourceDao.java b/security-admin/src/main/java/org/apache/ranger/db/XXServiceResourceDao.java index f87c0ae596..7c25d61f16 100644 --- a/security-admin/src/main/java/org/apache/ranger/db/XXServiceResourceDao.java +++ b/security-admin/src/main/java/org/apache/ranger/db/XXServiceResourceDao.java @@ -24,6 +24,7 @@ import javax.persistence.NoResultException; +import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang.StringUtils; import org.apache.ranger.authorization.utils.StringUtil; import org.apache.ranger.common.db.BaseDao; @@ -73,15 +74,32 @@ public XXServiceResource findByServiceAndResourceSignature(Long serviceId, Strin } public List findTaggedResourcesInServiceId(Long serviceId) { - if (serviceId == null) { - return new ArrayList(); - } - try { - return getEntityManager().createNamedQuery("XXServiceResource.findTaggedResourcesInServiceId", tClass) - .setParameter("serviceId", serviceId).getResultList(); - } catch (NoResultException e) { - return new ArrayList(); - } + List ret = new ArrayList<>(); + if (serviceId != null) { + List rows = null; + try { + rows = getEntityManager().createNamedQuery("XXServiceResource.findTaggedResourcesInServiceId", Object[].class) + .setParameter("serviceId", serviceId).getResultList(); + } catch (NoResultException e) { + // Nothing + } + if (CollectionUtils.isNotEmpty(rows)) { + for (Object[] row : rows) { + XXServiceResource xxServiceResource = new XXServiceResource(); + xxServiceResource.setId((Long) row[0]); + xxServiceResource.setGuid((String) row[1]); + xxServiceResource.setVersion((Long) row[2]); + xxServiceResource.setIsEnabled((Boolean) row[3]); + xxServiceResource.setResourceSignature((String) row[4]); + xxServiceResource.setServiceId((Long) row[5]); + xxServiceResource.setServiceResourceElements((String) row[6]); + xxServiceResource.setTags((String) row[7]); + + ret.add(xxServiceResource); + } + } + } + return ret; } public long countTaggedResourcesInServiceId(Long serviceId) { diff --git a/security-admin/src/main/java/org/apache/ranger/db/XXServiceResourceElementDao.java b/security-admin/src/main/java/org/apache/ranger/db/XXServiceResourceElementDao.java index 72fe2140b5..22a0e01527 100644 --- a/security-admin/src/main/java/org/apache/ranger/db/XXServiceResourceElementDao.java +++ b/security-admin/src/main/java/org/apache/ranger/db/XXServiceResourceElementDao.java @@ -72,16 +72,4 @@ public List findTaggedResourcesInServiceId(Long servic } } - public List findForServicePlugin(Long serviceId) { - if (serviceId == null) { - return new ArrayList(); - } - try { - return getEntityManager().createNamedQuery("XXServiceResourceElement.findForServicePlugin", tClass) - .setParameter("serviceId", serviceId) - .getResultList(); - } catch (NoResultException e) { - return new ArrayList(); - } - } } diff --git a/security-admin/src/main/java/org/apache/ranger/db/XXServiceResourceElementValueDao.java b/security-admin/src/main/java/org/apache/ranger/db/XXServiceResourceElementValueDao.java index e726d047c0..3170edd992 100644 --- a/security-admin/src/main/java/org/apache/ranger/db/XXServiceResourceElementValueDao.java +++ b/security-admin/src/main/java/org/apache/ranger/db/XXServiceResourceElementValueDao.java @@ -84,19 +84,6 @@ public List findTaggedResourcesInServiceId(Long s } } - @SuppressWarnings("unchecked") - public List findForServicePlugin(Long serviceId) { - if (serviceId == null) { - return new ArrayList(); - } - try { - return getEntityManager().createNamedQuery("XXServiceResourceElementValue.findForServicePlugin") - .setParameter("serviceId", serviceId).getResultList(); - } catch (NoResultException e) { - return new ArrayList(); - } - } - @SuppressWarnings("unchecked") public List findByResourceId(Long resourceId) { if (resourceId == null) { diff --git a/security-admin/src/main/java/org/apache/ranger/db/XXServiceVersionInfoDao.java b/security-admin/src/main/java/org/apache/ranger/db/XXServiceVersionInfoDao.java index 0098bff233..8e4376d630 100644 --- a/security-admin/src/main/java/org/apache/ranger/db/XXServiceVersionInfoDao.java +++ b/security-admin/src/main/java/org/apache/ranger/db/XXServiceVersionInfoDao.java @@ -23,6 +23,7 @@ import javax.persistence.NoResultException; import org.apache.commons.collections.CollectionUtils; +import org.apache.ranger.biz.ServiceDBStore; import org.apache.ranger.common.db.BaseDao; import org.apache.ranger.entity.XXServiceVersionInfo; @@ -30,6 +31,7 @@ */ public class XXServiceVersionInfoDao extends BaseDao { + /** * Default Constructor */ @@ -121,14 +123,14 @@ private void updateTagVersionAndTagUpdateTime(List service } for(XXServiceVersionInfo serviceVersionInfo : serviceVersionInfos) { - Long currentTagVersion = serviceVersionInfo.getTagVersion(); + final RangerDaoManager finaldaoManager = daoManager; + final Long finalServiceId = serviceVersionInfo.getServiceId(); + final ServiceDBStore.VERSION_TYPE versionType = ServiceDBStore.VERSION_TYPE.TAG_VERSION; - if(currentTagVersion == null) { - currentTagVersion = Long.valueOf(0); - } + Runnable serviceVersionUpdater = new ServiceDBStore.ServiceVersionUpdater(finaldaoManager, finalServiceId, versionType); - serviceVersionInfo.setTagVersion(currentTagVersion + 1); - serviceVersionInfo.setTagUpdateTime(updateTime); + daoManager.getRangerTransactionSynchronizationAdapter().executeOnTransactionCommit(serviceVersionUpdater); } + } } diff --git a/security-admin/src/main/java/org/apache/ranger/db/XXTagAttributeDao.java b/security-admin/src/main/java/org/apache/ranger/db/XXTagAttributeDao.java index e361b33bd3..a24e3dbbad 100644 --- a/security-admin/src/main/java/org/apache/ranger/db/XXTagAttributeDao.java +++ b/security-admin/src/main/java/org/apache/ranger/db/XXTagAttributeDao.java @@ -57,18 +57,6 @@ public List findByServiceId(Long serviceId) { } } - public List findForServicePlugin(Long serviceId) { - if (serviceId == null) { - return new ArrayList(); - } - try { - return getEntityManager().createNamedQuery("XXTagAttribute.findForServicePlugin", tClass) - .setParameter("serviceId", serviceId).getResultList(); - } catch (NoResultException e) { - return new ArrayList(); - } - } - public List findByServiceIdAndOwner(Long serviceId, Short owner) { if (serviceId == null) { return new ArrayList(); diff --git a/security-admin/src/main/java/org/apache/ranger/db/XXTagAttributeDefDao.java b/security-admin/src/main/java/org/apache/ranger/db/XXTagAttributeDefDao.java index 145399f1cf..294c22287b 100644 --- a/security-admin/src/main/java/org/apache/ranger/db/XXTagAttributeDefDao.java +++ b/security-admin/src/main/java/org/apache/ranger/db/XXTagAttributeDefDao.java @@ -57,18 +57,6 @@ public List findByServiceId(Long serviceId) { } } - public List findForServicePlugin(Long serviceId) { - if (serviceId == null) { - return new ArrayList(); - } - try { - return getEntityManager().createNamedQuery("XXTagAttributeDef.findForServicePlugin", tClass) - .setParameter("serviceId", serviceId).getResultList(); - } catch (NoResultException e) { - return new ArrayList(); - } - } - public List findByResourceId(Long resourceId) { if (resourceId == null) { return new ArrayList(); diff --git a/security-admin/src/main/java/org/apache/ranger/db/XXTagDao.java b/security-admin/src/main/java/org/apache/ranger/db/XXTagDao.java index 77428dbf9e..199a155a8f 100644 --- a/security-admin/src/main/java/org/apache/ranger/db/XXTagDao.java +++ b/security-admin/src/main/java/org/apache/ranger/db/XXTagDao.java @@ -124,19 +124,6 @@ public List findByServiceId(Long serviceId) { } } - public List findForServicePlugin(Long serviceId) { - if (serviceId == null) { - return new ArrayList(); - } - - try { - return getEntityManager().createNamedQuery("XXTag.findForServicePlugin", tClass) - .setParameter("serviceId", serviceId).getResultList(); - } catch (NoResultException e) { - return new ArrayList(); - } - } - public List findByServiceIdAndOwner(Long serviceId, Short owner) { if (serviceId == null) { return new ArrayList(); diff --git a/security-admin/src/main/java/org/apache/ranger/db/XXTagDefDao.java b/security-admin/src/main/java/org/apache/ranger/db/XXTagDefDao.java index dfd2fcb5d0..877344b2d5 100644 --- a/security-admin/src/main/java/org/apache/ranger/db/XXTagDefDao.java +++ b/security-admin/src/main/java/org/apache/ranger/db/XXTagDefDao.java @@ -24,6 +24,7 @@ import javax.persistence.NoResultException; +import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang.StringUtils; import org.apache.ranger.common.db.BaseDao; import org.apache.ranger.entity.XXTagDef; @@ -60,31 +61,33 @@ public XXTagDef findByName(String name) { } } - public List findByServiceId(Long serviceId) { - if (serviceId == null) { - return new ArrayList(); - } - - try { - return getEntityManager().createNamedQuery("XXTagDef.findByServiceId", tClass) - .setParameter("serviceId", serviceId).getResultList(); - } catch (NoResultException e) { - return new ArrayList(); - } - } - - public List findForServicePlugin(Long serviceId) { - if (serviceId == null) { - return new ArrayList(); - } - - try { - return getEntityManager().createNamedQuery("XXTagDef.findForServicePlugin", tClass) - .setParameter("serviceId", serviceId).getResultList(); - } catch (NoResultException e) { - return new ArrayList(); - } - } + public List findByServiceId(Long serviceId) { + List ret = new ArrayList<>(); + if (serviceId != null) { + List rows = null; + try { + rows = getEntityManager().createNamedQuery("XXTagDef.findByServiceId", Object[].class) + .setParameter("serviceId", serviceId).getResultList(); + } catch (NoResultException e) { + // Nothing + } + if (CollectionUtils.isNotEmpty(rows)) { + for (Object[] row : rows) { + XXTagDef xxTagDef = new XXTagDef(); + xxTagDef.setId((Long) row[0]); + xxTagDef.setGuid((String) row[1]); + xxTagDef.setVersion((Long) row[2]); + xxTagDef.setIsEnabled((Boolean) row[3]); + xxTagDef.setName((String) row[4]); + xxTagDef.setSource((String) row[5]); + xxTagDef.setTagAttrDefs((String) row[6]); + + ret.add(xxTagDef); + } + } + } + return ret; + } public List getAllNames() { try { diff --git a/security-admin/src/main/java/org/apache/ranger/db/XXTagResourceMapDao.java b/security-admin/src/main/java/org/apache/ranger/db/XXTagResourceMapDao.java index bbcd54618a..c16cad0b95 100644 --- a/security-admin/src/main/java/org/apache/ranger/db/XXTagResourceMapDao.java +++ b/security-admin/src/main/java/org/apache/ranger/db/XXTagResourceMapDao.java @@ -141,15 +141,4 @@ public List findByServiceId(Long serviceId) { } } - public List findForServicePlugin(Long serviceId) { - if (serviceId == null) { - return new ArrayList(); - } - try { - return getEntityManager().createNamedQuery("XXTagResourceMap.findForServicePlugin", tClass) - .setParameter("serviceId", serviceId).getResultList(); - } catch (NoResultException e) { - return new ArrayList(); - } - } } diff --git a/security-admin/src/main/java/org/apache/ranger/db/XXUserDao.java b/security-admin/src/main/java/org/apache/ranger/db/XXUserDao.java index 7af1bf96a3..4d0d8e4acc 100644 --- a/security-admin/src/main/java/org/apache/ranger/db/XXUserDao.java +++ b/security-admin/src/main/java/org/apache/ranger/db/XXUserDao.java @@ -17,10 +17,7 @@ * under the License. */ - package org.apache.ranger.db; - - -import java.util.List; +package org.apache.ranger.db; import javax.persistence.NoResultException; @@ -51,20 +48,6 @@ public XXUser findByUserName(String name) { return null; } - @SuppressWarnings("unchecked") - public List findByPolicyItemId(Long polItemId) { - if (polItemId == null) { - return null; - } - try { - return getEntityManager() - .createNamedQuery("XXUser.findByPolicyItemId") - .setParameter("polItemId", polItemId).getResultList(); - } catch (NoResultException e) { - return null; - } - } - public XXUser findByPortalUserId(Long portalUserId) { if (portalUserId == null) { return null; @@ -76,4 +59,5 @@ public XXUser findByPortalUserId(Long portalUserId) { return null; } } + } diff --git a/security-admin/src/main/java/org/apache/ranger/entity/XXDBBase.java b/security-admin/src/main/java/org/apache/ranger/entity/XXDBBase.java index 8405eb3e4c..4816b02f90 100644 --- a/security-admin/src/main/java/org/apache/ranger/entity/XXDBBase.java +++ b/security-admin/src/main/java/org/apache/ranger/entity/XXDBBase.java @@ -25,6 +25,7 @@ */ import java.util.Date; +import java.util.Objects; import javax.persistence.Column; import javax.persistence.EntityListeners; @@ -205,6 +206,11 @@ public String toString( ) { return str; } + @Override + public int hashCode() { + return Objects.hash(createTime, updateTime, addedByUserId, updatedByUserId); + } + /** * Checks for all attributes except referenced db objects * @return true if all attributes match diff --git a/security-admin/src/main/java/org/apache/ranger/entity/XXPolicyBase.java b/security-admin/src/main/java/org/apache/ranger/entity/XXPolicyBase.java index 69d28bb54d..6c2503234f 100644 --- a/security-admin/src/main/java/org/apache/ranger/entity/XXPolicyBase.java +++ b/security-admin/src/main/java/org/apache/ranger/entity/XXPolicyBase.java @@ -110,6 +110,10 @@ public abstract class XXPolicyBase extends XXDBBase { @Column(name = "is_audit_enabled") protected boolean isAuditEnabled; + + @Column(name = "policy_text") + protected String policyText; + /** * @return the gUID */ @@ -267,7 +271,15 @@ public void setPolicyType(Integer policyType) { this.policyType = policyType; } - /* + public void setPolicyText(String policyText) { + this.policyText = policyText; + } + + public String getPolicyText() { + return this.policyText; + } + + /* * (non-Javadoc) * * @see java.lang.Object#equals(java.lang.Object) diff --git a/security-admin/src/main/java/org/apache/ranger/entity/XXPolicyRefAccessType.java b/security-admin/src/main/java/org/apache/ranger/entity/XXPolicyRefAccessType.java new file mode 100644 index 0000000000..6af8f99f4b --- /dev/null +++ b/security-admin/src/main/java/org/apache/ranger/entity/XXPolicyRefAccessType.java @@ -0,0 +1,191 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ranger.entity; + +import javax.persistence.*; +import javax.xml.bind.annotation.XmlRootElement; +import java.util.Objects; + +@Entity +@Cacheable +@XmlRootElement +@Table(name = "x_policy_ref_access_type") +public class XXPolicyRefAccessType extends XXDBBase implements + java.io.Serializable { + private static final long serialVersionUID = 1L; + /** + * id of the XXPolicyRefAccessType + *
    + *
+ * + */ + @Id + @SequenceGenerator(name = "x_policy_ref_access_type_SEQ", sequenceName = "x_policy_ref_access_type_SEQ", allocationSize = 1) + @GeneratedValue(strategy = GenerationType.AUTO, generator = "x_policy_ref_access_type_SEQ") + @Column(name = "id") + protected Long id; + + /** + * policyId of the XXPolicyRefAccessType + *
    + *
+ * + */ + @Column(name = "policy_id") + protected Long policyId; + + /** + * accessDefId of the XXPolicyRefAccessType + *
    + *
+ * + */ + @Column(name = "access_def_id") + protected Long accessDefId; + + /** + * accessTypeName of the XXPolicyRefAccessType + *
    + *
+ * + */ + @Column(name = "access_type_name") + protected String accessTypeName; + + /** + * This method sets the value to the member attribute id . You + * cannot set null to the attribute. + * + * @param id + * Value to set member attribute id + */ + public void setId(Long id) { + this.id = id; + } + + /** + * Returns the value for the member attribute id + * + * @return Date - value of member attribute id . + */ + public Long getId() { + return this.id; + } + + /** + * This method sets the value to the member attribute policyId . + * You cannot set null to the attribute. + * + * @param policyId + * Value to set member attribute policyId + */ + public void setPolicyId(Long policyId) { + this.policyId = policyId; + } + + /** + * Returns the value for the member attribute policyId + * + * @return Date - value of member attribute policyId . + */ + public Long getPolicyId() { + return this.policyId; + } + + /** + * This method sets the value to the member attribute accessDefId . + * You cannot set null to the attribute. + * + * @param accessDefId + * Value to set member attribute accessDefId + */ + public void setAccessDefId(Long accessDefId) { + this.accessDefId = accessDefId; + } + + /** + * Returns the value for the member attribute accessDefId + * + * @return Date - value of member attribute accessDefId . + */ + public Long getAccessDefId() { + return accessDefId; + } + + /** + * This method sets the value to the member attribute accessTypeName . + * You cannot set null to the attribute. + * + * @param accessTypeName + * Value to set member attribute accessTypeName + */ + public void setAccessTypeName(String accessTypeName) { + this.accessTypeName = accessTypeName; + } + + /** + * Returns the value for the member attribute accessTypeName + * + * @return Date - value of member attribute accessTypeName . + */ + public String getAccessTypeName() { + return accessTypeName; + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), id, policyId, accessDefId, accessTypeName); + } + + /* + * (non-Javadoc) + * + * @see java.lang.Object#equals(java.lang.Object) + */ + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + + if (getClass() != obj.getClass()) { + return false; + } + + XXPolicyRefAccessType other = (XXPolicyRefAccessType) obj; + + return super.equals(obj) && + Objects.equals(id, other.id) && + Objects.equals(policyId, other.policyId) && + Objects.equals(accessDefId, other.accessDefId) && + Objects.equals(accessTypeName, other.accessTypeName); + } + + /* (non-Javadoc) + * @see java.lang.Object#toString() + */ + @Override + public String toString() { + return "XXPolicyRefAccessType [" + super.toString() + " id=" + id + ", policyId=" + policyId + ", accessDefId=" + + accessDefId + ", accessTypeName=" + accessTypeName + "]"; + } + + + +} \ No newline at end of file diff --git a/security-admin/src/main/java/org/apache/ranger/entity/XXPolicyRefCondition.java b/security-admin/src/main/java/org/apache/ranger/entity/XXPolicyRefCondition.java new file mode 100644 index 0000000000..4f4409d6af --- /dev/null +++ b/security-admin/src/main/java/org/apache/ranger/entity/XXPolicyRefCondition.java @@ -0,0 +1,191 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ranger.entity; + +import javax.persistence.*; +import javax.xml.bind.annotation.XmlRootElement; +import java.util.Objects; + +@Entity +@Cacheable +@XmlRootElement +@Table(name = "x_policy_ref_condition") +public class XXPolicyRefCondition extends XXDBBase implements + java.io.Serializable { + private static final long serialVersionUID = 1L; + /** + * id of the XXPolicyRefCondition + *
    + *
+ * + */ + @Id + @SequenceGenerator(name = "x_policy_ref_condition_SEQ", sequenceName = "x_policy_ref_condition_SEQ", allocationSize = 1) + @GeneratedValue(strategy = GenerationType.AUTO, generator = "x_policy_ref_condition_SEQ") + @Column(name = "id") + protected Long id; + + /** + * policyId of the XXPolicyRefCondition + *
    + *
+ * + */ + @Column(name = "policy_id") + protected Long policyId; + + /** + * conditionDefId of the XXPolicyRefCondition + *
    + *
+ * + */ + @Column(name = "condition_def_id") + protected Long conditionDefId; + + /** + * conditionName of the XXPolicyRefCondition + *
    + *
+ * + */ + @Column(name = "condition_name") + protected String conditionName; + + /** + * This method sets the value to the member attribute id . You + * cannot set null to the attribute. + * + * @param id + * Value to set member attribute id + */ + public void setId(Long id) { + this.id = id; + } + + /** + * Returns the value for the member attribute id + * + * @return Date - value of member attribute id . + */ + public Long getId() { + return this.id; + } + + /** + * This method sets the value to the member attribute policyId . + * You cannot set null to the attribute. + * + * @param policyId + * Value to set member attribute policyId + */ + public void setPolicyId(Long policyId) { + this.policyId = policyId; + } + + /** + * Returns the value for the member attribute policyId + * + * @return Date - value of member attribute policyId . + */ + public Long getPolicyId() { + return this.policyId; + } + + /** + * This method sets the value to the member attribute conditionDefId . + * You cannot set null to the attribute. + * + * @param conditionDefId + * Value to set member attribute conditionDefId + */ + public void setConditionDefId(Long conditionDefId) { + this.conditionDefId = conditionDefId; + } + + /** + * Returns the value for the member attribute conditionDefId + * + * @return Date - value of member attribute conditionDefId . + */ + public Long getConditionDefId() { + return conditionDefId; + } + + /** + * This method sets the value to the member attribute conditionName . + * You cannot set null to the attribute. + * + * @param conditionName + * Value to set member attribute conditionName + */ + public void setConditionName(String conditionName) { + this.conditionName = conditionName; + } + + /** + * Returns the value for the member attribute conditionName + * + * @return Date - value of member attribute conditionName . + */ + public String getConditionName() { + return conditionName; + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), id, policyId, conditionDefId, conditionName); + } + + /* + * (non-Javadoc) + * + * @see java.lang.Object#equals(java.lang.Object) + */ + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + + if (getClass() != obj.getClass()) { + return false; + } + + XXPolicyRefCondition other = (XXPolicyRefCondition) obj; + + return super.equals(obj) && + Objects.equals(id, other.id) && + Objects.equals(policyId, other.policyId) && + Objects.equals(conditionDefId, other.conditionDefId) && + Objects.equals(conditionName, other.conditionName); + } + + /* (non-Javadoc) + * @see java.lang.Object#toString() + */ + @Override + public String toString() { + return "XXPolicyRefCondition [" + super.toString() + " id=" + id + ", policyId=" + policyId + ", conditionDefId=" + + conditionDefId + ", conditionName=" + conditionName + "]"; + } + + + +} \ No newline at end of file diff --git a/security-admin/src/main/java/org/apache/ranger/entity/XXPolicyRefDataMaskType.java b/security-admin/src/main/java/org/apache/ranger/entity/XXPolicyRefDataMaskType.java new file mode 100644 index 0000000000..cb926740e9 --- /dev/null +++ b/security-admin/src/main/java/org/apache/ranger/entity/XXPolicyRefDataMaskType.java @@ -0,0 +1,192 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ranger.entity; + +import javax.persistence.*; +import javax.xml.bind.annotation.XmlRootElement; +import java.util.Objects; + +@Entity +@Cacheable +@XmlRootElement +@Table(name = "x_policy_ref_datamask_type") +public class XXPolicyRefDataMaskType extends XXDBBase implements + java.io.Serializable { + private static final long serialVersionUID = 1L; + /** + * id of the XXPolicyRefDataMaskType + *
    + *
+ * + */ + @Id + @SequenceGenerator(name = "x_policy_ref_datamask_type_SEQ", sequenceName = "x_policy_ref_datamask_type_SEQ", allocationSize = 1) + @GeneratedValue(strategy = GenerationType.AUTO, generator = "x_policy_ref_datamask_type_SEQ") + @Column(name = "id") + protected Long id; + + /** + * policyId of the XXPolicyRefDataMaskType + *
    + *
+ * + */ + @Column(name = "policy_id") + protected Long policyId; + + /** + * DatamaskDefId of the XXPolicyRefDataMaskType + *
    + *
+ * + */ + @Column(name = "datamask_def_id") + protected Long dataMaskDefId; + + /** + * dataMaskTypeName of the XXPolicyRefDataMaskType + *
    + *
+ * + */ + @Column(name = "datamask_type_name") + protected String dataMaskTypeName; + + /** + * This method sets the value to the member attribute id . You + * cannot set null to the attribute. + * + * @param id + * Value to set member attribute id + */ + public void setId(Long id) { + this.id = id; + } + + /** + * Returns the value for the member attribute id + * + * @return Date - value of member attribute id . + */ + public Long getId() { + return this.id; + } + + + /** + * This method sets the value to the member attribute policyId . + * You cannot set null to the attribute. + * + * @param policyId + * Value to set member attribute policyId + */ + public void setPolicyId(Long policyId) { + this.policyId = policyId; + } + + /** + * Returns the value for the member attribute policyId + * + * @return Date - value of member attribute policyId . + */ + public Long getPolicyId() { + return this.policyId; + } + + /** + * This method sets the value to the member attribute dataMaskDefId . + * You cannot set null to the attribute. + * + * @param dataMaskDefId + * Value to set member attribute dataMaskDefId + */ + public void setDataMaskDefId(Long dataMaskDefId) { + this.dataMaskDefId = dataMaskDefId; + } + + /** + * Returns the value for the member attribute dataMaskDefId + * + * @return Date - value of member attribute dataMaskDefId . + */ + public Long getDataMaskDefId() { + return dataMaskDefId; + } + + /** + * This method sets the value to the member attribute dataMaskTypeName . + * You cannot set null to the attribute. + * + * @param dataMaskTypeName + * Value to set member attribute dataMaskTypeName + */ + public void setDataMaskTypeName(String dataMaskTypeName) { + this.dataMaskTypeName = dataMaskTypeName; + } + + /** + * Returns the value for the member attribute dataMaskTypeName + * + * @return Date - value of member attribute dataMaskTypeName . + */ + public String getDataMaskTypeName() { + return dataMaskTypeName; + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), id, policyId, dataMaskDefId, dataMaskTypeName); + } + + /* + * (non-Javadoc) + * + * @see java.lang.Object#equals(java.lang.Object) + */ + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + + if (getClass() != obj.getClass()) { + return false; + } + + XXPolicyRefDataMaskType other = (XXPolicyRefDataMaskType) obj; + + return super.equals(obj) && + Objects.equals(id, other.id) && + Objects.equals(policyId, other.policyId) && + Objects.equals(dataMaskDefId, other.dataMaskDefId) && + Objects.equals(dataMaskTypeName, other.dataMaskTypeName); + } + + /* (non-Javadoc) + * @see java.lang.Object#toString() + */ + @Override + public String toString() { + return "XXPolicyRefDataMaskType [" + super.toString() + " id=" + id + ", policyId=" + policyId + ", dataMaskDefId=" + + dataMaskDefId + ", dataMaskTypeName=" + dataMaskTypeName + "]"; + } + + + +} \ No newline at end of file diff --git a/security-admin/src/main/java/org/apache/ranger/entity/XXPolicyRefGroup.java b/security-admin/src/main/java/org/apache/ranger/entity/XXPolicyRefGroup.java new file mode 100644 index 0000000000..32a1b9f24c --- /dev/null +++ b/security-admin/src/main/java/org/apache/ranger/entity/XXPolicyRefGroup.java @@ -0,0 +1,206 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.entity; + +import java.io.Serializable; +import java.util.Objects; + +import javax.persistence.Cacheable; +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; +import javax.persistence.Id; +import javax.persistence.SequenceGenerator; +import javax.persistence.Table; +import javax.xml.bind.annotation.XmlRootElement; + + +/** + * The persistent class for the x_policy_ref_group database table. + * + */ +@Entity +@Cacheable +@XmlRootElement +@Table(name="x_policy_ref_group") +public class XXPolicyRefGroup extends XXDBBase implements Serializable { + + private static final long serialVersionUID = 1L; + /** + * id of the XXPolicyRefGroup + *
    + *
+ * + */ + @Id + @SequenceGenerator(name = "x_policy_ref_group_SEQ", sequenceName = "x_policy_ref_group_SEQ", allocationSize = 1) + @GeneratedValue(strategy = GenerationType.AUTO, generator = "x_policy_ref_group_SEQ") + @Column(name = "id") + protected Long id; + + /** + * policyId of the XXPolicyRefGroup + *
    + *
+ * + */ + @Column(name = "policy_id") + protected Long policyId; + + /** + * groupId of the XXPolicyRefGroup + *
    + *
+ * + */ + @Column(name = "group_id") + protected Long groupId; + + /** + * groupName of the XXPolicyRefGroup + *
    + *
+ * + */ + @Column(name = "group_name") + protected String groupName; + + /** + * This method sets the value to the member attribute id . You + * cannot set null to the attribute. + * + * @param id + * Value to set member attribute id + */ + public void setId(Long id) { + this.id = id; + } + + /** + * Returns the value for the member attribute id + * + * @return Date - value of member attribute id . + */ + public Long getId() { + return this.id; + } + + /** + * This method sets the value to the member attribute policyId . + * You cannot set null to the attribute. + * + * @param policyId + * Value to set member attribute policyId + */ + public void setPolicyId(Long policyId) { + this.policyId = policyId; + } + + /** + * Returns the value for the member attribute policyId + * + * @return Date - value of member attribute policyId . + */ + public Long getPolicyId() { + return this.policyId; + } + + /** + * This method sets the value to the member attribute groupId . + * You cannot set null to the attribute. + * + * @param groupId + * Value to set member attribute groupId + */ + public void setGroupId(Long groupId) { + this.groupId = groupId; + } + + /** + * Returns the value for the member attribute groupId + * + * @return Date - value of member attribute groupId . + */ + public Long getGroupId() { + return groupId; + } + + /** + * This method sets the value to the member attribute groupName . + * You cannot set null to the attribute. + * + * @param groupName + * Value to set member attribute groupName + */ + public void setGroupName(String groupName) { + this.groupName = groupName; + } + + /** + * Returns the value for the member attribute groupName + * + * @return Date - value of member attribute groupName . + */ + public String getGroupName() { + return groupName; + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), id, policyId, groupId, groupName); + } + + /* + * (non-Javadoc) + * + * @see java.lang.Object#equals(java.lang.Object) + */ + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + + if (getClass() != obj.getClass()) { + return false; + } + + XXPolicyRefGroup other = (XXPolicyRefGroup) obj; + + return super.equals(obj) && + Objects.equals(id, other.id) && + Objects.equals(policyId, other.policyId) && + Objects.equals(groupId, other.groupId) && + Objects.equals(groupName, other.groupName); + } + + /* + * (non-Javadoc) + * + * @see java.lang.Object#toString() + */ + @Override + public String toString() { + return "XXPolicyRefGroup [" + super.toString() + " id=" + id + ", policyId=" + policyId + ", groupId=" + groupId + + ", groupName=" + groupName + "]"; + } + +} \ No newline at end of file diff --git a/security-admin/src/main/java/org/apache/ranger/entity/XXPolicyRefResource.java b/security-admin/src/main/java/org/apache/ranger/entity/XXPolicyRefResource.java new file mode 100644 index 0000000000..1150646210 --- /dev/null +++ b/security-admin/src/main/java/org/apache/ranger/entity/XXPolicyRefResource.java @@ -0,0 +1,191 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ranger.entity; + +import javax.persistence.*; +import javax.xml.bind.annotation.XmlRootElement; +import java.util.Objects; + +@Entity +@Cacheable +@XmlRootElement +@Table(name = "x_policy_ref_resource") +public class XXPolicyRefResource extends XXDBBase implements + java.io.Serializable { + private static final long serialVersionUID = 1L; + /** + * id of the XXPolicyRefResource + *
    + *
+ * + */ + @Id + @SequenceGenerator(name = "x_policy_ref_resource_SEQ", sequenceName = "x_policy_ref_resource_SEQ", allocationSize = 1) + @GeneratedValue(strategy = GenerationType.AUTO, generator = "x_policy_ref_resource_SEQ") + @Column(name = "id") + protected Long id; + + /** + * policyId of the XXPolicyRefResource + *
    + *
+ * + */ + @Column(name = "policy_id") + protected Long policyId; + + /** + * resourceDefId of the XXPolicyRefResource + *
    + *
+ * + */ + @Column(name = "resource_def_id") + protected Long resourceDefId; + + /** + * resource_name of the XXPolicyRefResource + *
    + *
+ * + */ + @Column(name = "resource_name") + protected String resourceName; + + /** + * This method sets the value to the member attribute id . You + * cannot set null to the attribute. + * + * @param id + * Value to set member attribute id + */ + public void setId(Long id) { + this.id = id; + } + + /** + * Returns the value for the member attribute id + * + * @return Date - value of member attribute id . + */ + public Long getId() { + return this.id; + } + + /** + * This method sets the value to the member attribute policyId . + * You cannot set null to the attribute. + * + * @param policyId + * Value to set member attribute policyId + */ + public void setPolicyId(Long policyId) { + this.policyId = policyId; + } + + /** + * Returns the value for the member attribute policyId + * + * @return Date - value of member attribute policyId . + */ + public Long getPolicyId() { + return this.policyId; + } + + /** + * This method sets the value to the member attribute resourceDefId . + * You cannot set null to the attribute. + * + * @param resourceDefId + * Value to set member attribute resourceDefId + */ + public void setResourceDefId(Long resourceDefId) { + this.resourceDefId = resourceDefId; + } + + /** + * Returns the value for the member attribute resourceDefId + * + * @return Date - value of member attribute resourceDefId . + */ + public Long getResourceDefId() { + return resourceDefId; + } + + /** + * This method sets the value to the member attribute resource_name . + * You cannot set null to the attribute. + * + * @param resourceName + * Value to set member attribute resource_name + */ + public void setResourceName(String resourceName) { + this.resourceName = resourceName; + } + + /** + * Returns the value for the member attribute resourceName + * + * @return Date - value of member attribute resourceName . + */ + public String getResourceName() { + return resourceName; + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), id, policyId, resourceDefId, resourceName); + } + + /* + * (non-Javadoc) + * + * @see java.lang.Object#equals(java.lang.Object) + */ + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + + if (getClass() != obj.getClass()) { + return false; + } + + XXPolicyRefResource other = (XXPolicyRefResource) obj; + + return super.equals(obj) && + Objects.equals(id, other.id) && + Objects.equals(policyId, other.policyId) && + Objects.equals(resourceDefId, other.resourceDefId) && + Objects.equals(resourceName, other.resourceName); + } + + /* (non-Javadoc) + * @see java.lang.Object#toString() + */ + @Override + public String toString() { + return "XXPolicyRefResource [" + super.toString() + " id=" + id + ", policyId=" + policyId + ", resourceDefId=" + + resourceDefId + ", resource_name=" + resourceName + "]"; + } + + + +} \ No newline at end of file diff --git a/security-admin/src/main/java/org/apache/ranger/entity/XXPolicyRefUser.java b/security-admin/src/main/java/org/apache/ranger/entity/XXPolicyRefUser.java new file mode 100644 index 0000000000..8dfb928336 --- /dev/null +++ b/security-admin/src/main/java/org/apache/ranger/entity/XXPolicyRefUser.java @@ -0,0 +1,191 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ranger.entity; + +import javax.persistence.*; +import javax.xml.bind.annotation.XmlRootElement; +import java.util.Objects; + +@Entity +@Cacheable +@XmlRootElement +@Table(name = "x_policy_ref_user") +public class XXPolicyRefUser extends XXDBBase implements + java.io.Serializable { + private static final long serialVersionUID = 1L; + /** + * id of the XXPolicyRefUser + *
    + *
+ * + */ + @Id + @SequenceGenerator(name = "x_policy_ref_user_SEQ", sequenceName = "x_policy_ref_user_SEQ", allocationSize = 1) + @GeneratedValue(strategy = GenerationType.AUTO, generator = "x_policy_ref_user_SEQ") + @Column(name = "id") + protected Long id; + + /** + * policyId of the XXPolicyRefUser + *
    + *
+ * + */ + @Column(name = "policy_id") + protected Long policyId; + + /** + * userId of the XXPolicyRefUser + *
    + *
+ * + */ + @Column(name = "user_id") + protected Long userId; + + /** + * userName of the XXPolicyRefUser + *
    + *
+ * + */ + @Column(name = "user_name") + protected String userName; + + /** + * This method sets the value to the member attribute id . You + * cannot set null to the attribute. + * + * @param id + * Value to set member attribute id + */ + public void setId(Long id) { + this.id = id; + } + + /** + * Returns the value for the member attribute id + * + * @return Date - value of member attribute id . + */ + public Long getId() { + return this.id; + } + + /** + * This method sets the value to the member attribute policyId . + * You cannot set null to the attribute. + * + * @param policyId + * Value to set member attribute policyId + */ + public void setPolicyId(Long policyId) { + this.policyId = policyId; + } + + /** + * Returns the value for the member attribute policyId + * + * @return Date - value of member attribute policyId . + */ + public Long getPolicyId() { + return this.policyId; + } + + /** + * This method sets the value to the member attribute userId . + * You cannot set null to the attribute. + * + * @param userId + * Value to set member attribute userId + */ + public void setUserId(Long userId) { + this.userId = userId; + } + + /** + * Returns the value for the member attribute userId + * + * @return Date - value of member attribute userId . + */ + public Long getUserId() { + return userId; + } + + /** + * This method sets the value to the member attribute userName . + * You cannot set null to the attribute. + * + * @param userName + * Value to set member attribute userName + */ + public void setUserName(String userName) { + this.userName = userName; + } + + /** + * Returns the value for the member attribute userName + * + * @return Date - value of member attribute userName . + */ + public String getUserName() { + return userName; + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), id, policyId, userId, userName); + } + + /* + * (non-Javadoc) + * + * @see java.lang.Object#equals(java.lang.Object) + */ + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + + if (getClass() != obj.getClass()) { + return false; + } + + XXPolicyRefUser other = (XXPolicyRefUser) obj; + + return super.equals(obj) && + Objects.equals(id, other.id) && + Objects.equals(policyId, other.policyId) && + Objects.equals(userId, other.userId) && + Objects.equals(userName, other.userName); + } + + /* (non-Javadoc) + * @see java.lang.Object#toString() + */ + @Override + public String toString() { + return "XXPolicyRefUser [" + super.toString() + " id=" + id + ", policyId=" + policyId + ", userId=" + + userId + ", userName=" + userName + "]"; + } + + + +} \ No newline at end of file diff --git a/security-admin/src/main/java/org/apache/ranger/entity/XXServiceResource.java b/security-admin/src/main/java/org/apache/ranger/entity/XXServiceResource.java index 961627a3c3..c784830ef3 100644 --- a/security-admin/src/main/java/org/apache/ranger/entity/XXServiceResource.java +++ b/security-admin/src/main/java/org/apache/ranger/entity/XXServiceResource.java @@ -63,6 +63,12 @@ public class XXServiceResource extends XXDBBase implements Serializable { @Column(name = "service_id") protected Long serviceId; + @Column(name = "service_resource_elements_text") + protected String serviceResourceElements; + + @Column(name = "tags_text") + protected String tags; + @Override public void setId(Long id) { this.id = id; @@ -148,6 +154,16 @@ public void setIsEnabled(Boolean isEnabled) { this.isEnabled = isEnabled; } + public String getServiceResourceElements() { return serviceResourceElements; } + + public void setServiceResourceElements(String serviceResourceElements) { + this.serviceResourceElements = serviceResourceElements; + } + + public String getTags() { return tags; } + + public void setTags(String tags) { this.tags = tags; } + @Override public int getMyClassType() { return AppConstants.CLASS_TYPE_XA_SERVICE_RESOURCE; @@ -168,6 +184,8 @@ public int hashCode() { result = prime * result + ((isEnabled == null) ? 0 : isEnabled.hashCode()); result = prime * result + ((resourceSignature == null) ? 0 : resourceSignature.hashCode()); result = prime * result + ((serviceId == null) ? 0 : serviceId.hashCode()); + result = prime * result + ((serviceResourceElements == null) ? 0 : serviceResourceElements.hashCode()); + result = prime * result + ((tags == null) ? 0 : tags.hashCode()); return result; } @@ -215,6 +233,16 @@ public boolean equals(Object obj) { return false; } else if (!version.equals(other.version)) return false; + if (serviceResourceElements == null) { + if (other.serviceResourceElements != null) + return false; + } else if (!serviceResourceElements.equals(other.serviceResourceElements)) + return false; + if (tags == null) { + if (other.tags != null) + return false; + } else if (!tags.equals(other.tags)) + return false; return true; } @@ -239,6 +267,8 @@ public StringBuilder toString(StringBuilder sb) { sb.append("isEnabled={").append(isEnabled).append("} "); sb.append("resourceSignature={").append(resourceSignature).append("} "); sb.append("serviceId={").append(serviceId).append("} "); + sb.append("serviceResourceElements={").append(serviceResourceElements).append("} "); + sb.append("tags={").append(tags).append("} "); sb.append(" }"); return sb; diff --git a/security-admin/src/main/java/org/apache/ranger/entity/XXTag.java b/security-admin/src/main/java/org/apache/ranger/entity/XXTag.java index 9155385ec2..432119c97c 100644 --- a/security-admin/src/main/java/org/apache/ranger/entity/XXTag.java +++ b/security-admin/src/main/java/org/apache/ranger/entity/XXTag.java @@ -60,6 +60,9 @@ public class XXTag extends XXDBBase implements Serializable { @Column(name = "owned_by") protected Short owner; + @Column(name = "tag_attrs_text") + protected String tagAttrs; + @Override public void setId(Long id) { this.id = id; @@ -118,7 +121,11 @@ public void setType(Long type) { public Short getOwner() { return owner; } public void setOwner(Short owner) { this.owner = owner; } - @Override + public String getTagAttrs() { return tagAttrs; } + + public void setTagAttrs(String tagAttrs) { this.tagAttrs = tagAttrs; } + + @Override public int getMyClassType() { return AppConstants.CLASS_TYPE_XA_TAG; } @@ -137,6 +144,7 @@ public int hashCode() { result = prime * result + ((id == null) ? 0 : id.hashCode()); result = prime * result + ((type == null) ? 0 : type.hashCode()); result = prime * result + ((owner == null) ? 0 : owner.hashCode()); + result = prime * result + ((tagAttrs == null) ? 0 : tagAttrs.hashCode()); return result; } @@ -179,6 +187,11 @@ public boolean equals(Object obj) { return false; } else if (!owner.equals(other.owner)) return false; + if (tagAttrs == null) { + if (other.tagAttrs != null) + return false; + } else if (!tagAttrs.equals(other.tagAttrs)) + return false; return true; } @@ -201,6 +214,7 @@ public StringBuilder toString(StringBuilder sb) { sb.append("guid={").append(guid).append("} "); sb.append("type={").append(type).append("} "); sb.append("owned_by={").append(owner).append("} "); + sb.append("tagAttrs={").append(tagAttrs).append("} "); sb.append(" }"); return sb; diff --git a/security-admin/src/main/java/org/apache/ranger/entity/XXTagDef.java b/security-admin/src/main/java/org/apache/ranger/entity/XXTagDef.java index 818908ba81..88a7633ea4 100644 --- a/security-admin/src/main/java/org/apache/ranger/entity/XXTagDef.java +++ b/security-admin/src/main/java/org/apache/ranger/entity/XXTagDef.java @@ -63,6 +63,9 @@ public class XXTagDef extends XXDBBase implements Serializable { @Column(name = "source") protected String source; + @Column(name = "tag_attrs_def_text") + protected String tagAttrDefs; + /** * @return the guid */ @@ -138,6 +141,10 @@ public void setSource(String source) { this.source = source; } + public String getTagAttrDefs() { return tagAttrDefs; } + + public void setTagAttrDefs(String tagAttrDefs) { this.tagAttrDefs = tagAttrDefs; } + @Override public void setId(Long id) { this.id = id; @@ -168,6 +175,7 @@ public int hashCode() { result = prime * result + ((name == null) ? 0 : name.hashCode()); result = prime * result + ((source == null) ? 0 : source.hashCode()); result = prime * result + ((version == null) ? 0 : version.hashCode()); + result = prime * result + ((tagAttrDefs == null) ? 0 : tagAttrDefs.hashCode()); return result; } @@ -215,6 +223,11 @@ public boolean equals(Object obj) { return false; } else if (!version.equals(other.version)) return false; + if (tagAttrDefs == null) { + if (other.tagAttrDefs != null) + return false; + } else if (!tagAttrDefs.equals(other.tagAttrDefs)) + return false; return true; } @@ -239,6 +252,7 @@ public StringBuilder toString(StringBuilder sb) { sb.append("isEnabled={").append(isEnabled).append("} "); sb.append("source={").append(source).append("} "); sb.append("name={").append(name).append("} "); + sb.append("tagAttrDefs={").append(tagAttrDefs).append("} "); sb.append(" }"); return sb; diff --git a/security-admin/src/main/java/org/apache/ranger/patch/PatchForUpdatingPolicyJson_J10019.java b/security-admin/src/main/java/org/apache/ranger/patch/PatchForUpdatingPolicyJson_J10019.java new file mode 100644 index 0000000000..873c2e7c38 --- /dev/null +++ b/security-admin/src/main/java/org/apache/ranger/patch/PatchForUpdatingPolicyJson_J10019.java @@ -0,0 +1,1125 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ranger.patch; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.ListIterator; +import java.util.Map; +import java.util.Set; + +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.MapUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.log4j.Logger; +import org.apache.ranger.authorization.utils.JsonUtils; +import org.apache.ranger.authorization.utils.StringUtil; +import org.apache.ranger.biz.PolicyRefUpdater; +import org.apache.ranger.biz.ServiceDBStore; +import org.apache.ranger.db.RangerDaoManager; +import org.apache.ranger.db.XXGroupDao; +import org.apache.ranger.db.XXPolicyDao; +import org.apache.ranger.db.XXPolicyRefAccessTypeDao; +import org.apache.ranger.db.XXPolicyRefConditionDao; +import org.apache.ranger.db.XXPolicyRefDataMaskTypeDao; +import org.apache.ranger.db.XXPolicyRefGroupDao; +import org.apache.ranger.db.XXPolicyRefResourceDao; +import org.apache.ranger.db.XXPolicyRefUserDao; +import org.apache.ranger.db.XXUserDao; +import org.apache.ranger.entity.XXAccessTypeDef; +import org.apache.ranger.entity.XXDataMaskTypeDef; +import org.apache.ranger.entity.XXGroup; +import org.apache.ranger.entity.XXPolicy; +import org.apache.ranger.entity.XXPolicyConditionDef; +import org.apache.ranger.entity.XXPolicyItem; +import org.apache.ranger.entity.XXPolicyItemAccess; +import org.apache.ranger.entity.XXPolicyItemCondition; +import org.apache.ranger.entity.XXPolicyItemDataMaskInfo; +import org.apache.ranger.entity.XXPolicyItemGroupPerm; +import org.apache.ranger.entity.XXPolicyItemRowFilterInfo; +import org.apache.ranger.entity.XXPolicyItemUserPerm; +import org.apache.ranger.entity.XXPolicyRefAccessType; +import org.apache.ranger.entity.XXPolicyRefCondition; +import org.apache.ranger.entity.XXPolicyRefDataMaskType; +import org.apache.ranger.entity.XXPolicyRefGroup; +import org.apache.ranger.entity.XXPolicyRefResource; +import org.apache.ranger.entity.XXPolicyRefUser; +import org.apache.ranger.entity.XXPolicyResource; +import org.apache.ranger.entity.XXPolicyResourceMap; +import org.apache.ranger.entity.XXPortalUser; +import org.apache.ranger.entity.XXResourceDef; +import org.apache.ranger.entity.XXService; +import org.apache.ranger.entity.XXServiceDef; +import org.apache.ranger.entity.XXUser; +import org.apache.ranger.plugin.model.RangerPolicy; +import org.apache.ranger.plugin.model.RangerPolicy.RangerDataMaskPolicyItem; +import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyItem; +import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyItemAccess; +import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyItemCondition; +import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyItemDataMaskInfo; +import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyItemRowFilterInfo; +import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyResource; +import org.apache.ranger.plugin.model.RangerPolicy.RangerRowFilterPolicyItem; +import org.apache.ranger.plugin.model.RangerService; +import org.apache.ranger.plugin.policyevaluator.RangerPolicyItemEvaluator; +import org.apache.ranger.plugin.util.RangerPerfTracer; +import org.apache.ranger.plugin.util.SearchFilter; +import org.apache.ranger.util.CLIUtil; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.stereotype.Component; +import org.springframework.transaction.PlatformTransactionManager; +import org.springframework.transaction.TransactionStatus; +import org.springframework.transaction.support.TransactionCallback; +import org.springframework.transaction.support.TransactionTemplate; + +/** + * Consolidates Ranger policy details into a JSON string and stores it into a + * column in x_policy table After running this patch Ranger policy can be + * completely read/saved into x_policy table and some related Ref tables (which + * maintain ID->String mapping for each policy). + * + */ +@Component +public class PatchForUpdatingPolicyJson_J10019 extends BaseLoader { + private static final Logger logger = Logger.getLogger(PatchForUpdatingPolicyJson_J10019.class); + + @Autowired + RangerDaoManager daoMgr; + + @Autowired + ServiceDBStore svcStore; + + @Autowired + @Qualifier(value = "transactionManager") + PlatformTransactionManager txManager; + + @Autowired + PolicyRefUpdater policyRefUpdater; + + private final Map groupIdMap = new HashMap<>(); + private final Map userIdMap = new HashMap<>(); + private final Map> resourceNameIdMap = new HashMap<>(); + private final Map> accessTypeIdMap = new HashMap<>(); + private final Map> conditionNameIdMap = new HashMap<>(); + private final Map> dataMaskTypeIdMap = new HashMap<>(); + + public static void main(String[] args) { + logger.info("main()"); + try { + PatchForUpdatingPolicyJson_J10019 loader = (PatchForUpdatingPolicyJson_J10019) CLIUtil.getBean(PatchForUpdatingPolicyJson_J10019.class); + + loader.init(); + + while (loader.isMoreToProcess()) { + loader.load(); + } + + logger.info("Load complete. Exiting!!!"); + + System.exit(0); + } catch (Exception e) { + logger.error("Error loading", e); + System.exit(1); + } + } + + @Override + public void init() throws Exception { + // Do Nothing + } + + @Override + public void execLoad() { + logger.info("==> PatchForUpdatingPolicyJson.execLoad()"); + + try { + updateRangerPolicyTableWithPolicyJson(); + } catch (Exception e) { + logger.error("Error while updateRangerPolicyTableWithPolicyJson()", e); + System.exit(1); + } + + logger.info("<== PatchForUpdatingPolicyJson.execLoad()"); + } + + @Override + public void printStats() { + logger.info("updateRangerPolicyTableWithPolicyJson data "); + } + + private void updateRangerPolicyTableWithPolicyJson() throws Exception { + logger.info("==> updateRangerPolicyTableWithPolicyJson() "); + + List allServices = svcStore.getServices(new SearchFilter()); + + if (CollectionUtils.isNotEmpty(allServices)) { + for (RangerService service : allServices) { + XXService dbService = daoMgr.getXXService().getById(service.getId()); + + logger.info("==> Port Policies of service(name=" + dbService.getName() + ")"); + + RangerPolicyRetriever policyRetriever = new RangerPolicyRetriever(daoMgr, txManager); + + List policies = policyRetriever.getServicePolicies(dbService); + + if (CollectionUtils.isNotEmpty(policies)) { + for (RangerPolicy policy : policies) { + policyRefUpdater.cleanupRefTables(policy); + portPolicy(service.getType(), policy); + } + } + } + } + + logger.info("<== updateRangerPolicyTableWithPolicyJson() "); + } + + private void portPolicy(String serviceType, RangerPolicy policy) throws Exception { + logger.info("==> portPolicy(id=" + policy.getId() + ")"); + + String policyText = JsonUtils.objectToJson(policy); + + if (StringUtils.isEmpty(policyText)) { + throw new Exception("Failed to convert policy to json string. Policy: [id=" + policy.getId() + "; name=" + policy.getName() + "; serviceType=" + serviceType + "]"); + } + + XXPolicyDao policyDao = daoMgr.getXXPolicy(); + XXPolicy dbBean = policyDao.getById(policy.getId()); + + dbBean.setPolicyText(policyText); + + policyDao.update(dbBean); + + try { + Set accesses = new HashSet<>(); + Set users = new HashSet<>(); + Set groups = new HashSet<>(); + Set conditions = new HashSet<>(); + Set dataMasks = new HashSet<>(); + + buildLists(policy.getPolicyItems(), accesses, conditions, users, groups); + buildLists(policy.getDenyPolicyItems(), accesses, conditions, users, groups); + buildLists(policy.getAllowExceptions(), accesses, conditions, users, groups); + buildLists(policy.getDenyExceptions(), accesses, conditions, users, groups); + buildLists(policy.getDataMaskPolicyItems(), accesses, conditions, users, groups); + buildLists(policy.getRowFilterPolicyItems(), accesses, conditions, users, groups); + + buildList(policy.getDataMaskPolicyItems(), dataMasks); + + addResourceDefRef(serviceType, policy); + addUserNameRef(policy.getId(), users); + addGroupNameRef(policy.getId(), groups); + addAccessDefRef(serviceType, policy.getId(), accesses); + addPolicyConditionDefRef(serviceType, policy.getId(), conditions); + addDataMaskDefRef(serviceType, policy.getId(), dataMasks); + } catch (Exception e) { + logger.error("portPoliry(id=" + policy.getId() +") failed!!"); + logger.error("Offending policy:" + policyText); + throw e; + } + + logger.info("<== portPolicy(id=" + policy.getId() + ")"); + } + + private void addResourceDefRef(String serviceType, RangerPolicy policy) throws Exception { + logger.info("==> addResourceDefRef(id=" + policy.getId() + ")"); + + Map serviceDefResourceNameIDMap = resourceNameIdMap.get(serviceType); + + if (serviceDefResourceNameIDMap == null) { + serviceDefResourceNameIDMap = new HashMap<>(); + + resourceNameIdMap.put(serviceType, serviceDefResourceNameIDMap); + + XXServiceDef dbServiceDef = daoMgr.getXXServiceDef().findByName(serviceType); + + for (XXResourceDef resourceDef : daoMgr.getXXResourceDef().findByServiceDefId(dbServiceDef.getId())) { + serviceDefResourceNameIDMap.put(resourceDef.getName(), resourceDef.getId()); + } + } + + Map policyResources = policy.getResources(); + + if (MapUtils.isNotEmpty(policyResources)) { + XXPolicyRefResourceDao policyRefResourceDao = daoMgr.getXXPolicyRefResource(); + Set resourceNames = policyResources.keySet(); + + for (String resourceName : resourceNames) { + Long resourceDefId = serviceDefResourceNameIDMap.get(resourceName); + + if (resourceDefId == null) { + throw new Exception(resourceName + ": unknown resource in policy [id=" + policy.getId() + "; name=" + policy.getName() + "; serviceType=" + serviceType + "]. Known resources: " + serviceDefResourceNameIDMap.keySet()); + } + + // insert policy-id, resourceDefId, resourceName into Ref table + XXPolicyRefResource policyRefResource = new XXPolicyRefResource(); + + policyRefResource.setPolicyId(policy.getId()); + policyRefResource.setResourceDefId(resourceDefId); + policyRefResource.setResourceName(resourceName); + + policyRefResourceDao.create(policyRefResource); + } + } + + logger.info("<== addResourceDefRef(id=" + policy.getId() + ")"); + } + + private void addUserNameRef(Long policyId, Set users) throws Exception { + logger.info("==> addUserNameRef(id=" + policyId + ")"); + + XXPolicyRefUserDao policyRefUserDao = daoMgr.getXXPolicyRefUser(); + XXUserDao userDao = daoMgr.getXXUser(); + + // insert policy-id, userName into Ref table + for (String user : users) { + Long userId = userIdMap.get(user); + + if (userId == null) { + XXUser userObject = userDao.findByUserName(user); + + if (userObject == null) { + throw new Exception(user + ": unknown user in policy [id=" + policyId + "]"); + } + + userId = userObject.getId(); + + userIdMap.put(user, userId); + } + + XXPolicyRefUser policyRefUser = new XXPolicyRefUser(); + + policyRefUser.setPolicyId(policyId); + policyRefUser.setUserName(user); + policyRefUser.setUserId(userId); + + policyRefUserDao.create(policyRefUser); + } + + logger.info("<== addUserNameRef(id=" + policyId + ")"); + } + + private void addGroupNameRef(Long policyId, Set groups) throws Exception { + logger.info("==> addGroupNameRef(id=" + policyId + ")"); + + // insert policy-id, groupName into Ref table + XXPolicyRefGroupDao policyRefGroupDao = daoMgr.getXXPolicyRefGroup(); + XXGroupDao groupDao = daoMgr.getXXGroup(); + + for (String group : groups) { + Long groupId = groupIdMap.get(group); + + if (groupId == null) { + XXGroup groupObject = groupDao.findByGroupName(group); + + if (groupObject == null) { + throw new Exception(group + ": unknown group in policy [id=" + policyId + "]"); + } + + groupId = groupObject.getId(); + + groupIdMap.put(group, groupId); + } + + XXPolicyRefGroup policyRefGroup = new XXPolicyRefGroup(); + + policyRefGroup.setPolicyId(policyId); + policyRefGroup.setGroupName(group); + policyRefGroup.setGroupId(groupId); + + policyRefGroupDao.create(policyRefGroup); + } + + logger.info("<== addGroupNameRef(id=" + policyId + ")"); + + } + + private void addAccessDefRef(String serviceType, Long policyId, Set accesses) throws Exception { + logger.info("==> addAccessDefRef(id=" + policyId + ")"); + // insert policy-id, accessName into Ref table + + Map serviceDefAccessTypeIDMap = accessTypeIdMap.get(serviceType); + + if (serviceDefAccessTypeIDMap == null) { + serviceDefAccessTypeIDMap = new HashMap<>(); + + accessTypeIdMap.put(serviceType, serviceDefAccessTypeIDMap); + + XXServiceDef dbServiceDef = daoMgr.getXXServiceDef().findByName(serviceType); + + for (XXAccessTypeDef accessTypeDef : daoMgr.getXXAccessTypeDef().findByServiceDefId(dbServiceDef.getId())) { + serviceDefAccessTypeIDMap.put(accessTypeDef.getName(), accessTypeDef.getId()); + } + } + + XXPolicyRefAccessTypeDao policyRefAccessTypeDao = daoMgr.getXXPolicyRefAccessType(); + + for (String access : accesses) { + Long accessTypeDefId = serviceDefAccessTypeIDMap.get(access); + + if (accessTypeDefId == null) { + throw new Exception(access + ": unknown accessType in policy [id=" + policyId + "; serviceType=" + serviceType + "]. Known accessTypes: " + serviceDefAccessTypeIDMap.keySet()); + } + + XXPolicyRefAccessType policyRefAccessType = new XXPolicyRefAccessType(); + + policyRefAccessType.setPolicyId(policyId); + policyRefAccessType.setAccessTypeName(access); + policyRefAccessType.setAccessDefId(accessTypeDefId); + + policyRefAccessTypeDao.create(policyRefAccessType); + } + + logger.info("<== addAccessDefRef(id=" + policyId + ")"); + } + + private void addPolicyConditionDefRef(String serviceType, Long policyId, Set conditions) throws Exception { + logger.info("==> addPolicyConditionDefRef(id=" + policyId + ")"); + // insert policy-id, conditionName into Ref table + + Map serviceDefConditionNameIDMap = conditionNameIdMap.get(serviceType); + + if (serviceDefConditionNameIDMap == null) { + serviceDefConditionNameIDMap = new HashMap<>(); + + conditionNameIdMap.put(serviceType, serviceDefConditionNameIDMap); + + XXServiceDef dbServiceDef = daoMgr.getXXServiceDef().findByName(serviceType); + + for (XXPolicyConditionDef conditionDef : daoMgr.getXXPolicyConditionDef().findByServiceDefId(dbServiceDef.getId())) { + serviceDefConditionNameIDMap.put(conditionDef.getName(), conditionDef.getId()); + } + } + + XXPolicyRefConditionDao policyRefConditionDao = daoMgr.getXXPolicyRefCondition(); + + for (String condition : conditions) { + Long conditionDefId = serviceDefConditionNameIDMap.get(condition); + + if (conditionDefId == null) { + throw new Exception(condition + ": unknown condition in policy [id=" + policyId + "; serviceType=" + serviceType + "]. Known conditions are: " + serviceDefConditionNameIDMap.keySet()); + } + + XXPolicyRefCondition policyRefCondition = new XXPolicyRefCondition(); + + policyRefCondition.setPolicyId(policyId); + policyRefCondition.setConditionName(condition); + policyRefCondition.setConditionDefId(conditionDefId); + + policyRefConditionDao.create(policyRefCondition); + } + + logger.info("<== addPolicyConditionDefRef(id=" + policyId + ")"); + } + + private void addDataMaskDefRef(String serviceType, Long policyId, Set datamasks) throws Exception { + logger.info("==> addDataMaskDefRef(id=" + policyId + ")"); + + // insert policy-id, datamaskName into Ref table + + Map serviceDefDataMaskTypeIDMap = dataMaskTypeIdMap.get(serviceType); + + if (serviceDefDataMaskTypeIDMap == null) { + serviceDefDataMaskTypeIDMap = new HashMap<>(); + + dataMaskTypeIdMap.put(serviceType, serviceDefDataMaskTypeIDMap); + + XXServiceDef dbServiceDef = daoMgr.getXXServiceDef().findByName(serviceType); + + for (XXDataMaskTypeDef dataMaskTypeDef : daoMgr.getXXDataMaskTypeDef().findByServiceDefId(dbServiceDef.getId())) { + serviceDefDataMaskTypeIDMap.put(dataMaskTypeDef.getName(), dataMaskTypeDef.getId()); + } + } + + XXPolicyRefDataMaskTypeDao policyRefDataMaskTypeDao = daoMgr.getXXPolicyRefDataMaskType(); + + for (String datamask : datamasks) { + Long dataMaskTypeId = serviceDefDataMaskTypeIDMap.get(datamask); + + if (dataMaskTypeId == null) { + throw new Exception(datamask + ": unknown dataMaskType in policy [id=" + policyId + "; serviceType=" + serviceType + "]. Known dataMaskTypes " + serviceDefDataMaskTypeIDMap.keySet()); + } + + XXPolicyRefDataMaskType policyRefDataMaskType = new XXPolicyRefDataMaskType(); + + policyRefDataMaskType.setPolicyId(policyId); + policyRefDataMaskType.setDataMaskTypeName(datamask); + policyRefDataMaskType.setDataMaskDefId(dataMaskTypeId); + + policyRefDataMaskTypeDao.create(policyRefDataMaskType); + } + + logger.info("<== addDataMaskDefRef(id=" + policyId + ")"); + + } + + private void buildLists(List policyItems, Set accesses, Set conditions, Set users, Set groups) { + for (RangerPolicyItem item : policyItems) { + for (RangerPolicyItemAccess policyAccess : item.getAccesses()) { + accesses.add(policyAccess.getType()); + } + + for (RangerPolicyItemCondition policyCondition : item.getConditions()) { + conditions.add(policyCondition.getType()); + } + + users.addAll(item.getUsers()); + groups.addAll(item.getGroups()); + } + } + + private void buildList(List dataMaskPolicyItems, Set dataMasks) { + for (RangerDataMaskPolicyItem datMaskPolicyItem : dataMaskPolicyItems) { + dataMasks.add(datMaskPolicyItem.getDataMaskInfo().getDataMaskType()); + } + } + + static private class RangerPolicyRetriever { + static final Log LOG = LogFactory.getLog(RangerPolicyRetriever.class); + static final Log PERF_LOG = RangerPerfTracer.getPerfLogger("db.RangerPolicyRetriever"); + + private final RangerDaoManager daoMgr; + private final LookupCache lookupCache = new LookupCache(); + + private final PlatformTransactionManager txManager; + private final TransactionTemplate txTemplate; + + RangerPolicyRetriever(RangerDaoManager daoMgr, PlatformTransactionManager txManager) { + this.daoMgr = daoMgr; + this.txManager = txManager; + + if (this.txManager != null) { + this.txTemplate = new TransactionTemplate(this.txManager); + + this.txTemplate.setReadOnly(true); + } else { + this.txTemplate = null; + } + } + + private class PolicyLoaderThread extends Thread { + final TransactionTemplate txTemplate; + final XXService xService; + List policies; + + PolicyLoaderThread(TransactionTemplate txTemplate, final XXService xService) { + this.txTemplate = txTemplate; + this.xService = xService; + } + + public List getPolicies() { + return policies; + } + + @Override + public void run() { + txTemplate.setReadOnly(true); + policies = txTemplate.execute(new TransactionCallback>() { + @Override + public List doInTransaction(TransactionStatus status) { + RetrieverContext ctx = new RetrieverContext(xService); + return ctx.getAllPolicies(); + } + }); + } + } + + public List getServicePolicies(final XXService xService) throws InterruptedException { + String serviceName = xService == null ? null : xService.getName(); + Long serviceId = xService == null ? null : xService.getId(); + + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerPolicyRetriever.getServicePolicies(serviceName=" + serviceName + ", serviceId=" + serviceId + ")"); + } + + List ret = null; + RangerPerfTracer perf = null; + + if (RangerPerfTracer.isPerfTraceEnabled(PERF_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_LOG, "RangerPolicyRetriever.getServicePolicies(serviceName=" + serviceName + ",serviceId=" + serviceId + ")"); + } + + if (xService != null) { + if (txTemplate == null) { + if (LOG.isDebugEnabled()) { + LOG.debug("Transaction Manager is null; Retrieving policies in the existing transaction"); + } + + RetrieverContext ctx = new RetrieverContext(xService); + + ret = ctx.getAllPolicies(); + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("Retrieving policies in a new, read-only transaction"); + } + + PolicyLoaderThread t = new PolicyLoaderThread(txTemplate, xService); + t.start(); + t.join(); + ret = t.getPolicies(); + } + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("RangerPolicyRetriever.getServicePolicies(xService=" + xService + "): invalid parameter"); + } + } + + RangerPerfTracer.log(perf); + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerPolicyRetriever.getServicePolicies(serviceName=" + serviceName + ", serviceId=" + serviceId + "): policyCount=" + (ret == null ? 0 : ret.size())); + } + + return ret; + } + + class LookupCache { + final Map userNames = new HashMap(); + final Map userScreenNames = new HashMap(); + final Map groupNames = new HashMap(); + final Map accessTypes = new HashMap(); + final Map conditions = new HashMap(); + final Map resourceDefs = new HashMap(); + final Map dataMasks = new HashMap(); + + String getUserName(Long userId) { + String ret = null; + + if (userId != null) { + ret = userNames.get(userId); + + if (ret == null) { + XXUser user = daoMgr.getXXUser().getById(userId); + + if (user != null) { + ret = user.getName(); // Name is `loginId` + + userNames.put(userId, ret); + } + } + } + + return ret; + } + + String getUserScreenName(Long userId) { + String ret = null; + + if (userId != null) { + ret = userScreenNames.get(userId); + + if (ret == null) { + XXPortalUser user = daoMgr.getXXPortalUser().getById(userId); + + if (user != null) { + ret = user.getPublicScreenName(); + + if (StringUtil.isEmpty(ret)) { + ret = user.getFirstName(); + + if (StringUtil.isEmpty(ret)) { + ret = user.getLoginId(); + } else { + if (!StringUtil.isEmpty(user.getLastName())) { + ret += (" " + user.getLastName()); + } + } + } + + if (ret != null) { + userScreenNames.put(userId, ret); + } + } + } + } + + return ret; + } + + String getGroupName(Long groupId) { + String ret = null; + + if (groupId != null) { + ret = groupNames.get(groupId); + + if (ret == null) { + XXGroup group = daoMgr.getXXGroup().getById(groupId); + + if (group != null) { + ret = group.getName(); + + groupNames.put(groupId, ret); + } + } + } + + return ret; + } + + String getAccessType(Long accessTypeId) { + String ret = null; + + if (accessTypeId != null) { + ret = accessTypes.get(accessTypeId); + + if (ret == null) { + XXAccessTypeDef xAccessType = daoMgr.getXXAccessTypeDef().getById(accessTypeId); + + if (xAccessType != null) { + ret = xAccessType.getName(); + + accessTypes.put(accessTypeId, ret); + } else { + LOG.warn("getAccessType(): Canot find name for accessTypeId " + accessTypeId + ". This will cause Ranger policy migration to fail. Please check if all service-defs are migrated correctly!"); + } + } + } + + return ret; + } + + String getConditionType(Long conditionDefId) { + String ret = null; + + if (conditionDefId != null) { + ret = conditions.get(conditionDefId); + + if (ret == null) { + XXPolicyConditionDef xPolicyConditionDef = daoMgr.getXXPolicyConditionDef() + .getById(conditionDefId); + + if (xPolicyConditionDef != null) { + ret = xPolicyConditionDef.getName(); + + conditions.put(conditionDefId, ret); + } + } + } + + return ret; + } + + String getResourceName(Long resourceDefId) { + String ret = null; + + if (resourceDefId != null) { + ret = resourceDefs.get(resourceDefId); + + if (ret == null) { + XXResourceDef xResourceDef = daoMgr.getXXResourceDef().getById(resourceDefId); + + if (xResourceDef != null) { + ret = xResourceDef.getName(); + + resourceDefs.put(resourceDefId, ret); + } + } + } + + return ret; + } + + String getDataMaskName(Long dataMaskDefId) { + String ret = null; + + if (dataMaskDefId != null) { + ret = dataMasks.get(dataMaskDefId); + + if (ret == null) { + XXDataMaskTypeDef xDataMaskDef = daoMgr.getXXDataMaskTypeDef().getById(dataMaskDefId); + + if (xDataMaskDef != null) { + ret = xDataMaskDef.getName(); + + dataMasks.put(dataMaskDefId, ret); + } + } + } + + return ret; + } + } + + static List asList(XXPolicy policy) { + List ret = new ArrayList(); + + if (policy != null) { + ret.add(policy); + } + + return ret; + } + + class RetrieverContext { + final XXService service; + final ListIterator iterPolicy; + final ListIterator iterResources; + final ListIterator iterResourceMaps; + final ListIterator iterPolicyItems; + final ListIterator iterUserPerms; + final ListIterator iterGroupPerms; + final ListIterator iterAccesses; + final ListIterator iterConditions; + final ListIterator iterDataMaskInfos; + final ListIterator iterRowFilterInfos; + + RetrieverContext(XXService xService) { + Long serviceId = xService == null ? null : xService.getId(); + List xPolicies = daoMgr.getXXPolicy().findByServiceId(serviceId); + + this.service = xService; + this.iterPolicy = xPolicies.listIterator(); + + List xResources = daoMgr.getXXPolicyResource().findByServiceId(serviceId); + List xResourceMaps = daoMgr.getXXPolicyResourceMap().findByServiceId(serviceId); + List xPolicyItems = daoMgr.getXXPolicyItem().findByServiceId(serviceId); + List xUserPerms = daoMgr.getXXPolicyItemUserPerm().findByServiceId(serviceId); + List xGroupPerms = daoMgr.getXXPolicyItemGroupPerm().findByServiceId(serviceId); + List xAccesses = daoMgr.getXXPolicyItemAccess().findByServiceId(serviceId); + List xConditions = daoMgr.getXXPolicyItemCondition().findByServiceId(serviceId); + List xDataMaskInfos = daoMgr.getXXPolicyItemDataMaskInfo().findByServiceId(serviceId); + List xRowFilterInfos = daoMgr.getXXPolicyItemRowFilterInfo().findByServiceId(serviceId); + + this.iterResources = xResources.listIterator(); + this.iterResourceMaps = xResourceMaps.listIterator(); + this.iterPolicyItems = xPolicyItems.listIterator(); + this.iterUserPerms = xUserPerms.listIterator(); + this.iterGroupPerms = xGroupPerms.listIterator(); + this.iterAccesses = xAccesses.listIterator(); + this.iterConditions = xConditions.listIterator(); + this.iterDataMaskInfos = xDataMaskInfos.listIterator(); + this.iterRowFilterInfos = xRowFilterInfos.listIterator(); + } + + RetrieverContext(XXPolicy xPolicy, XXService xService) { + Long policyId = xPolicy == null ? null : xPolicy.getId(); + List xPolicies = asList(xPolicy); + + this.service = xService; + this.iterPolicy = xPolicies.listIterator(); + + List xResources = daoMgr.getXXPolicyResource().findByPolicyId(policyId); + List xResourceMaps = daoMgr.getXXPolicyResourceMap().findByPolicyId(policyId); + List xPolicyItems = daoMgr.getXXPolicyItem().findByPolicyId(policyId); + List xUserPerms = daoMgr.getXXPolicyItemUserPerm().findByPolicyId(policyId); + List xGroupPerms = daoMgr.getXXPolicyItemGroupPerm().findByPolicyId(policyId); + List xAccesses = daoMgr.getXXPolicyItemAccess().findByPolicyId(policyId); + List xConditions = daoMgr.getXXPolicyItemCondition().findByPolicyId(policyId); + List xDataMaskInfos = daoMgr.getXXPolicyItemDataMaskInfo().findByPolicyId(policyId); + List xRowFilterInfos = daoMgr.getXXPolicyItemRowFilterInfo().findByPolicyId(policyId); + + this.iterResources = xResources.listIterator(); + this.iterResourceMaps = xResourceMaps.listIterator(); + this.iterPolicyItems = xPolicyItems.listIterator(); + this.iterUserPerms = xUserPerms.listIterator(); + this.iterGroupPerms = xGroupPerms.listIterator(); + this.iterAccesses = xAccesses.listIterator(); + this.iterConditions = xConditions.listIterator(); + this.iterDataMaskInfos = xDataMaskInfos.listIterator(); + this.iterRowFilterInfos = xRowFilterInfos.listIterator(); + } + + RangerPolicy getNextPolicy() { + RangerPolicy ret = null; + + if (iterPolicy.hasNext()) { + XXPolicy xPolicy = iterPolicy.next(); + + if (xPolicy != null) { + ret = new RangerPolicy(); + + ret.setId(xPolicy.getId()); + ret.setGuid(xPolicy.getGuid()); + ret.setIsEnabled(xPolicy.getIsEnabled()); + ret.setCreatedBy(lookupCache.getUserScreenName(xPolicy.getAddedByUserId())); + ret.setUpdatedBy(lookupCache.getUserScreenName(xPolicy.getUpdatedByUserId())); + ret.setCreateTime(xPolicy.getCreateTime()); + ret.setUpdateTime(xPolicy.getUpdateTime()); + ret.setVersion(xPolicy.getVersion()); + ret.setService(service == null ? null : service.getName()); + ret.setName(StringUtils.trim(xPolicy.getName())); + ret.setPolicyType(xPolicy.getPolicyType() == null ? RangerPolicy.POLICY_TYPE_ACCESS : xPolicy.getPolicyType()); + ret.setDescription(xPolicy.getDescription()); + ret.setResourceSignature(xPolicy.getResourceSignature()); + ret.setIsAuditEnabled(xPolicy.getIsAuditEnabled()); + + getResource(ret); + getPolicyItems(ret); + } + } + + return ret; + } + + List getAllPolicies() { + List ret = new ArrayList(); + + while (iterPolicy.hasNext()) { + RangerPolicy policy = getNextPolicy(); + + if (policy != null) { + ret.add(policy); + } + } + + if (!hasProcessedAll()) { + LOG.warn("getAllPolicies(): perhaps one or more policies got updated during retrieval. Falling back to secondary method"); + + ret = getAllPoliciesBySecondary(); + } + + return ret; + } + + List getAllPoliciesBySecondary() { + List ret = null; + + if (service != null) { + List xPolicies = daoMgr.getXXPolicy().findByServiceId(service.getId()); + + if (CollectionUtils.isNotEmpty(xPolicies)) { + ret = new ArrayList(xPolicies.size()); + + for (XXPolicy xPolicy : xPolicies) { + RetrieverContext ctx = new RetrieverContext(xPolicy, service); + + RangerPolicy policy = ctx.getNextPolicy(); + + if (policy != null) { + ret.add(policy); + } + } + } + } + + return ret; + } + + private boolean hasProcessedAll() { + boolean moreToProcess = iterPolicy.hasNext() || iterResources.hasNext() || iterResourceMaps.hasNext() + || iterPolicyItems.hasNext() || iterUserPerms.hasNext() || iterGroupPerms.hasNext() + || iterAccesses.hasNext() || iterConditions.hasNext() || iterDataMaskInfos.hasNext() + || iterRowFilterInfos.hasNext(); + + return !moreToProcess; + } + + private void getResource(RangerPolicy policy) { + while (iterResources.hasNext()) { + XXPolicyResource xResource = iterResources.next(); + + if (xResource.getPolicyid().equals(policy.getId())) { + RangerPolicyResource resource = new RangerPolicyResource(); + + resource.setIsExcludes(xResource.getIsexcludes()); + resource.setIsRecursive(xResource.getIsrecursive()); + + while (iterResourceMaps.hasNext()) { + XXPolicyResourceMap xResourceMap = iterResourceMaps.next(); + + if (xResourceMap.getResourceid().equals(xResource.getId())) { + resource.getValues().add(xResourceMap.getValue()); + } else { + if (iterResourceMaps.hasPrevious()) { + iterResourceMaps.previous(); + } + + break; + } + } + + policy.getResources().put(lookupCache.getResourceName(xResource.getResdefid()), resource); + } else if (xResource.getPolicyid().compareTo(policy.getId()) > 0) { + if (iterResources.hasPrevious()) { + iterResources.previous(); + } + + break; + } + } + } + + private void getPolicyItems(RangerPolicy policy) { + while (iterPolicyItems.hasNext()) { + XXPolicyItem xPolicyItem = iterPolicyItems.next(); + + if (xPolicyItem.getPolicyid().equals(policy.getId())) { + final RangerPolicyItem policyItem; + final RangerDataMaskPolicyItem dataMaskPolicyItem; + final RangerRowFilterPolicyItem rowFilterPolicyItem; + + if (xPolicyItem.getItemType() == RangerPolicyItemEvaluator.POLICY_ITEM_TYPE_DATAMASK) { + dataMaskPolicyItem = new RangerDataMaskPolicyItem(); + rowFilterPolicyItem = null; + policyItem = dataMaskPolicyItem; + } else if (xPolicyItem.getItemType() == RangerPolicyItemEvaluator.POLICY_ITEM_TYPE_ROWFILTER) { + dataMaskPolicyItem = null; + rowFilterPolicyItem = new RangerRowFilterPolicyItem(); + policyItem = rowFilterPolicyItem; + } else { + dataMaskPolicyItem = null; + rowFilterPolicyItem = null; + policyItem = new RangerPolicyItem(); + } + + while (iterAccesses.hasNext()) { + XXPolicyItemAccess xAccess = iterAccesses.next(); + + if (xAccess.getPolicyitemid().equals(xPolicyItem.getId())) { + policyItem.getAccesses().add(new RangerPolicyItemAccess(lookupCache.getAccessType(xAccess.getType()), xAccess.getIsallowed())); + } else { + if (iterAccesses.hasPrevious()) { + iterAccesses.previous(); + } + + break; + } + } + + while (iterUserPerms.hasNext()) { + XXPolicyItemUserPerm xUserPerm = iterUserPerms.next(); + + if (xUserPerm.getPolicyitemid().equals(xPolicyItem.getId())) { + String userName = lookupCache.getUserName(xUserPerm.getUserid()); + + if (userName != null) { + policyItem.getUsers().add(userName); + } + } else { + if (iterUserPerms.hasPrevious()) { + iterUserPerms.previous(); + } + + break; + } + } + + while (iterGroupPerms.hasNext()) { + XXPolicyItemGroupPerm xGroupPerm = iterGroupPerms.next(); + + if (xGroupPerm.getPolicyitemid().equals(xPolicyItem.getId())) { + String groupName = lookupCache.getGroupName(xGroupPerm.getGroupid()); + + if (groupName != null) { + policyItem.getGroups().add(groupName); + } + } else { + if (iterGroupPerms.hasPrevious()) { + iterGroupPerms.previous(); + } + + break; + } + } + + RangerPolicyItemCondition condition = null; + Long prevConditionType = null; + + while (iterConditions.hasNext()) { + XXPolicyItemCondition xCondition = iterConditions.next(); + + if (xCondition.getPolicyitemid().equals(xPolicyItem.getId())) { + if (!xCondition.getType().equals(prevConditionType)) { + condition = new RangerPolicyItemCondition(); + + condition.setType(lookupCache.getConditionType(xCondition.getType())); + condition.getValues().add(xCondition.getValue()); + + policyItem.getConditions().add(condition); + + prevConditionType = xCondition.getType(); + } else { + condition.getValues().add(xCondition.getValue()); + } + } else { + if (iterConditions.hasPrevious()) { + iterConditions.previous(); + } + + break; + } + } + + policyItem.setDelegateAdmin(xPolicyItem.getDelegateAdmin()); + + if (dataMaskPolicyItem != null) { + while (iterDataMaskInfos.hasNext()) { + XXPolicyItemDataMaskInfo xDataMaskInfo = iterDataMaskInfos.next(); + + if (xDataMaskInfo.getPolicyItemId().equals(xPolicyItem.getId())) { + dataMaskPolicyItem.setDataMaskInfo(new RangerPolicyItemDataMaskInfo(lookupCache.getDataMaskName(xDataMaskInfo.getType()), xDataMaskInfo.getConditionExpr(), xDataMaskInfo.getValueExpr())); + } else { + if (iterDataMaskInfos.hasPrevious()) { + iterDataMaskInfos.previous(); + } + + break; + } + } + } + + if (rowFilterPolicyItem != null) { + while (iterRowFilterInfos.hasNext()) { + XXPolicyItemRowFilterInfo xRowFilterInfo = iterRowFilterInfos.next(); + + if (xRowFilterInfo.getPolicyItemId().equals(xPolicyItem.getId())) { + rowFilterPolicyItem.setRowFilterInfo(new RangerPolicyItemRowFilterInfo(xRowFilterInfo.getFilterExpr())); + } else { + if (iterRowFilterInfos.hasPrevious()) { + iterRowFilterInfos.previous(); + } + + break; + } + } + } + + int itemType = xPolicyItem.getItemType() == null ? RangerPolicyItemEvaluator.POLICY_ITEM_TYPE_ALLOW : xPolicyItem.getItemType(); + + if (itemType == RangerPolicyItemEvaluator.POLICY_ITEM_TYPE_ALLOW) { + policy.getPolicyItems().add(policyItem); + } else if (itemType == RangerPolicyItemEvaluator.POLICY_ITEM_TYPE_DENY) { + policy.getDenyPolicyItems().add(policyItem); + } else if (itemType == RangerPolicyItemEvaluator.POLICY_ITEM_TYPE_ALLOW_EXCEPTIONS) { + policy.getAllowExceptions().add(policyItem); + } else if (itemType == RangerPolicyItemEvaluator.POLICY_ITEM_TYPE_DENY_EXCEPTIONS) { + policy.getDenyExceptions().add(policyItem); + } else if (itemType == RangerPolicyItemEvaluator.POLICY_ITEM_TYPE_DATAMASK) { + policy.getDataMaskPolicyItems().add(dataMaskPolicyItem); + } else if (itemType == RangerPolicyItemEvaluator.POLICY_ITEM_TYPE_ROWFILTER) { + policy.getRowFilterPolicyItems().add(rowFilterPolicyItem); + } else { // unknown itemType + LOG.warn("RangerPolicyRetriever.getPolicy(policyId=" + policy.getId() + "): ignoring unknown policyItemType " + itemType); + } + } else if (xPolicyItem.getPolicyid().compareTo(policy.getId()) > 0) { + if (iterPolicyItems.hasPrevious()) { + iterPolicyItems.previous(); + } + + break; + } + } + } + } + } +} diff --git a/security-admin/src/main/java/org/apache/ranger/patch/PatchForUpdatingTagsJson_J10020.java b/security-admin/src/main/java/org/apache/ranger/patch/PatchForUpdatingTagsJson_J10020.java new file mode 100644 index 0000000000..48ae5f9e8b --- /dev/null +++ b/security-admin/src/main/java/org/apache/ranger/patch/PatchForUpdatingTagsJson_J10020.java @@ -0,0 +1,788 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Consolidates Ranger policy details into a JSON string and stores it into a + * column in x_policy table After running this patch Ranger policy can be + * completely read/saved into x_policy table and some related Ref tables (which + * maintain ID->String mapping for each policy). + * + */ + +package org.apache.ranger.patch; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.ListIterator; +import java.util.Map; + +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.MapUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.log4j.Logger; +import org.apache.ranger.authorization.utils.StringUtil; +import org.apache.ranger.biz.ServiceDBStore; +import org.apache.ranger.biz.TagDBStore; +import org.apache.ranger.db.RangerDaoManager; +import org.apache.ranger.entity.XXPortalUser; +import org.apache.ranger.entity.XXResourceDef; +import org.apache.ranger.entity.XXService; +import org.apache.ranger.entity.XXServiceResource; +import org.apache.ranger.entity.XXServiceResourceElement; +import org.apache.ranger.entity.XXServiceResourceElementValue; +import org.apache.ranger.entity.XXTag; +import org.apache.ranger.entity.XXTagAttribute; +import org.apache.ranger.entity.XXTagAttributeDef; +import org.apache.ranger.entity.XXTagDef; +import org.apache.ranger.plugin.model.RangerPolicy; +import org.apache.ranger.plugin.model.RangerService; +import org.apache.ranger.plugin.model.RangerServiceResource; +import org.apache.ranger.plugin.model.RangerTag; +import org.apache.ranger.plugin.model.RangerTagDef; +import org.apache.ranger.plugin.util.SearchFilter; +import org.apache.ranger.service.RangerServiceResourceService; +import org.apache.ranger.service.RangerTagDefService; +import org.apache.ranger.service.RangerTagService; +import org.apache.ranger.util.CLIUtil; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.stereotype.Component; +import org.springframework.transaction.PlatformTransactionManager; +import org.springframework.transaction.TransactionStatus; +import org.springframework.transaction.support.TransactionCallback; +import org.springframework.transaction.support.TransactionTemplate; + +@Component +public class PatchForUpdatingTagsJson_J10020 extends BaseLoader { + + private static final Logger logger = Logger.getLogger(PatchForUpdatingTagsJson_J10020.class); + + @Autowired + RangerDaoManager daoMgr; + + @Autowired + ServiceDBStore svcStore; + + @Autowired + TagDBStore tagStore; + + @Autowired + @Qualifier(value = "transactionManager") + PlatformTransactionManager txManager; + + @Autowired + RangerTagDefService tagDefService; + + @Autowired + RangerTagService tagService; + + @Autowired + RangerServiceResourceService serviceResourceService; + + public static void main(String[] args) { + logger.info("main()"); + try { + PatchForUpdatingTagsJson_J10020 loader = (PatchForUpdatingTagsJson_J10020) CLIUtil + .getBean(PatchForUpdatingTagsJson_J10020.class); + + loader.init(); + + while (loader.isMoreToProcess()) { + loader.load(); + } + + logger.info("Load complete. Exiting!!!"); + + System.exit(0); + } catch (Exception e) { + logger.error("Error loading", e); + System.exit(1); + } + } + + @Override + public void init() throws Exception { + // Do Nothing + } + + @Override + public void execLoad() { + logger.info("==> PatchForUpdatingTagsJson.execLoad()"); + + try { + updateRangerTagsTablesWithTagsJson(); + } catch (Exception e) { + logger.error("Error while UpdateRangerTagsTablesWithTagsJson()", e); + System.exit(1); + } + + logger.info("<== PatchForUpdatingTagsJson.execLoad()"); + } + + @Override + public void printStats() { + logger.info("Update Ranger Tags Tables with Json data "); + } + + private void updateRangerTagsTablesWithTagsJson() throws Exception { + logger.info("==> updateRangerTagsTablesWithTagsJson() "); + + List allServices = svcStore.getServices(new SearchFilter()); + + if (CollectionUtils.isNotEmpty(allServices)) { + for (RangerService service : allServices) { + XXService dbService = daoMgr.getXXService().getById(service.getId()); + RangerTagDBRetriever tagsRetriever = new RangerTagDBRetriever(daoMgr, txManager, dbService); + Map tagDefs = tagsRetriever.getTagDefs(); + Map tags = tagsRetriever.getTags(); + List serviceResources = tagsRetriever.getServiceResources(); + + if (MapUtils.isNotEmpty(tagDefs)) { + logger.info("==> Port " + tagDefs.size() + " Tag Definitions for service(name=" + dbService.getName() + ")"); + + for (Map.Entry entry : tagDefs.entrySet()) { + RangerTagDef tagDef = entry.getValue(); + + portTagDef(tagDef); + } + } + + if (MapUtils.isNotEmpty(tags)) { + logger.info("==> Port " + tags.size() + " Tags for service(name=" + dbService.getName() + ")"); + + for (Map.Entry entry : tags.entrySet()) { + RangerTag tag = entry.getValue(); + + portTag(tag); + } + } + + if (CollectionUtils.isNotEmpty(serviceResources)) { + logger.info("==> Port " + serviceResources.size() + " Service Resources for service(name=" + dbService.getName() + ")"); + + for (RangerServiceResource serviceResource : serviceResources) { + portServiceResource(serviceResource); + } + } + } + } + + logger.info("<== updateRangerTagsTablesWithTagsJson() "); + } + + private void portTagDef(RangerTagDef tagDef) { + tagDefService.update(tagDef); + } + + private void portTag(RangerTag tag) { + tagService.update(tag); + } + + private void portServiceResource(RangerServiceResource serviceResource) throws Exception { + serviceResourceService.update(serviceResource); + tagStore.refreshServiceResource(serviceResource.getId()); + } + + private class RangerTagDBRetriever { + final Log LOG = LogFactory.getLog(RangerTagDBRetriever.class); + + private final RangerDaoManager daoMgr; + private final XXService xService; + private final RangerTagDBRetriever.LookupCache lookupCache; + private final PlatformTransactionManager txManager; + private final TransactionTemplate txTemplate; + private List serviceResources; + private Map tagDefs; + private Map tags; + + RangerTagDBRetriever(final RangerDaoManager daoMgr, final PlatformTransactionManager txManager, final XXService xService) throws InterruptedException { + this.daoMgr = daoMgr; + this.xService = xService; + this.lookupCache = new RangerTagDBRetriever.LookupCache(); + this.txManager = txManager; + + if (this.txManager != null) { + this.txTemplate = new TransactionTemplate(this.txManager); + this.txTemplate.setReadOnly(true); + } else { + this.txTemplate = null; + } + + if (this.daoMgr != null && this.xService != null) { + if (this.txTemplate == null) { + if (LOG.isDebugEnabled()) { + LOG.debug("Load Tags in the same thread and using an existing transaction"); + } + + if (!initializeTagCache(xService)) { + LOG.error("Failed to get tags for service:[" + xService.getName() + "] in the same thread and using an existing transaction"); + } + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("Load Tags in a separate thread and using a new transaction"); + } + + RangerTagDBRetriever.TagLoaderThread t = new RangerTagDBRetriever.TagLoaderThread(txTemplate, xService); + t.setDaemon(true); + t.start(); + t.join(); + } + + } + } + + List getServiceResources() { + return serviceResources; + } + + Map getTagDefs() { + return tagDefs; + } + + Map getTags() { + return tags; + } + + private boolean initializeTagCache(XXService xService) { + boolean ret; + RangerTagDBRetriever.TagRetrieverServiceResourceContext serviceResourceContext = new RangerTagDBRetriever.TagRetrieverServiceResourceContext(xService); + RangerTagDBRetriever.TagRetrieverTagDefContext tagDefContext = new RangerTagDBRetriever.TagRetrieverTagDefContext(xService); + RangerTagDBRetriever.TagRetrieverTagContext tagContext = new RangerTagDBRetriever.TagRetrieverTagContext(xService); + + serviceResources = serviceResourceContext.getAllServiceResources(); + tagDefs = tagDefContext.getAllTagDefs(); + tags = tagContext.getAllTags(); + + ret = true; + return ret; + } + + private List asList(T obj) { + List ret = new ArrayList<>(); + + if (obj != null) { + ret.add(obj); + } + + return ret; + } + + private class LookupCache { + final Map userScreenNames = new HashMap<>(); + final Map resourceDefs = new HashMap<>(); + + String getUserScreenName(Long userId) { + String ret = null; + + if (userId != null) { + ret = userScreenNames.get(userId); + + if (ret == null) { + XXPortalUser user = daoMgr.getXXPortalUser().getById(userId); + + if (user != null) { + ret = user.getPublicScreenName(); + + if (StringUtil.isEmpty(ret)) { + ret = user.getFirstName(); + + if (StringUtil.isEmpty(ret)) { + ret = user.getLoginId(); + } else { + if (!StringUtil.isEmpty(user.getLastName())) { + ret += (" " + user.getLastName()); + } + } + } + + if (ret != null) { + userScreenNames.put(userId, ret); + } + } + } + } + + return ret; + } + + String getResourceName(Long resourceDefId) { + String ret = null; + + if (resourceDefId != null) { + ret = resourceDefs.get(resourceDefId); + + if (ret == null) { + XXResourceDef xResourceDef = daoMgr.getXXResourceDef().getById(resourceDefId); + + if (xResourceDef != null) { + ret = xResourceDef.getName(); + + resourceDefs.put(resourceDefId, ret); + } + } + } + + return ret; + } + } + + private class TagLoaderThread extends Thread { + final TransactionTemplate txTemplate; + final XXService xService; + + TagLoaderThread(TransactionTemplate txTemplate, final XXService xService) { + this.txTemplate = txTemplate; + this.xService = xService; + } + + @Override + public void run() { + txTemplate.setReadOnly(true); + Boolean result = txTemplate.execute(new TransactionCallback() { + @Override + public Boolean doInTransaction(TransactionStatus status) { + boolean ret = initializeTagCache(xService); + + if (!ret) { + status.setRollbackOnly(); + LOG.error("Failed to get tags for service:[" + xService.getName() + "] in a new transaction"); + } + return ret; + } + }); + + if (LOG.isDebugEnabled()) { + LOG.debug("transaction result:[" + result +"]"); + } + } + } + + private class TagRetrieverServiceResourceContext { + final XXService service; + final ListIterator iterServiceResource; + final ListIterator iterServiceResourceElement; + final ListIterator iterServiceResourceElementValue; + + TagRetrieverServiceResourceContext(XXService xService) { + Long serviceId = xService == null ? null : xService.getId(); + List xServiceResources = daoMgr.getXXServiceResource().findByServiceId(serviceId); + List xServiceResourceElements = daoMgr.getXXServiceResourceElement().findTaggedResourcesInServiceId(serviceId); + List xServiceResourceElementValues = daoMgr.getXXServiceResourceElementValue().findTaggedResourcesInServiceId(serviceId); + + this.service = xService; + this.iterServiceResource = xServiceResources.listIterator(); + this.iterServiceResourceElement = xServiceResourceElements.listIterator(); + this.iterServiceResourceElementValue = xServiceResourceElementValues.listIterator(); + + } + + TagRetrieverServiceResourceContext(XXServiceResource xServiceResource, XXService xService) { + Long resourceId = xServiceResource == null ? null : xServiceResource.getId(); + List xServiceResources = asList(xServiceResource); + List xServiceResourceElements = daoMgr.getXXServiceResourceElement().findByResourceId(resourceId); + List xServiceResourceElementValues = daoMgr.getXXServiceResourceElementValue().findByResourceId(resourceId); + + this.service = xService; + this.iterServiceResource = xServiceResources.listIterator(); + this.iterServiceResourceElement = xServiceResourceElements.listIterator(); + this.iterServiceResourceElementValue = xServiceResourceElementValues.listIterator(); + } + + List getAllServiceResources() { + List ret = new ArrayList<>(); + + while (iterServiceResource.hasNext()) { + RangerServiceResource serviceResource = getNextServiceResource(); + + if (serviceResource != null) { + ret.add(serviceResource); + } + } + + if (!hasProcessedAll()) { + LOG.warn("getAllServiceResources(): perhaps one or more serviceResources got updated during retrieval. Using fallback ... "); + + ret = getServiceResourcesBySecondary(); + } + + return ret; + } + + RangerServiceResource getNextServiceResource() { + RangerServiceResource ret = null; + + if (iterServiceResource.hasNext()) { + XXServiceResource xServiceResource = iterServiceResource.next(); + + if (xServiceResource != null) { + ret = new RangerServiceResource(); + + ret.setId(xServiceResource.getId()); + ret.setGuid(xServiceResource.getGuid()); + ret.setIsEnabled(xServiceResource.getIsEnabled()); + ret.setCreatedBy(lookupCache.getUserScreenName(xServiceResource.getAddedByUserId())); + ret.setUpdatedBy(lookupCache.getUserScreenName(xServiceResource.getUpdatedByUserId())); + ret.setCreateTime(xServiceResource.getCreateTime()); + ret.setUpdateTime(xServiceResource.getUpdateTime()); + ret.setVersion(xServiceResource.getVersion()); + ret.setResourceSignature(xServiceResource.getResourceSignature()); + ret.setServiceName(xService.getName()); + + getServiceResourceElements(ret); + } + } + + return ret; + } + + void getServiceResourceElements(RangerServiceResource serviceResource) { + while (iterServiceResourceElement.hasNext()) { + XXServiceResourceElement xServiceResourceElement = iterServiceResourceElement.next(); + + if (xServiceResourceElement.getResourceId().equals(serviceResource.getId())) { + RangerPolicy.RangerPolicyResource resource = new RangerPolicy.RangerPolicyResource(); + + resource.setIsExcludes(xServiceResourceElement.getIsExcludes()); + resource.setIsRecursive(xServiceResourceElement.getIsRecursive()); + + while (iterServiceResourceElementValue.hasNext()) { + XXServiceResourceElementValue xServiceResourceElementValue = iterServiceResourceElementValue.next(); + + if (xServiceResourceElementValue.getResElementId().equals(xServiceResourceElement.getId())) { + resource.getValues().add(xServiceResourceElementValue.getValue()); + } else { + if (iterServiceResourceElementValue.hasPrevious()) { + iterServiceResourceElementValue.previous(); + } + + break; + } + } + + serviceResource.getResourceElements().put(lookupCache.getResourceName(xServiceResourceElement.getResDefId()), resource); + } else if (xServiceResourceElement.getResourceId().compareTo(serviceResource.getId()) > 0) { + if (iterServiceResourceElement.hasPrevious()) { + iterServiceResourceElement.previous(); + } + + break; + } + } + } + + boolean hasProcessedAll() { + boolean moreToProcess = iterServiceResource.hasNext() + || iterServiceResourceElement.hasNext() + || iterServiceResourceElementValue.hasNext(); + + return !moreToProcess; + } + + List getServiceResourcesBySecondary() { + List ret = null; + + if (service != null) { + List xServiceResources = daoMgr.getXXServiceResource().findTaggedResourcesInServiceId(service.getId()); + + if (CollectionUtils.isNotEmpty(xServiceResources)) { + ret = new ArrayList<>(xServiceResources.size()); + + for (XXServiceResource xServiceResource : xServiceResources) { + RangerTagDBRetriever.TagRetrieverServiceResourceContext ctx = new RangerTagDBRetriever.TagRetrieverServiceResourceContext(xServiceResource, service); + + RangerServiceResource serviceResource = ctx.getNextServiceResource(); + + if (serviceResource != null) { + ret.add(serviceResource); + } + } + } + } + return ret; + } + } + + private class TagRetrieverTagDefContext { + final XXService service; + final ListIterator iterTagDef; + final ListIterator iterTagAttributeDef; + + + TagRetrieverTagDefContext(XXService xService) { + Long serviceId = xService == null ? null : xService.getId(); + List xTagDefs = daoMgr.getXXTagDef().findByServiceId(serviceId); + List xTagAttributeDefs = daoMgr.getXXTagAttributeDef().findByServiceId(serviceId); + + this.service = xService; + this.iterTagDef = xTagDefs.listIterator(); + this.iterTagAttributeDef = xTagAttributeDefs.listIterator(); + } + + TagRetrieverTagDefContext(XXTagDef xTagDef, XXService xService) { + Long tagDefId = xTagDef == null ? null : xTagDef.getId(); + List xTagDefs = asList(xTagDef); + List xTagAttributeDefs = daoMgr.getXXTagAttributeDef().findByTagDefId(tagDefId); + + this.service = xService; + this.iterTagDef = xTagDefs.listIterator(); + this.iterTagAttributeDef = xTagAttributeDefs.listIterator(); + } + + Map getAllTagDefs() { + Map ret = new HashMap<>(); + + while (iterTagDef.hasNext()) { + RangerTagDef tagDef = getNextTagDef(); + + if (tagDef != null) { + ret.put(tagDef.getId(), tagDef); + } + } + + if (!hasProcessedAllTagDefs()) { + LOG.warn("getAllTagDefs(): perhaps one or more tag-definitions got updated during retrieval. Using fallback ... "); + + ret = getTagDefsBySecondary(); + } + + return ret; + } + + RangerTagDef getNextTagDef() { + RangerTagDef ret = null; + + if (iterTagDef.hasNext()) { + XXTagDef xTagDef = iterTagDef.next(); + + if (xTagDef != null) { + ret = new RangerTagDef(); + + ret.setId(xTagDef.getId()); + ret.setGuid(xTagDef.getGuid()); + ret.setIsEnabled(xTagDef.getIsEnabled()); + ret.setCreatedBy(lookupCache.getUserScreenName(xTagDef.getAddedByUserId())); + ret.setUpdatedBy(lookupCache.getUserScreenName(xTagDef.getUpdatedByUserId())); + ret.setCreateTime(xTagDef.getCreateTime()); + ret.setUpdateTime(xTagDef.getUpdateTime()); + ret.setVersion(xTagDef.getVersion()); + ret.setName(xTagDef.getName()); + ret.setSource(xTagDef.getSource()); + + getTagAttributeDefs(ret); + } + } + + return ret; + } + + void getTagAttributeDefs(RangerTagDef tagDef) { + while (iterTagAttributeDef.hasNext()) { + XXTagAttributeDef xTagAttributeDef = iterTagAttributeDef.next(); + + if (xTagAttributeDef.getTagDefId().equals(tagDef.getId())) { + RangerTagDef.RangerTagAttributeDef tagAttributeDef = new RangerTagDef.RangerTagAttributeDef(); + + tagAttributeDef.setName(xTagAttributeDef.getName()); + tagAttributeDef.setType(xTagAttributeDef.getType()); + + tagDef.getAttributeDefs().add(tagAttributeDef); + } else if (xTagAttributeDef.getTagDefId().compareTo(tagDef.getId()) > 0) { + if (iterTagAttributeDef.hasPrevious()) { + iterTagAttributeDef.previous(); + } + break; + } + } + } + + boolean hasProcessedAllTagDefs() { + boolean moreToProcess = iterTagAttributeDef.hasNext(); + + return !moreToProcess; + } + + Map getTagDefsBySecondary() { + Map ret = null; + + if (service != null) { + List xTagDefs = daoMgr.getXXTagDef().findByServiceId(service.getId()); + + if (CollectionUtils.isNotEmpty(xTagDefs)) { + ret = new HashMap<>(xTagDefs.size()); + + for (XXTagDef xTagDef : xTagDefs) { + TagRetrieverTagDefContext ctx = new TagRetrieverTagDefContext(xTagDef, service); + + RangerTagDef tagDef = ctx.getNextTagDef(); + + if (tagDef != null) { + ret.put(tagDef.getId(), tagDef); + } + } + } + } + return ret; + } + } + + private class TagRetrieverTagContext { + final XXService service; + final ListIterator iterTag; + final ListIterator iterTagAttribute; + + + TagRetrieverTagContext(XXService xService) { + Long serviceId = xService == null ? null : xService.getId(); + List xTags = daoMgr.getXXTag().findByServiceId(serviceId); + List xTagAttributes = daoMgr.getXXTagAttribute().findByServiceId(serviceId); + + this.service = xService; + this.iterTag = xTags.listIterator(); + this.iterTagAttribute = xTagAttributes.listIterator(); + + } + + TagRetrieverTagContext(XXTag xTag, XXService xService) { + Long tagId = xTag == null ? null : xTag.getId(); + List xTags = asList(xTag); + List xTagAttributes = daoMgr.getXXTagAttribute().findByTagId(tagId); + + this.service = xService; + this.iterTag = xTags.listIterator(); + this.iterTagAttribute = xTagAttributes.listIterator(); + } + + + Map getAllTags() { + Map ret = new HashMap<>(); + + while (iterTag.hasNext()) { + RangerTag tag = getNextTag(); + + if (tag != null) { + ret.put(tag.getId(), tag); + } + } + + if (!hasProcessedAllTags()) { + LOG.warn("getAllTags(): perhaps one or more tags got updated during retrieval. Using fallback ... "); + + ret = getTagsBySecondary(); + } + + return ret; + } + + RangerTag getNextTag() { + RangerTag ret = null; + + if (iterTag.hasNext()) { + XXTag xTag = iterTag.next(); + + if (xTag != null) { + ret = new RangerTag(); + + ret.setId(xTag.getId()); + ret.setGuid(xTag.getGuid()); + ret.setOwner(xTag.getOwner()); + ret.setCreatedBy(lookupCache.getUserScreenName(xTag.getAddedByUserId())); + ret.setUpdatedBy(lookupCache.getUserScreenName(xTag.getUpdatedByUserId())); + ret.setCreateTime(xTag.getCreateTime()); + ret.setUpdateTime(xTag.getUpdateTime()); + ret.setVersion(xTag.getVersion()); + + /* + Map mapOfOptions = JsonUtils.jsonToMapStringString(xTag.getOptions()); + + if (MapUtils.isNotEmpty(mapOfOptions)) { + String validityPeriodsStr = mapOfOptions.get(RangerTag.OPTION_TAG_VALIDITY_PERIODS); + + if (StringUtils.isNotEmpty(validityPeriodsStr)) { + List validityPeriods = JsonUtils.jsonToRangerValiditySchedule(validityPeriodsStr); + + ret.setValidityPeriods(validityPeriods); + } + } + */ + + Map tagDefs = getTagDefs(); + if (tagDefs != null) { + RangerTagDef tagDef = tagDefs.get(xTag.getType()); + if (tagDef != null) { + ret.setType(tagDef.getName()); + } + } + + getTagAttributes(ret); + } + } + + return ret; + } + + void getTagAttributes(RangerTag tag) { + while (iterTagAttribute.hasNext()) { + XXTagAttribute xTagAttribute = iterTagAttribute.next(); + + if (xTagAttribute.getTagId().equals(tag.getId())) { + String attributeName = xTagAttribute.getName(); + String attributeValue = xTagAttribute.getValue(); + + tag.getAttributes().put(attributeName, attributeValue); + } else if (xTagAttribute.getTagId().compareTo(tag.getId()) > 0) { + if (iterTagAttribute.hasPrevious()) { + iterTagAttribute.previous(); + } + break; + } + } + } + + boolean hasProcessedAllTags() { + boolean moreToProcess = iterTagAttribute.hasNext(); + + return !moreToProcess; + } + + Map getTagsBySecondary() { + Map ret = null; + + if (service != null) { + List xTags = daoMgr.getXXTag().findByServiceId(service.getId()); + + if (CollectionUtils.isNotEmpty(xTags)) { + ret = new HashMap<>(xTags.size()); + + for (XXTag xTag : xTags) { + TagRetrieverTagContext ctx = new TagRetrieverTagContext(xTag, service); + + RangerTag tag = ctx.getNextTag(); + + if (tag != null) { + ret.put(tag.getId(), tag); + } + } + } + } + return ret; + } + } + } +} + diff --git a/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java b/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java index 6685e1a588..b2a481b925 100644 --- a/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java +++ b/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java @@ -99,8 +99,8 @@ import org.apache.ranger.plugin.policyengine.RangerPolicyEngineImpl; import org.apache.ranger.plugin.policyengine.RangerPolicyEngineOptions; import org.apache.ranger.plugin.service.ResourceLookupContext; -import org.apache.ranger.plugin.store.PList; import org.apache.ranger.plugin.store.EmbeddedServiceDefsUtil; +import org.apache.ranger.plugin.store.PList; import org.apache.ranger.plugin.store.ServiceStore; import org.apache.ranger.plugin.util.GrantRevokeRequest; import org.apache.ranger.plugin.util.RangerAccessRequestUtil; @@ -1426,7 +1426,7 @@ public RangerPolicy createPolicy(RangerPolicy policy, @Context HttpServletReques String serviceName = request.getParameter(PARAM_SERVICE_NAME); String policyName = request.getParameter(PARAM_POLICY_NAME); String updateIfExists = request.getParameter(PARAM_UPDATE_IF_EXISTS); - + if (serviceName == null && policyName == null && updateIfExists != null && updateIfExists.equalsIgnoreCase("true")) { serviceName = (String) request.getAttribute(PARAM_SERVICE_NAME); @@ -1441,7 +1441,7 @@ public RangerPolicy createPolicy(RangerPolicy policy, @Context HttpServletReques policy.setName(StringUtils.trim(policyName)); } - if (updateIfExists != null && Boolean.valueOf(updateIfExists)) { + if (updateIfExists != null && Boolean.valueOf(updateIfExists)) { RangerPolicy existingPolicy = null; try { if(StringUtils.isNotEmpty(policy.getGuid())) { @@ -2282,7 +2282,7 @@ private void deletePoliciesProvidedInServiceMap( try { validator.validate(rangerPolicy.getId(), Action.DELETE); ensureAdminAccess(rangerPolicy.getService(), rangerPolicy.getResources()); - svcStore.deletePolicy(rangerPolicy); + svcStore.deletePolicy(rangerPolicy.getId()); totalDeletedPilicies = totalDeletedPilicies + 1; if (LOG.isDebugEnabled()) { LOG.debug("Policy " + rangerPolicy.getName() + " deleted successfully." ); @@ -2304,7 +2304,7 @@ private void deletePoliciesProvidedInServiceMap( } private void deletePoliciesForResource(List sourceServices, List destinationServices, - String resource, HttpServletRequest request, List exportPolicies) { + String resource, HttpServletRequest request, List exportPolicies) { int totalDeletedPilicies = 0; if (CollectionUtils.isNotEmpty(sourceServices) && CollectionUtils.isNotEmpty(destinationServices)) { Set exportedPolicyNames = new HashSet(); @@ -3012,8 +3012,8 @@ private List applyAdminAccessFilter(List policies) { if (isKmsService) { ret.addAll(listToFilter); } - } else if (isServiceAdminUser) { - ret.addAll(listToFilter); + } else if (isServiceAdminUser) { + ret.addAll(listToFilter); } continue; diff --git a/security-admin/src/main/java/org/apache/ranger/service/RangerAuditFields.java b/security-admin/src/main/java/org/apache/ranger/service/RangerAuditFields.java index 7223f109c9..79097cc779 100644 --- a/security-admin/src/main/java/org/apache/ranger/service/RangerAuditFields.java +++ b/security-admin/src/main/java/org/apache/ranger/service/RangerAuditFields.java @@ -17,15 +17,13 @@ package org.apache.ranger.service; -import org.apache.ranger.common.ContextUtil; -import org.apache.ranger.common.DateUtil; import org.apache.ranger.entity.XXDBBase; import org.springframework.stereotype.Component; @Component public class RangerAuditFields { - public T populateAuditFields(T xObj, T parentObj) { + public T populateAuditFields(T xObj, PARENT parentObj) { xObj.setCreateTime(parentObj.getCreateTime()); xObj.setUpdateTime(parentObj.getUpdateTime()); xObj.setAddedByUserId(parentObj.getAddedByUserId()); @@ -33,12 +31,4 @@ public T populateAuditFields(T xObj, T parentObj) { return xObj; } - public T populateAuditFieldsForCreate(T xObj) { - xObj.setCreateTime(DateUtil.getUTCDate()); - xObj.setUpdateTime(DateUtil.getUTCDate()); - xObj.setAddedByUserId(ContextUtil.getCurrentUserId()); - xObj.setUpdatedByUserId(ContextUtil.getCurrentUserId()); - return xObj; - } - } diff --git a/security-admin/src/main/java/org/apache/ranger/service/RangerPolicyServiceBase.java b/security-admin/src/main/java/org/apache/ranger/service/RangerPolicyServiceBase.java index 0195c72a76..09f15c0909 100644 --- a/security-admin/src/main/java/org/apache/ranger/service/RangerPolicyServiceBase.java +++ b/security-admin/src/main/java/org/apache/ranger/service/RangerPolicyServiceBase.java @@ -18,6 +18,7 @@ package org.apache.ranger.service; import org.apache.commons.lang.StringUtils; +import org.apache.ranger.authorization.utils.JsonUtils; import org.apache.ranger.common.GUIDUtil; import org.apache.ranger.common.MessageEnums; import org.apache.ranger.common.SearchField; @@ -49,20 +50,22 @@ public RangerPolicyServiceBase() { "XXService xSvc", "xSvc.id = obj.service")); searchFields .add(new SearchField(SearchFilter.IS_ENABLED, "obj.isEnabled", DATA_TYPE.BOOLEAN, SEARCH_TYPE.FULL)); - searchFields.add(new SearchField(SearchFilter.IS_RECURSIVE,"xPolRes.isRecursive",DATA_TYPE.BOOLEAN,SEARCH_TYPE.FULL, - "XXPolicyResource xPolRes","obj.id=xPolRes.policyId")); + //might need updation + /*searchFields.add(new SearchField(SearchFilter.IS_RECURSIVE,"xPolRes.isRecursive",DATA_TYPE.BOOLEAN,SEARCH_TYPE.FULL, + "XXPolicyResource xPolRes","obj.id=xPolRes.policyId"));*/ searchFields.add(new SearchField(SearchFilter.POLICY_ID, "obj.id", DATA_TYPE.INTEGER, SEARCH_TYPE.FULL)); searchFields.add(new SearchField(SearchFilter.POLICY_NAME, "obj.name", DATA_TYPE.STRING, SEARCH_TYPE.FULL)); searchFields.add(new SearchField(SearchFilter.GUID, "obj.guid", DATA_TYPE.STRING, SEARCH_TYPE.FULL)); searchFields.add(new SearchField(SearchFilter.USER, "xUser.name", DATA_TYPE.STRING, SEARCH_TYPE.FULL, - "XXUser xUser, XXPolicyItem xPolItem, XXPolicyItemUserPerm userPerm", "obj.id = xPolItem.policyId " - + "and userPerm.policyItemId = xPolItem.id and xUser.id = userPerm.userId")); + "XXUser xUser, XXPolicyRefUser refUser", "obj.id = refUser.policyId " + + "and xUser.id = refUser.userId")); searchFields.add(new SearchField(SearchFilter.GROUP, "xGrp.name", DATA_TYPE.STRING, SEARCH_TYPE.FULL, - "XXGroup xGrp, XXPolicyItem xPolItem, XXPolicyItemGroupPerm grpPerm", "obj.id = xPolItem.policyId " - + "and grpPerm.policyItemId = xPolItem.id and xGrp.id = grpPerm.groupId")); - searchFields.add(new SearchField(SearchFilter.POL_RESOURCE, "resMap.value", DATA_TYPE.STRING, + "XXGroup xGrp , XXPolicyRefGroup refGroup", "obj.id = refGroup.policyId " + + "and xGrp.id = refGroup.groupId")); + //might need updation + /*searchFields.add(new SearchField(SearchFilter.POL_RESOURCE, "resMap.value", DATA_TYPE.STRING, SEARCH_TYPE.PARTIAL, "XXPolicyResourceMap resMap, XXPolicyResource polRes", - "resMap.resourceId = polRes.id and polRes.policyId = obj.id")); + "resMap.resourceId = polRes.id and polRes.policyId = obj.id"));*/ searchFields.add(new SearchField(SearchFilter.POLICY_NAME_PARTIAL, "obj.name", DATA_TYPE.STRING, SEARCH_TYPE.PARTIAL)); searchFields.add(new SearchField(SearchFilter.POLICY_TYPE, "obj.policyType", DATA_TYPE.INTEGER, SEARCH_TYPE.FULL)); @@ -92,7 +95,7 @@ protected T mapViewToEntityBean(V vObj, T xObj, int OPERATION_CONTEXT) { xObj.setResourceSignature(vObj.getResourceSignature()); xObj.setIsAuditEnabled(vObj.getIsAuditEnabled()); xObj.setIsEnabled(vObj.getIsEnabled()); - + xObj.setPolicyText(JsonUtils.objectToJson(vObj)); return xObj; } diff --git a/security-admin/src/main/java/org/apache/ranger/service/RangerServiceResourceService.java b/security-admin/src/main/java/org/apache/ranger/service/RangerServiceResourceService.java index d7256802a6..f0cb8f4d20 100644 --- a/security-admin/src/main/java/org/apache/ranger/service/RangerServiceResourceService.java +++ b/security-admin/src/main/java/org/apache/ranger/service/RangerServiceResourceService.java @@ -19,113 +19,167 @@ package org.apache.ranger.service; +import java.lang.reflect.Type; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import com.google.gson.reflect.TypeToken; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.MapUtils; import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.ranger.authorization.utils.JsonUtils; +import org.apache.ranger.biz.RangerTagDBRetriever; import org.apache.ranger.common.SearchField; import org.apache.ranger.common.SearchField.DATA_TYPE; import org.apache.ranger.common.SearchField.SEARCH_TYPE; import org.apache.ranger.entity.XXServiceResource; +import org.apache.ranger.plugin.model.RangerPolicy; import org.apache.ranger.plugin.model.RangerServiceResource; +import org.apache.ranger.plugin.model.RangerTag; import org.apache.ranger.plugin.util.SearchFilter; import org.springframework.stereotype.Service; @Service public class RangerServiceResourceService extends RangerServiceResourceServiceBase { - private boolean serviceUpdateNeeded = true; + private static final Log LOG = LogFactory.getLog(RangerServiceResourceService.class); - public RangerServiceResourceService() { - searchFields.add(new SearchField(SearchFilter.TAG_RESOURCE_ID, "obj.id", DATA_TYPE.INTEGER, SEARCH_TYPE.FULL)); - searchFields.add(new SearchField(SearchFilter.TAG_SERVICE_ID, "obj.serviceId", DATA_TYPE.INTEGER, SEARCH_TYPE.FULL)); - searchFields.add(new SearchField(SearchFilter.TAG_RESOURCE_SIGNATURE, "obj.resourceSignature", DATA_TYPE.STRING, SEARCH_TYPE.FULL)); - } + private boolean serviceUpdateNeeded = true; - @Override - protected void validateForCreate(RangerServiceResource vObj) { + public static final Type subsumedDataType = new TypeToken>() {}.getType(); + public static final Type duplicatedDataType = new TypeToken>() {}.getType(); - } + public RangerServiceResourceService() { + searchFields.add(new SearchField(SearchFilter.TAG_RESOURCE_ID, "obj.id", DATA_TYPE.INTEGER, SEARCH_TYPE.FULL)); + searchFields.add(new SearchField(SearchFilter.TAG_SERVICE_ID, "obj.serviceId", DATA_TYPE.INTEGER, SEARCH_TYPE.FULL)); + searchFields.add(new SearchField(SearchFilter.TAG_RESOURCE_SIGNATURE, "obj.resourceSignature", DATA_TYPE.STRING, SEARCH_TYPE.FULL)); + } - @Override - protected void validateForUpdate(RangerServiceResource vObj, XXServiceResource entityObj) { - if (StringUtils.equals(entityObj.getGuid(), vObj.getGuid()) && - StringUtils.equals(entityObj.getResourceSignature(), vObj.getResourceSignature())) { - serviceUpdateNeeded = false; - } else { - serviceUpdateNeeded = true; - } - } + @Override + protected void validateForCreate(RangerServiceResource vObj) { - @Override - public RangerServiceResource postUpdate(XXServiceResource resource) { - RangerServiceResource ret = super.postUpdate(resource); + } - if (serviceUpdateNeeded) { - daoMgr.getXXServiceVersionInfo().updateServiceVersionInfoForServiceResourceUpdate(resource.getId(), resource.getUpdateTime()); - } + @Override + protected void validateForUpdate(RangerServiceResource vObj, XXServiceResource entityObj) { + if (StringUtils.equals(entityObj.getGuid(), vObj.getGuid()) && + StringUtils.equals(entityObj.getResourceSignature(), vObj.getResourceSignature())) { + serviceUpdateNeeded = false; + } else { + serviceUpdateNeeded = true; + } + } - return ret; - } + @Override + public RangerServiceResource postUpdate(XXServiceResource resource) { + RangerServiceResource ret = super.postUpdate(resource); - public RangerServiceResource getPopulatedViewObject(XXServiceResource xObj) { - return populateViewBean(xObj); - } + if (serviceUpdateNeeded) { + daoMgr.getXXServiceVersionInfo().updateServiceVersionInfoForServiceResourceUpdate(resource.getId(), resource.getUpdateTime()); + } - public RangerServiceResource getServiceResourceByGuid(String guid) { - RangerServiceResource ret = null; + return ret; + } - XXServiceResource xxServiceResource = daoMgr.getXXServiceResource().findByGuid(guid); - - if(xxServiceResource != null) { - ret = populateViewBean(xxServiceResource); - } + public RangerServiceResource getPopulatedViewObject(XXServiceResource xObj) { + return populateViewBean(xObj); + } - return ret; - } + public RangerServiceResource getServiceResourceByGuid(String guid) { + RangerServiceResource ret = null; - public List getByServiceId(Long serviceId) { - List ret = new ArrayList(); + XXServiceResource xxServiceResource = daoMgr.getXXServiceResource().findByGuid(guid); - List xxServiceResources = daoMgr.getXXServiceResource().findByServiceId(serviceId); + if (xxServiceResource != null) { + ret = populateViewBean(xxServiceResource); + } - if(CollectionUtils.isNotEmpty(xxServiceResources)) { - for(XXServiceResource xxServiceResource : xxServiceResources) { - RangerServiceResource serviceResource = populateViewBean(xxServiceResource); + return ret; + } - ret.add(serviceResource); - } - } + public List getByServiceId(Long serviceId) { + List ret = new ArrayList(); - return ret; - } + List xxServiceResources = daoMgr.getXXServiceResource().findByServiceId(serviceId); - public RangerServiceResource getByServiceAndResourceSignature(Long serviceId, String resourceSignature) { - RangerServiceResource ret = null; + if (CollectionUtils.isNotEmpty(xxServiceResources)) { + for (XXServiceResource xxServiceResource : xxServiceResources) { + RangerServiceResource serviceResource = populateViewBean(xxServiceResource); - XXServiceResource xxServiceResource = daoMgr.getXXServiceResource().findByServiceAndResourceSignature(serviceId, resourceSignature); - - if(xxServiceResource != null) { - ret = populateViewBean(xxServiceResource); - } + ret.add(serviceResource); + } + } - return ret; - } + return ret; + } - public List getTaggedResourcesInServiceId(Long serviceId) { - List ret = new ArrayList(); + public RangerServiceResource getByServiceAndResourceSignature(Long serviceId, String resourceSignature) { + RangerServiceResource ret = null; - List xxServiceResources = daoMgr.getXXServiceResource().findByServiceId(serviceId); - - if(CollectionUtils.isNotEmpty(xxServiceResources)) { - for(XXServiceResource xxServiceResource : xxServiceResources) { - RangerServiceResource serviceResource = populateViewBean(xxServiceResource); + XXServiceResource xxServiceResource = daoMgr.getXXServiceResource().findByServiceAndResourceSignature(serviceId, resourceSignature); - ret.add(serviceResource); - } - } + if (xxServiceResource != null) { + ret = populateViewBean(xxServiceResource); + } - return ret; - } + return ret; + } + + public List getTaggedResourcesInServiceId(Long serviceId) { + List ret = new ArrayList(); + + List xxServiceResources = daoMgr.getXXServiceResource().findByServiceId(serviceId); + + if (CollectionUtils.isNotEmpty(xxServiceResources)) { + for (XXServiceResource xxServiceResource : xxServiceResources) { + RangerServiceResource serviceResource = populateViewBean(xxServiceResource); + + ret.add(serviceResource); + } + } + + return ret; + } + + @Override + protected XXServiceResource mapViewToEntityBean(RangerServiceResource serviceResource, XXServiceResource xxServiceResource, int operationContext) { + XXServiceResource ret = super.mapViewToEntityBean(serviceResource, xxServiceResource, operationContext); + if (MapUtils.isNotEmpty(serviceResource.getResourceElements())) { + String serviceResourceElements = JsonUtils.mapToJson(serviceResource.getResourceElements()); + if (StringUtils.isNotEmpty(serviceResourceElements)) { + ret.setServiceResourceElements(serviceResourceElements); + } else { + LOG.info("Empty string representing serviceResourceElements in [" + ret + "]!!"); + } + } + + return ret; + } + + @Override + protected RangerServiceResource mapEntityToViewBean(RangerServiceResource serviceResource, XXServiceResource xxServiceResource) { + RangerServiceResource ret = super.mapEntityToViewBean(serviceResource, xxServiceResource); + if (StringUtils.isNotEmpty(xxServiceResource.getServiceResourceElements())) { + Map serviceResourceElements = + RangerTagDBRetriever.gsonBuilder.fromJson(xxServiceResource.getServiceResourceElements(), RangerServiceResourceService.subsumedDataType); + if (MapUtils.isNotEmpty(serviceResourceElements)) { + ret.setResourceElements(serviceResourceElements); + } else { + LOG.info("Empty serviceResourceElement in [" + ret + "]!!"); + } + } else { + LOG.info("Empty string representing serviceResourceElements in [" + xxServiceResource + "]!!"); + } + + return ret; + } + + @Override + Map getServiceResourceElements(XXServiceResource xxServiceResource) { + return new HashMap<>(); + } } diff --git a/security-admin/src/main/java/org/apache/ranger/service/RangerServiceResourceServiceBase.java b/security-admin/src/main/java/org/apache/ranger/service/RangerServiceResourceServiceBase.java index 6af682a811..329d3eeecb 100644 --- a/security-admin/src/main/java/org/apache/ranger/service/RangerServiceResourceServiceBase.java +++ b/security-admin/src/main/java/org/apache/ranger/service/RangerServiceResourceServiceBase.java @@ -73,26 +73,31 @@ protected V mapEntityToViewBean(V vObj, T xObj) { vObj.setServiceName(xService.getName()); - List resElementList = daoMgr.getXXServiceResourceElement().findByResourceId(xObj.getId()); - Map resourceElements = new HashMap(); + Map resourceElements = getServiceResourceElements(xObj); - for (XXServiceResourceElement resElement : resElementList) { - List resValueMapList = daoMgr.getXXServiceResourceElementValue().findValuesByResElementId(resElement.getId()); + vObj.setResourceElements(resourceElements); - XXResourceDef xResDef = daoMgr.getXXResourceDef().getById(resElement.getResDefId()); + return vObj; + } - RangerPolicyResource policyRes = new RangerPolicyResource(); - policyRes.setIsExcludes(resElement.getIsExcludes()); - policyRes.setIsRecursive(resElement.getIsRecursive()); - policyRes.setValues(resValueMapList); + Map getServiceResourceElements(T xObj) { + List resElementList = daoMgr.getXXServiceResourceElement().findByResourceId(xObj.getId()); + Map resourceElements = new HashMap(); - resourceElements.put(xResDef.getName(), policyRes); - } + for (XXServiceResourceElement resElement : resElementList) { + List resValueMapList = daoMgr.getXXServiceResourceElementValue().findValuesByResElementId(resElement.getId()); - vObj.setResourceElements(resourceElements); + XXResourceDef xResDef = daoMgr.getXXResourceDef().getById(resElement.getResDefId()); - return vObj; - } + RangerPolicyResource policyRes = new RangerPolicyResource(); + policyRes.setIsExcludes(resElement.getIsExcludes()); + policyRes.setIsRecursive(resElement.getIsRecursive()); + policyRes.setValues(resValueMapList); + + resourceElements.put(xResDef.getName(), policyRes); + } + return resourceElements; + } public PList searchServiceResources(SearchFilter searchFilter) { PList retList = new PList(); diff --git a/security-admin/src/main/java/org/apache/ranger/service/RangerTagDefService.java b/security-admin/src/main/java/org/apache/ranger/service/RangerTagDefService.java index 82eb252e6b..10c73f0d2e 100644 --- a/security-admin/src/main/java/org/apache/ranger/service/RangerTagDefService.java +++ b/security-admin/src/main/java/org/apache/ranger/service/RangerTagDefService.java @@ -23,6 +23,8 @@ import java.util.List; import org.apache.commons.collections.CollectionUtils; +import org.apache.ranger.authorization.utils.JsonUtils; +import org.apache.ranger.biz.RangerTagDBRetriever; import org.apache.ranger.common.SearchField; import org.apache.ranger.common.SearchField.DATA_TYPE; import org.apache.ranger.common.SearchField.SEARCH_TYPE; @@ -102,5 +104,27 @@ public List getTagDefsByServiceId(Long serviceId) { return ret; } + + @Override + protected RangerTagDef mapEntityToViewBean(RangerTagDef vObj, XXTagDef xObj) { + super.mapEntityToViewBean(vObj, xObj); + + List attributeDefs = RangerTagDBRetriever.gsonBuilder.fromJson(xObj.getTagAttrDefs(), RangerTagDBRetriever.subsumedDataType); + vObj.setAttributeDefs(attributeDefs); + + return vObj; + } + + @Override + protected XXTagDef mapViewToEntityBean(RangerTagDef vObj, XXTagDef xObj, int OPERATION_CONTEXT) { + super.mapViewToEntityBean(vObj, xObj, OPERATION_CONTEXT); + xObj.setTagAttrDefs(JsonUtils.listToJson(vObj.getAttributeDefs())); + return xObj; + } + + @Override + public List getAttributeDefForTagDef(XXTagDef xtagDef) { + return new ArrayList<>(); + } } diff --git a/security-admin/src/main/java/org/apache/ranger/service/RangerTagService.java b/security-admin/src/main/java/org/apache/ranger/service/RangerTagService.java index 28b9115fab..2fa883096c 100644 --- a/security-admin/src/main/java/org/apache/ranger/service/RangerTagService.java +++ b/security-admin/src/main/java/org/apache/ranger/service/RangerTagService.java @@ -19,13 +19,21 @@ package org.apache.ranger.service; +import java.lang.reflect.Type; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import com.google.gson.reflect.TypeToken; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.ranger.authorization.utils.JsonUtils; +import org.apache.ranger.biz.RangerTagDBRetriever; import org.apache.ranger.common.SearchField; import org.apache.ranger.common.SearchField.DATA_TYPE; import org.apache.ranger.common.SearchField.SEARCH_TYPE; +import org.apache.ranger.entity.XXServiceResource; import org.apache.ranger.entity.XXTag; import org.apache.ranger.plugin.model.RangerTag; import org.apache.ranger.plugin.util.SearchFilter; @@ -35,6 +43,8 @@ @Service public class RangerTagService extends RangerTagServiceBase { + public static final Type subsumedDataType = new TypeToken>() {}.getType(); + public RangerTagService() { searchFields.add(new SearchField(SearchFilter.TAG_ID, "obj.id", SearchField.DATA_TYPE.INTEGER, SearchField.SEARCH_TYPE.FULL)); searchFields.add(new SearchField(SearchFilter.TAG_DEF_ID, "obj.type", SearchField.DATA_TYPE.INTEGER, SearchField.SEARCH_TYPE.FULL)); @@ -95,13 +105,12 @@ public List getTagsByType(String name) { public List getTagsForResourceId(Long resourceId) { List ret = new ArrayList(); - List xxTags = daoMgr.getXXTag().findForResourceId(resourceId); - - if(CollectionUtils.isNotEmpty(xxTags)) { - for(XXTag xxTag : xxTags) { - RangerTag tag = populateViewBean(xxTag); + XXServiceResource serviceResourceEntity = daoMgr.getXXServiceResource().getById(resourceId); - ret.add(tag); + if (serviceResourceEntity != null) { + String tagsText = serviceResourceEntity.getTags(); + if (StringUtils.isNotEmpty(tagsText)) { + ret = RangerTagDBRetriever.gsonBuilder.fromJson(tagsText, RangerServiceResourceService.duplicatedDataType); } } @@ -111,13 +120,12 @@ public List getTagsForResourceId(Long resourceId) { public List getTagsForResourceGuid(String resourceGuid) { List ret = new ArrayList(); - List xxTags = daoMgr.getXXTag().findForResourceGuid(resourceGuid); - - if(CollectionUtils.isNotEmpty(xxTags)) { - for(XXTag xxTag : xxTags) { - RangerTag tag = populateViewBean(xxTag); + XXServiceResource serviceResourceEntity = daoMgr.getXXServiceResource().findByGuid(resourceGuid); - ret.add(tag); + if (serviceResourceEntity != null) { + String tagsText = serviceResourceEntity.getTags(); + if (StringUtils.isNotEmpty(tagsText)) { + ret = RangerTagDBRetriever.gsonBuilder.fromJson(tagsText, RangerServiceResourceService.duplicatedDataType); } } @@ -139,4 +147,25 @@ public List getTagsByServiceId(Long serviceId) { return ret; } + + @Override + protected RangerTag mapEntityToViewBean(RangerTag vObj, XXTag xObj) { + super.mapEntityToViewBean(vObj, xObj); + + Map attributes = RangerTagDBRetriever.gsonBuilder.fromJson(xObj.getTagAttrs(), RangerTagService.subsumedDataType); + vObj.setAttributes(attributes); + return vObj; + } + + @Override + protected XXTag mapViewToEntityBean(RangerTag vObj, XXTag xObj, int OPERATION_CONTEXT) { + super.mapViewToEntityBean(vObj, xObj, OPERATION_CONTEXT); + xObj.setTagAttrs(JsonUtils.mapToJson(vObj.getAttributes())); + return xObj; + } + + @Override + public Map getAttributesForTag(XXTag xTag) { + return new HashMap<>(); + } } \ No newline at end of file diff --git a/security-admin/src/main/resources/META-INF/jpa_named_queries.xml b/security-admin/src/main/resources/META-INF/jpa_named_queries.xml index 4a7055deea..cacc43feb4 100644 --- a/security-admin/src/main/resources/META-INF/jpa_named_queries.xml +++ b/security-admin/src/main/resources/META-INF/jpa_named_queries.xml @@ -199,22 +199,11 @@ - - select obj.name from XXUser obj, XXPolicyItemUserPerm polItemUser - where polItemUser.policyItemId = :polItemId and polItemUser.userId = obj.id - - select obj from XXUser obj, XXPortalUser portalUser where portalUser.id = :portalUserId and obj.name = portalUser.loginId - - select obj.name from XXGroup obj, XXPolicyItemGroupPerm polItemGrp - where polItemGrp.policyItemId = :polItemId and polItemGrp.groupId = obj.id - - - select obj from XXPolicyItem obj @@ -343,18 +332,6 @@ select obj from XXPolicyConditionDef obj where obj.defId = :serviceDefId and obj.name = :name order by obj.order - - select obj from XXPolicyConditionDef obj, XXPolicyItemCondition xPolItemCond - where xPolItemCond.policyItemId = :polItemId and obj.id = xPolItemCond.type order by obj.order - - - - select obj from XXPolicyConditionDef obj, XXPolicyItemCondition xPolItemCond - where xPolItemCond.policyItemId = :polItemId and obj.name = :name - and obj.id = xPolItemCond.type order by obj.order - - - select obj from XXContextEnricherDef obj where obj.defId = :serviceDefId order by obj.order @@ -466,10 +443,6 @@ - - select obj from XXPolicyResource obj where - obj.policyId = :polId and obj.resDefId = :resDefId - select obj from XXPolicyResource obj @@ -488,11 +461,6 @@ select obj from XXPolicyResource obj where obj.resDefId = :resDefId - - - - select obj from XXPolicyResourceMap obj where obj.resourceId = :polResId order by obj.order - select obj from XXPolicyResourceMap obj, XXPolicyResource res @@ -511,13 +479,6 @@ - - select obj from XXPolicyItemAccess obj where obj.policyItemId = :polItemId order by obj.order - - - - select obj from XXPolicyItemAccess obj where obj.type = :type - select obj from XXPolicyItemAccess obj, XXPolicyItem item @@ -534,21 +495,214 @@ order by item.policyId, obj.policyItemId, obj.order - - - - select obj from XXPolicyItemCondition obj where obj.policyItemId = :polItemId order by obj.order + + + + select obj from XXPolicyRefAccessType obj where obj.policyId = :policyId + + + + select obj from XXPolicyRefAccessType obj where obj.accessDefId = :accessDefId + + + + + select obj from XXPolicyRefCondition obj where obj.policyId = :policyId + + + + select obj from XXPolicyRefCondition obj where obj.conditionName = :conditionName + + + + + select obj from XXPolicyRefGroup obj where obj.policyId = :policyId + + + + select obj from XXPolicyRefGroup obj where obj.groupName = :groupName + + + + + select obj from XXPolicyRefGroup obj where obj.groupId = :groupId and obj.policyId = :policyId + + + + select obj from XXPolicyRefCondition obj where obj.conditionDefId = :conditionDefId and obj.policyId = :policyId + + + select obj from XXPolicyRefCondition obj where obj.conditionDefId = :conditionDefId + + + + select distinct obj from XXUser obj, XXPolicyRefUser ref where ref.policyId = :policyId and ref.userId = obj.id + and ref.userName != obj.name + + + + select distinct obj from XXAccessTypeDef obj, XXPolicyRefAccessType ref where ref.policyId = :policyId and ref.accessDefId = obj.id + and ref.accessTypeName != obj.name + + + + select distinct obj from XXDataMaskTypeDef obj, XXPolicyRefDataMaskType ref where ref.policyId = :policyId and ref.dataMaskDefId = obj.id + and ref.dataMaskTypeName != obj.name + + + + select distinct obj from XXResourceDef obj, XXPolicyRefResource ref where ref.policyId = :policyId and ref.resourceDefId = obj.id + and ref.resourceName != obj.name + + + + select groupRef.policyId, groupRef.groupName, grp.name + from XXPolicyRefGroup groupRef, XXGroup grp + where groupRef.policyId = :policy + and groupRef.groupId = grp.id + and groupRef.groupName != grp.name + + + + + select groupRef.policyId, groupRef.groupName, grp.name + from XXPolicy policy, XXPolicyRefGroup groupRef, XXGroup grp + where policy.service = :service + and groupRef.policyId = policy.id + and groupRef.groupId = grp.id + and groupRef.groupName != grp.name + + + + + select userRef.policyId, userRef.userName, user.name + from XXPolicyRefUser userRef, XXUser user + where userRef.policyId = :policy + and userRef.userId = user.id + and userRef.userName != user.name + + + + + select userRef.policyId, userRef.userName, user.name + from XXPolicy policy, XXPolicyRefUser userRef, XXUser user + where policy.service = :service + and userRef.policyId = policy.id + and userRef.userId = user.id + and userRef.userName != user.name + + + + + select accessRef.policyId, accessRef.accessTypeName, accessDef.name + from XXPolicyRefAccessType accessRef, XXAccessTypeDef accessDef + where accessRef.policyId = :policy + and accessRef.accessDefId = accessDef.id + and accessRef.accessTypeName != accessDef.name + + + + + select accessRef.policyId, accessRef.accessTypeName, accessDef.name + from XXPolicy policy, XXPolicyRefAccessType accessRef, XXAccessTypeDef accessDef + where policy.service = :service + and accessRef.policyId = policy.id + and accessRef.accessDefId = accessDef.id + and accessRef.accessTypeName != accessDef.name + + + + + select resourceRef.policyId, resourceRef.resourceName, resourceDef.name + from XXPolicyRefResource resourceRef, XXResourceDef resourceDef + where resourceRef.policyId = :policy + and resourceRef.resourceDefId = resourceDef.id + and resourceRef.resourceName != resourceDef.name + + + + + select resourceRef.policyId, resourceRef.resourceName, resourceDef.name + from XXPolicy policy, XXPolicyRefResource resourceRef, XXResourceDef resourceDef + where policy.service = :service + and resourceRef.policyId = policy.id + and resourceRef.resourceDefId = resourceDef.id + and resourceRef.resourceName != resourceDef.name + + + + + select dataMaskRef.policyId, dataMaskRef.dataMaskTypeName, dMaskDef.name + from XXPolicyRefDataMaskType dataMaskRef, XXDataMaskTypeDef dMaskDef + where dataMaskRef.policyId = :policy + and dataMaskRef.dataMaskDefId = dMaskDef.id + and dataMaskRef.dataMaskTypeName != dMaskDef.name + + + + + select dataMaskRef.policyId, dataMaskRef.dataMaskTypeName, dMaskDef.name + from XXPolicy policy, XXPolicyRefDataMaskType dataMaskRef, XXDataMaskTypeDef dMaskDef + where policy.service = :service + and dataMaskRef.policyId = policy.id + and dataMaskRef.dataMaskDefId = dMaskDef.id + and dataMaskRef.dataMaskTypeName != dMaskDef.name + + + + + select conditionRef.policyId, conditionRef.conditionName, condDef.name + from XXPolicyRefCondition conditionRef, XXPolicyConditionDef condDef + where conditionRef.policyId = :policy + and conditionRef.conditionDefId = condDef.id + and conditionRef.conditionName != condDef.name + + + + + select conditionRef.policyId, conditionRef.conditionName, condDef.name + from XXPolicy policy, XXPolicyRefCondition conditionRef, XXPolicyConditionDef condDef + where policy.service = :service + and conditionRef.policyId = policy.id + and conditionRef.conditionDefId = condDef.id + and conditionRef.conditionName != condDef.name + + + + + + + select obj from XXPolicyRefDataMaskType obj where obj.policyId = :policyId + + + + select obj from XXPolicyRefDataMaskType obj where obj.dataMaskTypeName = :dataMaskTypeName + + + + + select obj from XXPolicyRefResource obj where obj.policyId = :policyId - - select obj from XXPolicyItemCondition obj where - obj.policyItemId = :polItemId and obj.type = :polCondDefId order by obj.order + + select obj from XXPolicyRefResource obj where obj.resourceDefId = :resourceDefId - - select obj from XXPolicyItemCondition obj where obj.type = :polCondDefId + + + select obj from XXPolicyRefUser obj where obj.policyId = :policyId + + select obj from XXPolicyRefUser obj where obj.userName = :userName + + + + select obj from XXPolicyRefUser obj where obj.userId = :userId + + + + select obj from XXPolicyItemCondition obj, XXPolicyItem item where obj.policyItemId = item.id @@ -566,9 +720,6 @@ - - select obj from XXPolicyItemGroupPerm obj where obj.policyItemId = :polItemId order by obj.order - select obj from XXPolicyItemGroupPerm obj, XXPolicyItem item @@ -587,9 +738,6 @@ - - select obj from XXPolicyItemUserPerm obj where obj.policyItemId = :polItemId order by obj.order - select obj from XXPolicyItemUserPerm obj, XXPolicyItem item @@ -608,9 +756,6 @@ - - select obj from XXPolicyItemDataMaskInfo obj where obj.policyItemId = :polItemId - select obj from XXPolicyItemDataMaskInfo obj, XXPolicyItem item @@ -628,14 +773,7 @@ - - select obj from XXPolicyItemDataMaskInfo obj where obj.type = :type - - - - select obj from XXPolicyItemRowFilterInfo obj where obj.policyItemId = :polItemId - select obj from XXPolicyItemRowFilterInfo obj, XXPolicyItem item @@ -810,20 +948,6 @@ - - - select obj from XXServiceResourceElement obj where obj.resourceId in - (select serviceRes.id from XXServiceResource serviceRes, XXService service where service.id = :serviceId and service.tagService is not null and serviceRes.serviceId = service.id and serviceRes.id in - (select tagResMap.resourceId from XXTagResourceMap tagResMap, XXTag tag, XXTagDef tagDef - where tagResMap.tagId = tag.id and tag.type = tagDef.id and tagDef.name in - (select policyResMap.value from XXPolicyResourceMap policyResMap, XXPolicyResource policyRes, XXPolicy policy - where policy.service = service.tagService and policy.isEnabled = TRUE and policyRes.policyId = policy.id and policyResMap.resourceId = policyRes.id) - ) - ) - order by obj.resourceId, obj.id - - - select obj from XXServiceResourceElement obj where obj.resourceId in (select serviceresource.id from XXServiceResource serviceresource where serviceresource.serviceId = :serviceId) @@ -848,20 +972,6 @@ - - - select obj from XXServiceResourceElementValue obj, XXServiceResourceElement serviceResElem where obj.resElementId = serviceResElem.id and - serviceResElem.resourceId in - (select serviceRes.id from XXServiceResource serviceRes, XXService service where service.id = :serviceId and service.tagService is not null and serviceRes.serviceId = service.id and serviceRes.id in - (select tagResMap.resourceId from XXTagResourceMap tagResMap, XXTag tag, XXTagDef tagDef - where tagResMap.tagId = tag.id and tag.type = tagDef.id and tagDef.name in - (select policyResMap.value from XXPolicyResourceMap policyResMap, XXPolicyResource policyRes, XXPolicy policy - where policy.service = service.tagService and policy.isEnabled = TRUE and policyRes.policyId = policy.id and policyResMap.resourceId = policyRes.id) - ) - ) - order by serviceResElem.resourceId, serviceResElem.id - - select obj from XXServiceResourceElementValue obj, XXServiceResourceElement resElem where obj.resElementId = resElem.id and resElem.resourceId in (select res.id from XXServiceResource res where res.serviceId = :serviceId) @@ -909,19 +1019,6 @@ order by obj.resourceId - - - select obj from XXTagResourceMap obj, XXService service, XXServiceResource serviceRes where service.id = :serviceId and service.tagService is not null - and obj.resourceId = serviceRes.id and serviceRes.serviceId = :serviceId - and obj.tagId in - (select tag.id from XXTag tag, XXTagDef tagDef where tag.type = tagDef.id and tagDef.name in - (select policyResMap.value from XXPolicyResourceMap policyResMap, XXPolicyResource policyRes, XXPolicy policy - where policy.service = service.tagService and policy.isEnabled = TRUE and policyRes.policyId = policy.id and policyResMap.resourceId = policyRes.id) - ) - order by obj.resourceId - - - select obj.tagId from XXTagResourceMap obj where obj.resourceId = :resourceId order by obj.id @@ -935,22 +1032,14 @@ select obj from XXTagDef obj where obj.name = :name - - - select obj from XXTagDef obj where obj.id in - (select tag.type from XXTag tag, XXTagResourceMap tagRes, XXServiceResource resource where tag.id = tagRes.tagId and tagRes.resourceId = resource.id and resource.serviceId = :serviceId) - order by obj.id - - - - select obj from XXTagDef obj, XXService service where service.id = :serviceId and service.tagService is not null and - obj.name in - (select policyResMap.value from XXPolicyResourceMap policyResMap, XXPolicyResource policyRes, XXPolicy policy - where policy.service = service.tagService and policyRes.policyId = policy.id and policy.isEnabled = TRUE and policyResMap.resourceId = policyRes.id) + + select obj.id, obj.guid, obj.version, obj.isEnabled, obj.name, obj.source, obj.tagAttrDefs from XXTagDef obj where obj.id in + (select tag.type from XXTag tag, XXTagResourceMap tagRes, XXServiceResource resource where tag.id = tagRes.tagId and tagRes.resourceId = resource.id and resource.serviceId = :serviceId) order by obj.id + select obj from XXTagDef obj where obj.id in (select tag.type from XXTag tag, XXTagResourceMap tagRes where tag.id = tagRes.tagId and tagRes.resourceId = :resourceId) @@ -972,16 +1061,6 @@ order by obj.tagDefId - - select obj from XXTagAttributeDef obj where obj.tagDefId in - (select tagDef.id from XXTagDef tagDef, XXService service where service.id = :serviceId and service.tagService is not null and - tagDef.name in - (select policyResMap.value from XXPolicyResourceMap policyResMap, XXPolicyResource policyRes, XXPolicy policy - where policy.service = service.tagService and policy.isEnabled = TRUE and policyRes.policyId = policy.id and policyResMap.resourceId = policyRes.id) - ) - order by obj.tagDefId - - select obj from XXTagAttributeDef obj where obj.tagDefId in (select tag.type from XXTag tag, XXTagResourceMap tagRes where tag.id = tagRes.tagId and tagRes.resourceId = :resourceId) @@ -1017,16 +1096,6 @@ - - - select obj from XXTag obj, XXTagDef tagDef, XXService service where service.id = :serviceId and service.tagService is not null and - obj.type = tagDef.id and tagDef.name in - (select policyResMap.value from XXPolicyResourceMap policyResMap, XXPolicyResource policyRes, XXPolicy policy - where policy.service = service.tagService and policy.isEnabled = TRUE and policyRes.policyId = policy.id and policyResMap.resourceId = policyRes.id) - order by obj.id - - - select obj from XXTag obj where obj.owner = :owner and obj.id in (select tagRes.tagId from XXTagResourceMap tagRes, XXServiceResource resource where tagRes.resourceId = resource.id and resource.serviceId = :serviceId) @@ -1061,18 +1130,6 @@ - - - select obj from XXTagAttribute obj where obj.tagId in - (select tag.id from XXTag tag, XXTagDef tagDef, XXService service where service.id = :serviceId and service.tagService is not null and - tag.type = tagDef.id and tagDef.name in - (select policyResMap.value from XXPolicyResourceMap policyResMap, XXPolicyResource policyRes, XXPolicy policy - where policy.service = service.tagService and policy.isEnabled = TRUE and policyRes.policyId = policy.id and policyResMap.resourceId = policyRes.id) - ) - order by obj.tagId - - - select obj from XXServiceResource obj where obj.guid = :guid @@ -1084,9 +1141,9 @@ - select obj from XXServiceResource obj where obj.serviceId = :serviceId and obj.id in - (select tagResMap.resourceId from XXTagResourceMap tagResMap) - order by obj.id + select obj.id, obj.guid, obj.version, obj.isEnabled, obj.resourceSignature, obj.serviceId, obj.serviceResourceElements, obj.tags from XXServiceResource obj where obj.serviceId = :serviceId and obj.id in + (select tagResMap.resourceId from XXTagResourceMap tagResMap) + order by obj.id @@ -1096,18 +1153,6 @@ - - - select obj from XXServiceResource obj, XXService service where service.id = :serviceId and service.tagService is not null and obj.serviceId = service.id and obj.id in - (select tagResMap.resourceId from XXTagResourceMap tagResMap, XXTag tag, XXTagDef tagDef - where tagResMap.tagId = tag.id and tag.type = tagDef.id and tagDef.name in - (select policyResMap.value from XXPolicyResourceMap policyResMap, XXPolicyResource policyRes, XXPolicy policy - where policy.service = service.tagService and policy.isEnabled = TRUE and policyRes.policyId = policy.id and policyResMap.resourceId = policyRes.id) - ) - order by obj.id - - - select obj from XXServiceResource obj where obj.serviceId = :serviceId and obj.resourceSignature = :resourceSignature @@ -1137,12 +1182,12 @@ - select obj from XXPolicy obj, XXPolicyItem polItem,XXPolicyItemUserPerm polItemUserPerm where - obj.id = polItem.policyId and polItem.id = polItemUserPerm.policyItemId and polItemUserPerm.userId = :userId + select obj from XXPolicy obj, XXPolicyRefUser ref where + obj.id = ref.policyId and ref.userId = :userId - select obj from XXPolicy obj, XXPolicyItem polItem,XXPolicyItemGroupPerm polItemGroupPerm where - obj.id = polItem.policyId and polItem.id = polItemGroupPerm.policyItemId and polItemGroupPerm.groupId = :groupId + select obj from XXPolicy obj, XXPolicyRefGroup ref where + obj.id = ref.policyId and ref.groupId = :groupId select max(obj.id) from XXAccessAudit obj diff --git a/security-admin/src/test/java/org/apache/ranger/biz/TestServiceDBStore.java b/security-admin/src/test/java/org/apache/ranger/biz/TestServiceDBStore.java index 44523fb22d..ecb7571630 100644 --- a/security-admin/src/test/java/org/apache/ranger/biz/TestServiceDBStore.java +++ b/security-admin/src/test/java/org/apache/ranger/biz/TestServiceDBStore.java @@ -28,6 +28,7 @@ import org.apache.ranger.common.RangerFactory; import org.apache.ranger.common.StringUtil; import org.apache.ranger.common.UserSessionBase; +import org.apache.ranger.common.db.RangerTransactionSynchronizationAdapter; import org.apache.ranger.db.*; import org.apache.ranger.entity.*; import org.apache.ranger.plugin.model.RangerPolicy; @@ -45,7 +46,7 @@ import org.apache.ranger.plugin.model.RangerServiceDef.RangerPolicyConditionDef; import org.apache.ranger.plugin.model.RangerServiceDef.RangerResourceDef; import org.apache.ranger.plugin.model.RangerServiceDef.RangerServiceConfigDef; -//import org.apache.ranger.plugin.store.EmbeddedServiceDefsUtil; +import org.apache.ranger.plugin.store.EmbeddedServiceDefsUtil; import org.apache.ranger.plugin.store.PList; import org.apache.ranger.plugin.store.ServicePredicateUtil; import org.apache.ranger.plugin.util.SearchFilter; @@ -126,6 +127,15 @@ public class TestServiceDBStore { @Mock ServicePredicateUtil predicateUtil; + @Mock + PolicyRefUpdater policyRefUpdater; + + @Mock + AssetMgr assetMgr; + + @Mock + RangerTransactionSynchronizationAdapter transactionSynchronizationAdapter; + @Rule public ExpectedException thrown = ExpectedException.none(); @@ -137,6 +147,22 @@ public void setup() { .getCurrentUserSession(); currentUserSession.setUserAdmin(true); } + + private XXAccessTypeDef rangerKmsAccessTypes(String accessTypeName, int itemId) { + XXAccessTypeDef accessTypeDefObj = new XXAccessTypeDef(); + accessTypeDefObj.setAddedByUserId(Id); + accessTypeDefObj.setCreateTime(new Date()); + accessTypeDefObj.setDefid(Long.valueOf(itemId)); + accessTypeDefObj.setId(Long.valueOf(itemId)); + accessTypeDefObj.setItemId(Long.valueOf(itemId)); + accessTypeDefObj.setLabel(accessTypeName); + accessTypeDefObj.setName(accessTypeName); + accessTypeDefObj.setOrder(null); + accessTypeDefObj.setRbkeylabel(null); + accessTypeDefObj.setUpdatedByUserId(Id); + accessTypeDefObj.setUpdateTime(new Date()); + return accessTypeDefObj; + } private RangerServiceDef rangerServiceDef() { List configs = new ArrayList(); @@ -206,6 +232,28 @@ private RangerService rangerService() { return rangerService; } + + private RangerService rangerKMSService() { + Map configs = new HashMap(); + configs.put("username", "servicemgr"); + configs.put("password", "servicemgr"); + configs.put("provider", "kmsurl"); + + RangerService rangerService = new RangerService(); + rangerService.setId(Id); + rangerService.setConfigs(configs); + rangerService.setCreateTime(new Date()); + rangerService.setDescription("service kms policy"); + rangerService.setGuid("1427365526516_835_1"); + rangerService.setIsEnabled(true); + rangerService.setName("KMS_1"); + rangerService.setPolicyUpdateTime(new Date()); + rangerService.setType("7"); + rangerService.setUpdatedBy("Admin"); + rangerService.setUpdateTime(new Date()); + + return rangerService; + } private RangerPolicy rangerPolicy() { List accesses = new ArrayList(); @@ -603,6 +651,7 @@ public void test12updateServiceDef() throws Exception { @Test public void test13deleteServiceDef() throws Exception { + setup(); XXServiceDao xServiceDao = Mockito.mock(XXServiceDao.class); XXDataMaskTypeDefDao xDataMaskDefDao = Mockito.mock(XXDataMaskTypeDefDao.class); XXAccessTypeDefDao xAccessTypeDefDao = Mockito @@ -611,6 +660,12 @@ public void test13deleteServiceDef() throws Exception { .mock(XXAccessTypeDefGrantsDao.class); XXPolicyItemAccessDao xPolicyItemAccessDao = Mockito .mock(XXPolicyItemAccessDao.class); + XXPolicyRefAccessTypeDao xPolicyRefAccessTypeDao = Mockito + .mock(XXPolicyRefAccessTypeDao.class); + XXPolicyRefConditionDao xPolicyRefConditionDao = Mockito + .mock(XXPolicyRefConditionDao.class); + XXPolicyRefResourceDao xPolicyRefResourceDao = Mockito + .mock(XXPolicyRefResourceDao.class); XXContextEnricherDefDao xContextEnricherDefDao = Mockito .mock(XXContextEnricherDefDao.class); XXEnumDefDao xEnumDefDao = Mockito.mock(XXEnumDefDao.class); @@ -866,6 +921,40 @@ public void test13deleteServiceDef() throws Exception { policyItemUserPermObj.setUserId(Id); policyItemUserPermList.add(policyItemUserPermObj); + List policyRefAccessTypeList = new ArrayList(); + XXPolicyRefAccessType policyRefAccessType = new XXPolicyRefAccessType(); + policyRefAccessType.setId(Id); + policyRefAccessType.setAccessTypeName("myAccessType"); + policyRefAccessType.setPolicyId(Id); + policyRefAccessType.setCreateTime(new Date()); + policyRefAccessType.setUpdateTime(new Date()); + policyRefAccessType.setAddedByUserId(Id); + policyRefAccessType.setUpdatedByUserId(Id); + policyRefAccessTypeList.add(policyRefAccessType); + + List policyRefConditionsList = new ArrayList(); + XXPolicyRefCondition policyRefCondition = new XXPolicyRefCondition(); + policyRefCondition.setId(Id); + policyRefCondition.setAddedByUserId(Id); + policyRefCondition.setConditionDefId(Id); + policyRefCondition.setConditionName("myConditionName"); + policyRefCondition.setPolicyId(Id); + policyRefCondition.setUpdatedByUserId(Id); + policyRefCondition.setCreateTime(new Date()); + policyRefCondition.setUpdateTime(new Date()); + policyRefConditionsList.add(policyRefCondition); + + List policyRefResourcesList = new ArrayList(); + XXPolicyRefResource policyRefResource = new XXPolicyRefResource(); + policyRefResource.setAddedByUserId(Id); + policyRefResource.setCreateTime(new Date()); + policyRefResource.setId(Id); + policyRefResource.setPolicyId(Id); + policyRefResource.setResourceDefId(Id); + policyRefResource.setUpdateTime(new Date()); + policyRefResource.setResourceName("myresourceName"); + policyRefResourcesList.add(policyRefResource); + XXUser xUser = new XXUser(); xUser.setAddedByUserId(Id); xUser.setCreateTime(new Date()); @@ -878,6 +967,18 @@ public void test13deleteServiceDef() throws Exception { xUser.setUpdatedByUserId(Id); xUser.setUpdateTime(new Date()); + Mockito.when(daoManager.getXXPolicyRefAccessType()).thenReturn(xPolicyRefAccessTypeDao); + Mockito.when(xPolicyRefAccessTypeDao.findByAccessTypeDefId(Id)).thenReturn(policyRefAccessTypeList); + Mockito.when(xPolicyRefAccessTypeDao.remove(policyRefAccessType)).thenReturn(true); + + Mockito.when(daoManager.getXXPolicyRefCondition()).thenReturn(xPolicyRefConditionDao); + Mockito.when(xPolicyRefConditionDao.findByConditionDefId(Id)).thenReturn(policyRefConditionsList); + Mockito.when(xPolicyRefConditionDao.remove(policyRefCondition)).thenReturn(true); + + Mockito.when(daoManager.getXXPolicyRefResource()).thenReturn(xPolicyRefResourceDao); + Mockito.when(xPolicyRefResourceDao.findByResourceDefID(Id)).thenReturn(policyRefResourcesList); + Mockito.when(xPolicyRefResourceDao.remove(policyRefResource)).thenReturn(true); + Mockito.when(serviceDefService.read(Id)).thenReturn(rangerServiceDef); Mockito.when(daoManager.getXXService()).thenReturn(xServiceDao); Mockito.when(xServiceDao.findByServiceDefId(serviceDefId)).thenReturn(null); @@ -896,8 +997,8 @@ public void test13deleteServiceDef() throws Exception { Mockito.when(daoManager.getXXPolicyItemAccess()).thenReturn( xPolicyItemAccessDao); - Mockito.when(xPolicyItemAccessDao.findByType(accessTypeDefObj.getId())) - .thenReturn(policyItemAccessList); + /*Mockito.when(xPolicyItemAccessDao.findByType(accessTypeDefObj.getId())) + .thenReturn(policyItemAccessList);*/ Mockito.when(daoManager.getXXContextEnricherDef()).thenReturn( xContextEnricherDefDao); @@ -920,10 +1021,10 @@ public void test13deleteServiceDef() throws Exception { Mockito.when(daoManager.getXXPolicyItemCondition()).thenReturn( xPolicyItemConditionDao); - Mockito.when( + /*Mockito.when( xPolicyItemConditionDao .findByPolicyConditionDefId(policyConditionDefObj - .getId())).thenReturn(policyItemConditionList); + .getId())).thenReturn(policyItemConditionList);*/ Mockito.when(daoManager.getXXResourceDef()).thenReturn(xResourceDefDao); Mockito.when(xResourceDefDao.findByServiceDefId(serviceDefId)) @@ -931,14 +1032,14 @@ public void test13deleteServiceDef() throws Exception { Mockito.when(daoManager.getXXPolicyResource()).thenReturn( xPolicyResourceDao); - Mockito.when(xPolicyResourceDao.findByResDefId(resourceDef.getId())) - .thenReturn(policyResourceList); + /*Mockito.when(xPolicyResourceDao.findByResDefId(resourceDef.getId())) + .thenReturn(policyResourceList);*/ Mockito.when(daoManager.getXXPolicyResourceMap()).thenReturn( xPolicyResourceMapDao); - Mockito.when( + /*Mockito.when( xPolicyResourceMapDao.findByPolicyResId(policyResource.getId())) - .thenReturn(policyResourceMapList); + .thenReturn(policyResourceMapList);*/ Mockito.when(daoManager.getXXServiceConfigDef()).thenReturn( xServiceConfigDefDao); @@ -965,15 +1066,15 @@ public void test13deleteServiceDef() throws Exception { Mockito.when(daoManager.getXXPolicyItemGroupPerm()).thenReturn( xPolicyItemGroupPermDao); - Mockito.when( + /*Mockito.when( xPolicyItemGroupPermDao.findByPolicyItemId(policyItem.getId())) - .thenReturn(policyItemGroupPermlist); + .thenReturn(policyItemGroupPermlist);*/ Mockito.when(daoManager.getXXPolicyItemUserPerm()).thenReturn( policyItemUserPermDao); - Mockito.when( + /*Mockito.when( policyItemUserPermDao.findByPolicyItemId(policyItem.getId())) - .thenReturn(policyItemUserPermList); + .thenReturn(policyItemUserPermList);*/ svcServiceWithAssignedId.setPopulateExistingBaseFields(true); @@ -1179,8 +1280,8 @@ public void test19createService() throws Exception { Mockito.when(xUserMgr.createServiceConfigUser(userName)).thenReturn(vXUser); XXServiceConfigMap xConfMap = new XXServiceConfigMap(); - Mockito.when(rangerAuditFields.populateAuditFields(xConfMap, xService)) - .thenReturn(xService); + //Mockito.when(rangerAuditFields.populateAuditFields(xConfMap, xService)) + // .thenReturn(xService); Mockito.when(svcService.getPopulatedViewObject(xService)).thenReturn( rangerService); @@ -1196,10 +1297,10 @@ public void test19createService() throws Exception { ServiceDBStore spy = Mockito.spy(serviceDBStore); - Mockito.doNothing().when(spy).createDefaultPolicies(rangerService); + Mockito.doNothing().when(spy).createDefaultPolicies(xService, vXUser); spy.createService(rangerService); - + Mockito.verify(daoManager, Mockito.atLeast(1)).getXXService(); Mockito.verify(daoManager).getXXServiceConfigMap(); } @@ -1208,8 +1309,8 @@ public void test19createService() throws Exception { public void test20updateService() throws Exception { XXServiceDao xServiceDao = Mockito.mock(XXServiceDao.class); XXService xService = Mockito.mock(XXService.class); - XXServiceVersionInfoDao xServiceVersionInfoDao = Mockito.mock(XXServiceVersionInfoDao.class); - XXServiceVersionInfo xServiceVersionInfo = Mockito.mock(XXServiceVersionInfo.class); + //XXServiceVersionInfoDao xServiceVersionInfoDao = Mockito.mock(XXServiceVersionInfoDao.class); + //XXServiceVersionInfo xServiceVersionInfo = Mockito.mock(XXServiceVersionInfo.class); XXServiceConfigMapDao xServiceConfigMapDao = Mockito .mock(XXServiceConfigMapDao.class); XXServiceConfigDefDao xServiceConfigDefDao = Mockito @@ -1297,9 +1398,9 @@ public void test20updateService() throws Exception { Mockito.when(svcService.getPopulatedViewObject(xService)).thenReturn( rangerService); - Mockito.when(daoManager.getXXServiceVersionInfo()).thenReturn(xServiceVersionInfoDao); - Mockito.when(xServiceVersionInfoDao.findByServiceId(Id)).thenReturn(xServiceVersionInfo); - Mockito.when(xServiceVersionInfoDao.update(xServiceVersionInfo)).thenReturn(xServiceVersionInfo); + //Mockito.when(daoManager.getXXServiceVersionInfo()).thenReturn(xServiceVersionInfoDao); + //Mockito.when(xServiceVersionInfoDao.findByServiceId(Id)).thenReturn(xServiceVersionInfo); + //Mockito.when(xServiceVersionInfoDao.update(xServiceVersionInfo)).thenReturn(xServiceVersionInfo); RangerService dbRangerService = serviceDBStore .updateService(rangerService, options); @@ -1319,11 +1420,12 @@ public void test20updateService() throws Exception { @Test public void test21deleteService() throws Exception { + setup(); XXPolicyDao xPolicyDao = Mockito.mock(XXPolicyDao.class); XXServiceDao xServiceDao = Mockito.mock(XXServiceDao.class); XXService xService = Mockito.mock(XXService.class); - XXServiceVersionInfoDao xServiceVersionInfoDao = Mockito.mock(XXServiceVersionInfoDao.class); - XXServiceVersionInfo xServiceVersionInfo = Mockito.mock(XXServiceVersionInfo.class); + //XXServiceVersionInfoDao xServiceVersionInfoDao = Mockito.mock(XXServiceVersionInfoDao.class); + //XXServiceVersionInfo xServiceVersionInfo = Mockito.mock(XXServiceVersionInfo.class); XXPolicyItemDao xPolicyItemDao = Mockito.mock(XXPolicyItemDao.class); XXPolicyItemDataMaskInfoDao xxPolicyItemDataMaskInfoDao = Mockito.mock(XXPolicyItemDataMaskInfoDao.class); XXPolicyItemRowFilterInfoDao xxPolicyItemRowFilterInfoDao = Mockito.mock(XXPolicyItemRowFilterInfoDao.class); @@ -1494,9 +1596,9 @@ public void test21deleteService() throws Exception { Mockito.when(svcService.getPopulatedViewObject(xService)).thenReturn( rangerService); - Mockito.when(daoManager.getXXServiceVersionInfo()).thenReturn(xServiceVersionInfoDao); - Mockito.when(xServiceVersionInfoDao.findByServiceId(Id)).thenReturn(xServiceVersionInfo); - Mockito.when(xServiceVersionInfoDao.update(xServiceVersionInfo)).thenReturn(xServiceVersionInfo); + //Mockito.when(daoManager.getXXServiceVersionInfo()).thenReturn(xServiceVersionInfoDao); + //Mockito.when(xServiceVersionInfoDao.findByServiceId(Id)).thenReturn(xServiceVersionInfo); + //Mockito.when(xServiceVersionInfoDao.update(xServiceVersionInfo)).thenReturn(xServiceVersionInfo); Mockito.when(daoManager.getXXPolicyItem()).thenReturn(xPolicyItemDao); Mockito.when(xPolicyItemDao.findByPolicyId(policyItem.getId())) @@ -1571,6 +1673,7 @@ public void test21deleteService() throws Exception { Mockito.when(xUserDao.findByUserName(name)).thenReturn(xUser); Mockito.when(!bizUtil.hasAccess(xService, null)).thenReturn(true); serviceDBStore.deleteService(Id); + Mockito.verify(svcService).delete(rangerService); } @Test @@ -1696,15 +1799,14 @@ public void test25getPaginatedServiceDefs() throws Exception { } @Test - public void tess26createPolicy() throws Exception { + public void test26createPolicy() throws Exception { setup(); XXServiceDefDao xServiceDefDao = Mockito.mock(XXServiceDefDao.class); XXPolicy xPolicy = Mockito.mock(XXPolicy.class); XXPolicyDao xPolicyDao = Mockito.mock(XXPolicyDao.class); XXServiceDao xServiceDao = Mockito.mock(XXServiceDao.class); - XXServiceVersionInfoDao xServiceVersionInfoDao = Mockito.mock(XXServiceVersionInfoDao.class); + //XXServiceVersionInfoDao xServiceVersionInfoDao = Mockito.mock(XXServiceVersionInfoDao.class); XXService xService = Mockito.mock(XXService.class); - XXServiceVersionInfo xServiceVersionInfo = Mockito.mock(XXServiceVersionInfo.class); XXPolicyItemDao xPolicyItemDao = Mockito.mock(XXPolicyItemDao.class); XXServiceConfigDefDao xServiceConfigDefDao = Mockito .mock(XXServiceConfigDefDao.class); @@ -1865,9 +1967,6 @@ public void tess26createPolicy() throws Exception { Mockito.when(daoManager.getXXService()).thenReturn(xServiceDao); Mockito.when(xServiceDao.findByName(name)).thenReturn(xService); - Mockito.when(daoManager.getXXServiceVersionInfo()).thenReturn(xServiceVersionInfoDao); - Mockito.when(xServiceVersionInfoDao.findByServiceId(Id)).thenReturn(xServiceVersionInfo); - Mockito.when(xServiceVersionInfoDao.update(xServiceVersionInfo)).thenReturn(xServiceVersionInfo); Mockito.when(svcService.getPopulatedViewObject(xService)).thenReturn( rangerService); @@ -1885,7 +1984,8 @@ public void tess26createPolicy() throws Exception { rangerPolicy); Mockito.when(daoManager.getXXPolicy()).thenReturn(xPolicyDao); - Mockito.when(xPolicyDao.getById(Id)).thenReturn(xPolicy); + Mockito.when(xPolicyDao.getById(Id)).thenReturn(xPolicy);Mockito.doNothing().when(policyRefUpdater).createNewPolMappingForRefTable(rangerPolicy, xPolicy, xServiceDef); + Mockito.when(policyService.getPopulatedViewObject(xPolicy)).thenReturn(rangerPolicy); Mockito.when( rangerAuditFields.populateAuditFields( @@ -1968,14 +2068,13 @@ public void tess26createPolicy() throws Exception { .thenReturn(policyConditionDefObj); Mockito.when(!bizUtil.hasAccess(xService, null)).thenReturn(true); + //RangerTransactionSynchronizationAdapter spy = Mockito.spy(transactionSynchronizationAdapter); + //Mockito.doNothing().when(spy).executeOnTransactionCommit(Mockito.any(Runnable.class)); + RangerPolicy dbRangerPolicy = serviceDBStore.createPolicy(rangerPolicy); - Assert.assertNull(dbRangerPolicy); - Assert.assertEquals(Id, rangerPolicy.getId()); - Mockito.verify(daoManager).getXXServiceDef(); - Mockito.verify(policyService).create(rangerPolicy); - Mockito.verify(rangerAuditFields).populateAuditFields( - Mockito.isA(XXPolicyItem.class), Mockito.isA(XXPolicy.class)); - Mockito.verify(daoManager).getXXPolicyItem(); + + Assert.assertNotNull(dbRangerPolicy); + Assert.assertEquals(Id, dbRangerPolicy.getId()); } @Test @@ -2017,10 +2116,8 @@ public void tess28updatePolicy() throws Exception { XXPolicy xPolicy = Mockito.mock(XXPolicy.class); XXServiceDao xServiceDao = Mockito.mock(XXServiceDao.class); XXService xService = Mockito.mock(XXService.class); - XXServiceVersionInfoDao xServiceVersionInfoDao = Mockito.mock(XXServiceVersionInfoDao.class); XXServiceDefDao xServiceDefDao = Mockito.mock(XXServiceDefDao.class); XXServiceDef xServiceDef = Mockito.mock(XXServiceDef.class); - XXServiceVersionInfo xServiceVersionInfo = Mockito.mock(XXServiceVersionInfo.class); XXPolicyResourceDao xPolicyResourceDao = Mockito .mock(XXPolicyResourceDao.class); XXPolicyResourceMapDao xPolicyResourceMapDao = Mockito @@ -2029,10 +2126,8 @@ public void tess28updatePolicy() throws Exception { XXPolicyItem xPolicyItem = Mockito.mock(XXPolicyItem.class); XXServiceConfigDefDao xServiceConfigDefDao = Mockito .mock(XXServiceConfigDefDao.class); - XXServiceConfigMapDao xServiceConfigMapDao = Mockito - .mock(XXServiceConfigMapDao.class); - XXUserDao xUserDao = Mockito.mock(XXUserDao.class); - XXUser xUser = Mockito.mock(XXUser.class); + XXServiceVersionInfoDao serviceVersionInfoDao = Mockito.mock(XXServiceVersionInfoDao.class); + XXServiceVersionInfo serviceVersionInfoDbObj = Mockito.mock(XXServiceVersionInfo.class); RangerService rangerService = rangerService(); @@ -2091,10 +2186,6 @@ public void tess28updatePolicy() throws Exception { Mockito.when(svcService.getPopulatedViewObject(xService)).thenReturn( rangerService); - Mockito.when(daoManager.getXXServiceVersionInfo()).thenReturn(xServiceVersionInfoDao); - Mockito.when(xServiceVersionInfoDao.findByServiceId(Id)).thenReturn(xServiceVersionInfo); - Mockito.when(xServiceVersionInfoDao.update(xServiceVersionInfo)).thenReturn(xServiceVersionInfo); - Mockito.when(daoManager.getXXServiceDef()).thenReturn(xServiceDefDao); Mockito.when(xServiceDefDao.findByName(rangerService.getType())) .thenReturn(xServiceDef); @@ -2112,9 +2203,9 @@ public void tess28updatePolicy() throws Exception { Mockito.when(daoManager.getXXPolicyResourceMap()).thenReturn( xPolicyResourceMapDao); - Mockito.when( + /*Mockito.when( xPolicyResourceMapDao.findByPolicyResId(policyResourceMap - .getId())).thenReturn(policyResourceMapList); + .getId())).thenReturn(policyResourceMapList);*/ Mockito.when(daoManager.getXXPolicyItem()).thenReturn(xPolicyItemDao); @@ -2137,19 +2228,8 @@ public void tess28updatePolicy() throws Exception { Mockito.when(daoManager.getXXService()).thenReturn(xServiceDao); Mockito.when(xServiceDao.getById(rangerService.getId())).thenReturn( xService); - - Mockito.when(daoManager.getXXServiceConfigMap()).thenReturn( - xServiceConfigMapDao); - Mockito.when( - xServiceConfigMapDao.findByServiceId(rangerService.getId())) - .thenReturn(xConfMapList); - - Mockito.when( - rangerAuditFields.populateAuditFields( - Mockito.isA(XXServiceConfigMap.class), - Mockito.isA(XXService.class))).thenReturn(xConfMap); - Mockito.when(daoManager.getXXUser()).thenReturn(xUserDao); - Mockito.when(xUserDao.findByUserName(name)).thenReturn(xUser); + Mockito.when(daoManager.getXXServiceVersionInfo()).thenReturn(serviceVersionInfoDao); + Mockito.when(serviceVersionInfoDao.findByServiceId(rangerService.getId())).thenReturn(serviceVersionInfoDbObj); RangerPolicyResourceSignature signature = Mockito .mock(RangerPolicyResourceSignature.class); @@ -2173,9 +2253,6 @@ public void tess28updatePolicy() throws Exception { rangerPolicy.getIsEnabled()); Assert.assertEquals(dbRangerPolicy.getVersion(), rangerPolicy.getVersion()); - - Mockito.verify(rangerAuditFields).populateAuditFields( - Mockito.isA(XXPolicyItem.class), Mockito.isA(XXPolicy.class)); } @Test @@ -2183,9 +2260,6 @@ public void tess29deletePolicy() throws Exception { setup(); XXServiceDao xServiceDao = Mockito.mock(XXServiceDao.class); XXService xService = Mockito.mock(XXService.class); - XXServiceVersionInfoDao xServiceVersionInfoDao = Mockito.mock(XXServiceVersionInfoDao.class); - XXServiceVersionInfo xServiceVersionInfo = Mockito.mock(XXServiceVersionInfo.class); - XXPolicyItemDao xPolicyItemDao = Mockito.mock(XXPolicyItemDao.class); XXPolicyItemDataMaskInfoDao xPolicyItemDataMaskInfoDao = Mockito.mock(XXPolicyItemDataMaskInfoDao.class); XXPolicyItemRowFilterInfoDao xPolicyItemRowFilterInfoDao = Mockito.mock(XXPolicyItemRowFilterInfoDao.class); @@ -2337,26 +2411,26 @@ public void tess29deletePolicy() throws Exception { Mockito.when(daoManager.getXXPolicyItemCondition()).thenReturn( xPolicyItemConditionDao); - Mockito.when( + /*Mockito.when( xPolicyItemConditionDao.findByPolicyItemId(policyItemCondition - .getId())).thenReturn(policyItemConditionList); + .getId())).thenReturn(policyItemConditionList);*/ Mockito.when(daoManager.getXXPolicyItemGroupPerm()).thenReturn( xPolicyItemGroupPermDao); - Mockito.when( + /*Mockito.when( xPolicyItemGroupPermDao.findByPolicyItemId(policyItem.getId())) - .thenReturn(policyItemGroupPermList); + .thenReturn(policyItemGroupPermList);*/ Mockito.when(daoManager.getXXPolicyItemUserPerm()).thenReturn( xPolicyItemUserPermDao); - Mockito.when(xPolicyItemUserPermDao.findByPolicyItemId(Id)).thenReturn( - policyItemUserPermList); + /*Mockito.when(xPolicyItemUserPermDao.findByPolicyItemId(Id)).thenReturn( + policyItemUserPermList);*/ Mockito.when(daoManager.getXXPolicyItemAccess()).thenReturn( xPolicyItemAccessDao); - Mockito.when( + /*Mockito.when( xPolicyItemAccessDao.findByPolicyItemId(policyItemAccess - .getId())).thenReturn(policyItemAccessList); + .getId())).thenReturn(policyItemAccessList);*/ Mockito.when(daoManager.getXXPolicyResource()).thenReturn( xPolicyResourceDao); @@ -2365,16 +2439,16 @@ public void tess29deletePolicy() throws Exception { Mockito.when(daoManager.getXXPolicyResourceMap()).thenReturn( xPolicyResourceMapDao); - Mockito.when( + /*Mockito.when( xPolicyResourceMapDao.findByPolicyResId(policyResourceMap - .getId())).thenReturn(policyResourceMapList); + .getId())).thenReturn(policyResourceMapList);*/ Mockito.when(daoManager.getXXService()).thenReturn(xServiceDao); Mockito.when(xServiceDao.getById(Id)).thenReturn(xService); - Mockito.when(daoManager.getXXServiceVersionInfo()).thenReturn(xServiceVersionInfoDao); - Mockito.when(xServiceVersionInfoDao.findByServiceId(Id)).thenReturn(xServiceVersionInfo); - Mockito.when(xServiceVersionInfoDao.update(xServiceVersionInfo)).thenReturn(xServiceVersionInfo); + //Mockito.when(daoManager.getXXServiceVersionInfo()).thenReturn(xServiceVersionInfoDao); + //Mockito.when(xServiceVersionInfoDao.findByServiceId(Id)).thenReturn(xServiceVersionInfo); + //Mockito.when(xServiceVersionInfoDao.update(xServiceVersionInfo)).thenReturn(xServiceVersionInfo); Mockito.when(daoManager.getXXServiceConfigDef()).thenReturn( xServiceConfigDefDao); @@ -2637,4 +2711,131 @@ public void test40getPoliciesByResourceSignature() throws Exception { Assert.assertNotNull(policyList); Mockito.verify(daoManager).getXXPolicy(); } + + @Test + public void test41createKMSService() throws Exception { + XXServiceDefDao xServiceDefDao = Mockito.mock(XXServiceDefDao.class); + XXPolicy xPolicy = Mockito.mock(XXPolicy.class); + XXPolicyDao xPolicyDao = Mockito.mock(XXPolicyDao.class); + XXAccessTypeDefDao xAccessTypeDefDao = Mockito + .mock(XXAccessTypeDefDao.class); + XXServiceDao xServiceDao = Mockito.mock(XXServiceDao.class); + XXServiceConfigMapDao xServiceConfigMapDao = Mockito + .mock(XXServiceConfigMapDao.class); + XXUserDao xUserDao = Mockito.mock(XXUserDao.class); + XXServiceConfigDefDao xServiceConfigDefDao = Mockito + .mock(XXServiceConfigDefDao.class); + XXService xService = Mockito.mock(XXService.class); + XXUser xUser = Mockito.mock(XXUser.class); + XXServiceDef xServiceDef = Mockito.mock(XXServiceDef.class); + Mockito.when(daoManager.getXXServiceDef()).thenReturn(xServiceDefDao); + Mockito.when(xServiceDefDao.findByName("KMS_1")).thenReturn( + xServiceDef); + Mockito.when(xService.getName()).thenReturn( + "KMS_1"); + Mockito.when(xServiceDao.findByName("KMS_1")).thenReturn( + xService); + Mockito.when(!bizUtil.hasAccess(xService, null)).thenReturn(true); + + RangerService rangerService = rangerKMSService(); + VXUser vXUser = null; + String userName = "servicemgr"; + Mockito.when(xService.getType()).thenReturn(Long.valueOf(rangerService.getType())); + Mockito.when(xServiceDefDao.getById(Long.valueOf(rangerService.getType()))).thenReturn(xServiceDef); + Mockito.when(xServiceDef.getImplclassname()).thenReturn(EmbeddedServiceDefsUtil.KMS_IMPL_CLASS_NAME); + List svcConfDefList = new ArrayList(); + XXServiceConfigDef serviceConfigDefObj = new XXServiceConfigDef(); + serviceConfigDefObj.setId(Id); + serviceConfigDefObj.setType("7"); + svcConfDefList.add(serviceConfigDefObj); + Mockito.when(daoManager.getXXServiceConfigDef()).thenReturn( + xServiceConfigDefDao); + Mockito.when(xServiceConfigDefDao.findByServiceDefName(userName)) + .thenReturn(svcConfDefList); + + Mockito.when(svcService.create(rangerService)).thenReturn(rangerService); + + Mockito.when(daoManager.getXXService()).thenReturn(xServiceDao); + Mockito.when(xServiceDao.getById(rangerService.getId())).thenReturn( + xService); + Mockito.when(daoManager.getXXServiceConfigMap()).thenReturn( + xServiceConfigMapDao); + + Mockito.when(stringUtil.getValidUserName(userName)) + .thenReturn(userName); + Mockito.when(daoManager.getXXUser()).thenReturn(xUserDao); + Mockito.when(xUserDao.findByUserName(userName)).thenReturn(xUser); + + Mockito.when(xUserService.populateViewBean(xUser)).thenReturn(vXUser); + Mockito.when(xUserMgr.createServiceConfigUser(userName)).thenReturn(vXUser); + VXUser vXUserHdfs = new VXUser(); + vXUserHdfs.setName("hdfs"); + vXUserHdfs.setPassword("hdfs"); + Mockito.when(xUserMgr.createServiceConfigUser("hdfs")).thenReturn(vXUserHdfs); + VXUser vXUserHive = new VXUser(); + vXUserHive.setName("hive"); + vXUserHive.setPassword("hive"); + Mockito.when(xUserMgr.createServiceConfigUser("hive")).thenReturn(vXUserHive); + + XXServiceConfigMap xConfMap = new XXServiceConfigMap(); + //Mockito.when(rangerAuditFields.populateAuditFields(xConfMap, xService)) + //.thenReturn(xService); + + Mockito.when(svcService.getPopulatedViewObject(xService)).thenReturn( + rangerService); + + Mockito.when( + rangerAuditFields.populateAuditFields( + Mockito.isA(XXServiceConfigMap.class), + Mockito.isA(XXService.class))).thenReturn(xConfMap); + + Mockito.when(daoManager.getXXPolicy()).thenReturn(xPolicyDao); + + Mockito.when(xPolicyDao.getById(Id)).thenReturn(xPolicy); + + + List accessTypeDefList = new ArrayList(); + accessTypeDefList.add(rangerKmsAccessTypes("getmetadata", 7)); + accessTypeDefList.add(rangerKmsAccessTypes("generateeek", 8)); + accessTypeDefList.add(rangerKmsAccessTypes("decrypteek", 9)); + + RangerServiceDef ran = new RangerServiceDef(); + ran.setName("KMS Test"); + Mockito.when(serviceDefService.read(1L)).thenReturn(ran); + Long serviceDefId = ran.getId(); + + ServiceDBStore spy = Mockito.spy(serviceDBStore); + + Mockito.when(daoManager.getXXAccessTypeDef()).thenReturn( + xAccessTypeDefDao); + Mockito.when(xAccessTypeDefDao.findByServiceDefId(serviceDefId)) + .thenReturn(accessTypeDefList); + Mockito.when(spy.getServiceByName("KMS_1")).thenReturn( + rangerService); + Mockito.doNothing().when(spy).createDefaultPolicies(xService, vXUser); + + RangerPolicy policy = new RangerPolicy(); + RangerResourceDef resourceDef = new RangerResourceDef(); + resourceDef.setItemId(Id); + resourceDef.setName("keyname"); + resourceDef.setType("string"); + resourceDef.setType("string"); + resourceDef.setLabel("Key Name"); + resourceDef.setDescription("Key Name"); + + List resourceHierarchy = new ArrayList(); + resourceHierarchy.addAll(resourceHierarchy); + + spy.createService(rangerService); + vXUser = new VXUser(); + vXUser.setName(userName); + vXUser.setPassword(userName); + + spy.createDefaultPolicy(policy, xService, vXUser, resourceHierarchy); + + Mockito.verify(daoManager, Mockito.atLeast(1)).getXXService(); + Mockito.verify(daoManager).getXXServiceConfigMap(); + //Assert.assertNull(policy); + Assert.assertEquals(3, policy.getPolicyItems().size()); + } } diff --git a/security-admin/src/test/java/org/apache/ranger/rest/TestServiceREST.java b/security-admin/src/test/java/org/apache/ranger/rest/TestServiceREST.java index 68908b2057..7d407f798c 100644 --- a/security-admin/src/test/java/org/apache/ranger/rest/TestServiceREST.java +++ b/security-admin/src/test/java/org/apache/ranger/rest/TestServiceREST.java @@ -247,7 +247,7 @@ private RangerService rangerService() { return rangerService; } - private RangerPolicy rangerPolicy() { + RangerPolicy rangerPolicy() { List accesses = new ArrayList(); List users = new ArrayList(); List groups = new ArrayList(); diff --git a/security-admin/src/test/java/org/apache/ranger/service/TestRangerServiceDefService.java b/security-admin/src/test/java/org/apache/ranger/service/TestRangerServiceDefService.java index d65b0d6eea..f5fdef5db2 100644 --- a/security-admin/src/test/java/org/apache/ranger/service/TestRangerServiceDefService.java +++ b/security-admin/src/test/java/org/apache/ranger/service/TestRangerServiceDefService.java @@ -322,9 +322,6 @@ public void test3PopulateViewBean() { Mockito.when(daoManager.getXXPolicyConditionDef()).thenReturn( xPolicyConditionDefDao); - Mockito.when( - xPolicyConditionDefDao.findByPolicyItemId(xPolicyItem.getId())) - .thenReturn(xConditionDefList); Mockito.when(daoManager.getXXContextEnricherDef()).thenReturn( xContextEnricherDefDao); @@ -547,9 +544,6 @@ public void test4getAllServiceDefs() { Mockito.when(daoManager.getXXPolicyConditionDef()).thenReturn( xPolicyConditionDefDao); - Mockito.when( - xPolicyConditionDefDao.findByPolicyItemId(xPolicyItem.getId())) - .thenReturn(xConditionDefList); Mockito.when(daoManager.getXXContextEnricherDef()).thenReturn( xContextEnricherDefDao); @@ -745,9 +739,6 @@ public void test5getPopulatedViewObject() { Mockito.when(daoManager.getXXPolicyConditionDef()).thenReturn( xPolicyConditionDefDao); - Mockito.when( - xPolicyConditionDefDao.findByPolicyItemId(xPolicyItem.getId())) - .thenReturn(xConditionDefList); Mockito.when(daoManager.getXXContextEnricherDef()).thenReturn( xContextEnricherDefDao); diff --git a/security-admin/src/test/java/org/apache/ranger/service/TestRangerTagDefService.java b/security-admin/src/test/java/org/apache/ranger/service/TestRangerTagDefService.java index 2ca0fd2fba..6b4b9a2319 100644 --- a/security-admin/src/test/java/org/apache/ranger/service/TestRangerTagDefService.java +++ b/security-admin/src/test/java/org/apache/ranger/service/TestRangerTagDefService.java @@ -22,7 +22,6 @@ import org.apache.ranger.db.RangerDaoManager; import org.apache.ranger.db.XXServiceVersionInfoDao; -import org.apache.ranger.db.XXTagAttributeDefDao; import org.apache.ranger.db.XXTagDefDao; import org.apache.ranger.entity.XXTagAttributeDef; import org.apache.ranger.entity.XXTagDef; @@ -36,6 +35,7 @@ import org.mockito.Mock; import org.mockito.Mockito; import org.mockito.runners.MockitoJUnitRunner; +//import org.mockito.junit.MockitoJUnitRunner; @RunWith(MockitoJUnitRunner.class) @FixMethodOrder(MethodSorters.NAME_ASCENDING) @@ -81,21 +81,15 @@ public void test3postUpdate(){ xxTagAttributeDef.setName(name); tagAttrDefList.add(xxTagAttributeDef); - XXTagAttributeDefDao xxTagAttributeDefDao = Mockito.mock(XXTagAttributeDefDao.class); XXServiceVersionInfoDao xxServiceVersionInfoDao = Mockito.mock(XXServiceVersionInfoDao.class); - - Mockito.when(daoMgr.getXXTagAttributeDef()).thenReturn(xxTagAttributeDefDao); - Mockito.when(xxTagAttributeDefDao.findByTagDefId(tagDef.getId())).thenReturn(tagAttrDefList); - + Mockito.when(daoMgr.getXXServiceVersionInfo()).thenReturn(xxServiceVersionInfoDao); Mockito.doNothing().when(xxServiceVersionInfoDao).updateServiceVersionInfoForTagDefUpdate(tagDef.getId(), tagDef.getUpdateTime()); RangerTagDef result = rangerTagDefService.postUpdate(tagDef); Assert.assertEquals(result.getId(), tagAttrDefList.get(0).getId()); Assert.assertEquals(result.getName(), tagAttrDefList.get(0).getName()); - - Mockito.verify(daoMgr).getXXTagAttributeDef(); - Mockito.verify(xxTagAttributeDefDao).findByTagDefId(tagDef.getId()); + Mockito.verify(daoMgr).getXXServiceVersionInfo(); Mockito.verify(xxServiceVersionInfoDao).updateServiceVersionInfoForTagDefUpdate(tagDef.getId(), tagDef.getUpdateTime()); } @@ -117,19 +111,12 @@ public void test4getTagDefByGuid(){ xxTagAttributeDef.setName(name); tagAttrDefList.add(xxTagAttributeDef); - XXTagAttributeDefDao xxTagAttributeDefDao = Mockito.mock(XXTagAttributeDefDao.class); - - Mockito.when(daoMgr.getXXTagAttributeDef()).thenReturn(xxTagAttributeDefDao); - Mockito.when(xxTagAttributeDefDao.findByTagDefId(xxTagDef.getId())).thenReturn(tagAttrDefList); - RangerTagDef result = rangerTagDefService.getTagDefByGuid(guid); Assert.assertEquals(result.getId(), tagAttrDefList.get(0).getId()); Assert.assertEquals(result.getName(), tagAttrDefList.get(0).getName()); Mockito.verify(daoMgr).getXXTagDef(); Mockito.verify(xXTagDefDao).findByGuid(guid); - Mockito.verify(daoMgr).getXXTagAttributeDef(); - Mockito.verify(xxTagAttributeDefDao).findByTagDefId(xxTagDef.getId()); } @Test @@ -167,19 +154,12 @@ public void test6getTagDefByName(){ xxTagAttributeDef.setName(name); tagAttrDefList.add(xxTagAttributeDef); - XXTagAttributeDefDao xxTagAttributeDefDao = Mockito.mock(XXTagAttributeDefDao.class); - - Mockito.when(daoMgr.getXXTagAttributeDef()).thenReturn(xxTagAttributeDefDao); - Mockito.when(xxTagAttributeDefDao.findByTagDefId(xxTagDef.getId())).thenReturn(tagAttrDefList); - RangerTagDef result = rangerTagDefService.getTagDefByName(name); Assert.assertEquals(result.getId(), tagAttrDefList.get(0).getId()); Assert.assertEquals(result.getName(), tagAttrDefList.get(0).getName()); Mockito.verify(daoMgr).getXXTagDef(); Mockito.verify(xXTagDefDao).findByName(name); - Mockito.verify(daoMgr).getXXTagAttributeDef(); - Mockito.verify(xxTagAttributeDefDao).findByTagDefId(xxTagDef.getId()); } @Test @@ -215,19 +195,12 @@ public void test8getTagDefsByServiceId(){ xxTagAttributeDef.setName(name); tagAttrDefList.add(xxTagAttributeDef); - XXTagAttributeDefDao xxTagAttributeDefDao = Mockito.mock(XXTagAttributeDefDao.class); - - Mockito.when(daoMgr.getXXTagAttributeDef()).thenReturn(xxTagAttributeDefDao); - Mockito.when(xxTagAttributeDefDao.findByTagDefId(xxTagDef.getId())).thenReturn(tagAttrDefList); - List result = rangerTagDefService.getTagDefsByServiceId(serviceId); Assert.assertEquals(result.get(0).getId(), tagAttrDefList.get(0).getId()); Assert.assertEquals(result.get(0).getName(), tagAttrDefList.get(0).getName()); Mockito.verify(daoMgr).getXXTagDef(); Mockito.verify(xXTagDefDao).findByServiceId(serviceId); - Mockito.verify(daoMgr).getXXTagAttributeDef(); - Mockito.verify(xxTagAttributeDefDao).findByTagDefId(xxTagDef.getId()); } @@ -259,16 +232,8 @@ public void test10getPopulatedViewObject(){ xxTagAttributeDef.setName(name); tagAttrDefList.add(xxTagAttributeDef); - XXTagAttributeDefDao xxTagAttributeDefDao = Mockito.mock(XXTagAttributeDefDao.class); - - Mockito.when(daoMgr.getXXTagAttributeDef()).thenReturn(xxTagAttributeDefDao); - Mockito.when(xxTagAttributeDefDao.findByTagDefId(xxTagDef.getId())).thenReturn(tagAttrDefList); - RangerTagDef result = rangerTagDefService.getPopulatedViewObject(xxTagDef); Assert.assertEquals(result.getId(), tagAttrDefList.get(0).getId()); Assert.assertEquals(result.getName(), tagAttrDefList.get(0).getName()); - - Mockito.verify(daoMgr).getXXTagAttributeDef(); - Mockito.verify(xxTagAttributeDefDao).findByTagDefId(xxTagDef.getId()); } } From f16e5811f4a48c6bd2eab82d14759dcc17394575 Mon Sep 17 00:00:00 2001 From: Abhay Kulkarni Date: Thu, 18 Oct 2018 19:45:22 -0700 Subject: [PATCH 141/151] RANGER-2203, RANGER-2219: Review and update database schema for ranger policies and tag objects to minimize database queries/updates; back-port of RANGER-2186, RANGER-2195 - Part 2 --- .../db/sqlserver/optimized/current/ranger_core_db_sqlserver.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/security-admin/db/sqlserver/optimized/current/ranger_core_db_sqlserver.sql b/security-admin/db/sqlserver/optimized/current/ranger_core_db_sqlserver.sql index d33dae05d9..da726d8817 100644 --- a/security-admin/db/sqlserver/optimized/current/ranger_core_db_sqlserver.sql +++ b/security-admin/db/sqlserver/optimized/current/ranger_core_db_sqlserver.sql @@ -3276,7 +3276,7 @@ INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active) VALUES ('029',CURRENT_TIMESTAMP,'Ranger 0.7.0',CURRENT_TIMESTAMP,'localhost','Y'); INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active) VALUES ('033',CURRENT_TIMESTAMP,'Ranger 0.7.0',CURRENT_TIMESTAMP,'localhost','Y'); INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active) VALUES ('035',CURRENT_TIMESTAMP,'Ranger 0.7.0',CURRENT_TIMESTAMP,'localhost','Y'); -INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active) VALUES ('040',CURRENT_TIMESTAMP,'Ranger 0.7.0',CURRENT_TIMESTAMP,'localhost','Y'); +INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active) VALUES ('036',CURRENT_TIMESTAMP,'Ranger 0.7.0',CURRENT_TIMESTAMP,'localhost','Y'); INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active) VALUES ('DB_PATCHES',CURRENT_TIMESTAMP,'Ranger 0.7.0',CURRENT_TIMESTAMP,'localhost','Y'); INSERT INTO x_user_module_perm (user_id,module_id,create_time,update_time,added_by_id,upd_by_id,is_allowed) VALUES (1,3,CURRENT_TIMESTAMP,CURRENT_TIMESTAMP,1,1,1); INSERT INTO x_user_module_perm (user_id,module_id,create_time,update_time,added_by_id,upd_by_id,is_allowed) VALUES (1,1,CURRENT_TIMESTAMP,CURRENT_TIMESTAMP,1,1,1); From 61b69730225687508123dae489b542bc86bfbd60 Mon Sep 17 00:00:00 2001 From: Vishal Suvagia Date: Tue, 23 Oct 2018 16:33:48 +0530 Subject: [PATCH 142/151] RANGER-2251 : Need to provide options for making java heap size memory configurable in Ranger services. --- .../scripts/ranger-admin-services.sh | 8 +++-- kms/scripts/ranger-kms | 4 ++- security-admin/scripts/db_setup.py | 30 ++++++++++++------- security-admin/scripts/install.properties | 2 ++ tagsync/scripts/ranger-tagsync-services.sh | 3 ++ .../scripts/ranger-usersync-services.sh | 3 ++ 6 files changed, 36 insertions(+), 14 deletions(-) diff --git a/embeddedwebserver/scripts/ranger-admin-services.sh b/embeddedwebserver/scripts/ranger-admin-services.sh index 350826ff09..e5643a34b5 100755 --- a/embeddedwebserver/scripts/ranger-admin-services.sh +++ b/embeddedwebserver/scripts/ranger-admin-services.sh @@ -28,13 +28,12 @@ action=`echo $action | tr '[:lower:]' '[:upper:]'` realScriptPath=`readlink -f $0` realScriptDir=`dirname $realScriptPath` XAPOLICYMGR_DIR=`(cd $realScriptDir/..; pwd)` -max_memory=1g XAPOLICYMGR_EWS_DIR=${XAPOLICYMGR_DIR}/ews RANGER_JAAS_LIB_DIR="${XAPOLICYMGR_EWS_DIR}/ranger_jaas" RANGER_JAAS_CONF_DIR="${XAPOLICYMGR_EWS_DIR}/webapp/WEB-INF/classes/conf/ranger_jaas" -JAVA_OPTS=" ${JAVA_OPTS} -XX:MetaspaceSize=100m -XX:MaxMetaspaceSize=200m -Xmx${max_memory} -Xms1g -Xloggc:${XAPOLICYMGR_EWS_DIR}/logs/gc-worker.log -verbose:gc -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=1m -XX:+PrintGCDetails -XX:+PrintHeapAtGC -XX:+PrintGCDateStamps" -if [[ ${JAVA_OPTS} != *"-Duser.timezone"* ]] ;then export JAVA_OPTS=" ${JAVA_OPTS} -Duser.timezone=UTC" ;fi +ranger_admin_max_heap_size=1g + if [ -f ${XAPOLICYMGR_DIR}/ews/webapp/WEB-INF/classes/conf/java_home.sh ]; then . ${XAPOLICYMGR_DIR}/ews/webapp/WEB-INF/classes/conf/java_home.sh fi @@ -45,6 +44,9 @@ for custom_env_script in `find ${XAPOLICYMGR_DIR}/ews/webapp/WEB-INF/classes/con fi done +JAVA_OPTS=" ${JAVA_OPTS} -XX:MetaspaceSize=100m -XX:MaxMetaspaceSize=200m -Xmx${ranger_admin_max_heap_size} -Xms1g -Xloggc:${XAPOLICYMGR_EWS_DIR}/logs/gc-worker.log -verbose:gc -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=1m -XX:+PrintGCDetails -XX:+PrintHeapAtGC -XX:+PrintGCDateStamps" +if [[ ${JAVA_OPTS} != *"-Duser.timezone"* ]] ;then export JAVA_OPTS=" ${JAVA_OPTS} -Duser.timezone=UTC" ;fi + if [ "$JAVA_HOME" != "" ]; then export PATH=$JAVA_HOME/bin:$PATH fi diff --git a/kms/scripts/ranger-kms b/kms/scripts/ranger-kms index d1e33608f3..36c7533b56 100755 --- a/kms/scripts/ranger-kms +++ b/kms/scripts/ranger-kms @@ -33,7 +33,7 @@ RANGER_KMS_EWS_DIR=${RANGER_KMS_DIR}/ews RANGER_KMS_EWS_CONF_DIR="${RANGER_KMS_EWS_DIR}/webapp/WEB-INF/classes/conf" RANGER_KMS_EWS_LIB_DIR="${RANGER_KMS_EWS_DIR}/webapp/WEB-INF/classes/lib" -JAVA_OPTS=" ${JAVA_OPTS} -XX:MaxPermSize=256m -Xmx1024m -Xms1024m " +ranger_kms_max_heap_size=1g if [ -f ${RANGER_KMS_DIR}/ews/webapp/WEB-INF/classes/conf/java_home.sh ]; then . ${RANGER_KMS_DIR}/ews/webapp/WEB-INF/classes/conf/java_home.sh @@ -45,6 +45,8 @@ for custom_env_script in `find ${RANGER_KMS_DIR}/ews/webapp/WEB-INF/classes/conf fi done +JAVA_OPTS=" ${JAVA_OPTS} -XX:MetaspaceSize=100m -XX:MaxMetaspaceSize=256m -Xmx${ranger_kms_max_heap_size} -Xms1g " + if [ "$JAVA_HOME" != "" ]; then export PATH=$JAVA_HOME/bin:$PATH fi diff --git a/security-admin/scripts/db_setup.py b/security-admin/scripts/db_setup.py index 83463cfaf3..7b17f9ae11 100644 --- a/security-admin/scripts/db_setup.py +++ b/security-admin/scripts/db_setup.py @@ -37,7 +37,6 @@ retryPatchAfterSeconds=120 java_patch_regex="^Patch.*?J\d{5}.class$" is_unix = os_name == "LINUX" or os_name == "DARWIN" -max_memory='1g' if is_unix: RANGER_ADMIN_HOME = os.getenv("RANGER_ADMIN_HOME") @@ -94,6 +93,8 @@ def populate_global_dict(): value = '' value = value.strip() globalDict[key] = value + if 'ranger_admin_max_heap_size' not in globalDict: + globalDict['ranger_admin_max_heap_size']='1g' def jisql_log(query, db_password): if jisql_debug == True: @@ -486,6 +487,7 @@ def auditdb_operation(self, xa_db_host, audit_db_host, db_name, audit_db_name, d self.grant_audit_db_user(db_user, audit_db_name, audit_db_user, audit_db_password, db_password,TABLE_NAME) def execute_java_patches(self, xa_db_host, db_user, db_password, db_name): + global globalDict my_dict = {} version = "" className = "" @@ -556,7 +558,7 @@ def execute_java_patches(self, xa_db_host, db_user, db_password, db_name): path = os.path.join("%s","WEB-INF","classes","conf:%s","WEB-INF","classes","lib","*:%s","WEB-INF",":%s","META-INF",":%s","WEB-INF","lib","*:%s","WEB-INF","classes",":%s","WEB-INF","classes","META-INF:%s" )%(app_home ,app_home ,app_home, app_home, app_home, app_home ,app_home ,self.SQL_CONNECTOR_JAR) elif os_name == "WINDOWS": path = os.path.join("%s","WEB-INF","classes","conf;%s","WEB-INF","classes","lib","*;%s","WEB-INF",";%s","META-INF",";%s","WEB-INF","lib","*;%s","WEB-INF","classes",";%s","WEB-INF","classes","META-INF;%s" )%(app_home ,app_home ,app_home, app_home, app_home, app_home ,app_home ,self.SQL_CONNECTOR_JAR) - get_java_cmd = "%s -XX:MetaspaceSize=100m -XX:MaxMetaspaceSize=200m -Xmx%s -Xms1g -Dlogdir=%s -Dlog4j.configuration=db_patch.log4j.xml -cp %s org.apache.ranger.patch.%s"%(self.JAVA_BIN,max_memory,ranger_log,path,className) + get_java_cmd = "%s -XX:MetaspaceSize=100m -XX:MaxMetaspaceSize=200m -Xmx%s -Xms1g -Dlogdir=%s -Dlog4j.configuration=db_patch.log4j.xml -cp %s org.apache.ranger.patch.%s"%(self.JAVA_BIN,globalDict['ranger_admin_max_heap_size'],ranger_log,path,className) if is_unix: ret = subprocess.call(shlex.split(get_java_cmd)) elif os_name == "WINDOWS": @@ -1176,6 +1178,7 @@ def auditdb_operation(self, xa_db_host , audit_db_host , db_name ,audit_db_name, self.grant_audit_db_user( audit_db_name ,db_user, audit_db_user, db_password,audit_db_password) def execute_java_patches(self, xa_db_host, db_user, db_password, db_name): + global globalDict my_dict = {} version = "" className = "" @@ -1271,7 +1274,7 @@ def execute_java_patches(self, xa_db_host, db_user, db_password, db_name): path = os.path.join("%s","WEB-INF","classes","conf:%s","WEB-INF","classes","lib","*:%s","WEB-INF",":%s","META-INF",":%s","WEB-INF","lib","*:%s","WEB-INF","classes",":%s","WEB-INF","classes","META-INF:%s" )%(app_home ,app_home ,app_home, app_home, app_home, app_home ,app_home ,self.SQL_CONNECTOR_JAR) elif os_name == "WINDOWS": path = os.path.join("%s","WEB-INF","classes","conf;%s","WEB-INF","classes","lib","*;%s","WEB-INF",";%s","META-INF",";%s","WEB-INF","lib","*;%s","WEB-INF","classes",";%s","WEB-INF","classes","META-INF;%s" )%(app_home ,app_home ,app_home, app_home, app_home, app_home ,app_home ,self.SQL_CONNECTOR_JAR) - get_java_cmd = "%s -XX:MetaspaceSize=100m -XX:MaxMetaspaceSize=200m -Xmx%s -Xms1g -Djava.security.egd=file:///dev/urandom -Dlogdir=%s -Dlog4j.configuration=db_patch.log4j.xml -cp %s org.apache.ranger.patch.%s"%(self.JAVA_BIN,max_memory,ranger_log,path,className) + get_java_cmd = "%s -XX:MetaspaceSize=100m -XX:MaxMetaspaceSize=200m -Xmx%s -Xms1g -Djava.security.egd=file:///dev/urandom -Dlogdir=%s -Dlog4j.configuration=db_patch.log4j.xml -cp %s org.apache.ranger.patch.%s"%(self.JAVA_BIN,globalDict['ranger_admin_max_heap_size'],ranger_log,path,className) if is_unix: ret = subprocess.call(shlex.split(get_java_cmd)) elif os_name == "WINDOWS": @@ -1311,6 +1314,7 @@ def execute_java_patches(self, xa_db_host, db_user, db_password, db_name): sys.exit(1) def change_admin_default_password(self, xa_db_host, db_user, db_password, db_name,userName,oldPassword,newPassword): + global globalDict my_dict = {} version = "" className = "ChangePasswordUtil" @@ -1360,7 +1364,7 @@ def change_admin_default_password(self, xa_db_host, db_user, db_password, db_nam path = os.path.join("%s","WEB-INF","classes","conf:%s","WEB-INF","classes","lib","*:%s","WEB-INF",":%s","META-INF",":%s","WEB-INF","lib","*:%s","WEB-INF","classes",":%s","WEB-INF","classes","META-INF:%s" )%(app_home ,app_home ,app_home, app_home, app_home, app_home ,app_home ,self.SQL_CONNECTOR_JAR) elif os_name == "WINDOWS": path = os.path.join("%s","WEB-INF","classes","conf;%s","WEB-INF","classes","lib","*;%s","WEB-INF",";%s","META-INF",";%s","WEB-INF","lib","*;%s","WEB-INF","classes",";%s","WEB-INF","classes","META-INF;%s" )%(app_home ,app_home ,app_home, app_home, app_home, app_home ,app_home ,self.SQL_CONNECTOR_JAR) - get_java_cmd = "%s -XX:MetaspaceSize=100m -XX:MaxMetaspaceSize=200m -Xmx%s -Xms1g -Dlogdir=%s -Dlog4j.configuration=db_patch.log4j.xml -cp %s org.apache.ranger.patch.cliutil.%s %s %s %s -default"%(self.JAVA_BIN,max_memory,ranger_log,path,className,userName,oldPassword,newPassword) + get_java_cmd = "%s -XX:MetaspaceSize=100m -XX:MaxMetaspaceSize=200m -Xmx%s -Xms1g -Dlogdir=%s -Dlog4j.configuration=db_patch.log4j.xml -cp %s org.apache.ranger.patch.cliutil.%s %s %s %s -default"%(self.JAVA_BIN,globalDict['ranger_admin_max_heap_size'],ranger_log,path,className,userName,oldPassword,newPassword) if is_unix: status = subprocess.call(shlex.split(get_java_cmd)) elif os_name == "WINDOWS": @@ -1871,6 +1875,7 @@ def auditdb_operation(self, xa_db_host, audit_db_host, db_name, audit_db_name, d self.grant_audit_db_user(audit_db_name ,db_user, audit_db_user, db_password,audit_db_password) def execute_java_patches(self, xa_db_host, db_user, db_password, db_name): + global globalDict my_dict = {} version = "" className = "" @@ -1941,7 +1946,7 @@ def execute_java_patches(self, xa_db_host, db_user, db_password, db_name): path = os.path.join("%s","WEB-INF","classes","conf:%s","WEB-INF","classes","lib","*:%s","WEB-INF",":%s","META-INF",":%s","WEB-INF","lib","*:%s","WEB-INF","classes",":%s","WEB-INF","classes","META-INF:%s" )%(app_home ,app_home ,app_home, app_home, app_home, app_home ,app_home ,self.SQL_CONNECTOR_JAR) elif os_name == "WINDOWS": path = os.path.join("%s","WEB-INF","classes","conf;%s","WEB-INF","classes","lib","*;%s","WEB-INF",";%s","META-INF",";%s","WEB-INF","lib","*;%s","WEB-INF","classes",";%s","WEB-INF","classes","META-INF;%s" )%(app_home ,app_home ,app_home, app_home, app_home, app_home ,app_home ,self.SQL_CONNECTOR_JAR) - get_java_cmd = "%s -XX:MetaspaceSize=100m -XX:MaxMetaspaceSize=200m -Xmx%s -Xms1g -Dlogdir=%s -Dlog4j.configuration=db_patch.log4j.xml -cp %s org.apache.ranger.patch.%s"%(self.JAVA_BIN,max_memory,ranger_log,path,className) + get_java_cmd = "%s -XX:MetaspaceSize=100m -XX:MaxMetaspaceSize=200m -Xmx%s -Xms1g -Dlogdir=%s -Dlog4j.configuration=db_patch.log4j.xml -cp %s org.apache.ranger.patch.%s"%(self.JAVA_BIN,globalDict['ranger_admin_max_heap_size'],ranger_log,path,className) if is_unix: ret = subprocess.call(shlex.split(get_java_cmd)) elif os_name == "WINDOWS": @@ -1981,6 +1986,7 @@ def execute_java_patches(self, xa_db_host, db_user, db_password, db_name): sys.exit(1) def change_admin_default_password(self, xa_db_host, db_user, db_password, db_name,userName,oldPassword,newPassword): + global globalDict my_dict = {} version = "" className = "ChangePasswordUtil" @@ -2030,7 +2036,7 @@ def change_admin_default_password(self, xa_db_host, db_user, db_password, db_nam path = os.path.join("%s","WEB-INF","classes","conf:%s","WEB-INF","classes","lib","*:%s","WEB-INF",":%s","META-INF",":%s","WEB-INF","lib","*:%s","WEB-INF","classes",":%s","WEB-INF","classes","META-INF:%s" )%(app_home ,app_home ,app_home, app_home, app_home, app_home ,app_home ,self.SQL_CONNECTOR_JAR) elif os_name == "WINDOWS": path = os.path.join("%s","WEB-INF","classes","conf;%s","WEB-INF","classes","lib","*;%s","WEB-INF",";%s","META-INF",";%s","WEB-INF","lib","*;%s","WEB-INF","classes",";%s","WEB-INF","classes","META-INF;%s" )%(app_home ,app_home ,app_home, app_home, app_home, app_home ,app_home ,self.SQL_CONNECTOR_JAR) - get_java_cmd = "%s -XX:MetaspaceSize=100m -XX:MaxMetaspaceSize=200m -Xmx%s -Xms1g -Dlogdir=%s -Dlog4j.configuration=db_patch.log4j.xml -cp %s org.apache.ranger.patch.cliutil.%s %s %s %s -default"%(self.JAVA_BIN,max_memory,ranger_log,path,className,userName,oldPassword,newPassword) + get_java_cmd = "%s -XX:MetaspaceSize=100m -XX:MaxMetaspaceSize=200m -Xmx%s -Xms1g -Dlogdir=%s -Dlog4j.configuration=db_patch.log4j.xml -cp %s org.apache.ranger.patch.cliutil.%s %s %s %s -default"%(self.JAVA_BIN,globalDict['ranger_admin_max_heap_size'],ranger_log,path,className,userName,oldPassword,newPassword) if is_unix: status = subprocess.call(shlex.split(get_java_cmd)) elif os_name == "WINDOWS": @@ -2503,6 +2509,7 @@ def auditdb_operation(self, xa_db_host, audit_db_host, db_name, audit_db_name,db self.grant_audit_db_user( audit_db_name ,db_user, audit_db_user, db_password,audit_db_password,TABLE_NAME) def execute_java_patches(self, xa_db_host, db_user, db_password, db_name): + global globalDict my_dict = {} version = "" className = "" @@ -2573,7 +2580,7 @@ def execute_java_patches(self, xa_db_host, db_user, db_password, db_name): path = os.path.join("%s","WEB-INF","classes","conf:%s","WEB-INF","classes","lib","*:%s","WEB-INF",":%s","META-INF",":%s","WEB-INF","lib","*:%s","WEB-INF","classes",":%s","WEB-INF","classes","META-INF:%s" )%(app_home ,app_home ,app_home, app_home, app_home, app_home ,app_home ,self.SQL_CONNECTOR_JAR) elif os_name == "WINDOWS": path = os.path.join("%s","WEB-INF","classes","conf;%s","WEB-INF","classes","lib","*;%s","WEB-INF",";%s","META-INF",";%s","WEB-INF","lib","*;%s","WEB-INF","classes",";%s","WEB-INF","classes","META-INF;%s" )%(app_home ,app_home ,app_home, app_home, app_home, app_home ,app_home ,self.SQL_CONNECTOR_JAR) - get_java_cmd = "%s -XX:MetaspaceSize=100m -XX:MaxMetaspaceSize=200m -Xmx%s -Xms1g -Dlogdir=%s -Dlog4j.configuration=db_patch.log4j.xml -cp %s org.apache.ranger.patch.%s"%(self.JAVA_BIN,max_memory,ranger_log,path,className) + get_java_cmd = "%s -XX:MetaspaceSize=100m -XX:MaxMetaspaceSize=200m -Xmx%s -Xms1g -Dlogdir=%s -Dlog4j.configuration=db_patch.log4j.xml -cp %s org.apache.ranger.patch.%s"%(self.JAVA_BIN,globalDict['ranger_admin_max_heap_size'],ranger_log,path,className) if is_unix: ret = subprocess.call(shlex.split(get_java_cmd)) elif os_name == "WINDOWS": @@ -2613,6 +2620,7 @@ def execute_java_patches(self, xa_db_host, db_user, db_password, db_name): sys.exit(1) def change_admin_default_password(self, xa_db_host, db_user, db_password, db_name,userName,oldPassword,newPassword): + global globalDict my_dict = {} version = "" className = "ChangePasswordUtil" @@ -2662,7 +2670,7 @@ def change_admin_default_password(self, xa_db_host, db_user, db_password, db_nam path = os.path.join("%s","WEB-INF","classes","conf:%s","WEB-INF","classes","lib","*:%s","WEB-INF",":%s","META-INF",":%s","WEB-INF","lib","*:%s","WEB-INF","classes",":%s","WEB-INF","classes","META-INF:%s" )%(app_home ,app_home ,app_home, app_home, app_home, app_home ,app_home ,self.SQL_CONNECTOR_JAR) elif os_name == "WINDOWS": path = os.path.join("%s","WEB-INF","classes","conf;%s","WEB-INF","classes","lib","*;%s","WEB-INF",";%s","META-INF",";%s","WEB-INF","lib","*;%s","WEB-INF","classes",";%s","WEB-INF","classes","META-INF;%s" )%(app_home ,app_home ,app_home, app_home, app_home, app_home ,app_home ,self.SQL_CONNECTOR_JAR) - get_java_cmd = "%s -XX:MetaspaceSize=100m -XX:MaxMetaspaceSize=200m -Xmx%s -Xms1g -Dlogdir=%s -Dlog4j.configuration=db_patch.log4j.xml -cp %s org.apache.ranger.patch.cliutil.%s %s %s %s -default"%(self.JAVA_BIN,max_memory,ranger_log,path,className,userName,oldPassword,newPassword) + get_java_cmd = "%s -XX:MetaspaceSize=100m -XX:MaxMetaspaceSize=200m -Xmx%s -Xms1g -Dlogdir=%s -Dlog4j.configuration=db_patch.log4j.xml -cp %s org.apache.ranger.patch.cliutil.%s %s %s %s -default"%(self.JAVA_BIN,globalDict['ranger_admin_max_heap_size'],ranger_log,path,className,userName,oldPassword,newPassword) if is_unix: status = subprocess.call(shlex.split(get_java_cmd)) elif os_name == "WINDOWS": @@ -3127,6 +3135,7 @@ def auditdb_operation(self, xa_db_host, audit_db_host, db_name, audit_db_name,db self.grant_audit_db_user( audit_db_name ,db_user, audit_db_user, db_password,audit_db_password,TABLE_NAME) def execute_java_patches(self, xa_db_host, db_user, db_password, db_name): + global globalDict my_dict = {} version = "" className = "" @@ -3197,7 +3206,7 @@ def execute_java_patches(self, xa_db_host, db_user, db_password, db_name): path = os.path.join("%s","WEB-INF","classes","conf:%s","WEB-INF","classes","lib","*:%s","WEB-INF",":%s","META-INF",":%s","WEB-INF","lib","*:%s","WEB-INF","classes",":%s","WEB-INF","classes","META-INF:%s" )%(app_home ,app_home ,app_home, app_home, app_home, app_home ,app_home ,self.SQL_CONNECTOR_JAR) elif os_name == "WINDOWS": path = os.path.join("%s","WEB-INF","classes","conf;%s","WEB-INF","classes","lib","*;%s","WEB-INF",";%s","META-INF",";%s","WEB-INF","lib","*;%s","WEB-INF","classes",";%s","WEB-INF","classes","META-INF;%s" )%(app_home ,app_home ,app_home, app_home, app_home, app_home ,app_home ,self.SQL_CONNECTOR_JAR) - get_java_cmd = "%s -XX:MetaspaceSize=100m -XX:MaxMetaspaceSize=200m -Xmx%s -Xms1g -Dlogdir=%s -Dlog4j.configuration=db_patch.log4j.xml -cp %s org.apache.ranger.patch.%s"%(self.JAVA_BIN,max_memory,ranger_log,path,className) + get_java_cmd = "%s -XX:MetaspaceSize=100m -XX:MaxMetaspaceSize=200m -Xmx%s -Xms1g -Dlogdir=%s -Dlog4j.configuration=db_patch.log4j.xml -cp %s org.apache.ranger.patch.%s"%(self.JAVA_BIN,globalDict['ranger_admin_max_heap_size'],ranger_log,path,className) if is_unix: ret = subprocess.call(shlex.split(get_java_cmd)) elif os_name == "WINDOWS": @@ -3258,6 +3267,7 @@ def set_options(self, db_name, db_user, db_password, TABLE_NAME): ret = subprocessCallWithRetry(shlex.split(query)) def change_admin_default_password(self, xa_db_host, db_user, db_password, db_name,userName,oldPassword,newPassword): + global globalDict my_dict = {} version = "" className = "ChangePasswordUtil" @@ -3307,7 +3317,7 @@ def change_admin_default_password(self, xa_db_host, db_user, db_password, db_nam path = os.path.join("%s","WEB-INF","classes","conf:%s","WEB-INF","classes","lib","*:%s","WEB-INF",":%s","META-INF",":%s","WEB-INF","lib","*:%s","WEB-INF","classes",":%s","WEB-INF","classes","META-INF:%s" )%(app_home ,app_home ,app_home, app_home, app_home, app_home ,app_home ,self.SQL_CONNECTOR_JAR) elif os_name == "WINDOWS": path = os.path.join("%s","WEB-INF","classes","conf;%s","WEB-INF","classes","lib","*;%s","WEB-INF",";%s","META-INF",";%s","WEB-INF","lib","*;%s","WEB-INF","classes",";%s","WEB-INF","classes","META-INF;%s" )%(app_home ,app_home ,app_home, app_home, app_home, app_home ,app_home ,self.SQL_CONNECTOR_JAR) - get_java_cmd = "%s -XX:MetaspaceSize=100m -XX:MaxMetaspaceSize=200m -Xmx%s -Xms1g -Dlogdir=%s -Dlog4j.configuration=db_patch.log4j.xml -cp %s org.apache.ranger.patch.cliutil.%s %s %s %s -default"%(self.JAVA_BIN,max_memory,ranger_log,path,className,userName,oldPassword,newPassword) + get_java_cmd = "%s -XX:MetaspaceSize=100m -XX:MaxMetaspaceSize=200m -Xmx%s -Xms1g -Dlogdir=%s -Dlog4j.configuration=db_patch.log4j.xml -cp %s org.apache.ranger.patch.cliutil.%s %s %s %s -default"%(self.JAVA_BIN,globalDict['ranger_admin_max_heap_size'],ranger_log,path,className,userName,oldPassword,newPassword) if is_unix: status = subprocess.call(shlex.split(get_java_cmd)) elif os_name == "WINDOWS": diff --git a/security-admin/scripts/install.properties b/security-admin/scripts/install.properties index 687bd99fd3..d7f098b30f 100644 --- a/security-admin/scripts/install.properties +++ b/security-admin/scripts/install.properties @@ -221,6 +221,8 @@ JAVA_BIN='java' JAVA_VERSION_REQUIRED='1.7' JAVA_ORACLE='Java(TM) SE Runtime Environment' +ranger_admin_max_heap_size=1g + #mysql_create_user_file=${PWD}/db/mysql/create_dev_user.sql mysql_core_file=db/mysql/xa_core_db.sql mysql_audit_file=db/mysql/xa_audit_db.sql diff --git a/tagsync/scripts/ranger-tagsync-services.sh b/tagsync/scripts/ranger-tagsync-services.sh index 9a72316386..ed787be08f 100755 --- a/tagsync/scripts/ranger-tagsync-services.sh +++ b/tagsync/scripts/ranger-tagsync-services.sh @@ -26,6 +26,7 @@ realScriptPath=`readlink -f $0` realScriptDir=`dirname $realScriptPath` cd $realScriptDir cdir=`pwd` +ranger_tagsync_max_heap_size=1g for custom_env_script in `find ${cdir}/conf/ -name "ranger-tagsync-env*"`; do if [ -f $custom_env_script ]; then @@ -41,6 +42,8 @@ if [ -z "${UNIX_TAGSYNC_USER}" ]; then UNIX_TAGSYNC_USER=ranger fi +JAVA_OPTS=" ${JAVA_OPTS} -XX:MetaspaceSize=100m -XX:MaxMetaspaceSize=200m -Xmx${ranger_tagsync_max_heap_size} -Xms1g " + if [ "${action}" == "START" ]; then #Export JAVA_HOME diff --git a/unixauthservice/scripts/ranger-usersync-services.sh b/unixauthservice/scripts/ranger-usersync-services.sh index 36eefcdc9b..1fd8581d80 100644 --- a/unixauthservice/scripts/ranger-usersync-services.sh +++ b/unixauthservice/scripts/ranger-usersync-services.sh @@ -26,6 +26,7 @@ realScriptPath=`readlink -f $0` realScriptDir=`dirname $realScriptPath` cd $realScriptDir cdir=`pwd` +ranger_usersync_max_heap_size=1g for custom_env_script in `find ${cdir}/conf/ -name "ranger-usersync-env*"`; do if [ -f $custom_env_script ]; then @@ -40,6 +41,8 @@ if [ -z "${UNIX_USERSYNC_USER}" ]; then UNIX_USERSYNC_USER=ranger fi +JAVA_OPTS=" ${JAVA_OPTS} -XX:MetaspaceSize=100m -XX:MaxMetaspaceSize=200m -Xmx${ranger_usersync_max_heap_size} -Xms1g " + if [ "${action}" == "START" ]; then #Export JAVA_HOME From 37b1e1c236e6501b31fee89a0ca98377f3dd72fb Mon Sep 17 00:00:00 2001 From: Nikhil P Date: Tue, 23 Oct 2018 17:12:40 +0530 Subject: [PATCH 143/151] RANGER-2262 : Improvement of export to excel from report listing page for Oracle database --- .../main/java/org/apache/ranger/biz/ServiceDBStore.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java b/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java index a3e02612b1..773c2edffc 100644 --- a/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java +++ b/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java @@ -3649,12 +3649,16 @@ private void writeBookForPolicyItems(RangerPolicy policy, RangerPolicyItem polic if (CollectionUtils.isNotEmpty(groups)) { groupNames = groupNames + groups.toString(); StringTokenizer groupToken = new StringTokenizer(groupNames, "[]"); - groupNames = groupToken.nextToken().toString(); + while(groupToken.hasMoreTokens()) { + groupNames = groupToken.nextToken().toString(); + } } if (CollectionUtils.isNotEmpty(users)) { userNames = userNames + users.toString(); StringTokenizer userToken = new StringTokenizer(userNames, "[]"); - userNames = userToken.nextToken().toString(); + while(userToken.hasMoreTokens()) { + userNames = userToken.nextToken().toString(); + } } cell = row.createCell(3); cell.setCellValue(groupNames); From d81a7446afcfc557024db3615b9f5066fdd5e8d3 Mon Sep 17 00:00:00 2001 From: Abhay Kulkarni Date: Fri, 26 Oct 2018 17:27:21 -0700 Subject: [PATCH 144/151] RANGER-2268: Optimize policy and tags migration to new schema --- .../PatchForUpdatingPolicyJson_J10019.java | 53 ++++++++++- .../PatchForUpdatingTagsJson_J10020.java | 91 ++++++++++++++++++- 2 files changed, 139 insertions(+), 5 deletions(-) diff --git a/security-admin/src/main/java/org/apache/ranger/patch/PatchForUpdatingPolicyJson_J10019.java b/security-admin/src/main/java/org/apache/ranger/patch/PatchForUpdatingPolicyJson_J10019.java index 873c2e7c38..0f251de9dd 100644 --- a/security-admin/src/main/java/org/apache/ranger/patch/PatchForUpdatingPolicyJson_J10019.java +++ b/security-admin/src/main/java/org/apache/ranger/patch/PatchForUpdatingPolicyJson_J10019.java @@ -183,9 +183,22 @@ private void updateRangerPolicyTableWithPolicyJson() throws Exception { List policies = policyRetriever.getServicePolicies(dbService); if (CollectionUtils.isNotEmpty(policies)) { + TransactionTemplate txTemplate = new TransactionTemplate(txManager); + for (RangerPolicy policy : policies) { - policyRefUpdater.cleanupRefTables(policy); - portPolicy(service.getType(), policy); + XXPolicy xPolicy = daoMgr.getXXPolicy().getById(policy.getId()); + if (xPolicy != null && StringUtil.isEmpty(xPolicy.getPolicyText())) { + + PolicyUpdaterThread updaterThread = new PolicyUpdaterThread(txTemplate, service, policy); + updaterThread.setDaemon(true); + updaterThread.start(); + updaterThread.join(); + + String errorMsg = updaterThread.getErrorMsg(); + if (StringUtils.isNotEmpty(errorMsg)) { + throw new Exception(errorMsg); + } + } } } } @@ -194,6 +207,42 @@ private void updateRangerPolicyTableWithPolicyJson() throws Exception { logger.info("<== updateRangerPolicyTableWithPolicyJson() "); } + private class PolicyUpdaterThread extends Thread { + final TransactionTemplate txTemplate; + final RangerService service; + final RangerPolicy policy; + String errorMsg; + + PolicyUpdaterThread(TransactionTemplate txTemplate, final RangerService service, final RangerPolicy policy) { + this.txTemplate = txTemplate; + this.service = service; + this.policy = policy; + this.errorMsg = null; + } + + public String getErrorMsg() { + return errorMsg; + } + + @Override + public void run() { + errorMsg = txTemplate.execute(new TransactionCallback() { + @Override + public String doInTransaction(TransactionStatus status) { + String ret = null; + try { + policyRefUpdater.cleanupRefTables(policy); + portPolicy(service.getType(), policy); + } catch (Throwable e) { + logger.error("PortPolicy failed for policy:[" + policy + "]", e); + ret = e.toString(); + } + return ret; + } + }); + } + } + private void portPolicy(String serviceType, RangerPolicy policy) throws Exception { logger.info("==> portPolicy(id=" + policy.getId() + ")"); diff --git a/security-admin/src/main/java/org/apache/ranger/patch/PatchForUpdatingTagsJson_J10020.java b/security-admin/src/main/java/org/apache/ranger/patch/PatchForUpdatingTagsJson_J10020.java index 48ae5f9e8b..f76864e1dd 100644 --- a/security-admin/src/main/java/org/apache/ranger/patch/PatchForUpdatingTagsJson_J10020.java +++ b/security-admin/src/main/java/org/apache/ranger/patch/PatchForUpdatingTagsJson_J10020.java @@ -33,6 +33,7 @@ import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.MapUtils; +import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.log4j.Logger; @@ -40,6 +41,9 @@ import org.apache.ranger.biz.ServiceDBStore; import org.apache.ranger.biz.TagDBStore; import org.apache.ranger.db.RangerDaoManager; +import org.apache.ranger.db.XXServiceResourceDao; +import org.apache.ranger.db.XXTagDao; +import org.apache.ranger.db.XXTagDefDao; import org.apache.ranger.entity.XXPortalUser; import org.apache.ranger.entity.XXResourceDef; import org.apache.ranger.entity.XXService; @@ -146,6 +150,8 @@ private void updateRangerTagsTablesWithTagsJson() throws Exception { List allServices = svcStore.getServices(new SearchFilter()); if (CollectionUtils.isNotEmpty(allServices)) { + TransactionTemplate txTemplate = new TransactionTemplate(txManager); + for (RangerService service : allServices) { XXService dbService = daoMgr.getXXService().getById(service.getId()); RangerTagDBRetriever tagsRetriever = new RangerTagDBRetriever(daoMgr, txManager, dbService); @@ -153,13 +159,25 @@ private void updateRangerTagsTablesWithTagsJson() throws Exception { Map tags = tagsRetriever.getTags(); List serviceResources = tagsRetriever.getServiceResources(); + XXTagDefDao tagDefDao = daoMgr.getXXTagDef(); + XXTagDao tagDao = daoMgr.getXXTag(); + XXServiceResourceDao serviceResourceDao = daoMgr.getXXServiceResource(); + if (MapUtils.isNotEmpty(tagDefs)) { logger.info("==> Port " + tagDefs.size() + " Tag Definitions for service(name=" + dbService.getName() + ")"); for (Map.Entry entry : tagDefs.entrySet()) { RangerTagDef tagDef = entry.getValue(); + XXTagDef xTagDef = tagDefDao.getById(tagDef.getId()); + + if (xTagDef != null && StringUtils.isEmpty(xTagDef.getTagAttrDefs())) { - portTagDef(tagDef); + TagsUpdaterThread updaterThread = new TagsUpdaterThread(txTemplate, null, null, tagDef); + String errorMsg = runThread(updaterThread); + if (StringUtils.isNotEmpty(errorMsg)) { + throw new Exception(errorMsg); + } + } } } @@ -168,8 +186,15 @@ private void updateRangerTagsTablesWithTagsJson() throws Exception { for (Map.Entry entry : tags.entrySet()) { RangerTag tag = entry.getValue(); + XXTag xTag = tagDao.getById(tag.getId()); - portTag(tag); + if (xTag != null && StringUtils.isEmpty(xTag.getTagAttrs())) { + TagsUpdaterThread updaterThread = new TagsUpdaterThread(txTemplate, null, tag, null); + String errorMsg = runThread(updaterThread); + if (StringUtils.isNotEmpty(errorMsg)) { + throw new Exception(errorMsg); + } + } } } @@ -177,7 +202,16 @@ private void updateRangerTagsTablesWithTagsJson() throws Exception { logger.info("==> Port " + serviceResources.size() + " Service Resources for service(name=" + dbService.getName() + ")"); for (RangerServiceResource serviceResource : serviceResources) { - portServiceResource(serviceResource); + + XXServiceResource xServiceResource = serviceResourceDao.getById(serviceResource.getId()); + + if (xServiceResource != null && StringUtils.isEmpty(xServiceResource.getServiceResourceElements())) { + TagsUpdaterThread updaterThread = new TagsUpdaterThread(txTemplate, serviceResource, null, null); + String errorMsg = runThread(updaterThread); + if (StringUtils.isNotEmpty(errorMsg)) { + throw new Exception(errorMsg); + } + } } } } @@ -186,6 +220,57 @@ private void updateRangerTagsTablesWithTagsJson() throws Exception { logger.info("<== updateRangerTagsTablesWithTagsJson() "); } + private String runThread(TagsUpdaterThread updaterThread) throws Exception { + updaterThread.setDaemon(true); + updaterThread.start(); + updaterThread.join(); + return updaterThread.getErrorMsg(); + } + + private class TagsUpdaterThread extends Thread { + final TransactionTemplate txTemplate; + final RangerServiceResource serviceResource; + final RangerTag tag; + final RangerTagDef tagDef; + String errorMsg; + + TagsUpdaterThread(TransactionTemplate txTemplate, final RangerServiceResource serviceResource, final RangerTag tag, final RangerTagDef tagDef) { + this.txTemplate = txTemplate; + this.serviceResource = serviceResource; + this.tag = tag; + this.tagDef = tagDef; + this.errorMsg = null; + } + + public String getErrorMsg() { + return errorMsg; + } + + @Override + public void run() { + errorMsg = txTemplate.execute(new TransactionCallback() { + @Override + public String doInTransaction(TransactionStatus status) { + String ret = null; + try { + if (serviceResource != null) { + portServiceResource(serviceResource); + } + if (tag != null) { + portTag(tag); + } + if (tagDef != null) { + portTagDef(tagDef); + } + } catch (Throwable e) { + logger.error("Port failed :[serviceResource=" + serviceResource + ", tag=" + tag + ", tagDef=" + tagDef +"]", e); + ret = e.toString(); + } + return ret; + } + }); + } + } private void portTagDef(RangerTagDef tagDef) { tagDefService.update(tagDef); } From 216704aeadea8b0efdd2caddfeac083465b9c9da Mon Sep 17 00:00:00 2001 From: Abhay Kulkarni Date: Tue, 30 Oct 2018 15:36:23 -0700 Subject: [PATCH 145/151] RANGER-2272: Ensure that case of resource-definition names and access-type names in Ranger policy is the same as in service-definition after successful validation --- .../plugin/errors/ValidationErrorCode.java | 2 + .../validation/RangerPolicyValidator.java | 37 +++++++++---- .../validation/RangerServiceDefValidator.java | 2 +- .../model/validation/RangerValidator.java | 54 +++++++++++++++---- .../validation/TestRangerPolicyValidator.java | 4 +- .../model/validation/TestRangerValidator.java | 37 +------------ .../org/apache/ranger/biz/ServiceDBStore.java | 18 ++++++- 7 files changed, 94 insertions(+), 60 deletions(-) diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/errors/ValidationErrorCode.java b/agents-common/src/main/java/org/apache/ranger/plugin/errors/ValidationErrorCode.java index c2fe4ac778..25ed6b7c44 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/errors/ValidationErrorCode.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/errors/ValidationErrorCode.java @@ -61,6 +61,8 @@ public enum ValidationErrorCode { SERVICE_DEF_VALIDATION_ERR_ENUM_DEF_NO_VALUES(2018, "enum [{0}] does not have any elements"), SERVICE_DEF_VALIDATION_ERR_ENUM_DEF_INVALID_DEFAULT_INDEX(2019, "default index[{0}] for enum [{1}] is invalid"), SERVICE_DEF_VALIDATION_ERR_ENUM_DEF_NULL_ENUM_ELEMENT(2020, "An enum element in enum element collection of enum [{0}] is null"), + SERVICE_DEF_VALIDATION_ERR_INVALID_SERVICE_RESOURCE_LEVELS(2021, "Resource-def levels are not in increasing order in an hierarchy"), + SERVICE_DEF_VALIDATION_ERR_NOT_LOWERCASE_NAME(2022, "{0}:[{1}] Invalid resource name. Resource name should consist of only lowercase, hyphen or underscore characters"), // POLICY VALIDATION POLICY_VALIDATION_ERR_UNSUPPORTED_ACTION(3001, "Internal error: method signature isValid(Long) is only supported for DELETE"), diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerPolicyValidator.java b/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerPolicyValidator.java index e48e5e1e1b..6c93413b31 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerPolicyValidator.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerPolicyValidator.java @@ -449,7 +449,8 @@ boolean isValidResourceNames(final RangerPolicy policy, final List policyResources = getPolicyResources(policy); + convertPolicyResourceNamesToLower(policy); + Set policyResources = policy.getResources().keySet(); RangerServiceDefHelper defHelper = new RangerServiceDefHelper(serviceDef); Set> hierarchies = defHelper.getResourceHierarchies(policy.getPolicyType()); // this can be empty but not null! @@ -874,15 +875,20 @@ boolean isValidPolicyItemAccess(RangerPolicyItemAccess access, List validAccessTypes) { + String ret = null; + for (String validType : validAccessTypes) { + if (StringUtils.equalsIgnoreCase(accessType, validType)) { + ret = validType; + break; + } + } + return ret; + } } diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerServiceDefValidator.java b/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerServiceDefValidator.java index 709c396a58..54d526da93 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerServiceDefValidator.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerServiceDefValidator.java @@ -485,7 +485,7 @@ boolean isValidConfigType(String type, String configName, List failures, final Action action) { + public boolean isValidResources(RangerServiceDef serviceDef, List failures, final Action action) { if(LOG.isDebugEnabled()) { LOG.debug(String.format("==> RangerServiceDefValidator.isValidResources(%s, %s)", serviceDef, failures)); } diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerValidator.java b/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerValidator.java index 3ae02bfd60..1c9071bf0c 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerValidator.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerValidator.java @@ -33,6 +33,7 @@ import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.ranger.plugin.errors.ValidationErrorCode; import org.apache.ranger.plugin.model.RangerPolicy; import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyResource; import org.apache.ranger.plugin.model.RangerService; @@ -303,7 +304,7 @@ Set getAccessTypes(RangerServiceDef serviceDef) { if (StringUtils.isBlank(accessType)) { LOG.warn("Access type def name was null/empty/blank!"); } else { - accessTypes.add(accessType.toLowerCase()); + accessTypes.add(accessType); } } } @@ -409,22 +410,22 @@ Set getAllResourceNames(RangerServiceDef serviceDef) { } return resourceNames; } - + /** - * Returns the resource-types defined on the policy converted to lowe-case + * Converts, in place, the resources defined in the policy to have lower-case resource-def-names * @param policy * @return */ - Set getPolicyResources(RangerPolicy policy) { - if (policy == null || policy.getResources() == null || policy.getResources().isEmpty()) { - return new HashSet(); - } else { - Set result = new HashSet(); - for (String name : policy.getResources().keySet()) { - result.add(name.toLowerCase()); + + void convertPolicyResourceNamesToLower(RangerPolicy policy) { + Map lowerCasePolicyResources = new HashMap<>(); + if (policy.getResources() != null) { + for (Map.Entry entry : policy.getResources().entrySet()) { + String lowerCasekey = entry.getKey().toLowerCase(); + lowerCasePolicyResources.put(lowerCasekey, entry.getValue()); } - return result; } + policy.setResources(lowerCasePolicyResources); } Map getValidationRegExes(RangerServiceDef serviceDef) { @@ -582,6 +583,37 @@ boolean isUnique(Integer value, Set alreadySeen, String valueName, Stri return valid; } + /* + * Important: Resource-names are required to be lowercase. This is used in validating policy create/update operations. + * Ref: RANGER-2272 + */ + boolean isValidResourceName(final String value, final String valueContext, final List failures) { + boolean ret = true; + + if (value != null && !StringUtils.isEmpty(value)) { + int sz = value.length(); + + for(int i = 0; i < sz; ++i) { + char c = value.charAt(i); + if (!(Character.isLowerCase(c) || c == '-' || c == '_')) { // Allow only lowercase, hyphen or underscore characters + ret = false; + break; + } + } + } else { + ret = false; + } + if (!ret) { + ValidationErrorCode errorCode = ValidationErrorCode.SERVICE_DEF_VALIDATION_ERR_NOT_LOWERCASE_NAME; + failures.add(new ValidationFailureDetailsBuilder() + .errorCode(errorCode.getErrorCode()) + .field(value) + .becauseOf(errorCode.getMessage(valueContext, value)) + .build()); + } + return ret; + } + boolean isUnique(final String value, final Set alreadySeen, final String valueName, final String collectionName, final List failures) { return isUnique(value, null, alreadySeen, valueName, collectionName, failures); } diff --git a/agents-common/src/test/java/org/apache/ranger/plugin/model/validation/TestRangerPolicyValidator.java b/agents-common/src/test/java/org/apache/ranger/plugin/model/validation/TestRangerPolicyValidator.java index 97a3ea75f9..e55c6eb832 100644 --- a/agents-common/src/test/java/org/apache/ranger/plugin/model/validation/TestRangerPolicyValidator.java +++ b/agents-common/src/test/java/org/apache/ranger/plugin/model/validation/TestRangerPolicyValidator.java @@ -123,13 +123,13 @@ public void setUp() throws Exception { private final Object[][] policyResourceMap_good = new Object[][] { // resource-name, values, excludes, recursive { "db", new String[] { "db1", "db2" }, null, null }, - { "TBL", new String[] { "tbl1", "tbl2" }, true, false } // case should not matter + { "tbl", new String[] { "tbl1", "tbl2" }, true, false } // case matters - use only lowercase characters }; private final Object[][] policyResourceMap_goodMultipleHierarchies = new Object[][] { // resource-name, values, excludes, recursive { "db", new String[] { "db1", "db2" }, null, null }, - { "UDF", new String[] { "udf1", "udf2" }, true, false } // case should not matter + { "udf", new String[] { "udf1", "udf2" }, true, false } // case matters - use only lowercase characters }; private final Object[][] policyResourceMap_bad = new Object[][] { diff --git a/agents-common/src/test/java/org/apache/ranger/plugin/model/validation/TestRangerValidator.java b/agents-common/src/test/java/org/apache/ranger/plugin/model/validation/TestRangerValidator.java index fb8073fc47..a6e31511d2 100644 --- a/agents-common/src/test/java/org/apache/ranger/plugin/model/validation/TestRangerValidator.java +++ b/agents-common/src/test/java/org/apache/ranger/plugin/model/validation/TestRangerValidator.java @@ -33,7 +33,6 @@ import java.util.Set; import org.apache.ranger.plugin.model.RangerPolicy; -import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyResource; import org.apache.ranger.plugin.model.RangerService; import org.apache.ranger.plugin.model.RangerServiceDef; import org.apache.ranger.plugin.model.RangerServiceDef.RangerAccessTypeDef; @@ -46,8 +45,6 @@ import org.junit.Before; import org.junit.Test; -import com.google.common.collect.Maps; - public class TestRangerValidator { static class RangerValidatorForTest extends RangerValidator { @@ -270,8 +267,8 @@ public void test_getAccessTypes() { Assert.assertEquals(4, accessTypes.size()); Assert.assertTrue(accessTypes.contains("a")); Assert.assertTrue(accessTypes.contains("b ")); - Assert.assertTrue(accessTypes.contains(" c")); - Assert.assertTrue(accessTypes.contains(" d ")); + Assert.assertTrue(accessTypes.contains(" C")); + Assert.assertTrue(accessTypes.contains(" D ")); } @Test @@ -358,36 +355,6 @@ public void test_getValidationRegExes() { Assert.assertEquals("regex3", regExMap.get("f")); } - @Test - public void test_getPolicyResources() { - - Set result; - RangerPolicy policy = null; - // null policy - result = _validator.getPolicyResources(null); - Assert.assertTrue(result != null); - Assert.assertTrue(result.isEmpty()); - // null resource map - policy = mock(RangerPolicy.class); - when(policy.getResources()).thenReturn(null); - result = _validator.getPolicyResources(null); - Assert.assertTrue(result != null); - Assert.assertTrue(result.isEmpty()); - // empty resource map - Map input = Maps.newHashMap(); - when(policy.getResources()).thenReturn(input); - result = _validator.getPolicyResources(policy); - Assert.assertTrue(result != null); - Assert.assertTrue(result.isEmpty()); - // known resource map - input.put("r1", mock(RangerPolicyResource.class)); - input.put("R2", mock(RangerPolicyResource.class)); - result = _validator.getPolicyResources(policy); - Assert.assertEquals(2, result.size()); - Assert.assertTrue("r1", result.contains("r1")); - Assert.assertTrue("R2", result.contains("r2")); // result should lowercase the resource-names - } - @Test public void test_getIsAuditEnabled() { // null policy diff --git a/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java b/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java index 773c2edffc..52d0130430 100644 --- a/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java +++ b/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java @@ -64,6 +64,10 @@ import org.apache.ranger.common.ContextUtil; import org.apache.ranger.common.MessageEnums; import org.apache.ranger.common.db.RangerTransactionSynchronizationAdapter; +import org.apache.ranger.entity.*; +import org.apache.ranger.plugin.model.validation.RangerServiceDefValidator; +import org.apache.ranger.plugin.model.validation.RangerValidator; +import org.apache.ranger.plugin.model.validation.ValidationFailureDetails; import org.apache.ranger.plugin.policyengine.RangerPolicyEngine; import org.apache.ranger.plugin.policyresourcematcher.RangerDefaultPolicyResourceMatcher; import org.apache.ranger.plugin.policyresourcematcher.RangerPolicyResourceMatcher; @@ -353,9 +357,21 @@ public RangerServiceDef createServiceDef(RangerServiceDef serviceDef) throws Exc + serviceDef.getName() + " already exists", MessageEnums.ERROR_DUPLICATE_OBJECT); } - + List configs = serviceDef.getConfigs(); List resources = serviceDef.getResources(); + + if (CollectionUtils.isNotEmpty(resources)) { + RangerServiceDefValidator validator = new RangerServiceDefValidator(this); + List failures = new ArrayList<>(); + boolean isValidResources = validator.isValidResources(serviceDef, failures, RangerValidator.Action.CREATE); + if (!isValidResources) { + throw restErrorUtil.createRESTException("service-def with name: " + + serviceDef.getName() + " has invalid resources:[" + failures.toString() + "]", + MessageEnums.INVALID_INPUT_DATA); + } + } + List accessTypes = serviceDef.getAccessTypes(); List policyConditions = serviceDef.getPolicyConditions(); List contextEnrichers = serviceDef.getContextEnrichers(); From 39df85e72d6037ee80180719b1df84c8f9fdca4d Mon Sep 17 00:00:00 2001 From: Sailaja Polavarapu Date: Tue, 13 Nov 2018 16:22:01 -0800 Subject: [PATCH 146/151] RANGER-2049: Added support for doAs for Ranger REST APIs with Kerberized mode --- .../filter/RangerKRBAuthenticationFilter.java | 132 ++++++++++++++---- 1 file changed, 105 insertions(+), 27 deletions(-) diff --git a/security-admin/src/main/java/org/apache/ranger/security/web/filter/RangerKRBAuthenticationFilter.java b/security-admin/src/main/java/org/apache/ranger/security/web/filter/RangerKRBAuthenticationFilter.java index 11bc9e294c..519071e030 100644 --- a/security-admin/src/main/java/org/apache/ranger/security/web/filter/RangerKRBAuthenticationFilter.java +++ b/security-admin/src/main/java/org/apache/ranger/security/web/filter/RangerKRBAuthenticationFilter.java @@ -32,6 +32,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.Collections; import javax.servlet.Filter; import javax.servlet.FilterChain; @@ -53,6 +54,13 @@ import javax.servlet.http.HttpServletResponse; import org.apache.commons.collections.iterators.IteratorEnumeration; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.SaslRpcServer; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.authentication.server.AuthenticationToken; +import org.apache.hadoop.security.authorize.AuthorizationException; +import org.apache.hadoop.security.authorize.ProxyUsers; +import org.apache.hadoop.util.HttpExceptionUtils; import org.apache.ranger.biz.UserMgr; import org.apache.ranger.common.PropertiesUtil; import org.apache.ranger.common.RESTErrorUtil; @@ -98,6 +106,8 @@ public class RangerKRBAuthenticationFilter extends RangerKrbFilter { static final String RANGER_AUTH_TYPE = "hadoop.security.authentication"; static final String AUTH_COOKIE_NAME = "hadoop.auth"; static final String HOST_NAME = "ranger.service.host"; + static final String ALLOW_TRUSTED_PROXY = "ranger.authentication.allow.trustedproxy"; + static final String PROXY_PREFIX = "ranger.proxyuser."; private static final String KERBEROS_TYPE = "kerberos"; private static final String S_USER = "suser"; @@ -119,6 +129,7 @@ public void init(FilterConfig conf) throws ServletException { params.put(TOKEN_VALID_PARAM, PropertiesUtil.getProperty(TOKEN_VALID,"30")); params.put(COOKIE_DOMAIN_PARAM, PropertiesUtil.getProperty(COOKIE_DOMAIN, PropertiesUtil.getProperty(HOST_NAME, "localhost"))); params.put(COOKIE_PATH_PARAM, PropertiesUtil.getProperty(COOKIE_PATH, "/")); + params.put(ALLOW_TRUSTED_PROXY, PropertiesUtil.getProperty(ALLOW_TRUSTED_PROXY, "false")); try { params.put(PRINCIPAL_PARAM, SecureClientLogin.getPrincipal(PropertiesUtil.getProperty(PRINCIPAL,""), PropertiesUtil.getProperty(HOST_NAME))); } catch (IOException ignored) { @@ -153,6 +164,20 @@ public String getFilterName() { } }; super.init(myConf); + Configuration conf1 = this.getProxyuserConfiguration(); + ProxyUsers.refreshSuperUserGroupsConfiguration(conf1, PROXY_PREFIX); + } + + protected Configuration getProxyuserConfiguration() { + Configuration conf = new Configuration(false); + Map propertiesMap = PropertiesUtil.getPropertiesMap(); + for (String key : propertiesMap.keySet()) { + if (!key.startsWith(PROXY_PREFIX)) { + continue; + } + conf.set(key, propertiesMap.get(key)); + } + return conf; } @Override @@ -162,6 +187,7 @@ protected void doFilter(FilterChain filterChain, String authType = PropertiesUtil.getProperty(RANGER_AUTH_TYPE); String userName = null; boolean checkCookie = response.containsHeader("Set-Cookie"); + boolean allowTrustedProxy = PropertiesUtil.getBooleanProperty(ALLOW_TRUSTED_PROXY, false); if(checkCookie){ Collection authUserName = response.getHeaders("Set-Cookie"); if(authUserName != null){ @@ -200,46 +226,98 @@ protected void doFilter(FilterChain filterChain, userName = sessionUserName; } + if(LOG.isDebugEnabled()) { + LOG.debug("Remote user from request = " + request.getRemoteUser()); + } + if((isSpnegoEnable(authType) && (!StringUtils.isEmpty(userName)))){ Authentication existingAuth = SecurityContextHolder.getContext().getAuthentication(); if(existingAuth == null || !existingAuth.isAuthenticated()){ //--------------------------- To Create Ranger Session -------------------------------------- String rangerLdapDefaultRole = PropertiesUtil.getProperty("ranger.ldap.default.role", "ROLE_USER"); - //if we get the userName from the token then log into ranger using the same user - final List grantedAuths = new ArrayList<>(); - grantedAuths.add(new SimpleGrantedAuthority(rangerLdapDefaultRole)); - final UserDetails principal = new User(userName, "",grantedAuths); - final Authentication finalAuthentication = new UsernamePasswordAuthenticationToken(principal, "", grantedAuths); - WebAuthenticationDetails webDetails = new WebAuthenticationDetails(request); - ((AbstractAuthenticationToken) finalAuthentication).setDetails(webDetails); - RangerAuthenticationProvider authenticationProvider = new RangerAuthenticationProvider(); - Authentication authentication = authenticationProvider.authenticate(finalAuthentication); - authentication = getGrantedAuthority(authentication); - if(authentication != null && authentication.isAuthenticated()) { - if (request.getParameterMap().containsKey("doAs")) { - if(!response.isCommitted()) { + if(LOG.isDebugEnabled()) { + LOG.debug("Http headers: " + Collections.list(request.getHeaderNames()).toString()); + } + String doAsUser = request.getParameter("doAs"); + + if (allowTrustedProxy && doAsUser != null && !doAsUser.isEmpty()) { + if(LOG.isDebugEnabled()) { + LOG.debug("userPrincipal from request = " + request.getUserPrincipal() + " request paramerters = " + request.getParameterMap().keySet()); + } + AuthenticationToken authToken = (AuthenticationToken)request.getUserPrincipal(); + if(authToken != null && authToken != AuthenticationToken.ANONYMOUS) { + if(LOG.isDebugEnabled()) { + LOG.debug("remote user from authtoken = " + authToken.getUserName()); + } + UserGroupInformation ugi = UserGroupInformation.createRemoteUser(authToken.getUserName(), SaslRpcServer.AuthMethod.KERBEROS); + if(ugi != null) { + ugi = UserGroupInformation.createProxyUser(doAsUser, ugi); if(LOG.isDebugEnabled()) { - LOG.debug("Request contains unsupported parameter, doAs."); + LOG.debug("Real user from UGI = " + ugi.getRealUser().getShortUserName()); + } + + try { + ProxyUsers.authorize(ugi, request.getRemoteAddr()); + } catch (AuthorizationException ex) { + HttpExceptionUtils.createServletExceptionResponse(response, 403, ex); + if(LOG.isDebugEnabled()) { + LOG.debug("Authentication exception: " + ex.getMessage(), ex); + } else { + LOG.warn("Authentication exception: " + ex.getMessage()); + } + return; } - request.setAttribute("spnegoenabled", false); - response.sendError(HttpServletResponse.SC_FORBIDDEN, "Missing authentication token."); + final List grantedAuths = new ArrayList<>(); + grantedAuths.add(new SimpleGrantedAuthority(rangerLdapDefaultRole)); + final UserDetails principal = new User(doAsUser, "", grantedAuths); + final Authentication finalAuthentication = new UsernamePasswordAuthenticationToken(principal, "", grantedAuths); + WebAuthenticationDetails webDetails = new WebAuthenticationDetails(request); + ((AbstractAuthenticationToken) finalAuthentication).setDetails(webDetails); + SecurityContextHolder.getContext().setAuthentication(finalAuthentication); + request.setAttribute("spnegoEnabled", true); } + } - if(request.getParameterMap().containsKey("user.name")) { - if(!response.isCommitted()) { - if(LOG.isDebugEnabled()) { - LOG.debug("Request contains an unsupported parameter user.name"); + LOG.info("Logged into Ranger as doAsUser = " + doAsUser + ", by authenticatedUser=" + authToken.getUserName()); + + + }else { + //if we get the userName from the token then log into ranger using the same user + final List grantedAuths = new ArrayList<>(); + grantedAuths.add(new SimpleGrantedAuthority(rangerLdapDefaultRole)); + final UserDetails principal = new User(userName, "", grantedAuths); + final Authentication finalAuthentication = new UsernamePasswordAuthenticationToken(principal, "", grantedAuths); + WebAuthenticationDetails webDetails = new WebAuthenticationDetails(request); + ((AbstractAuthenticationToken) finalAuthentication).setDetails(webDetails); + RangerAuthenticationProvider authenticationProvider = new RangerAuthenticationProvider(); + Authentication authentication = authenticationProvider.authenticate(finalAuthentication); + authentication = getGrantedAuthority(authentication); + if (authentication != null && authentication.isAuthenticated()) { + if (request.getParameterMap().containsKey("doAs")) { + if (!response.isCommitted()) { + if (LOG.isDebugEnabled()) { + LOG.debug("Request contains unsupported parameter, doAs."); + } + request.setAttribute("spnegoenabled", false); + response.sendError(HttpServletResponse.SC_FORBIDDEN, "Missing authentication token."); + } + } + if (request.getParameterMap().containsKey("user.name")) { + if (!response.isCommitted()) { + if (LOG.isDebugEnabled()) { + LOG.debug("Request contains an unsupported parameter user.name"); + } + request.setAttribute("spnegoenabled", false); + response.sendError(HttpServletResponse.SC_FORBIDDEN, "Missing authentication token."); + } else { + LOG.info("Response seems to be already committed for user.name."); } - request.setAttribute("spnegoenabled", false); - response.sendError(HttpServletResponse.SC_FORBIDDEN, "Missing authentication token."); - } else { - LOG.info("Response seems to be already committed for user.name."); } } + SecurityContextHolder.getContext().setAuthentication(authentication); + request.setAttribute("spnegoEnabled", true); + LOG.info("Logged into Ranger as = " + userName); } - SecurityContextHolder.getContext().setAuthentication(authentication); - request.setAttribute("spnegoEnabled", true); - LOG.info("Logged into Ranger as = "+userName); filterChain.doFilter(request, response); }else{ try{ From ad9ae7656b79f54a5cb1d9918d859508a175e170 Mon Sep 17 00:00:00 2001 From: Sailaja Polavarapu Date: Fri, 16 Nov 2018 13:39:30 -0800 Subject: [PATCH 147/151] RANGER-2049: Fixed an issue where doAs User role is not set properly --- .../web/filter/RangerKRBAuthenticationFilter.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/security-admin/src/main/java/org/apache/ranger/security/web/filter/RangerKRBAuthenticationFilter.java b/security-admin/src/main/java/org/apache/ranger/security/web/filter/RangerKRBAuthenticationFilter.java index 519071e030..02020f2a9e 100644 --- a/security-admin/src/main/java/org/apache/ranger/security/web/filter/RangerKRBAuthenticationFilter.java +++ b/security-admin/src/main/java/org/apache/ranger/security/web/filter/RangerKRBAuthenticationFilter.java @@ -270,16 +270,16 @@ protected void doFilter(FilterChain filterChain, final List grantedAuths = new ArrayList<>(); grantedAuths.add(new SimpleGrantedAuthority(rangerLdapDefaultRole)); final UserDetails principal = new User(doAsUser, "", grantedAuths); - final Authentication finalAuthentication = new UsernamePasswordAuthenticationToken(principal, "", grantedAuths); + Authentication authentication = new UsernamePasswordAuthenticationToken(principal, "", grantedAuths); WebAuthenticationDetails webDetails = new WebAuthenticationDetails(request); - ((AbstractAuthenticationToken) finalAuthentication).setDetails(webDetails); - SecurityContextHolder.getContext().setAuthentication(finalAuthentication); + ((AbstractAuthenticationToken) authentication).setDetails(webDetails); + authentication = getGrantedAuthority(authentication); + SecurityContextHolder.getContext().setAuthentication(authentication); request.setAttribute("spnegoEnabled", true); + LOG.info("Logged into Ranger as doAsUser = " + doAsUser + ", by authenticatedUser=" + authToken.getUserName()); } } - LOG.info("Logged into Ranger as doAsUser = " + doAsUser + ", by authenticatedUser=" + authToken.getUserName()); - }else { //if we get the userName from the token then log into ranger using the same user From 40746add620ba22947f6eaf976567aeb39e5c55b Mon Sep 17 00:00:00 2001 From: Sailaja Polavarapu Date: Mon, 17 Dec 2018 12:48:19 -0800 Subject: [PATCH 148/151] RANGER-2074: Update netty version in ranger - ranger-0.7 branch --- pom.xml | 2 +- security-admin/pom.xml | 11 ++++++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/pom.xml b/pom.xml index c415e538d7..787e7d1a78 100644 --- a/pom.xml +++ b/pom.xml @@ -194,7 +194,7 @@ 3.0.2 1.8.4 5.1.31 - 3.6.2.Final + 3.10.5.Final 0.6 r239 2.3 diff --git a/security-admin/pom.xml b/security-admin/pom.xml index c20f277c1b..51d958e466 100644 --- a/security-admin/pom.xml +++ b/security-admin/pom.xml @@ -218,9 +218,14 @@ ${sun.jersey.core.version} - org.apache.hadoop - hadoop-mapreduce-client-core - ${hadoop.version} + org.apache.hadoop + hadoop-mapreduce-client-core + ${hadoop.version} + + + io.netty + netty + ${netty.version} com.sun.jersey.contribs From df1eb588465b4a8d31cfb04f5a62d8f9444e9cec Mon Sep 17 00:00:00 2001 From: Gautam Borad Date: Thu, 21 Feb 2019 11:52:17 +0530 Subject: [PATCH 149/151] RANGER-2331 Ranger-KMS - KeySecure HSM Integration --- kms/config/kms-webapp/dbks-site.xml | 51 ++++++ kms/scripts/DBMKTOKEYSECURE.sh | 19 ++ kms/scripts/KEYSECUREMKTOKMSDB.sh | 19 ++ kms/scripts/install.properties | 10 ++ kms/scripts/setup.sh | 81 +++++++++ .../hadoop/crypto/key/DBToKeySecure.java | 127 ++++++++++++++ .../hadoop/crypto/key/JKS2RangerUtil.java | 51 +++++- .../crypto/key/KeySecureToRangerDBMKUtil.java | 104 +++++++++++ .../hadoop/crypto/key/Ranger2JKSUtil.java | 60 ++++++- .../crypto/key/RangerKeyStoreProvider.java | 162 +++++++++++------- .../hadoop/crypto/key/RangerMasterKey.java | 11 ++ .../crypto/key/RangerSafenetKeySecure.java | 156 +++++++++++++++++ src/main/assembly/kms.xml | 2 + 13 files changed, 782 insertions(+), 71 deletions(-) create mode 100644 kms/scripts/DBMKTOKEYSECURE.sh create mode 100644 kms/scripts/KEYSECUREMKTOKMSDB.sh create mode 100644 kms/src/main/java/org/apache/hadoop/crypto/key/DBToKeySecure.java create mode 100644 kms/src/main/java/org/apache/hadoop/crypto/key/KeySecureToRangerDBMKUtil.java create mode 100644 kms/src/main/java/org/apache/hadoop/crypto/key/RangerSafenetKeySecure.java diff --git a/kms/config/kms-webapp/dbks-site.xml b/kms/config/kms-webapp/dbks-site.xml index 0e0f2eca9b..22480322c1 100755 --- a/kms/config/kms-webapp/dbks-site.xml +++ b/kms/config/kms-webapp/dbks-site.xml @@ -123,6 +123,57 @@ ranger.ks.kerberos.keytab + + + + + ranger.kms.keysecure.enabled + false + + + + + ranger.kms.keysecure.UserPassword.Authentication + true + + + + ranger.kms.keysecure.masterkey.name + safenetmasterkey + Safenet key secure master key name + + + ranger.kms.keysecure.login.username + user1 + Safenet key secure username + + + ranger.kms.keysecure.login.password + t1e2s3t4 + Safenet key secure user password + + + ranger.kms.keysecure.login.password.alias + ranger.ks.login.password + Safenet key secure user password + + + ranger.kms.keysecure.hostname + SunPKCS11-keysecurehn + Safenet key secure hostname + + + ranger.kms.keysecure.masterkey.size + 256 + key size + + + ranger.kms.keysecure.sunpkcs11.cfg.filepath + /opt/safenetConf/64/8.3.1/sunpkcs11.cfg + Location of Safenet key secure library configuration file + + + diff --git a/kms/scripts/DBMKTOKEYSECURE.sh b/kms/scripts/DBMKTOKEYSECURE.sh new file mode 100644 index 0000000000..c0aa6e58c2 --- /dev/null +++ b/kms/scripts/DBMKTOKEYSECURE.sh @@ -0,0 +1,19 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ------------------------------------------------------------------------------------- +RANGER_KMS_HOME=`dirname $0` +cp="${RANGER_KMS_HOME}/cred/lib/*:${RANGER_KMS_HOME}/./ews/webapp/WEB-INF/classes/conf/:${RANGER_KMS_HOME}/ews/webapp/config:${RANGER_KMS_HOME}/ews/lib/*:${RANGER_KMS_HOME}/ews/webapp/lib/*:${RANGER_KMS_HOME}/ews/webapp/META-INF" +java -cp "${cp}" org.apache.hadoop.crypto.key.DBToKeySecure ${1} ${2} ${3} ${4} diff --git a/kms/scripts/KEYSECUREMKTOKMSDB.sh b/kms/scripts/KEYSECUREMKTOKMSDB.sh new file mode 100644 index 0000000000..ba0a8a98b2 --- /dev/null +++ b/kms/scripts/KEYSECUREMKTOKMSDB.sh @@ -0,0 +1,19 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ------------------------------------------------------------------------------------- +RANGER_KMS_HOME=`dirname $0` +cp="${RANGER_KMS_HOME}/cred/lib/*:${RANGER_KMS_HOME}/./ews/webapp/WEB-INF/classes/conf/:${RANGER_KMS_HOME}/ews/webapp/config:${RANGER_KMS_HOME}/ews/lib/*:${RANGER_KMS_HOME}/ews/webapp/lib/*:${RANGER_KMS_HOME}/ews/webapp/META-INF" +java -cp "${cp}" org.apache.hadoop.crypto.key.KeySecureToRangerDBMKUtil ${1} diff --git a/kms/scripts/install.properties b/kms/scripts/install.properties index ddc779dad5..a4fda7e232 100755 --- a/kms/scripts/install.properties +++ b/kms/scripts/install.properties @@ -89,6 +89,16 @@ HSM_ENABLED=false HSM_PARTITION_NAME=par19 HSM_PARTITION_PASSWORD=S@fenet123 +#------------------------- Ranger SAFENET KEYSECURE CONFIG ------------------------------ +KEYSECURE_ENABLED=false +KEYSECURE_USER_PASSWORD_AUTHENTICATION=true +KEYSECURE_MASTERKEY_NAME=safenetkeysecure +KEYSECURE_USERNAME=user1 +KEYSECURE_PASSWORD=t1e2s3t4 +KEYSECURE_HOSTNAME=SunPKCS11-keysecurehn +KEYSECURE_MASTER_KEY_SIZE=256 +KEYSECURE_LIB_CONFIG_PATH=/opt/safenetConf/64/8.3.1/sunpkcs11.cfg + # # ------- UNIX User CONFIG ---------------- # diff --git a/kms/scripts/setup.sh b/kms/scripts/setup.sh index 2db05b8628..39afa5ae0c 100755 --- a/kms/scripts/setup.sh +++ b/kms/scripts/setup.sh @@ -98,6 +98,15 @@ HSM_ENABLED=$(get_prop 'HSM_ENABLED' $PROPFILE) HSM_PARTITION_NAME=$(get_prop 'HSM_PARTITION_NAME' $PROPFILE) HSM_PARTITION_PASSWORD=$(get_prop 'HSM_PARTITION_PASSWORD' $PROPFILE) +KEYSECURE_ENABLED=$(get_prop 'KEYSECURE_ENABLED' $PROPFILE) +KEYSECURE_USER_PASSWORD_AUTHENTICATION=$(get_prop 'KEYSECURE_USER_PASSWORD_AUTHENTICATION' $PROPFILE) +KEYSECURE_MASTERKEY_NAME=$(get_prop 'KEYSECURE_MASTERKEY_NAME' $PROPFILE) +KEYSECURE_USERNAME=$(get_prop 'KEYSECURE_USERNAME' $PROPFILE) +KEYSECURE_PASSWORD=$(get_prop 'KEYSECURE_PASSWORD' $PROPFILE) +KEYSECURE_HOSTNAME=$(get_prop 'KEYSECURE_HOSTNAME' $PROPFILE) +KEYSECURE_MASTER_KEY_SIZE=$(get_prop 'KEYSECURE_MASTER_KEY_SIZE' $PROPFILE) +KEYSECURE_LIB_CONFIG_PATH=$(get_prop 'KEYSECURE_LIB_CONFIG_PATH' $PROPFILE) + kms_principal=$(get_prop 'kms_principal' $PROPFILE) kms_keytab=$(get_prop 'kms_keytab' $PROPFILE) hadoop_conf=$(get_prop 'hadoop_conf' $PROPFILE) @@ -210,6 +219,17 @@ password_validation() { fi fi } + +password_validation_safenet_keysecure(){ + if [ -z "$1" ] + then + log "[I] Blank password is not allowed for" $2". Please enter valid password." + exit 1 + else + log "[I]" $2 "password validated." + fi +} + init_variables(){ curDt=`date '+%Y%m%d%H%M%S'` @@ -574,7 +594,11 @@ update_properties() { HSM_PARTITION_PASSWD="ranger.ks.hsm.partition.password" HSM_PARTITION_PASSWORD_ALIAS="ranger.kms.hsm.partition.password" + KEYSECURE_PASSWD="ranger.kms.keysecure.login.password" + KEYSECURE_PASSWORD_ALIAS="ranger.ks.login.password" + HSM_ENABLED=`echo $HSM_ENABLED | tr '[:lower:]' '[:upper:]'` + KEYSECURE_ENABLED=`echo $KEYSECURE_ENABLED | tr '[:lower:]' '[:upper:]'` if [ "${keystore}" != "" ] then @@ -600,6 +624,20 @@ update_properties() { updatePropertyToFilePy $propertyName $newPropertyValue $to_file fi + if [ "${KEYSECURE_ENABLED}" == "TRUE" ] + then + password_validation_safenet_keysecure "$KEYSECURE_PASSWORD" "KEYSECURE User Password" + $PYTHON_COMMAND_INVOKER ranger_credential_helper.py -l "cred/lib/*" -f "$keystore" -k "${KEYSECURE_PASSWORD_ALIAS}" -v "${KEYSECURE_PASSWORD}" -c 1 + + propertyName=ranger.kms.keysecure.login.password.alias + newPropertyValue="${KEYSECURE_PASSWORD_ALIAS}" + updatePropertyToFilePy $propertyName $newPropertyValue $to_file + + propertyName=ranger.kms.keysecure.login.password + newPropertyValue="_" + updatePropertyToFilePy $propertyName $newPropertyValue $to_file + fi + propertyName=ranger.ks.jpa.jdbc.credential.alias newPropertyValue="${DB_CREDENTIAL_ALIAS}" updatePropertyToFilePy $propertyName $newPropertyValue $to_file @@ -631,6 +669,10 @@ update_properties() { propertyName="${HSM_PARTITION_PASSWD}" newPropertyValue="${HSM_PARTITION_PASSWORD}" updatePropertyToFilePy $propertyName $newPropertyValue $to_file + + propertyName="${KEYSECURE_PASSWD}" + newPropertyValue="${KEYSECURE_PASSWORD}" + updatePropertyToFilePy $propertyName $newPropertyValue $to_file fi if test -f $keystore; then @@ -695,6 +737,45 @@ update_properties() { updatePropertyToFilePy $propertyName $newPropertyValue $to_file fi + ########### SAFENET KEYSECURE CONFIG ################# + + + if [ "${KEYSECURE_ENABLED}" != "TRUE" ] + then + propertyName=ranger.kms.keysecure.enabled + newPropertyValue="false" + updatePropertyToFilePy $propertyName $newPropertyValue $to_file + else + propertyName=ranger.kms.keysecure.enabled + newPropertyValue="true" + updatePropertyToFilePy $propertyName $newPropertyValue $to_file + + propertyName=ranger.kms.keysecure.UserPassword.Authentication + newPropertyValue="${KEYSECURE_USER_PASSWORD_AUTHENTICATION}" + updatePropertyToFilePy $propertyName $newPropertyValue $to_file + + propertyName=ranger.kms.keysecure.masterkey.name + newPropertyValue="${KEYSECURE_MASTERKEY_NAME}" + updatePropertyToFilePy $propertyName $newPropertyValue $to_file + + propertyName=ranger.kms.keysecure.login.username + newPropertyValue="${KEYSECURE_USERNAME}" + updatePropertyToFilePy $propertyName $newPropertyValue $to_file + + propertyName=ranger.kms.keysecure.hostname + newPropertyValue="${KEYSECURE_HOSTNAME}" + updatePropertyToFilePy $propertyName $newPropertyValue $to_file + + propertyName=ranger.kms.keysecure.masterkey.size + newPropertyValue="${KEYSECURE_MASTER_KEY_SIZE}" + updatePropertyToFilePy $propertyName $newPropertyValue $to_file + + propertyName=ranger.kms.keysecure.sunpkcs11.cfg.filepath + newPropertyValue="${KEYSECURE_LIB_CONFIG_PATH}" + updatePropertyToFilePy $propertyName $newPropertyValue $to_file + + fi + to_file_kms_site=$PWD/ews/webapp/WEB-INF/classes/conf/ranger-kms-site.xml if test -f $to_file_kms_site; then log "[I] $to_file_kms_site file found" diff --git a/kms/src/main/java/org/apache/hadoop/crypto/key/DBToKeySecure.java b/kms/src/main/java/org/apache/hadoop/crypto/key/DBToKeySecure.java new file mode 100644 index 0000000000..ab0baa1dd4 --- /dev/null +++ b/kms/src/main/java/org/apache/hadoop/crypto/key/DBToKeySecure.java @@ -0,0 +1,127 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.crypto.key; + +import java.io.IOException; +import org.apache.hadoop.conf.Configuration; +import org.apache.ranger.kms.dao.DaoManager; + +import com.sun.org.apache.xml.internal.security.utils.Base64; + +public class DBToKeySecure { + + private static final String ENCRYPTION_KEY = "ranger.db.encrypt.key.password"; + private static final String KEYSECURE_MASTERKEY_NAME = "ranger.kms.keysecure.masterkey.name"; + private static final String KEYSECURE_LOGIN = "ranger.kms.keysecure.login"; + private static final String CFGFILEPATH = "ranger.kms.keysecure.sunpkcs11.cfg.filepath"; + + public static void showUsage() { + System.err + .println("USAGE: java " + + DBToKeySecure.class.getName() + + " "); + } + + public static void main(String[] args) { + + if (args.length < 4) { + System.err.println("Invalid number of parameters found."); + showUsage(); + System.exit(1); + } else { + + Configuration conf = RangerKeyStoreProvider.getDBKSConf(); + + String keyName = args[0]; + if (keyName == null || keyName.trim().isEmpty()) { + System.err.println("Key Secure master key name not provided."); + showUsage(); + System.exit(1); + } + + String username = args[1]; + if (username == null || username.trim().isEmpty()) { + System.err.println("Key Secure username not provided."); + showUsage(); + System.exit(1); + } + String password = args[2]; + if (password == null || password.trim().isEmpty()) { + System.err.println("Key Secure password not provided."); + showUsage(); + System.exit(1); + } + + String cfgFilePath = args[3]; + if (cfgFilePath == null || cfgFilePath.trim().isEmpty()) { + System.err.println("sunpkcs11 Configuration File Path not provided"); + showUsage(); + System.exit(1); + } + + boolean result = new DBToKeySecure().doExportMKToKeySecure(keyName, username, password, cfgFilePath, conf); + if (result) { + System.out + .println("Master Key from Ranger KMS DB has been successfully imported into Key Secure."); + } else { + System.out + .println("Import of Master Key from DB has been unsuccessful."); + System.exit(1); + } + System.exit(0); + } + } + + private boolean doExportMKToKeySecure(String keyName, String username, String password, String cfgFilePath, Configuration conf) { + try { + String keySecureMKPassword = conf.get(ENCRYPTION_KEY); + if (keySecureMKPassword == null + || keySecureMKPassword.trim().equals("") + || keySecureMKPassword.trim().equals("_") + || keySecureMKPassword.trim().equals("crypted")) { + throw new IOException("Master Key Jceks does not exists"); + } + + conf.set(CFGFILEPATH, cfgFilePath); + conf.set(KEYSECURE_MASTERKEY_NAME, keyName); + conf.set(KEYSECURE_LOGIN,username + ":" + password); + + RangerKMSDB rangerkmsDb = new RangerKMSDB(conf); + DaoManager daoManager = rangerkmsDb.getDaoManager(); + String mkPassword = conf.get(ENCRYPTION_KEY); + + // Get Master Key from Ranger DB + RangerMasterKey rangerMasterKey = new RangerMasterKey(daoManager); + String mkey = rangerMasterKey.getMasterKey(mkPassword); + byte[] key = Base64.decode(mkey); + + if (conf != null) { + RangerSafenetKeySecure rangerSafenetKeySecure = new RangerSafenetKeySecure( + conf); + return rangerSafenetKeySecure.setMasterKey(password, key,conf); + } + + return false; + } catch (Throwable t) { + throw new RuntimeException( + "Unable to import Master key from Ranger DB to KeySecure ", + t); + } + + } + +} diff --git a/kms/src/main/java/org/apache/hadoop/crypto/key/JKS2RangerUtil.java b/kms/src/main/java/org/apache/hadoop/crypto/key/JKS2RangerUtil.java index 22dce0f5fa..ee0913f8c4 100644 --- a/kms/src/main/java/org/apache/hadoop/crypto/key/JKS2RangerUtil.java +++ b/kms/src/main/java/org/apache/hadoop/crypto/key/JKS2RangerUtil.java @@ -26,13 +26,22 @@ import java.security.KeyStore; import java.security.KeyStoreException; +import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; +import org.apache.ranger.credentialapi.CredentialReader; import org.apache.ranger.kms.dao.DaoManager; public class JKS2RangerUtil { private static final String DEFAULT_KEYSTORE_TYPE = "jceks"; private static final String ENCRYPTION_KEY = "ranger.db.encrypt.key.password"; + private static final String KEYSECURE_ENABLED = "ranger.kms.keysecure.enabled"; + private static final String KEYSECURE_USERNAME = "ranger.kms.keysecure.login.username"; + private static final String KEYSECURE_PASSWORD = "ranger.kms.keysecure.login.password"; + private static final String KEYSECURE_PASSWORD_ALIAS = "ranger.kms.keysecure.login.password.alias"; + private static final String KEYSECURE_LOGIN = "ranger.kms.keysecure.login"; + private static final String CREDENTIAL_PATH = "ranger.ks.jpa.jdbc.credential.provider.path"; + public static void showUsage() { System.err.println("USAGE: java " + JKS2RangerUtil.class.getName() + " [KeyStoreType]"); @@ -80,12 +89,31 @@ private void doImportKeysFromJKS(String keyStoreFileName, String keyStoreType) { Configuration conf = RangerKeyStoreProvider.getDBKSConf(); RangerKMSDB rangerkmsDb = new RangerKMSDB(conf); DaoManager daoManager = rangerkmsDb.getDaoManager(); - RangerKeyStore dbStore = new RangerKeyStore(daoManager); + RangerKeyStore dbStore= new RangerKeyStore(daoManager); + char[] masterKey; String password = conf.get(ENCRYPTION_KEY); - RangerMasterKey rangerMasterKey = new RangerMasterKey(daoManager); - rangerMasterKey.generateMasterKey(password); - char[] masterKey = rangerMasterKey.getMasterKey(password).toCharArray(); InputStream in = null; + + if (conf != null + && StringUtils.isNotEmpty(conf.get(KEYSECURE_ENABLED)) + && conf.get(KEYSECURE_ENABLED).equalsIgnoreCase("true")) { + + getFromJceks(conf, CREDENTIAL_PATH, KEYSECURE_PASSWORD_ALIAS, KEYSECURE_PASSWORD); + String keySecureLoginCred = conf.get(KEYSECURE_USERNAME).trim() + ":" + conf.get(KEYSECURE_PASSWORD); + conf.set(KEYSECURE_LOGIN, keySecureLoginCred); + + RangerSafenetKeySecure rangerSafenetKeySecure = new RangerSafenetKeySecure( + conf); + rangerSafenetKeySecure.generateMasterKey(password); + masterKey = rangerSafenetKeySecure.getMasterKey(password).toCharArray(); + } else { + RangerMasterKey rangerMasterKey = new RangerMasterKey( + daoManager); + rangerMasterKey.generateMasterKey(password); + masterKey = rangerMasterKey.getMasterKey(password) + .toCharArray(); + } + try { in = new FileInputStream(new File(keyStoreFileName)); dbStore.engineLoadKeyStoreFile(in, keyStorePassword, keyPassword, masterKey, keyStoreType); @@ -105,6 +133,21 @@ private void doImportKeysFromJKS(String keyStoreFileName, String keyStoreType) { throw new RuntimeException("Unable to import keys from [" + keyStoreFileName + "] due to exception.", t); } } + private static void getFromJceks(Configuration conf, String path, String alias, String key) { + + //update credential from keystore + if (conf != null) { + String pathValue = conf.get(path); + String aliasValue = conf.get(alias); + if (pathValue != null && aliasValue != null) { + String xaDBPassword = CredentialReader.getDecryptedString(pathValue.trim(), aliasValue.trim()); + if (xaDBPassword != null && !xaDBPassword.trim().isEmpty() && + !xaDBPassword.trim().equalsIgnoreCase("none")) { + conf.set(key, xaDBPassword); + } + } + } + } private char[] getPasswordFromConsole(String prompt) throws IOException { diff --git a/kms/src/main/java/org/apache/hadoop/crypto/key/KeySecureToRangerDBMKUtil.java b/kms/src/main/java/org/apache/hadoop/crypto/key/KeySecureToRangerDBMKUtil.java new file mode 100644 index 0000000000..538fde95e6 --- /dev/null +++ b/kms/src/main/java/org/apache/hadoop/crypto/key/KeySecureToRangerDBMKUtil.java @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.crypto.key; + +import org.apache.hadoop.conf.Configuration; +import org.apache.ranger.credentialapi.CredentialReader; +import org.apache.ranger.kms.dao.DaoManager; + +import com.sun.org.apache.xml.internal.security.utils.Base64; + +public class KeySecureToRangerDBMKUtil { + private static final String ENCRYPTION_KEY = "ranger.db.encrypt.key.password"; + private static final String KEYSECURE_USERNAME = "ranger.kms.keysecure.login.username"; + private static final String KEYSECURE_PASSWORD = "ranger.kms.keysecure.login.password"; + private static final String KEYSECURE_PASSWORD_ALIAS = "ranger.kms.keysecure.login.password.alias"; + private static final String KEYSECURE_LOGIN = "ranger.kms.keysecure.login"; + private static final String CREDENTIAL_PATH = "ranger.ks.jpa.jdbc.credential.provider.path"; + + public static void showUsage() { + System.err.println("USAGE: java " + KeySecureToRangerDBMKUtil.class.getName() + " "); + } + + public static void main(String[] args) { + + if (args.length != 1) { + System.err.println("Invalid number of parameters found."); + showUsage(); + System.exit(1); + } + else { + String kmsMKPassword = args[0]; + if (kmsMKPassword == null || kmsMKPassword.trim().isEmpty()) { + System.err.println("KMS master key password not provided"); + showUsage(); + System.exit(1); + } + + new KeySecureToRangerDBMKUtil().doImportMKFromKeySecure(kmsMKPassword); + System.out.println("Master Key from Key Secure has been successfully imported into Ranger KMS DB."); + } + } + + private void doImportMKFromKeySecure(String kmsMKPassword) { + try { + Configuration conf = RangerKeyStoreProvider.getDBKSConf(); + conf.set(ENCRYPTION_KEY, kmsMKPassword); + getFromJceks(conf, CREDENTIAL_PATH, KEYSECURE_PASSWORD_ALIAS, KEYSECURE_PASSWORD); + String keySecureLoginCred = conf.get(KEYSECURE_USERNAME).trim() + ":" + conf.get(KEYSECURE_PASSWORD); + conf.set(KEYSECURE_LOGIN, keySecureLoginCred); + + RangerKMSDB rangerkmsDb = new RangerKMSDB(conf); + DaoManager daoManager = rangerkmsDb.getDaoManager(); + String password = conf.get(ENCRYPTION_KEY); + + RangerSafenetKeySecure rangerSafenetKeySecure = new RangerSafenetKeySecure( + conf); + String mKey = rangerSafenetKeySecure.getMasterKey(password); + + byte[] key = Base64.decode(mKey); + + // Put Master Key in Ranger DB + RangerMasterKey rangerMasterKey = new RangerMasterKey(daoManager); + rangerMasterKey.generateMKFromKeySecureMK(password, key); + + } catch (Throwable t) { + throw new RuntimeException( + "Unable to migrate Master key from KeySecure to Ranger DB", + t); + + } + } + + private static void getFromJceks(Configuration conf, String path, String alias, String key) { + + //update credential from keystore + if (conf != null) { + String pathValue = conf.get(path); + String aliasValue = conf.get(alias); + if (pathValue != null && aliasValue != null) { + String xaDBPassword = CredentialReader.getDecryptedString(pathValue.trim(), aliasValue.trim()); + if (xaDBPassword != null && !xaDBPassword.trim().isEmpty() && + !xaDBPassword.trim().equalsIgnoreCase("none")) { + conf.set(key, xaDBPassword); + } + } + } + } + + +} diff --git a/kms/src/main/java/org/apache/hadoop/crypto/key/Ranger2JKSUtil.java b/kms/src/main/java/org/apache/hadoop/crypto/key/Ranger2JKSUtil.java index 1abbf8e3cf..3a5404111f 100644 --- a/kms/src/main/java/org/apache/hadoop/crypto/key/Ranger2JKSUtil.java +++ b/kms/src/main/java/org/apache/hadoop/crypto/key/Ranger2JKSUtil.java @@ -26,13 +26,21 @@ import java.security.KeyStore; import java.security.KeyStoreException; +import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; +import org.apache.ranger.credentialapi.CredentialReader; import org.apache.ranger.kms.dao.DaoManager; public class Ranger2JKSUtil { private static final String DEFAULT_KEYSTORE_TYPE = "jceks"; private static final String ENCRYPTION_KEY = "ranger.db.encrypt.key.password"; + private static final String KEYSECURE_ENABLED = "ranger.kms.keysecure.enabled"; + private static final String KEYSECURE_USERNAME = "ranger.kms.keysecure.login.username"; + private static final String KEYSECURE_PASSWORD = "ranger.kms.keysecure.login.password"; + private static final String KEYSECURE_PASSWORD_ALIAS = "ranger.kms.keysecure.login.password.alias"; + private static final String KEYSECURE_LOGIN = "ranger.kms.keysecure.login"; + private static final String CREDENTIAL_PATH = "ranger.ks.jpa.jdbc.credential.provider.path"; public static void showUsage() { System.err.println("USAGE: java " + Ranger2JKSUtil.class.getName() + " [KeyStoreType]"); @@ -79,23 +87,43 @@ private void doExportKeysFromJKS(String keyStoreFileName, String keyStoreType) { char[] keyStorePassword = getPasswordFromConsole("Enter Password for the keystore FILE :"); char[] keyPassword = getPasswordFromConsole("Enter Password for the KEY(s) stored in the keystore:"); Configuration conf = RangerKeyStoreProvider.getDBKSConf(); - RangerKMSDB rangerkmsDb = new RangerKMSDB(conf); + RangerKMSDB rangerkmsDb = new RangerKMSDB(conf); DaoManager daoManager = rangerkmsDb.getDaoManager(); - RangerKeyStore dbStore = new RangerKeyStore(daoManager); + RangerKeyStore dbStore= new RangerKeyStore(daoManager); + + char[] masterKey; String password = conf.get(ENCRYPTION_KEY); - RangerMasterKey rangerMasterKey = new RangerMasterKey(daoManager); - char[] masterKey = rangerMasterKey.getMasterKey(password).toCharArray(); + if (conf != null + && StringUtils.isNotEmpty(conf.get(KEYSECURE_ENABLED)) + && conf.get(KEYSECURE_ENABLED).equalsIgnoreCase("true")) { + + getFromJceks(conf, CREDENTIAL_PATH, KEYSECURE_PASSWORD_ALIAS, KEYSECURE_PASSWORD); + String keySecureLoginCred = conf.get(KEYSECURE_USERNAME).trim() + ":" + conf.get(KEYSECURE_PASSWORD); + conf.set(KEYSECURE_LOGIN, keySecureLoginCred); + + RangerSafenetKeySecure rangerSafenetKeySecure = new RangerSafenetKeySecure( + conf); + masterKey = rangerSafenetKeySecure.getMasterKey(password).toCharArray(); + + } else { + RangerMasterKey rangerMasterKey = new RangerMasterKey( + daoManager); + masterKey = rangerMasterKey.getMasterKey(password) + .toCharArray(); + } OutputStream out = null; try { out = new FileOutputStream(new File(keyStoreFileName)); - dbStore.engineLoadToKeyStoreFile(out, keyStorePassword, keyPassword, masterKey, keyStoreType); - } - finally { + dbStore.engineLoadToKeyStoreFile(out, keyStorePassword, + keyPassword, masterKey, keyStoreType); + } finally { if (out != null) { try { out.close(); } catch (Exception e) { - throw new RuntimeException("ERROR: Unable to close file stream for [" + keyStoreFileName + "]", e); + throw new RuntimeException( + "ERROR: Unable to close file stream for [" + + keyStoreFileName + "]", e); } } } @@ -134,4 +162,20 @@ private char[] getPasswordFromConsole(String prompt) throws IOException { } return ret.toCharArray(); } + + private static void getFromJceks(Configuration conf, String path, String alias, String key) { + + //update credential from keystore + if (conf != null) { + String pathValue = conf.get(path); + String aliasValue = conf.get(alias); + if (pathValue != null && aliasValue != null) { + String xaDBPassword = CredentialReader.getDecryptedString(pathValue.trim(), aliasValue.trim()); + if (xaDBPassword != null && !xaDBPassword.trim().isEmpty() && + !xaDBPassword.trim().equalsIgnoreCase("none")) { + conf.set(key, xaDBPassword); + } + } + } + } } diff --git a/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKeyStoreProvider.java b/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKeyStoreProvider.java index 267fcf08f1..448469b99c 100755 --- a/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKeyStoreProvider.java +++ b/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKeyStoreProvider.java @@ -68,6 +68,13 @@ public class RangerKeyStoreProvider extends KeyProvider{ private static final String HSM_ENABLED = "ranger.ks.hsm.enabled"; private static final String HSM_PARTITION_PASSWORD_ALIAS = "ranger.ks.hsm.partition.password.alias"; private static final String HSM_PARTITION_PASSWORD = "ranger.ks.hsm.partition.password"; + private static final String KEYSECURE_ENABLED = "ranger.kms.keysecure.enabled"; + + private static final String KEYSECURE_USERNAME = "ranger.kms.keysecure.login.username"; + private static final String KEYSECURE_PASSWORD_ALIAS = "ranger.kms.keysecure.login.password.alias"; + private static final String KEYSECURE_PASSWORD = "ranger.kms.keysecure.login.password"; + private static final String KEYSECURE_LOGIN = "ranger.kms.keysecure.login"; + private final RangerKeyStore dbStore; private char[] masterKey; @@ -80,37 +87,68 @@ public class RangerKeyStoreProvider extends KeyProvider{ public RangerKeyStoreProvider(Configuration conf) throws Throwable { super(conf); conf = getDBKSConf(); - getFromJceks(conf,CREDENTIAL_PATH, MK_CREDENTIAL_ALIAS, ENCRYPTION_KEY); - getFromJceks(conf,CREDENTIAL_PATH, DB_CREDENTIAL_ALIAS, DB_PASSWORD); - getFromJceks(conf,CREDENTIAL_PATH, HSM_PARTITION_PASSWORD_ALIAS, HSM_PARTITION_PASSWORD); + getFromJceks(conf, CREDENTIAL_PATH, MK_CREDENTIAL_ALIAS, ENCRYPTION_KEY); + getFromJceks(conf, CREDENTIAL_PATH, DB_CREDENTIAL_ALIAS, DB_PASSWORD); + getFromJceks(conf, CREDENTIAL_PATH, HSM_PARTITION_PASSWORD_ALIAS, + HSM_PARTITION_PASSWORD); RangerKMSDB rangerKMSDB = new RangerKMSDB(conf); daoManager = rangerKMSDB.getDaoManager(); - + RangerKMSMKI rangerMasterKey = null; String password = conf.get(ENCRYPTION_KEY); - if(password == null || password.trim().equals("") || password.trim().equals("_") || password.trim().equals("crypted")){ + if (password == null || password.trim().equals("") + || password.trim().equals("_") + || password.trim().equals("crypted")) { throw new IOException("Master Key Jceks does not exists"); } - if(StringUtils.isEmpty(conf.get(HSM_ENABLED)) || conf.get(HSM_ENABLED).equalsIgnoreCase("false")){ + if (StringUtils.isEmpty(conf.get(HSM_ENABLED)) + || conf.get(HSM_ENABLED).equalsIgnoreCase("false")) { rangerMasterKey = new RangerMasterKey(daoManager); - }else{ + } else { rangerMasterKey = new RangerHSM(conf); String partitionPasswd = conf.get(HSM_PARTITION_PASSWORD); - if(partitionPasswd == null || partitionPasswd.trim().equals("") || partitionPasswd.trim().equals("_") || partitionPasswd.trim().equals("crypted")){ + if (partitionPasswd == null || partitionPasswd.trim().equals("") + || partitionPasswd.trim().equals("_") + || partitionPasswd.trim().equals("crypted")) { throw new IOException("Partition Password doesn't exists"); } } - dbStore = new RangerKeyStore(daoManager); - rangerMasterKey.generateMasterKey(password); - //code to retrieve rangerMasterKey password - masterKey = rangerMasterKey.getMasterKey(password).toCharArray(); - if(masterKey == null){ - // Master Key does not exists - throw new IOException("Ranger MasterKey does not exists"); + + + if (conf != null && StringUtils.isNotEmpty(conf.get(KEYSECURE_ENABLED)) + && conf.get(KEYSECURE_ENABLED).equalsIgnoreCase("true")) { + getFromJceks(conf, CREDENTIAL_PATH, KEYSECURE_PASSWORD_ALIAS, KEYSECURE_PASSWORD); + String keySecureLoginCred = conf.get(KEYSECURE_USERNAME).trim() + ":" + conf.get(KEYSECURE_PASSWORD); + conf.set(KEYSECURE_LOGIN, keySecureLoginCred); + rangerMasterKey = new RangerSafenetKeySecure(conf); + + dbStore = new RangerKeyStore(daoManager); + // generate master key on key secure server + rangerMasterKey.generateMasterKey(password); + try { + masterKey = rangerMasterKey.getMasterKey(password) + .toCharArray(); + } catch (Exception ex) { + throw new Exception("Error while getting Safenet KeySecure master key " + ex); + } + + } else { + dbStore = new RangerKeyStore(daoManager); + rangerMasterKey.generateMasterKey(password); + // code to retrieve rangerMasterKey password + try { + masterKey = rangerMasterKey.getMasterKey(password) + .toCharArray(); + } catch (Exception ex) { + throw new Exception("Error while getting Ranger Master key " + ex); + } + } - reloadKeys(); + + reloadKeys(); ReadWriteLock lock = new ReentrantReadWriteLock(true); - readLock = lock.readLock(); + readLock = lock.readLock(); + } public static Configuration getDBKSConf() { @@ -164,7 +202,7 @@ public KeyVersion createKey(String name, byte[] material, Options options) throw new IOException("Wrong key length. Required " + options.getBitLength() + ", but got " + (8 * material.length)); } - cache.put(name, meta); + cache.put(name, meta); String versionName = buildVersionName(name, 0); return innerSetKeyVersion(name, versionName, material, meta.getCipher(), meta.getBitLength(), meta.getDescription(), meta.getVersions(), meta.getAttributes()); } @@ -205,7 +243,8 @@ public void deleteKey(String name) throws IOException { } catch (KeyStoreException e) { throw new IOException("Problem removing " + name + " from " + this, e); } - cache.remove(name); + cache.remove(name); + changed = true; } @@ -236,7 +275,8 @@ public void flush() throws IOException { } changed = false; }catch (IOException ioe) { - cache.clear(); + cache.clear(); + reloadKeys(); throw ioe; } @@ -245,37 +285,40 @@ public void flush() throws IOException { @Override public KeyVersion getKeyVersion(String versionName) throws IOException { readLock.lock(); - try { - SecretKeySpec key = null; - try { - if (!dbStore.engineContainsAlias(versionName)) { - dbStore.engineLoad(null, masterKey); - if (!dbStore.engineContainsAlias(versionName)) { - return null; - } + try { + SecretKeySpec key = null; + try { + if (!dbStore.engineContainsAlias(versionName)) { + dbStore.engineLoad(null, masterKey); + if (!dbStore.engineContainsAlias(versionName)) { + return null; + } + } + key = (SecretKeySpec) dbStore.engineGetKey(versionName, + masterKey); + } catch (NoSuchAlgorithmException e) { + + throw new IOException("Can't get algorithm for key " + key, e); + } catch (UnrecoverableKeyException e) { + throw new IOException("Can't recover key " + key, e); + } catch (CertificateException e) { + throw new IOException("Certificate exception storing key", e); } - key = (SecretKeySpec) dbStore.engineGetKey(versionName, masterKey); - } catch (NoSuchAlgorithmException e) { - throw new IOException("Can't get algorithm for key " + key, e); - } catch (UnrecoverableKeyException e) { - throw new IOException("Can't recover key " + key, e); - } - catch (CertificateException e) { - throw new IOException("Certificate exception storing key", e); + if (key == null) { + return null; + } else { + return new KeyVersion(getBaseName(versionName), versionName, + key.getEncoded()); + } + } finally { + readLock.unlock(); } - if (key == null) { - return null; - } else { - return new KeyVersion(getBaseName(versionName), versionName, key.getEncoded()); - } - } finally { - readLock.unlock(); - } } @Override public List getKeyVersions(String name) throws IOException { List list = new ArrayList(); + Metadata km = getMetadata(name); if (km != null) { int latestVersion = km.getVersions(); @@ -294,18 +337,18 @@ public List getKeyVersions(String name) throws IOException { @Override public List getKeys() throws IOException { - ArrayList list = new ArrayList(); - String alias = null; - reloadKeys(); - Enumeration e = dbStore.engineAliases(); - while (e.hasMoreElements()) { - alias = e.nextElement(); - // only include the metadata key names in the list of names - if (!alias.contains("@")) { - list.add(alias); - } - } - return list; + ArrayList list = new ArrayList(); + String alias = null; + reloadKeys(); + Enumeration e = dbStore.engineAliases(); + while (e.hasMoreElements()) { + alias = e.nextElement(); + // only include the metadata key names in the list of names + if (!alias.contains("@")) { + list.add(alias); + } + } + return list; } @Override @@ -326,7 +369,7 @@ public Metadata getMetadata(String name) throws IOException { Key key = dbStore.engineGetKey(name, masterKey); if(key != null){ Metadata meta = ((KeyMetadata) key).metadata; - cache.put(name, meta); + cache.put(name, meta); return meta; } } catch (NoSuchAlgorithmException e) { @@ -347,6 +390,7 @@ public Metadata getMetadata(String name) throws IOException { @Override public KeyVersion rollNewVersion(String name, byte[] material)throws IOException { reloadKeys(); + Metadata meta = getMetadata(name); if (meta == null) { throw new IOException("Key " + name + " not found"); @@ -378,8 +422,8 @@ private static void getFromJceks(Configuration conf, String path, String alias, private void reloadKeys() throws IOException { try { - cache.clear(); - loadKeys(masterKey); + cache.clear(); + loadKeys(masterKey); } catch (NoSuchAlgorithmException e) { throw new IOException("Can't load Keys"); }catch(CertificateException e){ diff --git a/kms/src/main/java/org/apache/hadoop/crypto/key/RangerMasterKey.java b/kms/src/main/java/org/apache/hadoop/crypto/key/RangerMasterKey.java index 5614c1640b..c0910a45cf 100755 --- a/kms/src/main/java/org/apache/hadoop/crypto/key/RangerMasterKey.java +++ b/kms/src/main/java/org/apache/hadoop/crypto/key/RangerMasterKey.java @@ -131,6 +131,17 @@ private SecretKey decryptMasterKeySK(byte masterKey[], String password) throws T return getMasterKeyFromBytes(masterKeyFromDBDecrypted); } + public boolean generateMKFromKeySecureMK(String password, byte[] key) throws Throwable{ + logger.info("Generating Master Key"); + String encryptedMasterKey = encryptMasterKey(password, key); + String savedKey = saveEncryptedMK(encryptedMasterKey, daoManager); + if(savedKey != null && !savedKey.trim().equals("")){ + logger.debug("Master Key Created with id = "+savedKey); + return true; + } + return false; + } + private byte[] getEncryptedMK() throws Base64DecodingException { logger.debug("Retrieving Encrypted Master Key from database"); try{ diff --git a/kms/src/main/java/org/apache/hadoop/crypto/key/RangerSafenetKeySecure.java b/kms/src/main/java/org/apache/hadoop/crypto/key/RangerSafenetKeySecure.java new file mode 100644 index 0000000000..70ec504493 --- /dev/null +++ b/kms/src/main/java/org/apache/hadoop/crypto/key/RangerSafenetKeySecure.java @@ -0,0 +1,156 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.crypto.key; + +import javax.crypto.KeyGenerator; +import javax.crypto.SecretKey; +import javax.crypto.spec.SecretKeySpec; + +import org.apache.hadoop.conf.Configuration; +import org.apache.log4j.Logger; + +import com.sun.org.apache.xml.internal.security.utils.Base64; + +import java.io.IOException; +import java.security.Key; +import java.security.KeyStore; +import java.security.KeyStoreException; +import java.security.NoSuchAlgorithmException; +import java.security.Provider; +import java.security.Security; +import java.security.cert.CertificateException; + +/** + * This Class is for HSM Keystore + */ +public class RangerSafenetKeySecure implements RangerKMSMKI { + + static final Logger logger = Logger.getLogger(RangerSafenetKeySecure.class); + + private final String alias; + private final KeyStore myStore; + private final String adp; + private final Provider provider; + private static final String MK_ALGO = "AES"; + private final int mkSize; + private static final int MK_KeySize = 256; + private String pkcs11CfgFilePath = null; + private static final String CFGFILEPATH = "ranger.kms.keysecure.sunpkcs11.cfg.filepath"; + private static final String MK_KEYSIZE = "ranger.kms.keysecure.masterkey.size"; + private static final String ALIAS = "ranger.kms.keysecure.masterkey.name"; + + private static final String KEYSECURE_LOGIN = "ranger.kms.keysecure.login"; + + public RangerSafenetKeySecure(Configuration conf) throws KeyStoreException, NoSuchAlgorithmException, CertificateException, IOException { + mkSize = conf.getInt(MK_KEYSIZE, MK_KeySize); + alias = conf.get(ALIAS, "RANGERMK"); + adp = conf.get(KEYSECURE_LOGIN); + pkcs11CfgFilePath = conf.get(CFGFILEPATH); + + try { + // Create a PKCS#11 session and initialize it + // using the sunPKCS11 config file + provider = new sun.security.pkcs11.SunPKCS11(pkcs11CfgFilePath); + Security.addProvider(provider); + myStore = KeyStore.getInstance("PKCS11", provider); + if(myStore != null){ + myStore.load(null, adp.toCharArray()); + }else{ + logger.error("Safenet Keysecure not found. Please verify the Ranger KMS Safenet Keysecure configuration setup."); + } + + } catch (NoSuchAlgorithmException nsae) { + throw new NoSuchAlgorithmException("Unexpected NoSuchAlgorithmException while loading keystore : " + + nsae.getMessage()); + } catch (CertificateException e) { + throw new CertificateException("Unexpected CertificateException while loading keystore : " + + e.getMessage()); + } catch (IOException e) { + throw new IOException("Unexpected IOException while loading keystore : " + + e.getMessage()); + } + } + + @Override + public boolean generateMasterKey(String password){ + if (myStore != null) { + KeyGenerator keyGen = null; + SecretKey aesKey = null; + try { + boolean result = myStore.containsAlias(alias); + + if (!result) { + keyGen = KeyGenerator.getInstance(MK_ALGO, provider); + + + keyGen.init(mkSize); + + aesKey = keyGen.generateKey(); + myStore.setKeyEntry(alias, aesKey, password.toCharArray(), + (java.security.cert.Certificate[]) null); + return true; + } else { + return true; + } + + } catch (Exception e) { + logger.error("generateMasterKey : Exception during Ranger Master Key Generation - " + + e); + return false; + } + } + return false; + } + + @Override + public String getMasterKey(String password) { + if (myStore != null) { + try { + boolean result = myStore.containsAlias(alias); + if (result) { + SecretKey key = (SecretKey) myStore.getKey(alias, + password.toCharArray()); + if (key != null) { + return Base64.encode(key.getEncoded()); + } + + } + } catch (Exception e) { + logger.error("getMasterKey : Exception searching for Ranger Master Key - " + + e.getMessage()); + } + } + return null; + } + + public boolean setMasterKey(String password, byte[] key, Configuration conf) { + if (myStore != null) { + try { + Key aesKey = new SecretKeySpec(key, MK_ALGO); + myStore.setKeyEntry(alias, aesKey, password.toCharArray(), + (java.security.cert.Certificate[]) null); + return true; + } catch (Exception e) { + logger.error("setMasterKey : Exception while setting Master Key - " + + e.getMessage()); + } + } + return false; + } + +} \ No newline at end of file diff --git a/src/main/assembly/kms.xml b/src/main/assembly/kms.xml index fca6a324e3..c156fe30f8 100755 --- a/src/main/assembly/kms.xml +++ b/src/main/assembly/kms.xml @@ -317,6 +317,8 @@ exportKeysToJCEKS.sh HSMMK2DB.sh DBMK2HSM.sh + DBMKTOKEYSECURE.sh + KEYSECUREMKTOKMSDB.sh 544 From b661ac4815d8c0fce04fb58756cb6729487451f1 Mon Sep 17 00:00:00 2001 From: Bhavik Patel Date: Tue, 27 Nov 2018 13:31:34 +0530 Subject: [PATCH 150/151] RANGER-2208: Code improvement to fetch User/Group information and Service Config details Signed-off-by: Pradeep --- .../java/org/apache/ranger/biz/XUserMgr.java | 71 +++++ .../org/apache/ranger/rest/ServiceREST.java | 73 ++++- .../org/apache/ranger/rest/XUserREST.java | 34 ++- .../org/apache/ranger/biz/TestXUserMgr.java | 280 +++++++++++++++++- .../apache/ranger/rest/TestServiceREST.java | 154 ++++++++++ .../org/apache/ranger/rest/TestXUserREST.java | 113 +++++++ 6 files changed, 715 insertions(+), 10 deletions(-) diff --git a/security-admin/src/main/java/org/apache/ranger/biz/XUserMgr.java b/security-admin/src/main/java/org/apache/ranger/biz/XUserMgr.java index 410e3f852a..54ae07e229 100644 --- a/security-admin/src/main/java/org/apache/ranger/biz/XUserMgr.java +++ b/security-admin/src/main/java/org/apache/ranger/biz/XUserMgr.java @@ -694,6 +694,12 @@ public VXGroupUser createXGroupUser(VXGroupUser vXGroupUser) { public VXUser getXUser(Long id) { VXUser vXUser=null; vXUser=xUserService.readResourceWithOutLogin(id); + if(vXUser != null){ + if(!hasAccessToGetUserInfo(vXUser)){ + logger.info("Logged-In user is not allowed to access requested user data."); + throw restErrorUtil.create403RESTException("Logged-In user is not allowed to access requested user data."); + } + } if(vXUser!=null && !hasAccessToModule(RangerConstants.MODULE_USER_GROUPS)){ vXUser=getMaskedVXUser(vXUser); } @@ -707,6 +713,27 @@ public VXGroupUser getXGroupUser(Long id) { public VXGroup getXGroup(Long id) { VXGroup vXGroup=null; + UserSessionBase userSession = ContextUtil.getCurrentUserSession(); + if (userSession != null && userSession.getLoginId() != null) { + VXUser loggedInVXUser = xUserService.getXUserByUserName(userSession + .getLoginId()); + if (loggedInVXUser != null) { + if (loggedInVXUser.getUserRoleList().size() == 1 + && loggedInVXUser.getUserRoleList().contains( + RangerConstants.ROLE_USER)) { + + List listGroupId = daoManager.getXXGroupUser() + .findGroupIdListByUserId(loggedInVXUser.getId()); + + if (!listGroupId.contains(id)) { + logger.info("Logged-In user is not allowed to access requested user data."); + throw restErrorUtil + .create403RESTException("Logged-In user is not allowed to access requested group data."); + } + + } + } + } vXGroup=xGroupService.readResourceWithOutLogin(id); if(vXGroup!=null && !hasAccessToModule(RangerConstants.MODULE_USER_GROUPS)){ vXGroup=getMaskedVXGroup(vXGroup); @@ -1734,6 +1761,31 @@ public VXGroupList searchXGroups(SearchCriteria searchCriteria) { searchCriteria.setSortBy("id"); vXGroupList=xGroupService.searchXGroups(searchCriteria); } + UserSessionBase userSession = ContextUtil.getCurrentUserSession(); + if (userSession != null && userSession.getLoginId() != null) { + VXUser loggedInVXUser = xUserService.getXUserByUserName(userSession + .getLoginId()); + if (loggedInVXUser != null) { + if (loggedInVXUser.getUserRoleList().size() == 1 + && loggedInVXUser.getUserRoleList().contains( + RangerConstants.ROLE_USER)) { + + List updatedList = new ArrayList(); + + List listGroupId = daoManager.getXXGroupUser() + .findGroupIdListByUserId(loggedInVXUser.getId()); + + for (VXGroup group : vXGroupList.getList()) { + if (listGroupId.contains(group.getId())) { + updatedList.add(group); + } + } + logger.info("Logged-In user having user role will be able to fetch his own groups details."); + vXGroupList.setVXGroups(updatedList); + + } + } + } if(vXGroupList!=null && !hasAccessToModule(RangerConstants.MODULE_USER_GROUPS)){ if(vXGroupList!=null && vXGroupList.getListSize()>0){ List listMasked=new ArrayList(); @@ -2245,6 +2297,25 @@ private void validatePassword(VXUser vXUser) { } } + private boolean hasAccessToGetUserInfo(VXUser requestedVXUser) { + UserSessionBase userSession = ContextUtil.getCurrentUserSession(); + if (userSession != null && userSession.getLoginId() != null) { + VXUser loggedInVXUser = xUserService.getXUserByUserName(userSession + .getLoginId()); + if (loggedInVXUser != null) { + if (loggedInVXUser.getUserRoleList().size() == 1 + && loggedInVXUser.getUserRoleList().contains( + RangerConstants.ROLE_USER)) { + + return requestedVXUser.getId().equals(loggedInVXUser.getId()) ? true : false; + + }else{ + return true; + } + } + } + return false; + } public void denySelfRoleChange(String userName) { UserSessionBase session = ContextUtil.getCurrentUserSession(); if (session != null && session.getXXPortalUser()!=null) { diff --git a/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java b/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java index b2a481b925..f12da80d7b 100644 --- a/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java +++ b/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java @@ -71,6 +71,7 @@ import org.apache.ranger.common.MessageEnums; import org.apache.ranger.common.PropertiesUtil; import org.apache.ranger.common.RESTErrorUtil; +import org.apache.ranger.common.RangerConstants; import org.apache.ranger.common.RangerSearchUtil; import org.apache.ranger.common.RangerValidatorFactory; import org.apache.ranger.common.ServiceUtil; @@ -113,6 +114,7 @@ import org.apache.ranger.service.RangerPolicyService; import org.apache.ranger.service.RangerServiceDefService; import org.apache.ranger.service.RangerServiceService; +import org.apache.ranger.service.XUserService; import org.apache.ranger.view.RangerExportPolicyList; import org.apache.ranger.view.RangerPluginInfoList; import org.apache.ranger.view.RangerPolicyList; @@ -120,6 +122,7 @@ import org.apache.ranger.view.RangerServiceList; import org.apache.ranger.view.VXResponse; import org.apache.ranger.view.VXString; +import org.apache.ranger.view.VXUser; import org.codehaus.jackson.map.ObjectMapper; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Scope; @@ -203,6 +206,9 @@ public class ServiceREST { @Autowired JSONUtil jsonUtil; + + @Autowired + XUserService xUserService; private RangerPolicyEngineOptions delegateAdminOptions; private RangerPolicyEngineOptions policySearchAdminOptions; @@ -807,6 +813,19 @@ public RangerService getService(@PathParam("id") Long id) { perf = RangerPerfTracer.getPerfTracer(PERF_LOG, "ServiceREST.getService(serviceId=" + id + ")"); } ret = svcStore.getService(id); + if (ret != null) { + UserSessionBase userSession = ContextUtil.getCurrentUserSession(); + if (userSession != null && userSession.getLoginId() != null) { + VXUser loggedInVXUser = xUserService.getXUserByUserName(userSession.getLoginId()); + if (loggedInVXUser != null) { + if (loggedInVXUser.getUserRoleList().size() == 1 + && loggedInVXUser.getUserRoleList().contains(RangerConstants.ROLE_USER)) { + + ret = hideCriticalServiceDetailsForRoleUser(ret); + } + } + } + } } catch(WebApplicationException excp) { throw excp; } catch(Throwable excp) { @@ -845,6 +864,19 @@ public RangerService getServiceByName(@PathParam("name") String name) { perf = RangerPerfTracer.getPerfTracer(PERF_LOG, "ServiceREST.getService(serviceName=" + name + ")"); } ret = svcStore.getServiceByName(name); + if (ret != null) { + UserSessionBase userSession = ContextUtil.getCurrentUserSession(); + if (userSession != null && userSession.getLoginId() != null) { + VXUser loggedInVXUser = xUserService.getXUserByUserName(userSession.getLoginId()); + if (loggedInVXUser != null) { + if (loggedInVXUser.getUserRoleList().size() == 1 + && loggedInVXUser.getUserRoleList().contains(RangerConstants.ROLE_USER)) { + + ret = hideCriticalServiceDetailsForRoleUser(ret); + } + } + } + } } catch(WebApplicationException excp) { throw excp; } catch(Throwable excp) { @@ -887,8 +919,29 @@ public RangerServiceList getServices(@Context HttpServletRequest request) { perf = RangerPerfTracer.getPerfTracer(PERF_LOG, "ServiceREST.getServices()"); } paginatedSvcs = svcStore.getPaginatedServices(filter); + if (paginatedSvcs != null && !paginatedSvcs.getList().isEmpty()) { + UserSessionBase userSession = ContextUtil.getCurrentUserSession(); + if (userSession != null && userSession.getLoginId() != null) { + VXUser loggedInVXUser = xUserService.getXUserByUserName(userSession.getLoginId()); + if (loggedInVXUser != null) { + if (loggedInVXUser.getUserRoleList().size() == 1 + && loggedInVXUser.getUserRoleList().contains(RangerConstants.ROLE_USER)) { + + List updateServiceList = new ArrayList(); + for (RangerService rangerService : paginatedSvcs.getList()) { + if (rangerService != null) { + updateServiceList.add(hideCriticalServiceDetailsForRoleUser(rangerService)); + } + } - if(paginatedSvcs != null) { + if (updateServiceList != null && !updateServiceList.isEmpty()) { + paginatedSvcs.setList(updateServiceList); + } + } + } + } + } + if (paginatedSvcs != null) { ret = new RangerServiceList(); ret.setServices(paginatedSvcs.getList()); @@ -3330,4 +3383,22 @@ private Map getOptions(HttpServletRequest request) { } return ret; } + + private RangerService hideCriticalServiceDetailsForRoleUser(RangerService rangerService) { + RangerService ret = rangerService; + + ret.setConfigs(null); + ret.setDescription(null); + ret.setCreatedBy(null); + ret.setUpdatedBy(null); + ret.setCreateTime(null); + ret.setUpdateTime(null); + ret.setPolicyVersion(null); + ret.setPolicyUpdateTime(null); + ret.setTagVersion(null); + ret.setTagUpdateTime(null); + ret.setVersion(null); + + return ret; + } } diff --git a/security-admin/src/main/java/org/apache/ranger/rest/XUserREST.java b/security-admin/src/main/java/org/apache/ranger/rest/XUserREST.java index 3f25506e01..ec1dc26078 100644 --- a/security-admin/src/main/java/org/apache/ranger/rest/XUserREST.java +++ b/security-admin/src/main/java/org/apache/ranger/rest/XUserREST.java @@ -17,11 +17,10 @@ * under the License. */ - package org.apache.ranger.rest; +package org.apache.ranger.rest; import java.util.HashMap; import java.util.List; -import java.util.Random; import javax.servlet.http.HttpServletRequest; import javax.ws.rs.DELETE; @@ -33,17 +32,20 @@ import javax.ws.rs.Produces; import javax.ws.rs.core.Context; +import org.apache.log4j.Logger; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang.StringUtils; import org.apache.ranger.biz.RangerBizUtil; import org.apache.ranger.biz.SessionMgr; import org.apache.ranger.biz.XUserMgr; +import org.apache.ranger.common.ContextUtil; import org.apache.ranger.common.MessageEnums; import org.apache.ranger.common.RESTErrorUtil; import org.apache.ranger.common.RangerConstants; import org.apache.ranger.common.SearchCriteria; import org.apache.ranger.common.SearchUtil; import org.apache.ranger.common.StringUtil; +import org.apache.ranger.common.UserSessionBase; import org.apache.ranger.common.annotation.RangerAnnotationClassName; import org.apache.ranger.common.annotation.RangerAnnotationJSMgrName; import org.apache.ranger.db.RangerDaoManager; @@ -150,6 +152,11 @@ public class XUserREST { @Autowired XResourceService xResourceService; + @Autowired + StringUtil stringUtil; + + static final Logger logger = Logger.getLogger(XUserMgr.class); + // Handle XGroup @GET @Path("/groups/{id}") @@ -372,9 +379,26 @@ public VXUserList searchXUsers(@Context HttpServletRequest request) { else if ((searchCriteria.getParamList().containsKey("name")) && userName!= null && userName.contains((String) searchCriteria.getParamList().get("name"))) { searchCriteria.addParam("name", userName); } - else { - String randomString = new Random().toString(); - searchCriteria.addParam("name", randomString); + } + + UserSessionBase userSession = ContextUtil.getCurrentUserSession(); + if (userSession != null && userSession.getLoginId() != null) { + VXUser loggedInVXUser = xUserService.getXUserByUserName(userSession + .getLoginId()); + if (loggedInVXUser != null) { + if (loggedInVXUser.getUserRoleList().size() == 1 + && loggedInVXUser.getUserRoleList().contains( + RangerConstants.ROLE_USER)) { + logger.info("Logged-In user having user role will be able to fetch his own user details."); + if (!searchCriteria.getParamList().containsKey("name")) { + searchCriteria.addParam("name", loggedInVXUser.getName()); + }else if(searchCriteria.getParamList().containsKey("name") + && !stringUtil.isEmpty(searchCriteria.getParamValue("name").toString()) + && !searchCriteria.getParamValue("name").toString().equalsIgnoreCase(loggedInVXUser.getName())){ + throw restErrorUtil.create403RESTException("Logged-In user is not allowed to access requested user data."); + } + + } } } diff --git a/security-admin/src/test/java/org/apache/ranger/biz/TestXUserMgr.java b/security-admin/src/test/java/org/apache/ranger/biz/TestXUserMgr.java index 826307e8ca..aeb8ea2815 100644 --- a/security-admin/src/test/java/org/apache/ranger/biz/TestXUserMgr.java +++ b/security-admin/src/test/java/org/apache/ranger/biz/TestXUserMgr.java @@ -24,6 +24,9 @@ import java.util.List; import java.util.Map; import java.util.Set; + +import javax.ws.rs.WebApplicationException; + import org.apache.ranger.common.RangerCommonEnums; import org.apache.ranger.common.RangerConstants; import org.apache.ranger.common.ContextUtil; @@ -105,6 +108,7 @@ public class TestXUserMgr { private static Long userId = 8L; + private static String userLoginID = "testuser"; private static Integer emptyValue; @@ -299,9 +303,29 @@ private RangerPolicy rangerPolicy() { return policy; } + + @Test public void test11CreateXUser() { - setup(); + destroySession(); + + RangerSecurityContext context = new RangerSecurityContext(); + context.setUserSession(new UserSessionBase()); + RangerContextHolder.setSecurityContext(context); + UserSessionBase currentUserSession = ContextUtil.getCurrentUserSession(); + currentUserSession.setUserAdmin(true); + XXPortalUser xXPortalUser = new XXPortalUser(); + xXPortalUser.setLoginId(userLoginID); + xXPortalUser.setId(userId); + currentUserSession.setXXPortalUser(xXPortalUser); + + VXUser loggedInUser = vxUser(); + List loggedInUserRole = new ArrayList(); + loggedInUserRole.add(RangerConstants.ROLE_ADMIN); + loggedInUser.setId(8L); + loggedInUser.setName("testuser"); + loggedInUser.setUserRoleList(loggedInUserRole); + VXUser vxUser = vxUser(); Collection userRoleList = new ArrayList(); userRoleList.add("test"); @@ -313,6 +337,7 @@ public void test11CreateXUser() { vXPortalUser.setUserRoleList(userRoleListVXPortaUser); Mockito.when(xUserService.createResource(vxUser)).thenReturn(vxUser); + Mockito.when(xUserService.getXUserByUserName("testuser")).thenReturn(loggedInUser); XXModuleDefDao value = Mockito.mock(XXModuleDefDao.class); Mockito.when(daoManager.getXXModuleDef()).thenReturn(value); @@ -338,7 +363,6 @@ public void test11CreateXUser() { Mockito.verify(userMgr).createDefaultAccountUser( (VXPortalUser) Mockito.anyObject()); - Mockito.verify(daoManager).getXXModuleDef(); Assert.assertNotNull(dbvxUser); Assert.assertEquals(userId, dbvxUser.getId()); Assert.assertEquals(dbvxUser.getDescription(), vxUser.getDescription()); @@ -1363,7 +1387,17 @@ public void test44getGroupsForUser() { @Test public void test45setUserRolesByExternalID() { - setup(); + destroySession(); + + RangerSecurityContext context = new RangerSecurityContext(); + context.setUserSession(new UserSessionBase()); + RangerContextHolder.setSecurityContext(context); + UserSessionBase currentUserSession = ContextUtil.getCurrentUserSession(); + currentUserSession.setUserAdmin(true); + XXPortalUser xXPortalUser = new XXPortalUser(); + xXPortalUser.setLoginId(userLoginID); + xXPortalUser.setId(userId); + currentUserSession.setXXPortalUser(xXPortalUser); XXPortalUserRoleDao xPortalUserRoleDao = Mockito .mock(XXPortalUserRoleDao.class); XXPortalUserDao userDao = Mockito.mock(XXPortalUserDao.class); @@ -1448,6 +1482,13 @@ public void test45setUserRolesByExternalID() { userPermission.setUserName("xyz"); userPermission.setOwner("admin"); + VXUser loggedInUser = vxUser(); + List loggedInUserRole = new ArrayList(); + loggedInUserRole.add(RangerConstants.ROLE_ADMIN); + loggedInUser.setId(8L); + loggedInUser.setName("testuser"); + loggedInUser.setUserRoleList(loggedInUserRole); + Mockito.when(daoManager.getXXPortalUserRole()).thenReturn( xPortalUserRoleDao); Mockito.when(xPortalUserRoleDao.findByUserId(userId)).thenReturn( @@ -1478,6 +1519,7 @@ public void test45setUserRolesByExternalID() { Mockito.when(xModuleDefDao.findByModuleId(Mockito.anyLong())) .thenReturn(xModuleDef); Mockito.when(xUserMgr.getXUser(userId)).thenReturn(vXUser); + Mockito.when(xUserService.getXUserByUserName("testuser")).thenReturn(loggedInUser); Mockito.when(userMgr.getUserProfileByLoginId(vXUser.getName())) .thenReturn(userProfile); VXStringList vXStringList = xUserMgr.setUserRolesByExternalID(userId, @@ -1609,7 +1651,17 @@ public void test46setUserRolesByName() { @Test public void test47getUserRolesByExternalID() { - setup(); + destroySession(); + + RangerSecurityContext context = new RangerSecurityContext(); + context.setUserSession(new UserSessionBase()); + RangerContextHolder.setSecurityContext(context); + UserSessionBase currentUserSession = ContextUtil.getCurrentUserSession(); + currentUserSession.setUserAdmin(true); + XXPortalUser xXPortalUser = new XXPortalUser(); + xXPortalUser.setLoginId(userLoginID); + xXPortalUser.setId(userId); + currentUserSession.setXXPortalUser(xXPortalUser); XXPortalUserRoleDao xPortalUserRoleDao = Mockito .mock(XXPortalUserRoleDao.class); XXPortalUserDao userDao = Mockito.mock(XXPortalUserDao.class); @@ -1694,6 +1746,13 @@ public void test47getUserRolesByExternalID() { userPermission.setUserName("xyz"); userPermission.setOwner("admin"); + VXUser loggedInUser = vxUser(); + List loggedInUserRole = new ArrayList(); + loggedInUserRole.add(RangerConstants.ROLE_ADMIN); + loggedInUser.setId(8L); + loggedInUser.setName("testuser"); + loggedInUser.setUserRoleList(loggedInUserRole); + Mockito.when(daoManager.getXXPortalUserRole()).thenReturn( xPortalUserRoleDao); Mockito.when(xPortalUserRoleDao.findByUserId(userId)).thenReturn( @@ -1726,6 +1785,7 @@ public void test47getUserRolesByExternalID() { Mockito.when(xUserMgr.getXUser(userId)).thenReturn(vXUser); Mockito.when(userMgr.getUserProfileByLoginId(vXUser.getName())) .thenReturn(userProfile); + Mockito.when(xUserService.getXUserByUserName("testuser")).thenReturn(loggedInUser); VXStringList vXStringList = xUserMgr.getUserRolesByExternalID(userId); Assert.assertNotNull(vXStringList); } @@ -1855,4 +1915,216 @@ public void test48getUserRolesByName() { .getLoginId()); Assert.assertNotNull(vXStringList); } + @Test + public void test49getAdminUserDetailsWithUserHavingUSER_ROLE() { + destroySession(); + + RangerSecurityContext context = new RangerSecurityContext(); + context.setUserSession(new UserSessionBase()); + RangerContextHolder.setSecurityContext(context); + UserSessionBase currentUserSession = ContextUtil.getCurrentUserSession(); + currentUserSession.setUserAdmin(false); + XXPortalUser xXPortalUser = new XXPortalUser(); + xXPortalUser.setLoginId(userLoginID); + xXPortalUser.setId(userId); + currentUserSession.setXXPortalUser(xXPortalUser); + + VXUser loggedInUser = vxUser(); + List loggedInUserRole = new ArrayList(); + loggedInUserRole.add(RangerConstants.ROLE_USER); + loggedInUser.setId(8L); + loggedInUser.setName("testuser"); + loggedInUser.setUserRoleList(loggedInUserRole); + + VXUser vxUser = vxUser(); + List userRole = new ArrayList(); + userRole.add(RangerConstants.ROLE_ADMIN); + vxUser.setId(5L); + vxUser.setName("test3"); + vxUser.setUserRoleList(userRole); + vxUser.setUserSource(RangerCommonEnums.USER_UNIX); + Mockito.when(xUserService.readResourceWithOutLogin(5L)).thenReturn(vxUser); + Mockito.when(xUserService.getXUserByUserName("testuser")).thenReturn(loggedInUser); + Mockito.when(restErrorUtil.create403RESTException("Logged-In user is not allowed to access requested user data.")).thenThrow(new WebApplicationException()); + thrown.expect(WebApplicationException.class); + xUserMgr.getXUser(5L); + } + + @Test + public void test50getKeyAdminUserDetailsWithUserHavingUSER_ROLE() { + destroySession(); + + RangerSecurityContext context = new RangerSecurityContext(); + context.setUserSession(new UserSessionBase()); + RangerContextHolder.setSecurityContext(context); + UserSessionBase currentUserSession = ContextUtil.getCurrentUserSession(); + currentUserSession.setUserAdmin(false); + XXPortalUser xXPortalUser = new XXPortalUser(); + xXPortalUser.setLoginId(userLoginID); + xXPortalUser.setId(userId); + currentUserSession.setXXPortalUser(xXPortalUser); + + VXUser loggedInUser = vxUser(); + List loggedInUserRole = new ArrayList(); + loggedInUserRole.add(RangerConstants.ROLE_USER); + loggedInUser.setId(8L); + loggedInUser.setName("testuser"); + loggedInUser.setUserRoleList(loggedInUserRole); + + VXUser vxUser = vxUser(); + List userRole = new ArrayList(); + userRole.add(RangerConstants.ROLE_KEY_ADMIN); + vxUser.setId(5L); + vxUser.setName("test3"); + vxUser.setUserRoleList(userRole); + vxUser.setUserSource(RangerCommonEnums.USER_UNIX); + Mockito.when(xUserService.readResourceWithOutLogin(5L)).thenReturn(vxUser); + Mockito.when(xUserService.getXUserByUserName("testuser")).thenReturn(loggedInUser); + Mockito.when(restErrorUtil.create403RESTException("Logged-In user is not allowed to access requested user data.")).thenThrow(new WebApplicationException()); + thrown.expect(WebApplicationException.class); + xUserMgr.getXUser(5L); + } + + + + @Test + public void test51getErrorWhenRoleUserFetchAnotherUserGroupInfo() { + destroySession(); + + RangerSecurityContext context = new RangerSecurityContext(); + context.setUserSession(new UserSessionBase()); + RangerContextHolder.setSecurityContext(context); + UserSessionBase currentUserSession = ContextUtil.getCurrentUserSession(); + currentUserSession.setUserAdmin(false); + XXPortalUser xXPortalUser = new XXPortalUser(); + xXPortalUser.setLoginId(userLoginID); + xXPortalUser.setId(userId); + currentUserSession.setXXPortalUser(xXPortalUser); + List permissionList = new ArrayList(); + permissionList.add(RangerConstants.MODULE_USER_GROUPS); + + List groupIdList = new ArrayList(); + groupIdList.add(2L); + + VXUser loggedInUser = vxUser(); + List loggedInUserRole = new ArrayList(); + loggedInUserRole.add(RangerConstants.ROLE_USER); + loggedInUser.setId(8L); + loggedInUser.setName("testuser"); + loggedInUser.setUserRoleList(loggedInUserRole); + loggedInUser.setGroupIdList(groupIdList); + + VXUser vxUser = vxUser(); + List userRole = new ArrayList(); + userRole.add(RangerConstants.ROLE_USER); + vxUser.setId(8L); + vxUser.setName("test3"); + vxUser.setUserRoleList(userRole); + vxUser.setUserSource(RangerCommonEnums.USER_UNIX); + + Mockito.when(xUserService.getXUserByUserName("testuser")).thenReturn(loggedInUser); + + XXGroupUserDao mockxxGroupUserDao = Mockito.mock(XXGroupUserDao.class); + + Mockito.when(daoManager.getXXGroupUser()).thenReturn(mockxxGroupUserDao); + Mockito.when(mockxxGroupUserDao.findGroupIdListByUserId(loggedInUser.getId())).thenReturn(groupIdList); + + Mockito.when(restErrorUtil.create403RESTException("Logged-In user is not allowed to access requested group data.")).thenThrow(new WebApplicationException()); + thrown.expect(WebApplicationException.class); + xUserMgr.getXGroup(5L); + } + + @Test + public void test52RoleUserWillFetchOnlyHisOwnGroupDetails() { + destroySession(); + + RangerSecurityContext context = new RangerSecurityContext(); + context.setUserSession(new UserSessionBase()); + RangerContextHolder.setSecurityContext(context); + UserSessionBase currentUserSession = ContextUtil.getCurrentUserSession(); + currentUserSession.setUserAdmin(false); + XXPortalUser xXPortalUser = new XXPortalUser(); + xXPortalUser.setLoginId(userLoginID); + xXPortalUser.setId(userId); + currentUserSession.setXXPortalUser(xXPortalUser); + List permissionList = new ArrayList(); + permissionList.add(RangerConstants.MODULE_USER_GROUPS); + + List groupIdList = new ArrayList(); + groupIdList.add(5L); + + VXGroup expectedVXGroup = new VXGroup(); + expectedVXGroup.setId(5L); + expectedVXGroup.setName("testGroup"); + + VXUser loggedInUser = vxUser(); + List loggedInUserRole = new ArrayList(); + loggedInUserRole.add(RangerConstants.ROLE_USER); + loggedInUser.setId(8L); + loggedInUser.setName("testuser"); + loggedInUser.setUserRoleList(loggedInUserRole); + loggedInUser.setGroupIdList(groupIdList); + + VXUser vxUser = vxUser(); + List userRole = new ArrayList(); + userRole.add(RangerConstants.ROLE_USER); + vxUser.setId(8L); + vxUser.setName("test3"); + vxUser.setUserRoleList(userRole); + vxUser.setUserSource(RangerCommonEnums.USER_UNIX); + Mockito.when(xGroupService.readResourceWithOutLogin(5L)).thenReturn(expectedVXGroup); + + VXGroup rcvVXGroup = xUserMgr.getXGroup(5L); + Assert.assertNotNull(rcvVXGroup); + Assert.assertEquals(expectedVXGroup.getId(), rcvVXGroup.getId()); + Assert.assertEquals(expectedVXGroup.getName(), rcvVXGroup.getName()); + } + @Test + public void test53getUserDetailsOfItsOwn() { + destroySession(); + + RangerSecurityContext context = new RangerSecurityContext(); + context.setUserSession(new UserSessionBase()); + RangerContextHolder.setSecurityContext(context); + UserSessionBase currentUserSession = ContextUtil.getCurrentUserSession(); + currentUserSession.setUserAdmin(false); + XXPortalUser xXPortalUser = new XXPortalUser(); + xXPortalUser.setLoginId(userLoginID); + xXPortalUser.setId(userId); + currentUserSession.setXXPortalUser(xXPortalUser); + List permissionList = new ArrayList(); + permissionList.add(RangerConstants.MODULE_USER_GROUPS); + + + VXUser loggedInUser = vxUser(); + List loggedInUserRole = new ArrayList(); + loggedInUserRole.add(RangerConstants.ROLE_USER); + loggedInUser.setId(8L); + loggedInUser.setName("testuser"); + loggedInUser.setUserRoleList(loggedInUserRole); + + VXUser vxUser = vxUser(); + List userRole = new ArrayList(); + userRole.add(RangerConstants.ROLE_USER); + vxUser.setId(8L); + vxUser.setName("test3"); + vxUser.setUserRoleList(userRole); + vxUser.setUserSource(RangerCommonEnums.USER_UNIX); + Mockito.when(xUserService.readResourceWithOutLogin(8L)).thenReturn(vxUser); + Mockito.when(xUserService.getXUserByUserName("testuser")).thenReturn(loggedInUser); + XXModuleDefDao mockxxModuleDefDao = Mockito.mock(XXModuleDefDao.class); + Mockito.when(daoManager.getXXModuleDef()).thenReturn(mockxxModuleDefDao); + Mockito.when(mockxxModuleDefDao.findAccessibleModulesByUserId(8L, 8L)).thenReturn(permissionList); + VXUser expectedVXUser = xUserMgr.getXUser(8L); + Assert.assertNotNull(expectedVXUser); + Assert.assertEquals(expectedVXUser.getName(), vxUser.getName()); + } + + + + public void destroySession() { + RangerSecurityContext context = new RangerSecurityContext(); + context.setUserSession(null); + RangerContextHolder.setSecurityContext(context); + } } diff --git a/security-admin/src/test/java/org/apache/ranger/rest/TestServiceREST.java b/security-admin/src/test/java/org/apache/ranger/rest/TestServiceREST.java index 7d407f798c..cee33c69d5 100644 --- a/security-admin/src/test/java/org/apache/ranger/rest/TestServiceREST.java +++ b/security-admin/src/test/java/org/apache/ranger/rest/TestServiceREST.java @@ -32,9 +32,12 @@ import org.apache.ranger.biz.AssetMgr; import org.apache.ranger.biz.RangerBizUtil; import org.apache.ranger.biz.ServiceDBStore; +import org.apache.ranger.common.RangerConstants; import org.apache.ranger.biz.ServiceMgr; import org.apache.ranger.biz.TagDBStore; import org.apache.ranger.biz.XUserMgr; +import org.apache.ranger.entity.XXPortalUser; +import org.apache.ranger.view.VXUser; import org.apache.ranger.common.ContextUtil; import org.apache.ranger.common.RESTErrorUtil; import org.apache.ranger.common.RangerSearchUtil; @@ -1433,4 +1436,155 @@ public void test43revoke() { assert(true); } + + @Test + public void test44getServiceWillOnlyReturnNameIdAndTypeForRoleUser() throws Exception { + RangerService actualService = rangerService(); + + String userLoginID = "testuser"; + Long userId = 8L; + + RangerSecurityContext context = new RangerSecurityContext(); + context.setUserSession(new UserSessionBase()); + RangerContextHolder.setSecurityContext(context); + UserSessionBase currentUserSession = ContextUtil.getCurrentUserSession(); + currentUserSession.setUserAdmin(false); + XXPortalUser xXPortalUser = new XXPortalUser(); + xXPortalUser.setLoginId(userLoginID); + xXPortalUser.setId(userId); + currentUserSession.setXXPortalUser(xXPortalUser); + + VXUser loggedInUser = new VXUser(); + List loggedInUserRole = new ArrayList(); + loggedInUserRole.add(RangerConstants.ROLE_USER); + loggedInUser.setId(8L); + loggedInUser.setName("testuser"); + loggedInUser.setUserRoleList(loggedInUserRole); + Mockito.when(xUserService.getXUserByUserName("testuser")).thenReturn(loggedInUser); + Mockito.when(svcStore.getService(Id)).thenReturn(actualService); + + RangerService service = serviceREST.getService(Id); + Assert.assertNotNull(service); + Mockito.verify(svcStore).getService(Id); + Assert.assertNull(service.getDescription()); + Assert.assertTrue(service.getConfigs().isEmpty()); + Assert.assertEquals(service.getId(), Id); + Assert.assertEquals(service.getName(), "HDFS_1"); + Assert.assertEquals(service.getType(), "1"); + } + + @Test + public void test45getServiceByNameWillOnlyReturnNameIdAndTypeForRoleUser() throws Exception { + RangerService actualService = rangerService(); + + String userLoginID = "testuser"; + Long userId = 8L; + + RangerSecurityContext context = new RangerSecurityContext(); + context.setUserSession(new UserSessionBase()); + RangerContextHolder.setSecurityContext(context); + UserSessionBase currentUserSession = ContextUtil.getCurrentUserSession(); + currentUserSession.setUserAdmin(false); + XXPortalUser xXPortalUser = new XXPortalUser(); + xXPortalUser.setLoginId(userLoginID); + xXPortalUser.setId(userId); + currentUserSession.setXXPortalUser(xXPortalUser); + + VXUser loggedInUser = new VXUser(); + List loggedInUserRole = new ArrayList(); + loggedInUserRole.add(RangerConstants.ROLE_USER); + loggedInUser.setId(8L); + loggedInUser.setName("testuser"); + loggedInUser.setUserRoleList(loggedInUserRole); + Mockito.when(xUserService.getXUserByUserName("testuser")).thenReturn(loggedInUser); + Mockito.when(svcStore.getServiceByName(actualService.getName())).thenReturn(actualService); + + RangerService service = serviceREST.getServiceByName(actualService.getName()); + Assert.assertNotNull(service); + Mockito.verify(svcStore).getServiceByName(actualService.getName()); + Assert.assertNull(service.getDescription()); + Assert.assertTrue(service.getConfigs().isEmpty()); + Assert.assertEquals(service.getId(), Id); + Assert.assertEquals(service.getName(), "HDFS_1"); + Assert.assertEquals(service.getType(), "1"); + } + + @Test + public void test46getServices() throws Exception{ + HttpServletRequest request = Mockito.mock(HttpServletRequest.class); + PList paginatedSvcs = new PList(); + RangerService svc1 = rangerService(); + + String userLoginID = "testuser"; + Long userId = 8L; + + RangerSecurityContext context = new RangerSecurityContext(); + context.setUserSession(new UserSessionBase()); + RangerContextHolder.setSecurityContext(context); + UserSessionBase currentUserSession = ContextUtil.getCurrentUserSession(); + currentUserSession.setUserAdmin(false); + XXPortalUser xXPortalUser = new XXPortalUser(); + xXPortalUser.setLoginId(userLoginID); + xXPortalUser.setId(userId); + currentUserSession.setXXPortalUser(xXPortalUser); + + VXUser loggedInUser = new VXUser(); + List loggedInUserRole = new ArrayList(); + loggedInUserRole.add(RangerConstants.ROLE_USER); + loggedInUser.setId(8L); + loggedInUser.setName("testuser"); + loggedInUser.setUserRoleList(loggedInUserRole); + + Map configs = new HashMap(); + configs.put("username", "servicemgr"); + configs.put("password", "servicemgr"); + configs.put("namenode", "servicemgr"); + configs.put("hadoop.security.authorization", "No"); + configs.put("hadoop.security.authentication", "Simple"); + configs.put("hadoop.security.auth_to_local", ""); + configs.put("dfs.datanode.kerberos.principal", ""); + configs.put("dfs.namenode.kerberos.principal", ""); + configs.put("dfs.secondary.namenode.kerberos.principal", ""); + configs.put("hadoop.rpc.protection", "Privacy"); + configs.put("commonNameForCertificate", ""); + + RangerService svc2 = new RangerService(); + svc2.setId(9L); + svc2.setConfigs(configs); + svc2.setCreateTime(new Date()); + svc2.setDescription("service policy"); + svc2.setGuid("1427365526516_835_1"); + svc2.setIsEnabled(true); + svc2.setName("YARN_1"); + svc2.setPolicyUpdateTime(new Date()); + svc2.setType("yarn"); + svc2.setUpdatedBy("Admin"); + svc2.setUpdateTime(new Date()); + + List rangerServiceList = new ArrayList(); + rangerServiceList.add(svc1); + rangerServiceList.add(svc2); + + paginatedSvcs.setList(rangerServiceList); + + SearchFilter filter = new SearchFilter(); + Mockito.when(searchUtil.getSearchFilter(request, svcService.sortFields)).thenReturn(filter); + Mockito.when(svcStore.getPaginatedServices(filter)).thenReturn(paginatedSvcs); + Mockito.when(xUserService.getXUserByUserName("testuser")).thenReturn(loggedInUser); + RangerServiceList retServiceList = serviceREST.getServices(request); + Assert.assertNotNull(retServiceList); + Assert.assertNull(retServiceList.getServices().get(0).getDescription()); + Assert.assertTrue(retServiceList.getServices().get(0).getConfigs().isEmpty()); + Assert.assertNull(retServiceList.getServices().get(1).getDescription()); + Assert.assertTrue(retServiceList.getServices().get(1).getConfigs().isEmpty()); + Assert.assertEquals(retServiceList.getServices().get(0).getId(), Id); + Assert.assertEquals(retServiceList.getServices().get(0).getName(), "HDFS_1"); + Assert.assertEquals(retServiceList.getServices().get(0).getType(), "1"); + + Assert.assertEquals(retServiceList.getServices().get(1).getId(), svc2.getId()); + Assert.assertEquals(retServiceList.getServices().get(1).getName(), "YARN_1"); + Assert.assertEquals(retServiceList.getServices().get(1).getType(), "yarn"); + + + } } diff --git a/security-admin/src/test/java/org/apache/ranger/rest/TestXUserREST.java b/security-admin/src/test/java/org/apache/ranger/rest/TestXUserREST.java index fceda025dd..1b0e750616 100644 --- a/security-admin/src/test/java/org/apache/ranger/rest/TestXUserREST.java +++ b/security-admin/src/test/java/org/apache/ranger/rest/TestXUserREST.java @@ -35,8 +35,13 @@ import org.apache.ranger.common.AppConstants; import org.apache.ranger.common.RangerConstants; import org.apache.ranger.common.StringUtil; +import org.apache.ranger.common.UserSessionBase; +import org.apache.ranger.security.context.RangerContextHolder; +import org.apache.ranger.security.context.RangerSecurityContext; +import org.apache.ranger.entity.XXPortalUser; import org.apache.ranger.db.RangerDaoManager; import org.apache.ranger.db.XXGroupDao; +import org.apache.ranger.common.ContextUtil; import org.apache.ranger.entity.XXResource; import org.apache.ranger.plugin.model.RangerPolicy; import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyItem; @@ -129,6 +134,7 @@ public class TestXUserREST { @Mock SearchCriteria searchCriteria; @Mock XGroupService xGroupService; @Mock SearchUtil searchUtil; + @Mock StringUtil stringUtil; @Mock VXLong vXLong; @Mock HttpServletRequest request; @Mock VXUser vXUser1; @@ -1982,6 +1988,113 @@ public void test112deleteUsersByUserNameNull() { } + @SuppressWarnings({ "unchecked", "static-access" }) + @Test + public void test113ErrorWhenRoleUserIsTryingToFetchAnotherUserDetails() { + + destroySession(); + String userLoginID = "testuser"; + Long userId = 8L; + + RangerSecurityContext context = new RangerSecurityContext(); + context.setUserSession(new UserSessionBase()); + RangerContextHolder.setSecurityContext(context); + UserSessionBase currentUserSession = ContextUtil.getCurrentUserSession(); + currentUserSession.setUserAdmin(false); + XXPortalUser xXPortalUser = new XXPortalUser(); + xXPortalUser.setLoginId(userLoginID); + xXPortalUser.setId(userId); + currentUserSession.setXXPortalUser(xXPortalUser); + + VXUser loggedInUser = createVXUser(); + List loggedInUserRole = new ArrayList(); + loggedInUserRole.add(RangerConstants.ROLE_USER); + loggedInUser.setId(8L); + loggedInUser.setName("testuser"); + loggedInUser.setUserRoleList(loggedInUserRole); + + HttpServletRequest request = Mockito.mock(HttpServletRequest.class); + SearchCriteria testSearchCriteria=createsearchCriteria(); + testSearchCriteria.addParam("name", "admin"); + + Mockito.when(searchUtil.extractCommonCriterias((HttpServletRequest)Mockito.any(), (List)Mockito.any())).thenReturn(testSearchCriteria); + + Mockito.when(searchUtil.extractCommonCriterias(request, xUserService.sortFields)).thenReturn(testSearchCriteria); + Mockito.when(searchUtil.extractString(request, testSearchCriteria, "emailAddress", "Email Address",null)).thenReturn(""); + Mockito.when(searchUtil.extractInt(request, testSearchCriteria, "userSource", "User Source")).thenReturn(1); + Mockito.when(searchUtil.extractInt(request, testSearchCriteria, "isVisible", "User Visibility")).thenReturn(1); + Mockito.when(searchUtil.extractInt(request, testSearchCriteria, "status", "User Status")).thenReturn(1); + Mockito.when(searchUtil.extractStringList(request, testSearchCriteria, "userRoleList", "User Role List", "userRoleList", null,null)).thenReturn(new ArrayList()); + + Mockito.when(xUserService.getXUserByUserName("testuser")).thenReturn(loggedInUser); + Mockito.when(restErrorUtil.create403RESTException("Logged-In user is not allowed to access requested user data.")).thenThrow(new WebApplicationException()); + thrown.expect(WebApplicationException.class); + + xUserRest.searchXUsers(request); + } + + @SuppressWarnings({ "unchecked", "static-access" }) + @Test + public void test114RoleUserWillGetOnlyHisOwnUserDetails() { + + destroySession(); + String userLoginID = "testuser"; + Long userId = 8L; + + RangerSecurityContext context = new RangerSecurityContext(); + context.setUserSession(new UserSessionBase()); + RangerContextHolder.setSecurityContext(context); + UserSessionBase currentUserSession = ContextUtil.getCurrentUserSession(); + currentUserSession.setUserAdmin(false); + XXPortalUser xXPortalUser = new XXPortalUser(); + xXPortalUser.setLoginId(userLoginID); + xXPortalUser.setId(userId); + currentUserSession.setXXPortalUser(xXPortalUser); + + VXUser loggedInUser = createVXUser(); + List loggedInUserRole = new ArrayList(); + loggedInUserRole.add(RangerConstants.ROLE_USER); + loggedInUser.setId(8L); + loggedInUser.setName("testuser"); + loggedInUser.setUserRoleList(loggedInUserRole); + + VXUserList expecteUserList = new VXUserList(); + VXUser expectedUser = new VXUser(); + expectedUser.setId(8L); + expectedUser.setName("testuser"); + List userList = new ArrayList(); + userList.add(expectedUser); + expecteUserList.setVXUsers(userList); + + HttpServletRequest request = Mockito.mock(HttpServletRequest.class); + SearchCriteria testSearchCriteria=createsearchCriteria(); + + Mockito.when(searchUtil.extractCommonCriterias((HttpServletRequest)Mockito.any(), (List)Mockito.any())).thenReturn(testSearchCriteria); + + Mockito.when(searchUtil.extractCommonCriterias(request, xUserService.sortFields)).thenReturn(testSearchCriteria); + Mockito.when(searchUtil.extractString(request, testSearchCriteria, "emailAddress", "Email Address",null)).thenReturn(""); + Mockito.when(searchUtil.extractInt(request, testSearchCriteria, "userSource", "User Source")).thenReturn(1); + Mockito.when(searchUtil.extractInt(request, testSearchCriteria, "isVisible", "User Visibility")).thenReturn(1); + Mockito.when(searchUtil.extractInt(request, testSearchCriteria, "status", "User Status")).thenReturn(1); + Mockito.when(searchUtil.extractStringList(request, testSearchCriteria, "userRoleList", "User Role List", "userRoleList", null,null)).thenReturn(new ArrayList()); + + Mockito.when(xUserService.getXUserByUserName("testuser")).thenReturn(loggedInUser); + Mockito.when(xUserMgr.searchXUsers(testSearchCriteria)).thenReturn(expecteUserList); + VXUserList gotVXUserList=xUserRest.searchXUsers(request); + + assertEquals(gotVXUserList.getList().size(), 1); + assertEquals(gotVXUserList.getList().get(0).getId(), expectedUser.getId()); + assertEquals(gotVXUserList.getList().get(0).getName(), expectedUser.getName()); + } + + + public void destroySession() { + RangerSecurityContext context = new RangerSecurityContext(); + context.setUserSession(null); + RangerContextHolder.setSecurityContext(context); + } + + private HashMap creategroupVisibilityMap() { From c1b63415493481c342c4f44d1da55d6b64dbbdee Mon Sep 17 00:00:00 2001 From: Bhavik Patel Date: Tue, 27 Nov 2018 14:07:26 +0530 Subject: [PATCH 151/151] RANGER-2273 : Allow service admin and delegated admin user to view list of users and groups though they have 'USER' role Signed-off-by: Pradeep --- .../java/org/apache/ranger/biz/XUserMgr.java | 96 +++++++++++++++++++ .../org/apache/ranger/rest/XUserREST.java | 59 ++++++++++++ .../security/context/RangerAPIList.java | 2 + .../scripts/views/policies/PermissionList.js | 8 +- .../scripts/views/reports/UserAccessLayout.js | 12 +-- 5 files changed, 167 insertions(+), 10 deletions(-) diff --git a/security-admin/src/main/java/org/apache/ranger/biz/XUserMgr.java b/security-admin/src/main/java/org/apache/ranger/biz/XUserMgr.java index 54ae07e229..9d48531c34 100644 --- a/security-admin/src/main/java/org/apache/ranger/biz/XUserMgr.java +++ b/security-admin/src/main/java/org/apache/ranger/biz/XUserMgr.java @@ -23,11 +23,14 @@ import java.util.Collection; import java.util.HashMap; import java.util.HashSet; +import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang.StringUtils; import org.apache.ranger.common.ContextUtil; import org.apache.ranger.common.GUIDUtil; import org.apache.ranger.common.RangerCommonEnums; @@ -1799,6 +1802,99 @@ public VXGroupList searchXGroups(SearchCriteria searchCriteria) { return vXGroupList; } + + public VXGroupList lookupXGroups(SearchCriteria searchCriteria) { + VXGroupList ret = null; + + try { + HashMap searchParams = searchCriteria.getParamList(); + String nameToLookFor = searchParams != null ? (String) searchParams.get("name") : null; + VXGroup exactMatch = null; + + if (StringUtils.isEmpty(searchCriteria.getSortBy())) { + searchCriteria.setSortBy(nameToLookFor != null ? "name" : "id"); + } + + if(nameToLookFor != null) { + exactMatch = getGroupByGroupName(nameToLookFor); + + for (Map.Entry entry : searchParams.entrySet()) { + if(exactMatch == null) { + break; + } + + String paramName = entry.getKey(); + Object paramValue = entry.getValue(); + + switch (paramName.toLowerCase()) { + case "isvisible": + if (!Objects.equals(exactMatch.getIsVisible(), paramValue)) { + exactMatch = null; + } + break; + + case "groupsource": + if (!Objects.equals(exactMatch.getGroupSource(), paramValue)) { + exactMatch = null; + } + break; + + default: + // ignore + break; + } + } + } + + VXGroupList searchResult = xGroupService.searchXGroups(searchCriteria); + + if (exactMatch != null && exactMatch.getId() != null) { + List groups = searchResult.getList(); + + if (!groups.isEmpty()) { // remove exactMatch from groups if it is present + boolean removed = false; + + for (Iterator iter = groups.iterator(); iter.hasNext(); ) { + VXGroup group = iter.next(); + + if (group != null && exactMatch.getId().equals(group.getId())) { + iter.remove(); + removed = true; + + break; + } + } + + if (!removed) { // remove the last entry, if exactMatch was not removed above - to accomodate for add() below + groups.remove(groups.size() - 1); + } + } + + groups.add(0, exactMatch); + + ret = new VXGroupList(groups); + + ret.setStartIndex(searchCriteria.getStartIndex()); + ret.setTotalCount(searchResult.getTotalCount()); + ret.setPageSize(searchCriteria.getMaxRows()); + ret.setSortBy(searchCriteria.getSortBy()); + ret.setSortType(searchCriteria.getSortType()); + } else { + ret = searchResult; + } + } catch (Exception e) { + logger.error("Error getting the exact match of group =>"+e); + } + + if (ret != null && ret.getListSize() > 0 && !hasAccessToModule(RangerConstants.MODULE_USER_GROUPS)) { + for(VXGroup vXGroup : ret.getList()) { + getMaskedVXGroup(vXGroup); + } + } + + return ret; + } + public Collection getMaskedCollection(Collection listunMasked){ List listMasked=new ArrayList(); if(listunMasked!=null) { diff --git a/security-admin/src/main/java/org/apache/ranger/rest/XUserREST.java b/security-admin/src/main/java/org/apache/ranger/rest/XUserREST.java index ec1dc26078..d06827deba 100644 --- a/security-admin/src/main/java/org/apache/ranger/rest/XUserREST.java +++ b/security-admin/src/main/java/org/apache/ranger/rest/XUserREST.java @@ -19,6 +19,7 @@ package org.apache.ranger.rest; +import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -1175,4 +1176,62 @@ public void deleteSingleGroupByGroupId(@Context HttpServletRequest request, @Pat xUserMgr.deleteXGroup(groupId, forceDelete); } } + + @GET + @Path("/lookup/users") + @Produces({ "application/xml", "application/json" }) + @PreAuthorize("@rangerPreAuthSecurityHandler.isAPIAccessible(\"" + RangerAPIList.GET_USERS_LOOKUP + "\")") + public VXStringList getUsersLookup(@Context HttpServletRequest request) { + SearchCriteria searchCriteria = searchUtil.extractCommonCriterias(request, xUserService.sortFields); + VXStringList ret = new VXStringList(); + List vXList = new ArrayList<>(); + searchUtil.extractString(request, searchCriteria, "name", "User name", null); + searchUtil.extractInt(request, searchCriteria, "isVisible", "User Visibility"); + try { + VXUserList vXUserList = xUserMgr.searchXUsers(searchCriteria); + VXString VXString = null; + for (VXUser vxUser : vXUserList.getList()) { + VXString = new VXString(); + VXString.setValue(vxUser.getName()); + vXList.add(VXString); + } + ret.setVXStrings(vXList); + ret.setPageSize(vXUserList.getPageSize()); + ret.setTotalCount(vXUserList.getTotalCount()); + ret.setSortType(vXUserList.getSortType()); + ret.setSortBy(vXUserList.getSortBy()); + } catch (Throwable excp) { + throw restErrorUtil.createRESTException(excp.getMessage()); + } + return ret; + } + + @GET + @Path("/lookup/groups") + @Produces({ "application/xml", "application/json" }) + @PreAuthorize("@rangerPreAuthSecurityHandler.isAPIAccessible(\"" + RangerAPIList.GET_GROUPS_LOOKUP + "\")") + public VXStringList getGroupsLookup(@Context HttpServletRequest request) { + VXStringList ret = new VXStringList(); + SearchCriteria searchCriteria = searchUtil.extractCommonCriterias(request, xGroupService.sortFields); + List vXList = new ArrayList<>(); + searchUtil.extractString(request, searchCriteria, "name", "group name", null); + searchUtil.extractInt(request, searchCriteria, "isVisible", "Group Visibility"); + try { + VXGroupList vXGroupList = xUserMgr.lookupXGroups(searchCriteria); + for (VXGroup vxGroup : vXGroupList.getList()) { + VXString VXString = new VXString(); + VXString.setValue(vxGroup.getName()); + vXList.add(VXString); + } + ret.setVXStrings(vXList); + ret.setPageSize(vXGroupList.getPageSize()); + ret.setTotalCount(vXGroupList.getTotalCount()); + ret.setSortType(vXGroupList.getSortType()); + ret.setSortBy(vXGroupList.getSortBy()); + } catch (Throwable excp) { + throw restErrorUtil.createRESTException(excp.getMessage()); + } + return ret; + } + } diff --git a/security-admin/src/main/java/org/apache/ranger/security/context/RangerAPIList.java b/security-admin/src/main/java/org/apache/ranger/security/context/RangerAPIList.java index 460c7fda20..da45338708 100644 --- a/security-admin/src/main/java/org/apache/ranger/security/context/RangerAPIList.java +++ b/security-admin/src/main/java/org/apache/ranger/security/context/RangerAPIList.java @@ -137,6 +137,8 @@ public class RangerAPIList { public static final String MODIFY_GROUPS_VISIBILITY = "XUserREST.modifyGroupsVisibility"; public static final String DELETE_X_GROUP = "XUserREST.deleteXGroup"; public static final String SEARCH_X_GROUPS = "XUserREST.searchXGroups"; + public static final String GET_USERS_LOOKUP = "XUserREST.getUsersLookup"; + public static final String GET_GROUPS_LOOKUP = "XUserREST.getGroupsLookup"; public static final String COUNT_X_GROUPS = "XUserREST.countXGroups"; public static final String GET_X_USER = "XUserREST.getXUser"; public static final String SECURE_GET_X_USER = "XUserREST.secureGetXUser"; diff --git a/security-admin/src/main/webapp/scripts/views/policies/PermissionList.js b/security-admin/src/main/webapp/scripts/views/policies/PermissionList.js index 9184675144..742f0a77e6 100644 --- a/security-admin/src/main/webapp/scripts/views/policies/PermissionList.js +++ b/security-admin/src/main/webapp/scripts/views/policies/PermissionList.js @@ -185,7 +185,7 @@ define(function(require) { createDropDown :function($select, typeGroup){ var that = this, tags = [], placeholder = (typeGroup) ? 'Select Group' : 'Select User', - searchUrl = (typeGroup) ? "service/xusers/groups" : "service/xusers/users"; + searchUrl = (typeGroup) ? "service/xusers/lookup/groups" : "service/xusers/lookup/users"; if(this.model.has('editMode') && !_.isEmpty($select.val())){ var temp = this.model.attributes[ (typeGroup) ? 'groupName': 'userName']; _.each(temp , function(name){ @@ -211,11 +211,11 @@ define(function(require) { var results = [] , selectedVals = []; //Get selected values of groups/users dropdown selectedVals = that.getSelectedValues($select, typeGroup); - if(data.resultSize != "0"){ + if(data.totalCount != "0"){ if(typeGroup){ - results = data.vXGroups.map(function(m, i){ return {id : _.escape(m.name), text: _.escape(m.name) }; }); + results = data.vXStrings.map(function(m){ return {id : _.escape(m.value), text: _.escape(m.value) }; }); } else { - results = data.vXUsers.map(function(m, i){ return {id : _.escape(m.name), text: _.escape(m.name) }; }); + results = data.vXStrings.map(function(m){ return {id : _.escape(m.value), text: _.escape(m.value) }; }); } if(!_.isEmpty(selectedVals)){ results = XAUtil.filterResultByText(results, selectedVals); diff --git a/security-admin/src/main/webapp/scripts/views/reports/UserAccessLayout.js b/security-admin/src/main/webapp/scripts/views/reports/UserAccessLayout.js index 90f44c19ad..74bbd5f31b 100644 --- a/security-admin/src/main/webapp/scripts/views/reports/UserAccessLayout.js +++ b/security-admin/src/main/webapp/scripts/views/reports/UserAccessLayout.js @@ -629,7 +629,7 @@ define(function(require) {'use strict'; callback(data); }, ajax: { - url: "service/xusers/groups", + url: "service/xusers/lookup/groups", dataType: 'json', data: function (term, page) { return {name : term}; @@ -638,8 +638,8 @@ define(function(require) {'use strict'; var results = [],selectedVals = []; if(!_.isEmpty(that.ui.userGroup.val())) selectedVals = that.ui.userGroup.val().split(','); - if(data.resultSize != "0"){ - results = data.vXGroups.map(function(m, i){ return {id : m.name, text: _.escape(m.name) }; }); + if(data.totalCount != "0"){ + results = data.vXStrings.map(function(m){ return {id : m.value, text: _.escape(m.value) }; }); if(!_.isEmpty(selectedVals)) results = XAUtil.filterResultByIds(results, selectedVals); return {results : results}; @@ -683,7 +683,7 @@ define(function(require) {'use strict'; callback(data); }, ajax: { - url: "service/xusers/users", + url: "service/xusers/lookup/users", dataType: 'json', data: function (term, page) { return {name : term}; @@ -692,8 +692,8 @@ define(function(require) {'use strict'; var results = [],selectedVals=[]; if(!_.isEmpty(that.ui.userName.select2('val'))) selectedVals = that.ui.userName.select2('val'); - if(data.resultSize != "0"){ - results = data.vXUsers.map(function(m, i){ return {id : m.name, text: _.escape(m.name) }; }); + if(data.totalCount != "0"){ + results = data.vXStrings.map(function(m){ return {id : m.value, text: _.escape(m.value) }; }); if(!_.isEmpty(selectedVals)) results = XAUtil.filterResultByIds(results, selectedVals); return {results : results};