diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/destination/HDFSAuditDestination.java b/agents-audit/src/main/java/org/apache/ranger/audit/destination/HDFSAuditDestination.java index 889b6ffdc7..5cf7e08fe4 100644 --- a/agents-audit/src/main/java/org/apache/ranger/audit/destination/HDFSAuditDestination.java +++ b/agents-audit/src/main/java/org/apache/ranger/audit/destination/HDFSAuditDestination.java @@ -63,6 +63,7 @@ public class HDFSAuditDestination extends AuditDestination { private String logFolder; private PrintWriter logWriter = null; + volatile FSDataOutputStream ostream = null; // output stream wrapped in logWriter private String currentFileName; @@ -169,6 +170,7 @@ public PrintWriter run() throws Exception { addDeferredCount(events.size()); out.close(); logWriter = null; + ostream = null; return false; } } catch (Throwable t) { @@ -178,7 +180,7 @@ public PrintWriter run() throws Exception { } finally { logger.info("Flushing HDFS audit. Event Size:" + events.size()); if (out != null) { - out.flush(); + flush(); } } addSuccessCount(events.size()); @@ -187,10 +189,22 @@ public PrintWriter run() throws Exception { @Override public void flush() { - if ( logWriter != null) { - logWriter.flush(); - logger.info("Flush HDFS audit logs completed....."); - } + logger.info("Flush called. name=" + getName()); + if (ostream != null) { + try { + synchronized (this) { + if (ostream != null) + // 1) PrinterWriter does not have bufferring of its own so + // we need to flush its underlying stream + // 2) HDFS flush() does not really flush all the way to disk. + ostream.hflush(); + logger.info("Flush HDFS audit logs completed....."); + } + } catch (IOException e) { + logger.error("Error on flushing log writer: " + e.getMessage() + + "\nException will be ignored. name=" + getName() + ", fileName=" + currentFileName); + } + } } /* @@ -246,6 +260,7 @@ synchronized public void stop() { + getName() + ", fileName=" + currentFileName); } logWriter = null; + ostream = null; } logStatus(); } @@ -290,7 +305,7 @@ synchronized private PrintWriter getLogFileStream() throws Exception { // Create the file to write logger.info("Creating new log file. hdfPath=" + fullPath); - FSDataOutputStream ostream = fileSystem.create(hdfPath); + ostream = fileSystem.create(hdfPath); logWriter = new PrintWriter(ostream); currentFileName = fullPath; } @@ -341,6 +356,7 @@ private void closeFileIfNeeded() throws FileNotFoundException, IOException { } logWriter = null; + ostream = null; currentFileName = null; if (!rollOverByDuration) { diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/provider/AuditProviderFactory.java b/agents-audit/src/main/java/org/apache/ranger/audit/provider/AuditProviderFactory.java index b37011e6f2..43107ba5e4 100644 --- a/agents-audit/src/main/java/org/apache/ranger/audit/provider/AuditProviderFactory.java +++ b/agents-audit/src/main/java/org/apache/ranger/audit/provider/AuditProviderFactory.java @@ -23,6 +23,7 @@ import java.util.Properties; import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -73,6 +74,7 @@ public class AuditProviderFactory { private AuditHandler mProvider = null; private String componentAppType = ""; private boolean mInitDone = false; + private JVMShutdownHook jvmShutdownHook = null; private AuditProviderFactory() { LOG.info("AuditProviderFactory: creating.."); @@ -106,6 +108,16 @@ public boolean isInitDone() { return mInitDone; } + /** + * call shutdown hook to provide a way to + * shutdown gracefully in addition to the ShutdownHook mechanism + */ + public void shutdown() { + if (isInitDone() && jvmShutdownHook != null) { + jvmShutdownHook.run(); + } + } + public synchronized void init(Properties props, String appType) { LOG.info("AuditProviderFactory: initializing.."); @@ -463,7 +475,7 @@ private AuditHandler getDefaultProvider() { private void installJvmSutdownHook(Properties props) { int shutdownHookMaxWaitSeconds = MiscUtil.getIntProperty(props, AUDIT_SHUTDOWN_HOOK_MAX_WAIT_SEC, AUDIT_SHUTDOWN_HOOK_MAX_WAIT_SEC_DEFAULT); - JVMShutdownHook jvmShutdownHook = new JVMShutdownHook(mProvider, shutdownHookMaxWaitSeconds); + jvmShutdownHook = new JVMShutdownHook(mProvider, shutdownHookMaxWaitSeconds); ShutdownHookManager.get().addShutdownHook(jvmShutdownHook, RANGER_AUDIT_SHUTDOWN_HOOK_PRIORITY); } @@ -503,6 +515,7 @@ private static class JVMShutdownHook extends Thread { final Semaphore doneCleanup = new Semaphore(0); final Thread cleanupThread; final int maxWait; + final AtomicBoolean done = new AtomicBoolean(false); public JVMShutdownHook(AuditHandler provider, int maxWait) { this.maxWait = maxWait; @@ -513,6 +526,10 @@ public JVMShutdownHook(AuditHandler provider, int maxWait) { } public void run() { + if (!done.compareAndSet(false, true)) { + LOG.info("==> JVMShutdownHook.run() already done by another thread"); + return; + } LOG.info("==> JVMShutdownHook.run()"); LOG.info("JVMShutdownHook: Signalling async audit cleanup to start."); startCleanup.release(); diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/provider/LocalFileLogBuffer.java b/agents-audit/src/main/java/org/apache/ranger/audit/provider/LocalFileLogBuffer.java index 56a24ed61d..6011dbc558 100644 --- a/agents-audit/src/main/java/org/apache/ranger/audit/provider/LocalFileLogBuffer.java +++ b/agents-audit/src/main/java/org/apache/ranger/audit/provider/LocalFileLogBuffer.java @@ -144,6 +144,8 @@ public void start(LogDestination destination) { mDispatcherThread = new DestinationDispatcherThread(this, destination, mLogger); + mDispatcherThread.setDaemon(true); + mDispatcherThread.start(); mLogger.debug("<== LocalFileLogBuffer.start()"); diff --git a/agents-common/src/main/java/org/apache/hadoop/security/SecureClientLogin.java b/agents-common/src/main/java/org/apache/hadoop/security/SecureClientLogin.java index e6b3387f97..0f0da96c70 100644 --- a/agents-common/src/main/java/org/apache/hadoop/security/SecureClientLogin.java +++ b/agents-common/src/main/java/org/apache/hadoop/security/SecureClientLogin.java @@ -49,6 +49,7 @@ public synchronized static Subject loginUserFromKeytab(String user, String path) SecureClientLoginConfiguration loginConf = new SecureClientLoginConfiguration(true, user, path); LoginContext login = new LoginContext("hadoop-keytab-kerberos", subject, null, loginConf); subject.getPrincipals().add(new User(user, AuthenticationMethod.KERBEROS, login)); + login.logout(); login.login(); return login.getSubject(); } catch (LoginException le) { @@ -63,6 +64,7 @@ public synchronized static Subject loginUserFromKeytab(String user, String path, LoginContext login = new LoginContext("hadoop-keytab-kerberos", subject, null, loginConf); KerberosName.setRules(nameRules); subject.getPrincipals().add(new User(user, AuthenticationMethod.KERBEROS, login)); + login.logout(); login.login(); return login.getSubject(); } catch (LoginException le) { @@ -71,16 +73,16 @@ public synchronized static Subject loginUserFromKeytab(String user, String path, } public synchronized static Subject loginUserWithPassword(String user, String password) throws IOException { - String tmpPass = password; try { Subject subject = new Subject(); SecureClientLoginConfiguration loginConf = new SecureClientLoginConfiguration(false, user, password); LoginContext login = new LoginContext("hadoop-keytab-kerberos", subject, null, loginConf); subject.getPrincipals().add(new User(user, AuthenticationMethod.KERBEROS, login)); + login.logout(); login.login(); return login.getSubject(); } catch (LoginException le) { - throw new IOException("Login failure for " + user + " using password " + tmpPass.replaceAll(".","*"), le); + throw new IOException("Login failure for " + user + " using password ****", le); } } diff --git a/agents-common/src/main/java/org/apache/ranger/admin/client/RangerAdminRESTClient.java b/agents-common/src/main/java/org/apache/ranger/admin/client/RangerAdminRESTClient.java index a32db46d4a..974634792c 100644 --- a/agents-common/src/main/java/org/apache/ranger/admin/client/RangerAdminRESTClient.java +++ b/agents-common/src/main/java/org/apache/ranger/admin/client/RangerAdminRESTClient.java @@ -31,6 +31,7 @@ import org.apache.ranger.admin.client.datatype.RESTResponse; import org.apache.ranger.audit.provider.MiscUtil; import org.apache.ranger.authorization.hadoop.config.RangerConfiguration; +import org.apache.ranger.authorization.utils.StringUtil; import org.apache.ranger.plugin.util.GrantRevokeRequest; import org.apache.ranger.plugin.util.RangerRESTClient; import org.apache.ranger.plugin.util.RangerRESTUtils; @@ -80,11 +81,18 @@ public void init(String serviceName, String appId, String propertyPrefix) { this.serviceName = serviceName; this.pluginId = restUtils.getPluginId(serviceName, appId); - String url = RangerConfiguration.getInstance().get(propertyPrefix + ".policy.rest.url"); + String url = ""; + String tmpUrl = RangerConfiguration.getInstance().get(propertyPrefix + ".policy.rest.url"); String sslConfigFileName = RangerConfiguration.getInstance().get(propertyPrefix + ".policy.rest.ssl.config.file"); clusterName = RangerConfiguration.getInstance().get(propertyPrefix + ".ambari.cluster.name", ""); int restClientConnTimeOutMs = RangerConfiguration.getInstance().getInt(propertyPrefix + ".policy.rest.client.connection.timeoutMs", 120 * 1000); int restClientReadTimeOutMs = RangerConfiguration.getInstance().getInt(propertyPrefix + ".policy.rest.client.read.timeoutMs", 30 * 1000); + if (!StringUtil.isEmpty(tmpUrl)) { + url = tmpUrl.trim(); + } + if (url.endsWith("/")) { + url = url.substring(0, url.length() - 1); + } init(url, sslConfigFileName, restClientConnTimeOutMs , restClientReadTimeOutMs); } diff --git a/agents-common/src/main/java/org/apache/ranger/authorization/hadoop/constants/RangerHadoopConstants.java b/agents-common/src/main/java/org/apache/ranger/authorization/hadoop/constants/RangerHadoopConstants.java index 83f720a41f..6d9fe26a47 100644 --- a/agents-common/src/main/java/org/apache/ranger/authorization/hadoop/constants/RangerHadoopConstants.java +++ b/agents-common/src/main/java/org/apache/ranger/authorization/hadoop/constants/RangerHadoopConstants.java @@ -21,7 +21,9 @@ public class RangerHadoopConstants { public static final String RANGER_ADD_HDFS_PERMISSION_PROP = "xasecure.add-hadoop-authorization"; + public static final String RANGER_OPTIMIZE_SUBACCESS_AUTHORIZATION_PROP = "ranger.optimize-subaccess-authorization" ; public static final boolean RANGER_ADD_HDFS_PERMISSION_DEFAULT = false; + public static final boolean RANGER_OPTIMIZE_SUBACCESS_AUTHORIZATION_DEFAULT = false ; public static final String READ_ACCCESS_TYPE = "read"; public static final String WRITE_ACCCESS_TYPE = "write"; public static final String EXECUTE_ACCCESS_TYPE = "execute"; diff --git a/agents-common/src/main/java/org/apache/ranger/authorization/utils/JsonUtils.java b/agents-common/src/main/java/org/apache/ranger/authorization/utils/JsonUtils.java new file mode 100644 index 0000000000..98d9c0a6ff --- /dev/null +++ b/agents-common/src/main/java/org/apache/ranger/authorization/utils/JsonUtils.java @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ranger.authorization.utils; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.reflect.TypeToken; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.MapUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +import java.lang.reflect.Type; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class JsonUtils { + private static final Log LOG = LogFactory.getLog(JsonUtils.class); + + private static final HashMap MAP_STRING_STRING = new HashMap<>(); + + private static final Gson gson; + + static { + gson = new GsonBuilder().setDateFormat("yyyyMMdd-HH:mm:ss.SSS-Z") + .create(); + } + + public static String mapToJson(Map map) { + String ret = null; + if (MapUtils.isNotEmpty(map)) { + try { + ret = gson.toJson(map); + } catch (Exception e) { + LOG.error("Invalid input data: ", e); + } + } + return ret; + } + + public static String listToJson(List list) { + String ret = null; + if (CollectionUtils.isNotEmpty(list)) { + try { + ret = gson.toJson(list); + } catch (Exception e) { + LOG.error("Invalid input data: ", e); + } + } + return ret; + } + + public static String objectToJson(Object object) { + String ret = null; + + if(object != null) { + try { + ret = gson.toJson(object); + } catch(Exception excp) { + LOG.warn("objectToJson() failed to convert object to Json", excp); + } + } + + return ret; + } + + public static T jsonToObject(String jsonStr, Class clz) { + T ret = null; + + if(StringUtils.isNotEmpty(jsonStr)) { + try { + ret = gson.fromJson(jsonStr, clz); + } catch(Exception excp) { + LOG.warn("jsonToObject() failed to convert json to object: " + jsonStr, excp); + } + } + + return ret; + } + + public static Map jsonToMapStringString(String jsonStr) { + Map ret = null; + + if(StringUtils.isNotEmpty(jsonStr)) { + try { + ret = gson.fromJson(jsonStr, MAP_STRING_STRING.getClass()); + } catch(Exception excp) { + LOG.warn("jsonToObject() failed to convert json to object: " + jsonStr, excp); + } + } + + return ret; + } + +} diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/audit/RangerMultiResourceAuditHandler.java b/agents-common/src/main/java/org/apache/ranger/plugin/audit/RangerMultiResourceAuditHandler.java index 839618e53f..fdf0054fe2 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/audit/RangerMultiResourceAuditHandler.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/audit/RangerMultiResourceAuditHandler.java @@ -43,7 +43,7 @@ public void logAuthzAudit(AuthzAuditEvent auditEvent) { @Override public void logAuthzAudits(Collection auditEvents) { - auditEvents.addAll(auditEvents); + this.auditEvents.addAll(auditEvents); } public void flushAudit() { diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/conditionevaluator/RangerScriptConditionEvaluator.java b/agents-common/src/main/java/org/apache/ranger/plugin/conditionevaluator/RangerScriptConditionEvaluator.java index 48ffc38253..5febf956d9 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/conditionevaluator/RangerScriptConditionEvaluator.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/conditionevaluator/RangerScriptConditionEvaluator.java @@ -24,12 +24,14 @@ import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.ranger.plugin.contextenricher.RangerTagForEval; import org.apache.ranger.plugin.policyengine.RangerAccessRequest; import javax.script.Bindings; import javax.script.ScriptEngine; import javax.script.ScriptEngineManager; import javax.script.ScriptException; +import java.util.Collections; import java.util.List; import java.util.Map; @@ -90,11 +92,15 @@ public boolean isMatched(RangerAccessRequest request) { RangerAccessRequest readOnlyRequest = request.getReadOnlyCopy(); - RangerScriptExecutionContext context = new RangerScriptExecutionContext(readOnlyRequest); + RangerScriptExecutionContext context = new RangerScriptExecutionContext(readOnlyRequest); + RangerTagForEval currentTag = context.getCurrentTag(); + Map tagAttribs = currentTag != null ? currentTag.getAttributes() : Collections.emptyMap(); Bindings bindings = scriptEngine.createBindings(); bindings.put("ctx", context); + bindings.put("tag", currentTag); + bindings.put("tagAttr", tagAttribs); if (LOG.isDebugEnabled()) { LOG.debug("RangerScriptConditionEvaluator.isMatched(): script={" + script + "}"); diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/conditionevaluator/RangerScriptExecutionContext.java b/agents-common/src/main/java/org/apache/ranger/plugin/conditionevaluator/RangerScriptExecutionContext.java index acd96be187..415d7fd542 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/conditionevaluator/RangerScriptExecutionContext.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/conditionevaluator/RangerScriptExecutionContext.java @@ -368,23 +368,23 @@ private Set getAllTags() { return ret; } - public void logDebug(String msg) { + public void logDebug(Object msg) { LOG.debug(msg); } - public void logInfo(String msg) { + public void logInfo(Object msg) { LOG.info(msg); } - public void logWarn(String msg) { + public void logWarn(Object msg) { LOG.warn(msg); } - public void logError(String msg) { + public void logError(Object msg) { LOG.error(msg); } - public void logFatal(String msg) { + public void logFatal(Object msg) { LOG.fatal(msg); } } diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerServiceResourceMatcher.java b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerServiceResourceMatcher.java index ecddf75e2e..f9bbb12b14 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerServiceResourceMatcher.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerServiceResourceMatcher.java @@ -28,9 +28,13 @@ import org.apache.ranger.plugin.resourcematcher.RangerResourceMatcher; import org.apache.ranger.plugin.util.ServiceDefUtil; +import java.io.Serializable; +import java.util.Comparator; import java.util.Map; public class RangerServiceResourceMatcher implements RangerPolicyResourceEvaluator { + public static final Comparator ID_COMPARATOR = new IdComparator(); + private final RangerServiceResource serviceResource; private final RangerPolicyResourceMatcher policyResourceMatcher; private final Integer leafResourceLevel; @@ -74,4 +78,11 @@ public RangerPolicyResourceMatcher.MatchType getMatchType(RangerAccessResource r RangerServiceDef getServiceDef() { return policyResourceMatcher != null ? policyResourceMatcher.getServiceDef() : null; } + + static class IdComparator implements Comparator, Serializable { + @Override + public int compare(RangerServiceResourceMatcher me, RangerServiceResourceMatcher other) { + return Long.compare(me.getId(), other.getId()); + } + } } diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java index 9a577197d1..8634bf27ce 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java @@ -27,9 +27,11 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.ranger.authorization.hadoop.config.RangerConfiguration; +import org.apache.ranger.plugin.model.RangerPolicy; import org.apache.ranger.plugin.model.RangerServiceDef; import org.apache.ranger.plugin.model.RangerServiceResource; import org.apache.ranger.plugin.model.RangerTag; +import org.apache.ranger.plugin.model.validation.RangerServiceDefHelper; import org.apache.ranger.plugin.policyengine.RangerAccessRequest; import org.apache.ranger.plugin.policyengine.RangerAccessResource; import org.apache.ranger.plugin.policyresourcematcher.RangerDefaultPolicyResourceMatcher; @@ -46,6 +48,7 @@ import java.io.Reader; import java.io.Writer; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -57,10 +60,13 @@ public class RangerTagEnricher extends RangerAbstractContextEnricher { private static final Log LOG = LogFactory.getLog(RangerTagEnricher.class); private static final Log PERF_CONTEXTENRICHER_INIT_LOG = RangerPerfTracer.getPerfLogger("contextenricher.init"); + private static final Log PERF_TRIE_OP_LOG = RangerPerfTracer.getPerfLogger("resourcetrie.retrieval"); + public static final String TAG_REFRESHER_POLLINGINTERVAL_OPTION = "tagRefresherPollingInterval"; public static final String TAG_RETRIEVER_CLASSNAME_OPTION = "tagRetrieverClassName"; public static final String TAG_DISABLE_TRIE_PREFILTER_OPTION = "disableTrieLookupPrefilter"; + public static final int[] allPolicyTypes = new int[] {RangerPolicy.POLICY_TYPE_ACCESS, RangerPolicy.POLICY_TYPE_DATAMASK, RangerPolicy.POLICY_TYPE_ROWFILTER}; private RangerTagRefresher tagRefresher = null; private RangerTagRetriever tagRetriever = null; @@ -148,6 +154,52 @@ public void enrich(RangerAccessRequest request) { } } + /* + * This class implements a cache of result of look-up of keyset of policy-resources for each of the collections of hierarchies + * for policy types: access, datamask and rowfilter. If a keyset is examined for validity in a hierarchy of a policy-type, + * then that record is maintained in this cache for later look-up. + * + * The basic idea is that with a large number of tagged service-resources, this cache will speed up performance as well as put + * a cap on the upper bound because it is expected that the cardinality of set of all possible keysets for all resource-def + * combinations in a service-def will be much smaller than the number of service-resources. + */ + + static private class ResourceHierarchies { + private final Map, Boolean> accessHierarchies = new HashMap<>(); + private final Map, Boolean> dataMaskHierarchies = new HashMap<>(); + private final Map, Boolean> rowFilterHierarchies = new HashMap<>(); + + public Boolean isValidHierarchy(int policyType, Collection resourceKeys) { + switch (policyType) { + case RangerPolicy.POLICY_TYPE_ACCESS: + return accessHierarchies.get(resourceKeys); + case RangerPolicy.POLICY_TYPE_DATAMASK: + return dataMaskHierarchies.get(resourceKeys); + case RangerPolicy.POLICY_TYPE_ROWFILTER: + return rowFilterHierarchies.get(resourceKeys); + default: + return null; + } + } + + public void addHierarchy(int policyType, Collection resourceKeys, Boolean isValid) { + switch (policyType) { + case RangerPolicy.POLICY_TYPE_ACCESS: + accessHierarchies.put(resourceKeys, isValid); + break; + case RangerPolicy.POLICY_TYPE_DATAMASK: + dataMaskHierarchies.put(resourceKeys, isValid); + break; + case RangerPolicy.POLICY_TYPE_ROWFILTER: + rowFilterHierarchies.put(resourceKeys, isValid); + break; + default: + LOG.error("unknown policy-type " + policyType); + break; + } + } + } + public void setServiceTags(final ServiceTags serviceTags) { if (serviceTags == null || CollectionUtils.isEmpty(serviceTags.getServiceResources())) { @@ -157,29 +209,52 @@ public void setServiceTags(final ServiceTags serviceTags) { List resourceMatchers = new ArrayList(); + RangerServiceDefHelper serviceDefHelper = new RangerServiceDefHelper(serviceDef, false); + List serviceResources = serviceTags.getServiceResources(); - if (CollectionUtils.isNotEmpty(serviceResources)) { + ResourceHierarchies hierarchies = new ResourceHierarchies(); + + for (RangerServiceResource serviceResource : serviceResources) { + final Collection resourceKeys = serviceResource.getResourceElements().keySet(); + + for (int policyType : allPolicyTypes) { + Boolean isValidHierarchy = hierarchies.isValidHierarchy(policyType, resourceKeys); - for (RangerServiceResource serviceResource : serviceResources) { - RangerDefaultPolicyResourceMatcher matcher = new RangerDefaultPolicyResourceMatcher(); + if (isValidHierarchy == null) { // hierarchy not yet validated + isValidHierarchy = Boolean.FALSE; - matcher.setServiceDef(this.serviceDef); - matcher.setPolicyResources(serviceResource.getResourceElements()); + for (List hierarchy : serviceDefHelper.getResourceHierarchies(policyType)) { + if (serviceDefHelper.hierarchyHasAllResources(hierarchy, resourceKeys)) { + isValidHierarchy = Boolean.TRUE; - if (LOG.isDebugEnabled()) { - LOG.debug("RangerTagEnricher.setServiceTags() - Initializing matcher with (resource=" + serviceResource - + ", serviceDef=" + this.serviceDef.getName() + ")"); + break; + } + } + hierarchies.addHierarchy(policyType, resourceKeys, isValidHierarchy); } - matcher.init(); - RangerServiceResourceMatcher serviceResourceMatcher = new RangerServiceResourceMatcher(serviceResource, matcher); - resourceMatchers.add(serviceResourceMatcher); - } + if (isValidHierarchy) { + RangerDefaultPolicyResourceMatcher matcher = new RangerDefaultPolicyResourceMatcher(); + matcher.setServiceDef(this.serviceDef); + matcher.setPolicyResources(serviceResource.getResourceElements(), policyType); + + if (LOG.isDebugEnabled()) { + LOG.debug("RangerTagEnricher.setServiceTags() - Initializing matcher with (resource=" + serviceResource + + ", serviceDef=" + this.serviceDef.getName() + ")"); + + } + matcher.init(); + + RangerServiceResourceMatcher serviceResourceMatcher = new RangerServiceResourceMatcher(serviceResource, matcher); + resourceMatchers.add(serviceResourceMatcher); + } + } } + Map> serviceResourceTrie = null; if (!disableTrieLookupPrefilter) { @@ -199,6 +274,10 @@ public void setServiceTags(final ServiceTags serviceTags) { } } + protected Long getServiceTagsVersion() { + return enrichedServiceTags != null ? enrichedServiceTags.getServiceTags().getTagVersion() : null; + } + @Override public boolean preCleanup() { boolean ret = true; @@ -247,10 +326,11 @@ private Set findMatchingTags(final RangerAccessRequest request if (request.isAccessTypeAny()) { isMatched = matchType != RangerPolicyResourceMatcher.MatchType.NONE; } else if (request.getResourceMatchingScope() == RangerAccessRequest.ResourceMatchingScope.SELF_OR_DESCENDANTS) { - isMatched = matchType == RangerPolicyResourceMatcher.MatchType.SELF || matchType == RangerPolicyResourceMatcher.MatchType.DESCENDANT; + isMatched = matchType != RangerPolicyResourceMatcher.MatchType.NONE; } else { isMatched = matchType == RangerPolicyResourceMatcher.MatchType.SELF || matchType == RangerPolicyResourceMatcher.MatchType.ANCESTOR; } + if (isMatched) { if (ret == null) { ret = new HashSet(); @@ -283,18 +363,23 @@ private List getEvaluators(RangerAccessResource re List ret = null; - final List serviceResourceMatchers = enrichedServiceTags.getServiceResourceMatchers(); final Map> serviceResourceTrie = enrichedServiceTags.getServiceResourceTrie(); - if (resource == null || resource.getKeys() == null || resource.getKeys().size() == 0 || serviceResourceTrie == null) { - ret = serviceResourceMatchers; + if (resource == null || resource.getKeys() == null || resource.getKeys().isEmpty() || serviceResourceTrie == null) { + ret = enrichedServiceTags.getServiceResourceMatchers(); } else { + RangerPerfTracer perf = null; + + if(RangerPerfTracer.isPerfTraceEnabled(PERF_TRIE_OP_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_TRIE_OP_LOG, "RangerTagEnricher.getEvaluators(resource=" + resource.getAsString() + ")"); + } + Set resourceKeys = resource.getKeys(); + List> serviceResourceMatchersList = null; + List smallestList = null; if (CollectionUtils.isNotEmpty(resourceKeys)) { - boolean isRetModifiable = false; - for (String resourceName : resourceKeys) { RangerResourceTrie trie = serviceResourceTrie.get(resourceName); @@ -302,39 +387,45 @@ private List getEvaluators(RangerAccessResource re continue; } - List resourceEvaluators = trie.getEvaluatorsForResource(resource.getValue(resourceName)); - - if (CollectionUtils.isEmpty(resourceEvaluators)) { // no policies for this resource, bail out - ret = null; - } else if (ret == null) { // initialize ret with policies found for this resource - ret = resourceEvaluators; - } else { // remove matchers from ret that are not in resourceEvaluators - if (isRetModifiable) { - ret.retainAll(resourceEvaluators); - } else { - final List shorterList; - final List longerList; - - if (ret.size() < resourceEvaluators.size()) { - shorterList = ret; - longerList = resourceEvaluators; - } else { - shorterList = resourceEvaluators; - longerList = ret; - } + List serviceResourceMatchers = trie.getEvaluatorsForResource(resource.getValue(resourceName)); - ret = new ArrayList<>(shorterList); - ret.retainAll(longerList); - isRetModifiable = true; - } + if (CollectionUtils.isEmpty(serviceResourceMatchers)) { // no policies for this resource, bail out + serviceResourceMatchersList = null; + smallestList = null; + break; } - if (CollectionUtils.isEmpty(ret)) { // if no matcher exists, bail out and return empty list - ret = null; - break; + if (smallestList == null) { + smallestList = serviceResourceMatchers; + } else { + if (serviceResourceMatchersList == null) { + serviceResourceMatchersList = new ArrayList<>(); + serviceResourceMatchersList.add(smallestList); + } + serviceResourceMatchersList.add(serviceResourceMatchers); + + if (smallestList.size() > serviceResourceMatchers.size()) { + smallestList = serviceResourceMatchers; + } } } + if (serviceResourceMatchersList != null) { + ret = new ArrayList<>(smallestList); + for (List serviceResourceMatchers : serviceResourceMatchersList) { + if (serviceResourceMatchers != smallestList) { + // remove policies from ret that are not in serviceResourceMatchers + ret.retainAll(serviceResourceMatchers); + if (CollectionUtils.isEmpty(ret)) { // if no policy exists, bail out and return empty list + ret = null; + break; + } + } + } + } else { + ret = smallestList; + } } + RangerPerfTracer.logAlways(perf); } if(ret == null) { diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagForEval.java b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagForEval.java index b8f5b429c2..e31efa3c54 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagForEval.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagForEval.java @@ -33,7 +33,7 @@ @JsonAutoDetect(fieldVisibility=JsonAutoDetect.Visibility.ANY) @JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL) -@JsonIgnoreProperties(ignoreUnknown=true, value="matchType") +@JsonIgnoreProperties(ignoreUnknown=true) @XmlRootElement @XmlAccessorType(XmlAccessType.FIELD) diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/errors/ValidationErrorCode.java b/agents-common/src/main/java/org/apache/ranger/plugin/errors/ValidationErrorCode.java index d0f015dc45..25ed6b7c44 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/errors/ValidationErrorCode.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/errors/ValidationErrorCode.java @@ -34,6 +34,7 @@ public enum ValidationErrorCode { SERVICE_VALIDATION_ERR_INVALID_SERVICE_ID(1005, "No service found for id [{0}]"), SERVICE_VALIDATION_ERR_INVALID_SERVICE_NAME(1006, "Missing service name"), SERVICE_VALIDATION_ERR_SERVICE_NAME_CONFICT(1007, "Duplicate service name: name=[{0}]"), + SERVICE_VALIDATION_ERR_SPECIAL_CHARACTERS_SERVICE_NAME(3031, "Name should not start with space, it should be less than 256 characters and special characters are not allowed(except _ - and space). : name=[{0}]"), SERVICE_VALIDATION_ERR_ID_NAME_CONFLICT(1008, "Duplicate service name: name=[{0}], id=[{1}]"), SERVICE_VALIDATION_ERR_MISSING_SERVICE_DEF(1009, "Missing service def"), SERVICE_VALIDATION_ERR_INVALID_SERVICE_DEF(1010, "Service def not found: service-def-name=[{0}]"), @@ -60,6 +61,8 @@ public enum ValidationErrorCode { SERVICE_DEF_VALIDATION_ERR_ENUM_DEF_NO_VALUES(2018, "enum [{0}] does not have any elements"), SERVICE_DEF_VALIDATION_ERR_ENUM_DEF_INVALID_DEFAULT_INDEX(2019, "default index[{0}] for enum [{1}] is invalid"), SERVICE_DEF_VALIDATION_ERR_ENUM_DEF_NULL_ENUM_ELEMENT(2020, "An enum element in enum element collection of enum [{0}] is null"), + SERVICE_DEF_VALIDATION_ERR_INVALID_SERVICE_RESOURCE_LEVELS(2021, "Resource-def levels are not in increasing order in an hierarchy"), + SERVICE_DEF_VALIDATION_ERR_NOT_LOWERCASE_NAME(2022, "{0}:[{1}] Invalid resource name. Resource name should consist of only lowercase, hyphen or underscore characters"), // POLICY VALIDATION POLICY_VALIDATION_ERR_UNSUPPORTED_ACTION(3001, "Internal error: method signature isValid(Long) is only supported for DELETE"), diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerPolicy.java b/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerPolicy.java index 140a59d3cd..b44c7db175 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerPolicy.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerPolicy.java @@ -70,6 +70,7 @@ public class RangerPolicy extends RangerBaseModelObject implements java.io.Seria private List denyExceptions = null; private List dataMaskPolicyItems = null; private List rowFilterPolicyItems = null; + private String serviceType = null; /** @@ -211,6 +212,14 @@ public void setIsAuditEnabled(Boolean isAuditEnabled) { this.isAuditEnabled = isAuditEnabled == null ? Boolean.TRUE : isAuditEnabled; } + public String getServiceType() { + return serviceType; + } + + public void setServiceType(String serviceType) { + this.serviceType = serviceType; + } + /** * @return the resources */ @@ -415,6 +424,7 @@ public StringBuilder toString(StringBuilder sb) { sb.append("description={").append(description).append("} "); sb.append("resourceSignature={").append(resourceSignature).append("} "); sb.append("isAuditEnabled={").append(isAuditEnabled).append("} "); + sb.append("serviceType={").append(serviceType).append("} "); sb.append("resources={"); if(resources != null) { diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerServiceDef.java b/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerServiceDef.java index ad5108bd43..3803c58863 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerServiceDef.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerServiceDef.java @@ -113,6 +113,7 @@ public void updateFrom(RangerServiceDef other) { setLabel(other.getLabel()); setDescription(other.getDescription()); setConfigs(other.getConfigs()); + setOptions(other.getOptions()); setResources(other.getResources()); setAccessTypes(other.getAccessTypes()); setPolicyConditions(other.getPolicyConditions()); diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerPolicyValidator.java b/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerPolicyValidator.java index 0c82b7edbb..6c93413b31 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerPolicyValidator.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerPolicyValidator.java @@ -153,72 +153,67 @@ boolean isValid(RangerPolicy policy, Action action, boolean isAdmin, List policies = getPolicies(serviceName, policyName); - if (CollectionUtils.isNotEmpty(policies)) { - if (policies.size() > 1) { - ValidationErrorCode error = ValidationErrorCode.POLICY_VALIDATION_ERR_POLICY_NAME_MULTIPLE_POLICIES_WITH_SAME_NAME; - failures.add(new ValidationFailureDetailsBuilder() - .field("name") - .isAnInternalError() - .becauseOf(error.getMessage(policyName)) - .errorCode(error.getErrorCode()) - .build()); - valid = false; - } else if (action == Action.CREATE) { // size == 1 - ValidationErrorCode error = ValidationErrorCode.POLICY_VALIDATION_ERR_POLICY_NAME_CONFLICT; - failures.add(new ValidationFailureDetailsBuilder() - .field("policy name") - .isSemanticallyIncorrect() - .becauseOf(error.getMessage(policies.iterator().next().getId(), serviceName)) - .errorCode(error.getErrorCode()) - .build()); - valid = false; - } else if (!policies.iterator().next().getId().equals(id)) { // size == 1 && action == UPDATE - ValidationErrorCode error = ValidationErrorCode.POLICY_VALIDATION_ERR_POLICY_NAME_CONFLICT; - failures.add(new ValidationFailureDetailsBuilder() - .field("id/name") + service = getService(serviceName); + if (service == null) { + ValidationErrorCode error = ValidationErrorCode.POLICY_VALIDATION_ERR_INVALID_SERVICE_NAME; + failures.add(new ValidationFailureDetailsBuilder() + .field("service name") .isSemanticallyIncorrect() - .becauseOf(error.getMessage(policies.iterator().next().getId(), serviceName)) + .becauseOf(error.getMessage(serviceName)) .errorCode(error.getErrorCode()) .build()); - valid = false; - } + valid = false; + } else { + serviceNameValid = true; } } - RangerService service = null; - boolean serviceNameValid = false; - if (StringUtils.isBlank(serviceName)) { + + if (StringUtils.isBlank(policyName)) { ValidationErrorCode error = ValidationErrorCode.POLICY_VALIDATION_ERR_MISSING_FIELD; failures.add(new ValidationFailureDetailsBuilder() - .field("service name") + .field("name") .isMissing() - .becauseOf(error.getMessage("service name")) + .becauseOf(error.getMessage("name")) .errorCode(error.getErrorCode()) .build()); valid = false; } else { - service = getService(serviceName); - if (service == null) { - ValidationErrorCode error = ValidationErrorCode.POLICY_VALIDATION_ERR_INVALID_SERVICE_NAME; - failures.add(new ValidationFailureDetailsBuilder() - .field("service name") - .isSemanticallyIncorrect() - .becauseOf(error.getMessage(serviceName)) - .errorCode(error.getErrorCode()) - .build()); - valid = false; - } else { - serviceNameValid = true; + if (service != null) { + Long policyId = getPolicyId(service.getId(), policyName); + if (policyId != null) { + if (action == Action.CREATE) { + ValidationErrorCode error = ValidationErrorCode.POLICY_VALIDATION_ERR_POLICY_NAME_CONFLICT; + failures.add(new ValidationFailureDetailsBuilder() + .field("policy name") + .isSemanticallyIncorrect() + .becauseOf(error.getMessage(policyId, serviceName)) + .errorCode(error.getErrorCode()) + .build()); + valid = false; + } else if (!policyId.equals(id)) { // action == UPDATE + ValidationErrorCode error = ValidationErrorCode.POLICY_VALIDATION_ERR_POLICY_NAME_CONFLICT; + failures.add(new ValidationFailureDetailsBuilder() + .field("id/name") + .isSemanticallyIncorrect() + .becauseOf(error.getMessage(policyId, serviceName)) + .errorCode(error.getErrorCode()) + .build()); + valid = false; + } + } } } @@ -454,7 +449,8 @@ boolean isValidResourceNames(final RangerPolicy policy, final List policyResources = getPolicyResources(policy); + convertPolicyResourceNamesToLower(policy); + Set policyResources = policy.getResources().keySet(); RangerServiceDefHelper defHelper = new RangerServiceDefHelper(serviceDef); Set> hierarchies = defHelper.getResourceHierarchies(policy.getPolicyType()); // this can be empty but not null! @@ -879,15 +875,20 @@ boolean isValidPolicyItemAccess(RangerPolicyItemAccess access, List validAccessTypes) { + String ret = null; + for (String validType : validAccessTypes) { + if (StringUtils.equalsIgnoreCase(accessType, validType)) { + ret = validType; + break; + } + } + return ret; + } } diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerServiceDefHelper.java b/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerServiceDefHelper.java index 210eb3d5e1..13a1669f73 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerServiceDefHelper.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerServiceDefHelper.java @@ -180,7 +180,7 @@ public Set> getResourceHierarchies(Integer policyType, C Set> ret = new HashSet>(); for (List hierarchy : getResourceHierarchies(policyType)) { - if (getAllResourceNames(hierarchy).containsAll(keys)) { + if (hierarchyHasAllResources(hierarchy, keys)) { ret.add(hierarchy); } } @@ -188,6 +188,28 @@ public Set> getResourceHierarchies(Integer policyType, C return ret; } + public boolean hierarchyHasAllResources(List hierarchy, Collection resourceNames) { + boolean foundAllResourceKeys = true; + + for (String resourceKey : resourceNames) { + boolean found = false; + + for (RangerResourceDef resourceDef : hierarchy) { + if (resourceDef.getName().equals(resourceKey)) { + found = true; + break; + } + } + + if (!found) { + foundAllResourceKeys = false; + break; + } + } + + return foundAllResourceKeys; + } + public Set getMandatoryResourceNames(List hierarchy) { Set result = new HashSet(hierarchy.size()); for (RangerResourceDef resourceDef : hierarchy) { diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerServiceDefValidator.java b/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerServiceDefValidator.java index 79ac6741e7..54d526da93 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerServiceDefValidator.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerServiceDefValidator.java @@ -26,6 +26,7 @@ import java.util.Map; import java.util.Objects; import java.util.Set; +import java.util.HashMap; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang.StringUtils; @@ -40,6 +41,7 @@ import org.apache.ranger.plugin.model.RangerServiceDef.RangerPolicyConditionDef; import org.apache.ranger.plugin.model.RangerServiceDef.RangerResourceDef; import org.apache.ranger.plugin.model.RangerServiceDef.RangerServiceConfigDef; +import org.apache.ranger.plugin.model.RangerServiceDef.RangerDataMaskTypeDef; import org.apache.ranger.plugin.store.ServiceStore; import com.google.common.collect.ImmutableSet; @@ -130,8 +132,8 @@ boolean isValid(final RangerServiceDef serviceDef, final Action action, final Li Long id = serviceDef.getId(); valid = isValidServiceDefId(id, action, failures) && valid; valid = isValidServiceDefName(serviceDef.getName(), id, action, failures) && valid; - valid = isValidAccessTypes(serviceDef.getAccessTypes(), failures) && valid; - if (isValidResources(serviceDef, failures)) { + valid = isValidAccessTypes(serviceDef.getId(), serviceDef.getAccessTypes(), failures, action) && valid; + if (isValidResources(serviceDef, failures, action)) { // Semantic check of resource graph can only be done if resources are "syntactically" valid valid = isValidResourceGraph(serviceDef, failures) && valid; } else { @@ -144,7 +146,8 @@ boolean isValid(final RangerServiceDef serviceDef, final Action action, final Li } else { valid = false; } - valid = isValidPolicyConditions(serviceDef.getPolicyConditions(), failures) && valid; + valid = isValidPolicyConditions(serviceDef.getId(), serviceDef.getPolicyConditions(), failures, action) && valid; + valid = isValidDataMaskTypes(serviceDef.getId(), serviceDef.getDataMaskDef().getMaskTypes(), failures, action) && valid; } if(LOG.isDebugEnabled()) { @@ -231,11 +234,10 @@ boolean isValidServiceDefName(String name, Long id, final Action action, final L return valid; } - boolean isValidAccessTypes(final List accessTypeDefs, final List failures) { + boolean isValidAccessTypes(final Long serviceDefId, final List accessTypeDefs, final List failures, final Action action) { if(LOG.isDebugEnabled()) { LOG.debug(String.format("==> RangerServiceDefValidator.isValidAccessTypes(%s, %s)", accessTypeDefs, failures)); } - boolean valid = true; if (CollectionUtils.isEmpty(accessTypeDefs)) { ValidationErrorCode error = ValidationErrorCode.SERVICE_DEF_VALIDATION_ERR_MISSING_FIELD; @@ -247,13 +249,32 @@ boolean isValidAccessTypes(final List accessTypeDefs, final .build()); valid = false; } else { + Map existingAccessTypeIDNameMap = new HashMap<>(); + if (action == Action.UPDATE) { + List existingAccessTypes = this.getServiceDef(serviceDefId).getAccessTypes(); + for (RangerAccessTypeDef existingAccessType : existingAccessTypes) { + existingAccessTypeIDNameMap.put(existingAccessType.getItemId(), existingAccessType.getName()); + } + } + LOG.debug("accessType names from db = " + existingAccessTypeIDNameMap.values()); + List defsWithImpliedGrants = new ArrayList(); Set accessNames = new HashSet(); Set ids = new HashSet(); for (RangerAccessTypeDef def : accessTypeDefs) { String name = def.getName(); + Long itemId = def.getItemId(); + LOG.debug("accessType name from input = " + name); valid = isUnique(name, accessNames, "access type name", "access types", failures) && valid; valid = isUnique(def.getItemId(), ids, "access type itemId", "access types", failures) && valid; + if (action == Action.UPDATE) { + if (existingAccessTypeIDNameMap.get(itemId) != null && !existingAccessTypeIDNameMap.get(itemId).equals(name)) { + ValidationErrorCode error; + error = ValidationErrorCode.SERVICE_DEF_VALIDATION_ERR_SERVICE_DEF_NAME_CONFICT; + failures.add((new ValidationFailureDetailsBuilder()).field("access type name").isSemanticallyIncorrect().errorCode(error.getErrorCode()).becauseOf(String.format("changing %s[%s] in %s is not supported", "access type name", name, "access types")).build()); + valid = false; + } + } if (CollectionUtils.isNotEmpty(def.getImpliedGrants())) { defsWithImpliedGrants.add(def); } @@ -295,7 +316,7 @@ boolean isValidAccessTypes(final List accessTypeDefs, final return valid; } - boolean isValidPolicyConditions(List policyConditions, List failures) { + boolean isValidPolicyConditions(Long serviceDefId, List policyConditions, List failures, final Action action) { if(LOG.isDebugEnabled()) { LOG.debug(String.format("==> RangerServiceDefValidator.isValidPolicyConditions(%s, %s)", policyConditions, failures)); } @@ -304,12 +325,31 @@ boolean isValidPolicyConditions(List policyConditions, if (CollectionUtils.isEmpty(policyConditions)) { LOG.debug("Configs collection was null/empty! ok"); } else { + Map existingPolicyCondIDNameMap = new HashMap<>(); + if (action == Action.UPDATE) { + List existingPolicyConditions = this.getServiceDef(serviceDefId).getPolicyConditions(); + for (RangerPolicyConditionDef existingPolicyCondition : existingPolicyConditions) { + existingPolicyCondIDNameMap.put(existingPolicyCondition.getItemId(), existingPolicyCondition.getName()); + } + } + LOG.debug("policy condition names from db = " + existingPolicyCondIDNameMap.values()); + Set ids = new HashSet(); Set names = new HashSet(); for (RangerPolicyConditionDef conditionDef : policyConditions) { - valid = isUnique(conditionDef.getItemId(), ids, "policy condition def itemId", "policy condition defs", failures) && valid; String name = conditionDef.getName(); + Long itemId = conditionDef.getItemId(); + LOG.debug("policy condition name from input = " + name); + valid = isUnique(itemId, ids, "policy condition def itemId", "policy condition defs", failures) && valid; valid = isUnique(name, names, "policy condition def name", "policy condition defs", failures) && valid; + if (action == Action.UPDATE) { + if (existingPolicyCondIDNameMap.get(itemId) != null && !existingPolicyCondIDNameMap.get(itemId).equals(name)) { + ValidationErrorCode error; + error = ValidationErrorCode.SERVICE_DEF_VALIDATION_ERR_SERVICE_DEF_NAME_CONFICT; + failures.add((new ValidationFailureDetailsBuilder()).field("policy condition def name").isSemanticallyIncorrect().errorCode(error.getErrorCode()).becauseOf(String.format("changing %s[%s] in %s is not supported", "policy condition def name", name, "policy condition defs")).build()); + valid = false; + } + } if (StringUtils.isBlank(conditionDef.getEvaluator())) { ValidationErrorCode error = ValidationErrorCode.SERVICE_DEF_VALIDATION_ERR_POLICY_CONDITION_NULL_EVALUATOR; failures.add(new ValidationFailureDetailsBuilder() @@ -445,7 +485,7 @@ boolean isValidConfigType(String type, String configName, List failures) { + public boolean isValidResources(RangerServiceDef serviceDef, List failures, final Action action) { if(LOG.isDebugEnabled()) { LOG.debug(String.format("==> RangerServiceDefValidator.isValidResources(%s, %s)", serviceDef, failures)); } @@ -462,14 +502,34 @@ boolean isValidResources(RangerServiceDef serviceDef, List existingResourceIDNameMap = new HashMap<>(); + if (action == Action.UPDATE) { + List existingResources = this.getServiceDef(serviceDef.getId()).getResources(); + for (RangerResourceDef existingResource : existingResources) { + existingResourceIDNameMap.put(existingResource.getItemId(), existingResource.getName()); + } + } + LOG.debug("resource names from db = " + existingResourceIDNameMap.values()); + Set names = new HashSet(resources.size()); Set ids = new HashSet(resources.size()); for (RangerResourceDef resource : resources) { /* * While id is the natural key, name is a surrogate key. At several places code expects resource name to be unique within a service. */ - valid = isUnique(resource.getName(), names, "resource name", "resources", failures) && valid; - valid = isUnique(resource.getItemId(), ids, "resource itemId", "resources", failures) && valid; + String name = resource.getName(); + Long itemId = resource.getItemId(); + LOG.debug("resource name from input = " + name); + valid = isUnique(name, names, "resource name", "resources", failures) && valid; + valid = isUnique(itemId, ids, "resource itemId", "resources", failures) && valid; + if (action == Action.UPDATE) { + if (existingResourceIDNameMap.get(itemId) != null && !existingResourceIDNameMap.get(itemId).equals(name)) { + ValidationErrorCode error; + error = ValidationErrorCode.SERVICE_DEF_VALIDATION_ERR_SERVICE_DEF_NAME_CONFICT; + failures.add((new ValidationFailureDetailsBuilder()).field("resource name").isSemanticallyIncorrect().errorCode(error.getErrorCode()).becauseOf(String.format("changing %s[%s] in %s is not supported", "resource name", name, "resources")).build()); + valid = false; + } + } } } @@ -611,4 +671,47 @@ boolean isValidEnumElements(List enumElementsDefs, List dataMaskTypes, List failures, final Action action) { + if(LOG.isDebugEnabled()) { + LOG.debug(String.format("==> RangerServiceDefValidator.isValidDataMaskTypes(%s, %s)", dataMaskTypes, failures)); + } + boolean valid = true; + + if (CollectionUtils.isEmpty(dataMaskTypes)) { + LOG.debug("Configs collection was null/empty! ok"); + } else { + Map existingDataMaskTypeIDNameMap = new HashMap<>(); + if (action == Action.UPDATE) { + List existingDataMaskTypes = this.getServiceDef(serviceDefId).getDataMaskDef().getMaskTypes(); + for (RangerDataMaskTypeDef existingDataMaskType : existingDataMaskTypes) { + existingDataMaskTypeIDNameMap.put(existingDataMaskType.getItemId(), existingDataMaskType.getName()); + } + } + LOG.debug("data mask type names from db = " + existingDataMaskTypeIDNameMap.values()); + + Set ids = new HashSet(); + Set names = new HashSet(); + for (RangerDataMaskTypeDef dataMaskType : dataMaskTypes) { + String name = dataMaskType.getName(); + Long itemId = dataMaskType.getItemId(); + LOG.debug("data mask type name from input = " + name); + valid = isUnique(itemId, ids, "data mask type def itemId", "data mask type defs", failures) && valid; + valid = isUnique(name, names, "data mask type def name", "data mask type defs", failures) && valid; + if (action == Action.UPDATE) { + if (existingDataMaskTypeIDNameMap.get(itemId) != null && !existingDataMaskTypeIDNameMap.get(itemId).equals(name)) { + ValidationErrorCode error; + error = ValidationErrorCode.SERVICE_DEF_VALIDATION_ERR_SERVICE_DEF_NAME_CONFICT; + failures.add((new ValidationFailureDetailsBuilder()).field("data mask type def name").isSemanticallyIncorrect().errorCode(error.getErrorCode()).becauseOf(String.format("changing %s[%s] in %s is not supported", "data mask type def name", name, "data mask type defs")).build()); + valid = false; + } + } + } + } + + if(LOG.isDebugEnabled()) { + LOG.debug(String.format("<== RangerServiceDefValidator.isValidDataMaskTypes(%s, %s): %s", dataMaskTypes, failures, valid)); + } + return valid; + } } diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerServiceValidator.java b/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerServiceValidator.java index d3efdc9b55..48b9d28912 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerServiceValidator.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerServiceValidator.java @@ -22,7 +22,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Set; - +import java.util.regex.Pattern; import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -35,9 +35,10 @@ import com.google.common.collect.Sets; public class RangerServiceValidator extends RangerValidator { - private static final Log LOG = LogFactory.getLog(RangerServiceValidator.class); + static final public String VALIDATION_SERVICE_NAME = "^[a-zA-Z0-9_-][a-zA-Z0-9\\s_-]{0,254}"; + static Pattern serviceNameCompiledRegEx; public RangerServiceValidator(ServiceStore store) { super(store); } @@ -151,9 +152,8 @@ boolean isValid(RangerService service, Action action, List getPolicies(final String serviceName, final String policyName) { + Long getPolicyId(final Long serviceId, final String policyName) { if(LOG.isDebugEnabled()) { - LOG.debug("==> RangerValidator.getPolicies(" + serviceName + ", " + policyName + ")"); + LOG.debug("==> RangerValidator.getPolicyId(" + serviceId + ", " + policyName + ")"); } - List policies = null; + Long policyId = null; try { - SearchFilter filter = new SearchFilter(); - if (StringUtils.isNotBlank(policyName)) { - filter.setParam(SearchFilter.POLICY_NAME, policyName); - } - filter.setParam(SearchFilter.SERVICE_NAME, serviceName); - - policies = _store.getPolicies(filter); + policyId = _store.getPolicyId(serviceId, policyName); + } catch (Exception e) { LOG.debug("Encountred exception while retrieving service from service store!", e); } - + if(LOG.isDebugEnabled()) { - int count = policies == null ? 0 : policies.size(); - LOG.debug("<== RangerValidator.getPolicies(" + serviceName + ", " + policyName + "): count[" + count + "], " + policies); + LOG.debug("<== RangerValidator.getPolicyId(" + serviceId + ", " + policyName + "): policy-id[" + policyId + "]"); } - return policies; + return policyId; } List getPoliciesForResourceSignature(String serviceName, String policySignature) { @@ -309,7 +304,7 @@ Set getAccessTypes(RangerServiceDef serviceDef) { if (StringUtils.isBlank(accessType)) { LOG.warn("Access type def name was null/empty/blank!"); } else { - accessTypes.add(accessType.toLowerCase()); + accessTypes.add(accessType); } } } @@ -415,22 +410,22 @@ Set getAllResourceNames(RangerServiceDef serviceDef) { } return resourceNames; } - + /** - * Returns the resource-types defined on the policy converted to lowe-case + * Converts, in place, the resources defined in the policy to have lower-case resource-def-names * @param policy * @return */ - Set getPolicyResources(RangerPolicy policy) { - if (policy == null || policy.getResources() == null || policy.getResources().isEmpty()) { - return new HashSet(); - } else { - Set result = new HashSet(); - for (String name : policy.getResources().keySet()) { - result.add(name.toLowerCase()); + + void convertPolicyResourceNamesToLower(RangerPolicy policy) { + Map lowerCasePolicyResources = new HashMap<>(); + if (policy.getResources() != null) { + for (Map.Entry entry : policy.getResources().entrySet()) { + String lowerCasekey = entry.getKey().toLowerCase(); + lowerCasePolicyResources.put(lowerCasekey, entry.getValue()); } - return result; } + policy.setResources(lowerCasePolicyResources); } Map getValidationRegExes(RangerServiceDef serviceDef) { @@ -588,6 +583,37 @@ boolean isUnique(Integer value, Set alreadySeen, String valueName, Stri return valid; } + /* + * Important: Resource-names are required to be lowercase. This is used in validating policy create/update operations. + * Ref: RANGER-2272 + */ + boolean isValidResourceName(final String value, final String valueContext, final List failures) { + boolean ret = true; + + if (value != null && !StringUtils.isEmpty(value)) { + int sz = value.length(); + + for(int i = 0; i < sz; ++i) { + char c = value.charAt(i); + if (!(Character.isLowerCase(c) || c == '-' || c == '_')) { // Allow only lowercase, hyphen or underscore characters + ret = false; + break; + } + } + } else { + ret = false; + } + if (!ret) { + ValidationErrorCode errorCode = ValidationErrorCode.SERVICE_DEF_VALIDATION_ERR_NOT_LOWERCASE_NAME; + failures.add(new ValidationFailureDetailsBuilder() + .errorCode(errorCode.getErrorCode()) + .field(value) + .becauseOf(errorCode.getMessage(valueContext, value)) + .build()); + } + return ret; + } + boolean isUnique(final String value, final Set alreadySeen, final String valueName, final String collectionName, final List failures) { return isUnique(value, null, alreadySeen, valueName, collectionName, failures); } diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngine.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngine.java index b758d693d8..d9b0298a12 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngine.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngine.java @@ -75,6 +75,8 @@ public interface RangerPolicyEngine { List getAllowedPolicies(String user, Set userGroups, String accessType); + List getMatchingPolicies(RangerAccessResource resource); + RangerResourceAccessInfo getResourceAccessInfo(RangerAccessRequest request); void reorderPolicyEvaluators(); diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineCache.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineCache.java index 51f2142e32..58fbffd7ec 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineCache.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineCache.java @@ -19,7 +19,6 @@ package org.apache.ranger.plugin.policyengine; -import java.util.Collections; import java.util.HashMap; import java.util.Map; @@ -28,20 +27,12 @@ import org.apache.ranger.plugin.store.ServiceStore; import org.apache.ranger.plugin.util.ServicePolicies; -public class RangerPolicyEngineCache { +class RangerPolicyEngineCache { private static final Log LOG = LogFactory.getLog(RangerPolicyEngineCache.class); - private static final RangerPolicyEngineCache sInstance = new RangerPolicyEngineCache(); + private final Map policyEngineCache = new HashMap(); - private final Map policyEngineCache = Collections.synchronizedMap(new HashMap()); - - private RangerPolicyEngineOptions options = null; - - public static RangerPolicyEngineCache getInstance() { - return sInstance; - } - - public RangerPolicyEngine getPolicyEngine(String serviceName, ServiceStore svcStore) { + synchronized final RangerPolicyEngine getPolicyEngine(String serviceName, ServiceStore svcStore, RangerPolicyEngineOptions options) { RangerPolicyEngine ret = null; if(serviceName != null) { @@ -55,9 +46,9 @@ public RangerPolicyEngine getPolicyEngine(String serviceName, ServiceStore svcSt if(policies != null) { if(ret == null) { - ret = addPolicyEngine(policies); + ret = addPolicyEngine(policies, options); } else if(policies.getPolicyVersion() != null && !policies.getPolicyVersion().equals(policyVersion)) { - ret = addPolicyEngine(policies); + ret = addPolicyEngine(policies, options); } } } catch(Exception excp) { @@ -69,15 +60,7 @@ public RangerPolicyEngine getPolicyEngine(String serviceName, ServiceStore svcSt return ret; } - public RangerPolicyEngineOptions getPolicyEngineOptions() { - return options; - } - - public void setPolicyEngineOptions(RangerPolicyEngineOptions options) { - this.options = options; - } - - private RangerPolicyEngine addPolicyEngine(ServicePolicies policies) { + private RangerPolicyEngine addPolicyEngine(ServicePolicies policies, RangerPolicyEngineOptions options) { RangerPolicyEngine ret = new RangerPolicyEngineImpl("ranger-admin", policies, options); policyEngineCache.put(policies.getServiceName(), ret); diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineCacheForEngineOptions.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineCacheForEngineOptions.java new file mode 100644 index 0000000000..ca6a2a3956 --- /dev/null +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineCacheForEngineOptions.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.plugin.policyengine; + +import org.apache.ranger.plugin.store.ServiceStore; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +public class RangerPolicyEngineCacheForEngineOptions { + + private static volatile RangerPolicyEngineCacheForEngineOptions sInstance = null; + + private final Map policyEngineCacheForEngineOptions = Collections.synchronizedMap(new HashMap()); + + public static RangerPolicyEngineCacheForEngineOptions getInstance() { + RangerPolicyEngineCacheForEngineOptions ret = sInstance; + if (ret == null) { + synchronized (RangerPolicyEngineCacheForEngineOptions.class) { + ret = sInstance; + if (ret == null) { + sInstance = new RangerPolicyEngineCacheForEngineOptions(); + ret = sInstance; + } + } + } + return ret; + } + + public final RangerPolicyEngine getPolicyEngine(String serviceName, ServiceStore svcStore, RangerPolicyEngineOptions options) { + + RangerPolicyEngineCache policyEngineCache; + + synchronized (this) { + policyEngineCache = policyEngineCacheForEngineOptions.get(options); + if (policyEngineCache == null) { + policyEngineCache = new RangerPolicyEngineCache(); + policyEngineCacheForEngineOptions.put(options, policyEngineCache); + } + } + return policyEngineCache.getPolicyEngine(serviceName, svcStore, options); + } +} + diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineImpl.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineImpl.java index a359d010fe..ab99b051cf 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineImpl.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineImpl.java @@ -31,6 +31,7 @@ import org.apache.ranger.plugin.model.RangerServiceDef; import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyResource; import org.apache.ranger.plugin.policyevaluator.RangerPolicyEvaluator; +import org.apache.ranger.plugin.policyresourcematcher.RangerPolicyResourceMatcher; import org.apache.ranger.plugin.util.RangerAccessRequestUtil; import org.apache.ranger.plugin.util.RangerPerfTracer; import org.apache.ranger.plugin.util.ServicePolicies; @@ -296,7 +297,9 @@ public RangerAccessResult isAccessAllowed(RangerAccessRequest request, RangerAcc RangerPerfTracer perf = null; if(RangerPerfTracer.isPerfTraceEnabled(PERF_POLICYENGINE_REQUEST_LOG)) { - perf = RangerPerfTracer.getPerfTracer(PERF_POLICYENGINE_REQUEST_LOG, "RangerPolicyEngine.isAccessAllowed(requestHashCode=" + Integer.toHexString(System.identityHashCode(request)) + ")"); + String requestHashCode = Integer.toHexString(System.identityHashCode(request)); + perf = RangerPerfTracer.getPerfTracer(PERF_POLICYENGINE_REQUEST_LOG, "RangerPolicyEngine.isAccessAllowed(requestHashCode=" + requestHashCode + ")"); + LOG.info("RangerPolicyEngineImpl.isAccessAllowed(" + requestHashCode + ", " + request + ")"); } RangerAccessResult ret = isAccessAllowedNoAudit(request); @@ -417,7 +420,7 @@ public boolean isAccessAllowed(RangerAccessResource resource, String user, Set getAllowedPolicies(String user, Set userGroups return ret; } + @Override + public List getMatchingPolicies(RangerAccessResource resource) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerPolicyEngineImpl.getMatchingPolicies(" + resource + ")"); + } + + List ret = new ArrayList<>(); + + RangerAccessRequestImpl request = new RangerAccessRequestImpl(resource, RangerPolicyEngine.ANY_ACCESS, null, null); + + preProcess(request); + + if (hasTagPolicies()) { + Set tags = RangerAccessRequestUtil.getRequestTagsFromContext(request.getContext()); + + if (CollectionUtils.isNotEmpty(tags)) { + for (RangerTagForEval tag : tags) { + RangerAccessRequest tagEvalRequest = new RangerTagAccessRequest(tag, tagPolicyRepository.getServiceDef(), request); + RangerAccessResource tagResource = tagEvalRequest.getResource(); + List accessPolicyEvaluators = tagPolicyRepository.getLikelyMatchPolicyEvaluators(tagResource); + List dataMaskPolicyEvaluators = tagPolicyRepository.getLikelyMatchDataMaskPolicyEvaluators(tagResource); + List rowFilterPolicyEvaluators = tagPolicyRepository.getLikelyMatchRowFilterPolicyEvaluators(tagResource); + + List[] likelyEvaluators = new List[] { accessPolicyEvaluators, dataMaskPolicyEvaluators, rowFilterPolicyEvaluators }; + + for (List evaluators : likelyEvaluators) { + for (RangerPolicyEvaluator evaluator : evaluators) { + RangerPolicyResourceMatcher matcher = evaluator.getPolicyResourceMatcher(); + if (matcher != null && matcher.isMatch(tagResource, RangerPolicyResourceMatcher.MatchScope.SELF_OR_ANCESTOR_OR_DESCENDANT, null)) { + ret.add(evaluator.getPolicy()); + } + } + } + } + } + } + + if (hasResourcePolicies()) { + List accessPolicyEvaluators = policyRepository.getLikelyMatchPolicyEvaluators(resource); + List dataMaskPolicyEvaluators = policyRepository.getLikelyMatchDataMaskPolicyEvaluators(resource); + List rowFilterPolicyEvaluators = policyRepository.getLikelyMatchRowFilterPolicyEvaluators(resource); + + List[] likelyEvaluators = new List[] { accessPolicyEvaluators, dataMaskPolicyEvaluators, rowFilterPolicyEvaluators }; + + for (List evaluators : likelyEvaluators) { + for (RangerPolicyEvaluator evaluator : evaluators) { + RangerPolicyResourceMatcher matcher = evaluator.getPolicyResourceMatcher(); + if (matcher != null && matcher.isMatch(resource, RangerPolicyResourceMatcher.MatchScope.SELF_OR_ANCESTOR_OR_DESCENDANT, null)) { + ret.add(evaluator.getPolicy()); + } + } + } + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerPolicyEngineImpl.getMatchingPolicies(" + resource + ") : " + ret.size()); + } + return ret; + } + @Override public RangerResourceAccessInfo getResourceAccessInfo(RangerAccessRequest request) { if(LOG.isDebugEnabled()) { @@ -558,7 +621,7 @@ public RangerResourceAccessInfo getResourceAccessInfo(RangerAccessRequest reques for (RangerTagForEval tag : tags) { RangerAccessRequest tagEvalRequest = new RangerTagAccessRequest(tag, tagPolicyRepository.getServiceDef(), request); - List evaluators = tagPolicyRepository.getPolicyEvaluators(tagEvalRequest.getResource()); + List evaluators = tagPolicyRepository.getLikelyMatchPolicyEvaluators(tagEvalRequest.getResource()); for (RangerPolicyEvaluator evaluator : evaluators) { evaluator.getResourceAccessInfo(tagEvalRequest, ret); @@ -567,7 +630,7 @@ public RangerResourceAccessInfo getResourceAccessInfo(RangerAccessRequest reques } } - List resPolicyEvaluators = policyRepository.getPolicyEvaluators(request.getResource()); + List resPolicyEvaluators = policyRepository.getLikelyMatchPolicyEvaluators(request.getResource()); if(CollectionUtils.isNotEmpty(resPolicyEvaluators)) { for (RangerPolicyEvaluator evaluator : resPolicyEvaluators) { @@ -615,7 +678,7 @@ protected RangerAccessResult isAccessAllowedNoAudit(RangerAccessRequest request) ret.setIsAccessDetermined(false); // discard allowed result by tag-policies, to evaluate resource policies for possible deny } - List evaluators = policyRepository.getPolicyEvaluators(request.getResource()); + List evaluators = policyRepository.getLikelyMatchPolicyEvaluators(request.getResource()); for (RangerPolicyEvaluator evaluator : evaluators) { ret.incrementEvaluatedPoliciesCount(); evaluator.evaluate(request, ret); @@ -673,7 +736,7 @@ protected void isAccessAllowedForTagPolicies(final RangerAccessRequest request, } tagEvalResult.setAuditResultFrom(result); - List evaluators = tagPolicyRepository.getPolicyEvaluators(tagEvalRequest.getResource()); + List evaluators = tagPolicyRepository.getLikelyMatchPolicyEvaluators(tagEvalRequest.getResource()); for (RangerPolicyEvaluator evaluator : evaluators) { result.incrementEvaluatedPoliciesCount(); @@ -751,7 +814,7 @@ RangerDataMaskResult evalDataMaskPoliciesNoAudit(RangerAccessRequest request) { if (evaluateResourcePolicies) { boolean findAuditByResource = !ret.getIsAuditedDetermined(); boolean foundInCache = findAuditByResource && policyRepository.setAuditEnabledFromCache(request, ret); - List evaluators = policyRepository.getDataMaskPolicyEvaluators(request.getResource()); + List evaluators = policyRepository.getLikelyMatchDataMaskPolicyEvaluators(request.getResource()); for (RangerPolicyEvaluator evaluator : evaluators) { ret.incrementEvaluatedPoliciesCount(); @@ -789,7 +852,7 @@ protected void evalDataMaskPoliciesForTagPolicies(final RangerAccessRequest requ if (CollectionUtils.isNotEmpty(tagEvaluators)) { Set tags = RangerAccessRequestUtil.getRequestTagsFromContext(request.getContext()); - List dataMaskEvaluators = tagPolicyRepository.getDataMaskPolicyEvaluators(tags); + List dataMaskEvaluators = tagPolicyRepository.getLikelyMatchDataMaskPolicyEvaluators(tags); if (CollectionUtils.isNotEmpty(dataMaskEvaluators)) { for (PolicyEvaluatorForTag dataMaskEvaluator : dataMaskEvaluators) { @@ -862,7 +925,7 @@ RangerRowFilterResult evalRowFilterPoliciesNoAudit(RangerAccessRequest request) if (evaluateResourcePolicies) { boolean findAuditByResource = !ret.getIsAuditedDetermined(); boolean foundInCache = findAuditByResource && policyRepository.setAuditEnabledFromCache(request, ret); - List evaluators = policyRepository.getRowFilterPolicyEvaluators(request.getResource()); + List evaluators = policyRepository.getLikelyMatchRowFilterPolicyEvaluators(request.getResource()); for (RangerPolicyEvaluator evaluator : evaluators) { ret.incrementEvaluatedPoliciesCount(); @@ -894,7 +957,7 @@ protected void evalRowFilterPoliciesForTagPolicies(final RangerAccessRequest req if (CollectionUtils.isNotEmpty(tagEvaluators)) { Set tags = RangerAccessRequestUtil.getRequestTagsFromContext(request.getContext()); - List rowFilterEvaluators = tagPolicyRepository.getRowFilterPolicyEvaluators(tags); + List rowFilterEvaluators = tagPolicyRepository.getLikelyMatchRowFilterPolicyEvaluators(tags); if (CollectionUtils.isNotEmpty(rowFilterEvaluators)) { for (PolicyEvaluatorForTag rowFilterEvaluator : rowFilterEvaluators) { diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineOptions.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineOptions.java index 2b2cf9b375..4c63f3b77e 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineOptions.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineOptions.java @@ -19,15 +19,133 @@ package org.apache.ranger.plugin.policyengine; +import org.apache.hadoop.conf.Configuration; import org.apache.ranger.plugin.policyevaluator.RangerPolicyEvaluator; public class RangerPolicyEngineOptions { - public String evaluatorType = RangerPolicyEvaluator.EVALUATOR_TYPE_AUTO; - public boolean cacheAuditResults = true; + public String evaluatorType = RangerPolicyEvaluator.EVALUATOR_TYPE_AUTO; + public boolean disableContextEnrichers = false; public boolean disableCustomConditions = false; - public boolean disableTagPolicyEvaluation = true; - public boolean evaluateDelegateAdminOnly = false; + public boolean disableTagPolicyEvaluation = false; public boolean disableTrieLookupPrefilter = false; + public boolean cacheAuditResults = true; + public boolean evaluateDelegateAdminOnly = false; + public boolean enableTagEnricherWithLocalRefresher = false; + public boolean optimizeTrieForRetrieval = false; + + public void configureForPlugin(Configuration conf, String propertyPrefix) { + disableContextEnrichers = conf.getBoolean(propertyPrefix + ".policyengine.option.disable.context.enrichers", false); + disableCustomConditions = conf.getBoolean(propertyPrefix + ".policyengine.option.disable.custom.conditions", false); + disableTagPolicyEvaluation = conf.getBoolean(propertyPrefix + ".policyengine.option.disable.tagpolicy.evaluation", false); + disableTrieLookupPrefilter = conf.getBoolean(propertyPrefix + ".policyengine.option.disable.trie.lookup.prefilter", false); + + cacheAuditResults = conf.getBoolean(propertyPrefix + ".policyengine.option.cache.audit.results", true); + + if (!disableTrieLookupPrefilter) { + cacheAuditResults = false; + } + evaluateDelegateAdminOnly = false; + enableTagEnricherWithLocalRefresher = false; + optimizeTrieForRetrieval = conf.getBoolean(propertyPrefix + ".policyengine.option.optimize.trie.for.retrieval", false); + + } + + public void configureDefaultRangerAdmin(Configuration conf, String propertyPrefix) { + disableContextEnrichers = conf.getBoolean(propertyPrefix + ".policyengine.option.disable.context.enrichers", true); + disableCustomConditions = conf.getBoolean(propertyPrefix + ".policyengine.option.disable.custom.conditions", true); + disableTagPolicyEvaluation = conf.getBoolean(propertyPrefix + ".policyengine.option.disable.tagpolicy.evaluation", true); + disableTrieLookupPrefilter = conf.getBoolean(propertyPrefix + ".policyengine.option.disable.trie.lookup.prefilter", false); + + cacheAuditResults = false; + evaluateDelegateAdminOnly = false; + enableTagEnricherWithLocalRefresher = false; + optimizeTrieForRetrieval = conf.getBoolean(propertyPrefix + ".policyengine.option.optimize.trie.for.retrieval", false); + + } + + public void configureDelegateAdmin(Configuration conf, String propertyPrefix) { + disableContextEnrichers = conf.getBoolean(propertyPrefix + ".policyengine.option.disable.context.enrichers", true); + disableCustomConditions = conf.getBoolean(propertyPrefix + ".policyengine.option.disable.custom.conditions", true); + disableTagPolicyEvaluation = conf.getBoolean(propertyPrefix + ".policyengine.option.disable.tagpolicy.evaluation", true); + disableTrieLookupPrefilter = conf.getBoolean(propertyPrefix + ".policyengine.option.disable.trie.lookup.prefilter", false); + optimizeTrieForRetrieval = conf.getBoolean(propertyPrefix + ".policyengine.option.optimize.trie.for.retrieval", false); + + + cacheAuditResults = false; + evaluateDelegateAdminOnly = true; + enableTagEnricherWithLocalRefresher = false; + + } + + public void configureRangerAdminForPolicySearch(Configuration conf, String propertyPrefix) { + disableContextEnrichers = conf.getBoolean(propertyPrefix + ".policyengine.option.disable.context.enrichers", true); + disableCustomConditions = conf.getBoolean(propertyPrefix + ".policyengine.option.disable.custom.conditions", true); + disableTagPolicyEvaluation = conf.getBoolean(propertyPrefix + ".policyengine.option.disable.tagpolicy.evaluation", false); + disableTrieLookupPrefilter = conf.getBoolean(propertyPrefix + ".policyengine.option.disable.trie.lookup.prefilter", false); + optimizeTrieForRetrieval = conf.getBoolean(propertyPrefix + ".policyengine.option.optimize.trie.for.retrieval", false); + + + cacheAuditResults = false; + evaluateDelegateAdminOnly = false; + enableTagEnricherWithLocalRefresher = true; + } + + /* + * There is no need to implement these, as the options are predefined in a component ServiceREST and hence + * guaranteed to be unique objects. That implies that the default equals and hashCode should suffice. + */ + + @Override + public boolean equals(Object other) { + boolean ret = false; + if (other instanceof RangerPolicyEngineOptions) { + RangerPolicyEngineOptions that = (RangerPolicyEngineOptions) other; + ret = this.disableContextEnrichers == that.disableContextEnrichers + && this.disableCustomConditions == that.disableCustomConditions + && this.disableTagPolicyEvaluation == that.disableTagPolicyEvaluation + && this.disableTrieLookupPrefilter == that.disableTrieLookupPrefilter + && this.cacheAuditResults == that.cacheAuditResults + && this.evaluateDelegateAdminOnly == that.evaluateDelegateAdminOnly + && this.enableTagEnricherWithLocalRefresher == that.enableTagEnricherWithLocalRefresher + && this.optimizeTrieForRetrieval == that.optimizeTrieForRetrieval; + } + return ret; + } + + @Override + public int hashCode() { + int ret = 0; + ret += disableContextEnrichers ? 1 : 0; + ret *= 2; + ret += disableCustomConditions ? 1 : 0; + ret *= 2; + ret += disableTagPolicyEvaluation ? 1 : 0; + ret *= 2; + ret += disableTrieLookupPrefilter ? 1 : 0; + ret *= 2; + ret += cacheAuditResults ? 1 : 0; + ret *= 2; + ret += evaluateDelegateAdminOnly ? 1 : 0; + ret *= 2; + ret += enableTagEnricherWithLocalRefresher ? 1 : 0; + ret *= 2; + ret += optimizeTrieForRetrieval ? 1 : 0; + ret *= 2; + return ret; + } + + @Override + public String toString() { + return "PolicyEngineOptions: {" + + " evaluatorType: " + evaluatorType + + ", cacheAuditResult: " + cacheAuditResults + + ", disableContextEnrichers: " + disableContextEnrichers + + ", disableCustomConditions: " + disableContextEnrichers + + ", disableTrieLookupPrefilter: " + disableTrieLookupPrefilter + + ", optimizeTrieForRetrieval: " + optimizeTrieForRetrieval + + " }"; + + } } diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java index bdbdd13f14..a41e08f61a 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java @@ -25,6 +25,7 @@ import org.apache.commons.logging.LogFactory; import org.apache.ranger.authorization.hadoop.config.RangerConfiguration; import org.apache.ranger.plugin.contextenricher.RangerContextEnricher; +import org.apache.ranger.plugin.contextenricher.RangerTagEnricher; import org.apache.ranger.plugin.contextenricher.RangerTagForEval; import org.apache.ranger.plugin.model.RangerPolicy; import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyItemDataMaskInfo; @@ -51,6 +52,7 @@ class RangerPolicyRepository { private static final Log LOG = LogFactory.getLog(RangerPolicyRepository.class); private static final Log PERF_CONTEXTENRICHER_INIT_LOG = RangerPerfTracer.getPerfLogger("contextenricher.init"); + private static final Log PERF_TRIE_OP_LOG = RangerPerfTracer.getPerfLogger("resourcetrie.retrieval"); enum AuditModeEnum { AUDIT_ALL, AUDIT_NONE, AUDIT_DEFAULT @@ -145,9 +147,9 @@ boolean getIsAudited() { dataMaskResourceTrie = null; rowFilterResourceTrie = null; } else { - policyResourceTrie = createResourceTrieMap(policyEvaluators); - dataMaskResourceTrie = createResourceTrieMap(dataMaskPolicyEvaluators); - rowFilterResourceTrie = createResourceTrieMap(rowFilterPolicyEvaluators); + policyResourceTrie = createResourceTrieMap(policyEvaluators, options.optimizeTrieForRetrieval); + dataMaskResourceTrie = createResourceTrieMap(dataMaskPolicyEvaluators, options.optimizeTrieForRetrieval); + rowFilterResourceTrie = createResourceTrieMap(rowFilterPolicyEvaluators, options.optimizeTrieForRetrieval); } } @@ -190,9 +192,9 @@ boolean getIsAudited() { dataMaskResourceTrie = null; rowFilterResourceTrie = null; } else { - policyResourceTrie = createResourceTrieMap(policyEvaluators); - dataMaskResourceTrie = createResourceTrieMap(dataMaskPolicyEvaluators); - rowFilterResourceTrie = createResourceTrieMap(rowFilterPolicyEvaluators); + policyResourceTrie = createResourceTrieMap(policyEvaluators, options.optimizeTrieForRetrieval); + dataMaskResourceTrie = createResourceTrieMap(dataMaskPolicyEvaluators, options.optimizeTrieForRetrieval); + rowFilterResourceTrie = createResourceTrieMap(rowFilterPolicyEvaluators, options.optimizeTrieForRetrieval); } } @@ -216,23 +218,23 @@ List getPolicyEvaluators() { return policyEvaluators; } - List getPolicyEvaluators(RangerAccessResource resource) { + List getLikelyMatchPolicyEvaluators(RangerAccessResource resource) { String resourceStr = resource == null ? null : resource.getAsString(); - return policyResourceTrie == null || StringUtils.isEmpty(resourceStr) ? getPolicyEvaluators() : getPolicyEvaluators(policyResourceTrie, resource); + return policyResourceTrie == null || StringUtils.isEmpty(resourceStr) ? getPolicyEvaluators() : getLikelyMatchPolicyEvaluators(policyResourceTrie, resource); } List getDataMaskPolicyEvaluators() { return dataMaskPolicyEvaluators; } - List getDataMaskPolicyEvaluators(RangerAccessResource resource) { + List getLikelyMatchDataMaskPolicyEvaluators(RangerAccessResource resource) { String resourceStr = resource == null ? null : resource.getAsString(); - return dataMaskResourceTrie == null || StringUtils.isEmpty(resourceStr) ? getDataMaskPolicyEvaluators() : getPolicyEvaluators(dataMaskResourceTrie, resource); + return dataMaskResourceTrie == null || StringUtils.isEmpty(resourceStr) ? getDataMaskPolicyEvaluators() : getLikelyMatchPolicyEvaluators(dataMaskResourceTrie, resource); } - List getDataMaskPolicyEvaluators(Set tags) { + List getLikelyMatchDataMaskPolicyEvaluators(Set tags) { return getSortedPolicyEvaluatorsForTags(tags, RangerPolicy.POLICY_TYPE_DATAMASK); } @@ -240,24 +242,31 @@ List getRowFilterPolicyEvaluators() { return rowFilterPolicyEvaluators; } - List getRowFilterPolicyEvaluators(RangerAccessResource resource) { + List getLikelyMatchRowFilterPolicyEvaluators(RangerAccessResource resource) { String resourceStr = resource == null ? null : resource.getAsString(); - return rowFilterResourceTrie == null || StringUtils.isEmpty(resourceStr) ? getRowFilterPolicyEvaluators() : getPolicyEvaluators(rowFilterResourceTrie, resource); + return rowFilterResourceTrie == null || StringUtils.isEmpty(resourceStr) ? getRowFilterPolicyEvaluators() : getLikelyMatchPolicyEvaluators(rowFilterResourceTrie, resource); } - List getRowFilterPolicyEvaluators(Set tags) { + List getLikelyMatchRowFilterPolicyEvaluators(Set tags) { return getSortedPolicyEvaluatorsForTags(tags, RangerPolicy.POLICY_TYPE_ROWFILTER); } AuditModeEnum getAuditModeEnum() { return auditModeEnum; } - private List getPolicyEvaluators(Map resourceTrie, RangerAccessResource resource) { + private List getLikelyMatchPolicyEvaluators(Map resourceTrie, RangerAccessResource resource) { List ret = null; Set resourceKeys = resource == null ? null : resource.getKeys(); + RangerPerfTracer perf = null; + + if(RangerPerfTracer.isPerfTraceEnabled(PERF_TRIE_OP_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_TRIE_OP_LOG, "RangerPolicyRepository.getLikelyMatchEvaluators(resource=" + resource.getAsString() + ")"); + } + if(CollectionUtils.isNotEmpty(resourceKeys)) { - boolean isRetModifiable = false; + List> resourceEvaluatorsList = null; + List smallestList = null; for(String resourceName : resourceKeys) { RangerResourceTrie trie = resourceTrie.get(resourceName); @@ -269,34 +278,41 @@ private List getPolicyEvaluators(Map resourceEvaluators = trie.getEvaluatorsForResource(resource.getValue(resourceName)); if(CollectionUtils.isEmpty(resourceEvaluators)) { // no policies for this resource, bail out - ret = null; - } else if(ret == null) { // initialize ret with policies found for this resource - ret = resourceEvaluators; - } else { // remove policies from ret that are not in resourceEvaluators - if(isRetModifiable) { - ret.retainAll(resourceEvaluators); - } else { - final List shorterList; - final List longerList; - - if (ret.size() < resourceEvaluators.size()) { - shorterList = ret; - longerList = resourceEvaluators; - } else { - shorterList = resourceEvaluators; - longerList = ret; - } + resourceEvaluatorsList = null; + smallestList = null; + break; + } + + if (smallestList == null) { + smallestList = resourceEvaluators; + } else { + if (resourceEvaluatorsList == null) { + resourceEvaluatorsList = new ArrayList<>(); + resourceEvaluatorsList.add(smallestList); + } + resourceEvaluatorsList.add(resourceEvaluators); - ret = new ArrayList<>(shorterList); - ret.retainAll(longerList); - isRetModifiable = true; + if (smallestList.size() > resourceEvaluators.size()) { + smallestList = resourceEvaluators; } } + } - if(CollectionUtils.isEmpty(ret)) { // if no policy exists, bail out and return empty list - ret = null; - break; + if (resourceEvaluatorsList != null) { + ret = new ArrayList<>(smallestList); + for (List resourceEvaluators : resourceEvaluatorsList) { + if (resourceEvaluators != smallestList) { + // remove policies from ret that are not in resourceEvaluators + ret.retainAll(resourceEvaluators); + + if (CollectionUtils.isEmpty(ret)) { // if no policy exists, bail out and return empty list + ret = null; + break; + } + } } + } else { + ret = smallestList; } } @@ -304,8 +320,10 @@ private List getPolicyEvaluators(Map contextEnrichers = new ArrayList(); - if (CollectionUtils.isNotEmpty(this.policyEvaluators)) { - if (!options.disableContextEnrichers && !CollectionUtils.isEmpty(serviceDef.getContextEnrichers())) { + if (CollectionUtils.isNotEmpty(this.policyEvaluators) || CollectionUtils.isNotEmpty(this.dataMaskPolicyEvaluators) + || CollectionUtils.isNotEmpty(this.rowFilterPolicyEvaluators)) { + if (CollectionUtils.isNotEmpty(serviceDef.getContextEnrichers())) { for (RangerServiceDef.RangerContextEnricherDef enricherDef : serviceDef.getContextEnrichers()) { if (enricherDef == null) { continue; } + if (!options.disableContextEnrichers || options.enableTagEnricherWithLocalRefresher && StringUtils.equals(enricherDef.getEnricher(), RangerTagEnricher.class.getName())) { + // This will be true only if the engine is initialized within ranger-admin + RangerServiceDef.RangerContextEnricherDef contextEnricherDef = enricherDef; - RangerContextEnricher contextEnricher = buildContextEnricher(enricherDef); + if (options.enableTagEnricherWithLocalRefresher && StringUtils.equals(enricherDef.getEnricher(), RangerTagEnricher.class.getName())) { + contextEnricherDef = new RangerServiceDef.RangerContextEnricherDef(enricherDef.getItemId(), enricherDef.getName(), "org.apache.ranger.common.RangerAdminTagEnricher", null); + } + + RangerContextEnricher contextEnricher = buildContextEnricher(contextEnricherDef); - if (contextEnricher != null) { - contextEnrichers.add(contextEnricher); + if (contextEnricher != null) { + contextEnrichers.add(contextEnricher); + } } } } @@ -610,7 +640,7 @@ private void init(RangerPolicyEngineOptions options) { LOG.debug("dataMask policy evaluation order: #" + (++order) + " - policy id=" + policy.getId() + "; name=" + policy.getName() + "; evalOrder=" + policyEvaluator.getEvalOrder()); } - LOG.debug("rowFilter policy evaluation order: " + this.dataMaskPolicyEvaluators.size() + " policies"); + LOG.debug("rowFilter policy evaluation order: " + this.rowFilterPolicyEvaluators.size() + " policies"); order = 0; for(RangerPolicyEvaluator policyEvaluator : this.rowFilterPolicyEvaluators) { RangerPolicy policy = policyEvaluator.getPolicy(); @@ -785,21 +815,15 @@ void reorderPolicyEvaluators() { LOG.debug("==> reorderEvaluators()"); } - if(policyResourceTrie != null) { - reorderPolicyEvaluators(policyResourceTrie); - } else { + if(policyResourceTrie == null) { policyEvaluators = getReorderedPolicyEvaluators(policyEvaluators); } - if(dataMaskResourceTrie != null) { - reorderPolicyEvaluators(dataMaskResourceTrie); - } else { + if(dataMaskResourceTrie == null) { dataMaskPolicyEvaluators = getReorderedPolicyEvaluators(dataMaskPolicyEvaluators); } - if(rowFilterResourceTrie != null) { - reorderPolicyEvaluators(rowFilterResourceTrie); - } else { + if(rowFilterResourceTrie == null) { rowFilterPolicyEvaluators = getReorderedPolicyEvaluators(rowFilterPolicyEvaluators); } @@ -808,18 +832,6 @@ void reorderPolicyEvaluators() { } } - private void reorderPolicyEvaluators(Map trieMap) { - if(trieMap != null) { - for(Map.Entry entry : trieMap.entrySet()) { - RangerResourceTrie trie = entry.getValue(); - - if(trie != null) { - trie.reorderEvaluators(); - } - } - } - } - private List getReorderedPolicyEvaluators(List evaluators) { List ret = evaluators; @@ -833,14 +845,14 @@ private List getReorderedPolicyEvaluators(List createResourceTrieMap(List evaluators) { + private Map createResourceTrieMap(List evaluators, boolean optimizeTrieForRetrieval) { final Map ret; if (CollectionUtils.isNotEmpty(evaluators) && serviceDef != null && CollectionUtils.isNotEmpty(serviceDef.getResources())) { ret = new HashMap(); for (RangerServiceDef.RangerResourceDef resourceDef : serviceDef.getResources()) { - ret.put(resourceDef.getName(), new RangerResourceTrie(resourceDef, evaluators)); + ret.put(resourceDef.getName(), new RangerResourceTrie(resourceDef, evaluators, RangerPolicyEvaluator.EVAL_ORDER_COMPARATOR, optimizeTrieForRetrieval)); } } else { ret = null; @@ -874,6 +886,32 @@ public StringBuilder toString(StringBuilder sb) { } } } + sb.append("} "); + + sb.append("dataMaskPolicyEvaluators={"); + + if (this.dataMaskPolicyEvaluators != null) { + for (RangerPolicyEvaluator policyEvaluator : dataMaskPolicyEvaluators) { + if (policyEvaluator != null) { + sb.append(policyEvaluator).append(" "); + } + } + } + sb.append("} "); + + sb.append("rowFilterPolicyEvaluators={"); + + if (this.rowFilterPolicyEvaluators != null) { + for (RangerPolicyEvaluator policyEvaluator : rowFilterPolicyEvaluators) { + if (policyEvaluator != null) { + sb.append(policyEvaluator).append(" "); + } + } + } + sb.append("} "); + + sb.append("contextEnrichers={"); + if (contextEnrichers != null) { for (RangerContextEnricher contextEnricher : contextEnrichers) { if (contextEnricher != null) { @@ -881,6 +919,7 @@ public StringBuilder toString(StringBuilder sb) { } } } + sb.append("} "); sb.append("} "); diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerTagAccessRequest.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerTagAccessRequest.java index dbdcacd11f..cf590f9aaa 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerTagAccessRequest.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerTagAccessRequest.java @@ -52,6 +52,7 @@ public RangerTagAccessRequest(RangerTagForEval resourceTag, RangerServiceDef tag super.setRemoteIPAddress(request.getRemoteIPAddress()); super.setForwardedAddresses(request.getForwardedAddresses()); super.setSessionId(request.getSessionId()); + super.setResourceMatchingScope(request.getResourceMatchingScope()); } public RangerPolicyResourceMatcher.MatchType getMatchType() { return matchType; diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerDefaultPolicyEvaluator.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerDefaultPolicyEvaluator.java index 71c076d03f..3ebc8b225e 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerDefaultPolicyEvaluator.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerDefaultPolicyEvaluator.java @@ -156,7 +156,7 @@ public void init(RangerPolicy policy, RangerServiceDef serviceDef, RangerPolicyE @Override public void evaluate(RangerAccessRequest request, RangerAccessResult result) { if (LOG.isDebugEnabled()) { - LOG.debug("==> RangerDefaultPolicyEvaluator.evaluate(" + request + ", " + result + ")"); + LOG.debug("==> RangerDefaultPolicyEvaluator.evaluate(policyId=" + getPolicy().getId() + ", " + request + ", " + result + ")"); } RangerPerfTracer perf = null; @@ -169,21 +169,25 @@ public void evaluate(RangerAccessRequest request, RangerAccessResult result) { if (request != null && result != null) { if (!result.getIsAccessDetermined() || !result.getIsAuditedDetermined()) { - RangerPolicyResourceMatcher.MatchType matchType = resourceMatcher != null ? resourceMatcher.getMatchType(request.getResource(), request.getContext()) : RangerPolicyResourceMatcher.MatchType.NONE; + RangerPolicyResourceMatcher.MatchType matchType; + + if (RangerTagAccessRequest.class.isInstance(request)) { + matchType = ((RangerTagAccessRequest) request).getMatchType(); + } else { + matchType = resourceMatcher != null ? resourceMatcher.getMatchType(request.getResource(), request.getContext()) : RangerPolicyResourceMatcher.MatchType.NONE; + } final boolean isMatched; + if (request.isAccessTypeAny()) { isMatched = matchType != RangerPolicyResourceMatcher.MatchType.NONE; } else if (request.getResourceMatchingScope() == RangerAccessRequest.ResourceMatchingScope.SELF_OR_DESCENDANTS) { - isMatched = matchType == RangerPolicyResourceMatcher.MatchType.SELF || matchType == RangerPolicyResourceMatcher.MatchType.DESCENDANT; + isMatched = matchType != RangerPolicyResourceMatcher.MatchType.NONE; } else { - isMatched = matchType == RangerPolicyResourceMatcher.MatchType.SELF || matchType == RangerPolicyResourceMatcher.MatchType.ANCESTOR; + isMatched = matchType == RangerPolicyResourceMatcher.MatchType.SELF || matchType == RangerPolicyResourceMatcher.MatchType.ANCESTOR; } if (isMatched) { - if (RangerTagAccessRequest.class.isInstance(request)) { - matchType = ((RangerTagAccessRequest) request).getMatchType(); - } if (!result.getIsAuditedDetermined()) { if (isAuditEnabled()) { result.setIsAudited(true); @@ -202,7 +206,7 @@ public void evaluate(RangerAccessRequest request, RangerAccessResult result) { RangerPerfTracer.log(perf); if(LOG.isDebugEnabled()) { - LOG.debug("<== RangerDefaultPolicyEvaluator.evaluate(" + request + ", " + result + ")"); + LOG.debug("<== RangerDefaultPolicyEvaluator.evaluate(policyId=" + getPolicy().getId() + ", " + request + ", " + result + ")"); } } @@ -410,17 +414,15 @@ public void getResourceAccessInfo(RangerAccessRequest request, RangerResourceAcc if(LOG.isDebugEnabled()) { LOG.debug("==> RangerDefaultPolicyEvaluator.getResourceAccessInfo(" + request + ", " + result + ")"); } - RangerPolicyResourceMatcher.MatchType matchType = resourceMatcher != null ? resourceMatcher.getMatchType(request.getResource(), request.getContext()) : RangerPolicyResourceMatcher.MatchType.NONE; - - final boolean isMatched; - if (request.isAccessTypeAny()) { - isMatched = matchType != RangerPolicyResourceMatcher.MatchType.NONE; - } else if (request.getResourceMatchingScope() == RangerAccessRequest.ResourceMatchingScope.SELF_OR_DESCENDANTS) { - isMatched = matchType == RangerPolicyResourceMatcher.MatchType.SELF || matchType == RangerPolicyResourceMatcher.MatchType.DESCENDANT; - } else { - isMatched = matchType == RangerPolicyResourceMatcher.MatchType.SELF || matchType == RangerPolicyResourceMatcher.MatchType.ANCESTOR; + RangerPolicyResourceMatcher.MatchType matchType; + if (RangerTagAccessRequest.class.isInstance(request)) { + matchType = ((RangerTagAccessRequest) request).getMatchType(); + } else { + matchType = resourceMatcher != null ? resourceMatcher.getMatchType(request.getResource(), request.getContext()) : RangerPolicyResourceMatcher.MatchType.NONE; } + final boolean isMatched = matchType != RangerPolicyResourceMatcher.MatchType.NONE; + if (isMatched) { if (CollectionUtils.isNotEmpty(allowEvaluators)) { @@ -470,7 +472,6 @@ public void getResourceAccessInfo(RangerAccessRequest request, RangerResourceAcc } } - protected void evaluatePolicyItems(RangerAccessRequest request, RangerAccessResult result, boolean isResourceMatch) { if(LOG.isDebugEnabled()) { LOG.debug("==> RangerDefaultPolicyEvaluator.evaluatePolicyItems(" + request + ", " + result + ", " + isResourceMatch + ")"); @@ -486,7 +487,7 @@ protected void evaluatePolicyItems(RangerAccessRequest request, RangerAccessResu RangerPolicy policy = getPolicy(); if(matchedPolicyItem.getPolicyItemType() == RangerPolicyItemEvaluator.POLICY_ITEM_TYPE_DENY) { - if(isResourceMatch) { + if(isResourceMatch || !request.isAccessTypeAny()) { result.setIsAllowed(false); result.setPolicyId(policy.getId()); result.setReason(matchedPolicyItem.getComments()); diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerPolicyEvaluator.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerPolicyEvaluator.java index 191ad98c86..7165594fb8 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerPolicyEvaluator.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerPolicyEvaluator.java @@ -39,6 +39,8 @@ public interface RangerPolicyEvaluator extends RangerPolicyResourceEvaluator { + Comparator EVAL_ORDER_COMPARATOR = new RangerPolicyEvaluator.PolicyEvalOrderComparator(); + String EVALUATOR_TYPE_AUTO = "auto"; String EVALUATOR_TYPE_OPTIMIZED = "optimized"; String EVALUATOR_TYPE_CACHED = "cached"; @@ -96,11 +98,14 @@ public int compare(RangerPolicyEvaluator me, RangerPolicyEvaluator other) { result = 1; } else { result = Long.compare(other.getUsageCount(), me.getUsageCount()); + if (result == 0) { result = Integer.compare(me.getEvalOrder(), other.getEvalOrder()); } } + return result; } } + } diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyresourcematcher/RangerDefaultPolicyResourceMatcher.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyresourcematcher/RangerDefaultPolicyResourceMatcher.java index be10b95466..8f1e102556 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyresourcematcher/RangerDefaultPolicyResourceMatcher.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyresourcematcher/RangerDefaultPolicyResourceMatcher.java @@ -46,7 +46,7 @@ public class RangerDefaultPolicyResourceMatcher implements RangerPolicyResourceM private static final Log LOG = LogFactory.getLog(RangerDefaultPolicyResourceMatcher.class); protected RangerServiceDef serviceDef = null; - protected RangerPolicy policy = null; + protected int policyType; protected Map policyResources = null; private Map matchers = null; @@ -70,14 +70,23 @@ public void setServiceDef(RangerServiceDef serviceDef) { @Override public void setPolicy(RangerPolicy policy) { - this.policy = policy; - setPolicyResources(policy == null ? null : policy.getResources()); + if (policy == null) { + setPolicyResources(null, RangerPolicy.POLICY_TYPE_ACCESS); + } else { + setPolicyResources(policy.getResources(), policy.getPolicyType() == null ? RangerPolicy.POLICY_TYPE_ACCESS : policy.getPolicyType()); + } } @Override public void setPolicyResources(Map policyResources) { + setPolicyResources(policyResources, RangerPolicy.POLICY_TYPE_ACCESS); + } + + @Override + public void setPolicyResources(Map policyResources, int policyType) { this.policyResources = policyResources; + this.policyType = policyType; } @Override @@ -98,7 +107,6 @@ public void init() { Set policyResourceKeySet = policyResources.keySet(); RangerServiceDefHelper serviceDefHelper = new RangerServiceDefHelper(serviceDef, false); - int policyType = policy != null && policy.getPolicyType() != null ? policy.getPolicyType() : RangerPolicy.POLICY_TYPE_ACCESS; Set> validResourceHierarchies = serviceDefHelper.getResourceHierarchies(policyType); for (List validResourceHierarchy : validResourceHierarchies) { @@ -371,6 +379,10 @@ public boolean isMatch(RangerPolicy policy, MatchScope scope, Map resources = policy.getResources(); if (MapUtils.isNotEmpty(resources)) { @@ -539,7 +551,6 @@ private boolean isValid(RangerAccessResource resource) { aValidHierarchy = firstValidResourceDefHierarchy; } else { RangerServiceDefHelper serviceDefHelper = new RangerServiceDefHelper(serviceDef, false); - int policyType = policy != null && policy.getPolicyType() != null ? policy.getPolicyType() : RangerPolicy.POLICY_TYPE_ACCESS; Set> validResourceHierarchies = serviceDefHelper.getResourceHierarchies(policyType); for (List resourceHierarchy : validResourceHierarchies) { diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyresourcematcher/RangerPolicyResourceEvaluator.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyresourcematcher/RangerPolicyResourceEvaluator.java index 181863c38c..7d43b4b8c5 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyresourcematcher/RangerPolicyResourceEvaluator.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyresourcematcher/RangerPolicyResourceEvaluator.java @@ -23,8 +23,6 @@ import org.apache.ranger.plugin.model.RangerPolicy; import org.apache.ranger.plugin.resourcematcher.RangerResourceMatcher; -import java.io.Serializable; -import java.util.Comparator; import java.util.Map; public interface RangerPolicyResourceEvaluator { @@ -37,11 +35,4 @@ public interface RangerPolicyResourceEvaluator { RangerResourceMatcher getResourceMatcher(String resourceName); Integer getLeafResourceLevel(); - - class IdComparator implements Comparator, Serializable { - @Override - public int compare(RangerPolicyResourceEvaluator me, RangerPolicyResourceEvaluator other) { - return Long.compare(me.getId(), other.getId()); - } - } } diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyresourcematcher/RangerPolicyResourceMatcher.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyresourcematcher/RangerPolicyResourceMatcher.java index b4dc2c509a..9cc4bd6213 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyresourcematcher/RangerPolicyResourceMatcher.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyresourcematcher/RangerPolicyResourceMatcher.java @@ -37,6 +37,8 @@ enum MatchType { NONE, SELF, DESCENDANT, ANCESTOR }; void setPolicyResources(Map policyResources); + void setPolicyResources(Map policyResources, int policyType); + void init(); RangerServiceDef getServiceDef(); diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/resourcematcher/RangerPathResourceMatcher.java b/agents-common/src/main/java/org/apache/ranger/plugin/resourcematcher/RangerPathResourceMatcher.java index 98c3c42714..480522fa52 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/resourcematcher/RangerPathResourceMatcher.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/resourcematcher/RangerPathResourceMatcher.java @@ -227,6 +227,9 @@ abstract class RecursiveMatcher extends ResourceMatcher { } String getStringToCompare(String policyValue) { + if (StringUtils.isEmpty(policyValue)) { + return policyValue; + } return (policyValue.lastIndexOf(levelSeparatorChar) == policyValue.length()-1) ? policyValue.substring(0, policyValue.length()-1) : policyValue; } @@ -242,9 +245,10 @@ boolean isMatch(String resourceValue, Map evalContext) { final String noSeparator; if (getNeedsDynamicEval()) { - noSeparator = getStringToCompare(getExpandedValue(evalContext)); + String expandedPolicyValue = getExpandedValue(evalContext); + noSeparator = expandedPolicyValue != null ? getStringToCompare(expandedPolicyValue) : null; } else { - if (valueWithoutSeparator == null) { + if (valueWithoutSeparator == null && value != null) { valueWithoutSeparator = getStringToCompare(value); valueWithSeparator = valueWithoutSeparator + Character.toString(levelSeparatorChar); } @@ -253,7 +257,7 @@ boolean isMatch(String resourceValue, Map evalContext) { boolean ret = StringUtils.equals(resourceValue, noSeparator); - if (!ret) { + if (!ret && noSeparator != null) { final String withSeparator = getNeedsDynamicEval() ? noSeparator + Character.toString(levelSeparatorChar) : valueWithSeparator; ret = StringUtils.startsWith(resourceValue, withSeparator); } @@ -273,9 +277,10 @@ boolean isMatch(String resourceValue, Map evalContext) { final String noSeparator; if (getNeedsDynamicEval()) { - noSeparator = getStringToCompare(getExpandedValue(evalContext)); + String expandedPolicyValue = getExpandedValue(evalContext); + noSeparator = expandedPolicyValue != null ? getStringToCompare(expandedPolicyValue) : null; } else { - if (valueWithoutSeparator == null) { + if (valueWithoutSeparator == null && value != null) { valueWithoutSeparator = getStringToCompare(value); valueWithSeparator = valueWithoutSeparator + Character.toString(levelSeparatorChar); } @@ -284,7 +289,7 @@ boolean isMatch(String resourceValue, Map evalContext) { boolean ret = StringUtils.equalsIgnoreCase(resourceValue, noSeparator); - if (!ret) { + if (!ret && noSeparator != null) { final String withSeparator = getNeedsDynamicEval() ? noSeparator + Character.toString(levelSeparatorChar) : valueWithSeparator; ret = StringUtils.startsWithIgnoreCase(resourceValue, withSeparator); } diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java b/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java index 927dcc2b7c..87ceeec99a 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java @@ -148,6 +148,8 @@ public void init() { policyEngineOptions.disableTagPolicyEvaluation = RangerConfiguration.getInstance().getBoolean(propertyPrefix + ".policyengine.option.disable.tagpolicy.evaluation", false); policyEngineOptions.disableTrieLookupPrefilter = RangerConfiguration.getInstance().getBoolean(propertyPrefix + ".policyengine.option.disable.trie.lookup.prefilter", false); + LOG.info(policyEngineOptions); + RangerAdminClient admin = createAdminClient(serviceName, appId, propertyPrefix); refresher = new PolicyRefresher(this, serviceType, appId, serviceName, admin, pollingIntervalMs, cacheDir); @@ -163,7 +165,7 @@ public void init() { LOG.debug(propertyPrefix + ".policy.policyReorderInterval:" + policyReorderIntervalMs); } - if (policyReorderIntervalMs > 0) { + if (policyEngineOptions.disableTrieLookupPrefilter && policyReorderIntervalMs > 0) { policyEngineRefreshTimer = new Timer("PolicyEngineRefreshTimer", true); try { policyEngineRefreshTimer.schedule(new PolicyEngineRefresher(this), policyReorderIntervalMs, policyReorderIntervalMs); @@ -176,8 +178,7 @@ public void init() { policyEngineRefreshTimer = null; } } else { - LOG.info("Policies will NOT be reordered based on number of evaluations because " - + propertyPrefix + ".policy.policyReorderInterval is set to a negative number[" + policyReorderIntervalMs +"]"); + LOG.info("Policies will NOT be reordered based on number of evaluations"); } } diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/store/AbstractServiceStore.java b/agents-common/src/main/java/org/apache/ranger/plugin/store/AbstractServiceStore.java index 2ee786ca1a..bee7520900 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/store/AbstractServiceStore.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/store/AbstractServiceStore.java @@ -23,11 +23,13 @@ import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.ranger.authorization.hadoop.config.RangerConfiguration; import org.apache.ranger.plugin.model.RangerBaseModelObject; import org.apache.ranger.plugin.model.RangerPolicy; import org.apache.ranger.plugin.model.RangerService; import org.apache.ranger.plugin.model.RangerServiceDef; import org.apache.ranger.plugin.util.SearchFilter; +import org.apache.ranger.services.tag.RangerServiceTag; import java.util.ArrayList; import java.util.Collection; @@ -40,6 +42,10 @@ public abstract class AbstractServiceStore implements ServiceStore { public static final String COMPONENT_ACCESSTYPE_SEPARATOR = ":"; + private static final String AUTOPROPAGATE_ROWFILTERDEF_TO_TAG_PROP = "ranger.servicedef.autopropagate.rowfilterdef.to.tag"; + + private static final boolean AUTOPROPAGATE_ROWFILTERDEF_TO_TAG_PROP_DEFAULT = false; + private static final int MAX_ACCESS_TYPES_IN_SERVICE_DEF = 1000; // when a service-def is updated, the updated service-def should be made available to plugins @@ -132,7 +138,7 @@ protected void postDelete(RangerBaseModelObject obj) throws Exception { } } - protected final long getNextVersion(Long currentVersion) { + public static long getNextVersion(Long currentVersion) { return currentVersion == null ? 1L : currentVersion + 1; } @@ -303,6 +309,10 @@ private void updateTagServiceDefForUpdatingAccessTypes(RangerServiceDef serviceD updateNeeded = true; } + boolean resourceUpdated = updateResourceInTagServiceDef(tagServiceDef); + + updateNeeded = updateNeeded || resourceUpdated; + if (updateNeeded) { try { updateServiceDef(tagServiceDef); @@ -344,8 +354,11 @@ private void updateTagServiceDefForDeletingAccessTypes(String serviceDefName) th tagServiceDef.getAccessTypes().removeAll(accessTypes); updateTagServiceDefForDeletingDataMaskDef(tagServiceDef, serviceDefName); + updateTagServiceDefForDeletingRowFilterDef(tagServiceDef, serviceDefName); + updateResourceInTagServiceDef(tagServiceDef); + try { updateServiceDef(tagServiceDef); LOG.info("AbstractServiceStore.updateTagServiceDefForDeletingAccessTypes -- updated TAG service def with " + serviceDefName + " access types"); @@ -502,19 +515,22 @@ private boolean updateTagServiceDefForUpdatingRowFilterDef(RangerServiceDef tagS } boolean ret = false; - RangerServiceDef.RangerRowFilterDef svcRowFilterDef = serviceDef.getRowFilterDef(); - RangerServiceDef.RangerRowFilterDef tagRowFilterDef = tagServiceDef.getRowFilterDef(); + boolean autopropagateRowfilterdefToTag = RangerConfiguration.getInstance().getBoolean(AUTOPROPAGATE_ROWFILTERDEF_TO_TAG_PROP, AUTOPROPAGATE_ROWFILTERDEF_TO_TAG_PROP_DEFAULT); - List svcDefAccessTypes = svcRowFilterDef.getAccessTypes(); - List tagDefAccessTypes = tagRowFilterDef.getAccessTypes(); + if (autopropagateRowfilterdefToTag) { + RangerServiceDef.RangerRowFilterDef svcRowFilterDef = serviceDef.getRowFilterDef(); + RangerServiceDef.RangerRowFilterDef tagRowFilterDef = tagServiceDef.getRowFilterDef(); - boolean tagRowFilterAccessTypesUpdated = updateTagAccessTypeDefs(svcDefAccessTypes, tagDefAccessTypes, itemIdOffset, prefix); + List svcDefAccessTypes = svcRowFilterDef.getAccessTypes(); + List tagDefAccessTypes = tagRowFilterDef.getAccessTypes(); - if (tagRowFilterAccessTypesUpdated) { - tagRowFilterDef.setAccessTypes(tagDefAccessTypes); - ret = true; - } + boolean tagRowFilterAccessTypesUpdated = updateTagAccessTypeDefs(svcDefAccessTypes, tagDefAccessTypes, itemIdOffset, prefix); + if (tagRowFilterAccessTypesUpdated) { + tagRowFilterDef.setAccessTypes(tagDefAccessTypes); + ret = true; + } + } if (LOG.isDebugEnabled()) { LOG.debug("<== AbstractServiceStore.updateTagServiceDefForUpdatingRowFilterDef(" + serviceDef.getName() + ") : " + ret); } @@ -548,4 +564,56 @@ private void updateTagServiceDefForDeletingRowFilterDef(RangerServiceDef tagServ LOG.debug("<== AbstractServiceStore.updateTagServiceDefForDeletingRowFilterDef(" + serviceDefName + ")"); } } -} \ No newline at end of file + + private boolean updateResourceInTagServiceDef(RangerServiceDef tagServiceDef) throws Exception { + if (LOG.isDebugEnabled()) { + LOG.debug("==> AbstractServiceStore.updateResourceInTagServiceDef(" + tagServiceDef + ")"); + } + boolean ret = false; + + RangerServiceDef.RangerResourceDef tagResource = new RangerServiceDef.RangerResourceDef(); + tagResource.setName(RangerServiceTag.TAG_RESOURCE_NAME); + List resources = new ArrayList<>(); + resources.add(tagResource); + + RangerServiceDef.RangerDataMaskDef dataMaskDef = tagServiceDef.getDataMaskDef(); + + if (dataMaskDef != null) { + if (CollectionUtils.isNotEmpty(dataMaskDef.getAccessTypes())) { + if (CollectionUtils.isEmpty(dataMaskDef.getResources())) { + dataMaskDef.setResources(resources); + ret = true; + } + } else { + if (CollectionUtils.isNotEmpty(dataMaskDef.getResources())) { + dataMaskDef.setResources(null); + ret = true; + } + } + } + + RangerServiceDef.RangerRowFilterDef rowFilterDef = tagServiceDef.getRowFilterDef(); + + if (rowFilterDef != null) { + boolean autopropagateRowfilterdefToTag = RangerConfiguration.getInstance().getBoolean(AUTOPROPAGATE_ROWFILTERDEF_TO_TAG_PROP, AUTOPROPAGATE_ROWFILTERDEF_TO_TAG_PROP_DEFAULT); + if (autopropagateRowfilterdefToTag) { + if (CollectionUtils.isNotEmpty(rowFilterDef.getAccessTypes())) { + if (CollectionUtils.isEmpty(rowFilterDef.getResources())) { + rowFilterDef.setResources(resources); + ret = true; + } + } else { + if (CollectionUtils.isNotEmpty(rowFilterDef.getResources())) { + rowFilterDef.setResources(null); + ret = true; + } + } + } + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== AbstractServiceStore.updateResourceInTagServiceDef(" + tagServiceDef + ") : " + ret); + } + return ret; + } +} diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/store/AbstractTagStore.java b/agents-common/src/main/java/org/apache/ranger/plugin/store/AbstractTagStore.java index 90c1da6e6b..5750030dd2 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/store/AbstractTagStore.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/store/AbstractTagStore.java @@ -33,6 +33,11 @@ final public void setServiceStore(ServiceStore svcStore) { this.svcStore = svcStore; } + @Override + final public ServiceStore getServiceStore() { + return svcStore; + } + } diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/store/EmbeddedServiceDefsUtil.java b/agents-common/src/main/java/org/apache/ranger/plugin/store/EmbeddedServiceDefsUtil.java index de12f8939e..7abcedecfc 100755 --- a/agents-common/src/main/java/org/apache/ranger/plugin/store/EmbeddedServiceDefsUtil.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/store/EmbeddedServiceDefsUtil.java @@ -63,6 +63,7 @@ public class EmbeddedServiceDefsUtil { public static final String EMBEDDED_SERVICEDEF_NIFI_NAME = "nifi"; public static final String EMBEDDED_SERVICEDEF_ATLAS_NAME = "atlas"; public static final String EMBEDDED_SERVICEDEF_WASB_NAME = "wasb"; + public static final String EMBEDDED_SERVICEDEF_ABFS_NAME = "abfs"; public static final String PROPERTY_CREATE_EMBEDDED_SERVICE_DEFS = "ranger.service.store.create.embedded.service-defs"; @@ -93,6 +94,7 @@ public class EmbeddedServiceDefsUtil { private RangerServiceDef nifiServiceDef = null; private RangerServiceDef atlasServiceDef = null; private RangerServiceDef wasbServiceDef = null; + private RangerServiceDef abfsServiceDef = null; private RangerServiceDef tagServiceDef = null; @@ -133,6 +135,7 @@ public void init(ServiceStore store) { tagServiceDef = getOrCreateServiceDef(store, EMBEDDED_SERVICEDEF_TAG_NAME); wasbServiceDef = getOrCreateServiceDef(store, EMBEDDED_SERVICEDEF_WASB_NAME); + abfsServiceDef = getOrCreateServiceDef(store, EMBEDDED_SERVICEDEF_ABFS_NAME); // Ensure that tag service def is updated with access types of all service defs @@ -192,6 +195,8 @@ public long getAtlasServiceDefId() { public long getWasbServiceDefId() { return getId(wasbServiceDef); } + public long getAbfsServiceDefId() { return getId(abfsServiceDef); } + public RangerServiceDef getEmbeddedServiceDef(String defType) throws Exception { RangerServiceDef serviceDef=null; if(StringUtils.isNotEmpty(defType)){ diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/store/ServiceStore.java b/agents-common/src/main/java/org/apache/ranger/plugin/store/ServiceStore.java index 89c33269bc..9924cb4c40 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/store/ServiceStore.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/store/ServiceStore.java @@ -20,6 +20,7 @@ package org.apache.ranger.plugin.store; import java.util.List; +import java.util.Map; import org.apache.ranger.plugin.model.RangerPolicy; import org.apache.ranger.plugin.model.RangerService; @@ -28,6 +29,9 @@ import org.apache.ranger.plugin.util.ServicePolicies; public interface ServiceStore { + + String OPTION_FORCE_RENAME = "forceRename"; + void init() throws Exception; RangerServiceDef createServiceDef(RangerServiceDef serviceDef) throws Exception; @@ -48,7 +52,7 @@ public interface ServiceStore { RangerService createService(RangerService service) throws Exception; - RangerService updateService(RangerService service) throws Exception; + RangerService updateService(RangerService service, Map options) throws Exception; void deleteService(Long id) throws Exception; @@ -70,6 +74,8 @@ public interface ServiceStore { List getPolicies(SearchFilter filter) throws Exception; + Long getPolicyId(final Long serviceId, final String policyName); + PList getPaginatedPolicies(SearchFilter filter) throws Exception; List getPoliciesByResourceSignature(String serviceName, String policySignature, Boolean isPolicyEnabled) throws Exception; diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/store/TagStore.java b/agents-common/src/main/java/org/apache/ranger/plugin/store/TagStore.java index 3c5a43ba74..fe4b278179 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/store/TagStore.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/store/TagStore.java @@ -34,6 +34,7 @@ public interface TagStore { void setServiceStore(ServiceStore svcStore); + ServiceStore getServiceStore(); RangerTagDef createTagDef(RangerTagDef tagDef) throws Exception; @@ -83,6 +84,8 @@ public interface TagStore { RangerServiceResource updateServiceResource(RangerServiceResource resource) throws Exception; + void refreshServiceResource(Long resourceId) throws Exception; + void deleteServiceResource(Long id) throws Exception; void deleteServiceResourceByGuid(String guid) throws Exception; diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPerfCollectorTracer.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPerfCollectorTracer.java index d899c6f5dd..353f7daec6 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPerfCollectorTracer.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPerfCollectorTracer.java @@ -22,13 +22,16 @@ import org.apache.commons.logging.Log; public class RangerPerfCollectorTracer extends RangerPerfTracer { + private final long startTimeNanos; public RangerPerfCollectorTracer(Log logger, String tag, String data) { super(logger, tag, data); + startTimeNanos = System.nanoTime(); } @Override public void log() { - PerfDataRecorder.recordStatistic(tag, getElapsedTime()); + // Collect elapsed time in microseconds + PerfDataRecorder.recordStatistic(tag, ((System.nanoTime() - startTimeNanos) + 500) / 1000); } } diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerRESTClient.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerRESTClient.java index 9395f28bc9..c8b8935453 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerRESTClient.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerRESTClient.java @@ -221,6 +221,10 @@ public boolean verify(String urlHostName, SSLSession session) { return client; } + public void resetClient(){ + client = null; + } + private void init() { try { gsonBuilder = new GsonBuilder().setDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSZ").setPrettyPrinting().create(); diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerResourceTrie.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerResourceTrie.java index f2c4c3a401..0eb58e7e42 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerResourceTrie.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerResourceTrie.java @@ -21,8 +21,10 @@ import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.ranger.authorization.hadoop.config.RangerConfiguration; import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyResource; import org.apache.ranger.plugin.model.RangerServiceDef; import org.apache.ranger.plugin.policyresourcematcher.RangerPolicyResourceEvaluator; @@ -30,28 +32,54 @@ import org.apache.ranger.plugin.resourcematcher.RangerResourceMatcher; import java.util.ArrayList; -import java.util.Collections; +import java.util.Collection; +import java.util.Comparator; import java.util.HashMap; import java.util.List; import java.util.Map; - +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; public class RangerResourceTrie { private static final Log LOG = LogFactory.getLog(RangerResourceTrie.class); + private static final Log PERF_TRIE_INIT_LOG = RangerPerfTracer.getPerfLogger("resourcetrie.init"); + private static final Log PERF_TRIE_OP_LOG = RangerPerfTracer.getPerfLogger("resourcetrie.op"); private static final String DEFAULT_WILDCARD_CHARS = "*?"; + private static final String TRIE_BUILDER_THREAD_COUNT = "ranger.policyengine.trie.builder.thread.count"; - private final String resourceName; - private final boolean optIgnoreCase; - private final boolean optWildcard; - private final String wildcardChars; - private final TrieNode root; + private final String resourceName; + private final boolean optIgnoreCase; + private final boolean optWildcard; + private final String wildcardChars; + private final TrieNode root; + private final Comparator comparator; + private final boolean isOptimizedForRetrieval; public RangerResourceTrie(RangerServiceDef.RangerResourceDef resourceDef, List evaluators) { + this(resourceDef, evaluators, null, true); + } + + public RangerResourceTrie(RangerServiceDef.RangerResourceDef resourceDef, List evaluators, Comparator comparator, boolean isOptimizedForRetrieval) { if(LOG.isDebugEnabled()) { - LOG.debug("==> RangerResourceTrie(" + resourceDef.getName() + ", evaluatorCount=" + evaluators.size() + ")"); + LOG.debug("==> RangerResourceTrie(" + resourceDef.getName() + ", evaluatorCount=" + evaluators.size() + ", isOptimizedForRetrieval=" + isOptimizedForRetrieval + ")"); + } + + RangerPerfTracer perf = null; + + if(RangerPerfTracer.isPerfTraceEnabled(PERF_TRIE_INIT_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_TRIE_INIT_LOG, "RangerResourceTrie.init(name=" + resourceDef.getName() + ")"); + } + + int builderThreadCount = RangerConfiguration.getInstance().getInt(TRIE_BUILDER_THREAD_COUNT, 1); + + if (builderThreadCount < 1) { + builderThreadCount = 1; } + LOG.info("builderThreadCount is set to ["+ builderThreadCount +"]"); + PERF_TRIE_INIT_LOG.info("builderThreadCount is set to ["+ builderThreadCount +"]"); + Map matcherOptions = resourceDef.getMatcherOptions(); boolean optReplaceTokens = RangerAbstractResourceMatcher.getOptionReplaceTokens(matcherOptions); @@ -72,48 +100,269 @@ public RangerResourceTrie(RangerServiceDef.RangerResourceDef resourceDef, List tmpRoot = buildTrie(resourceDef, evaluators, comparator, builderThreadCount); + + if (builderThreadCount > 1 && tmpRoot == null) { // if multi-threaded trie-creation failed, build using a single thread + this.root = buildTrie(resourceDef, evaluators, comparator, 1); + } else { + this.root = tmpRoot; + } + + RangerPerfTracer.logAlways(perf); + + if (PERF_TRIE_INIT_LOG.isDebugEnabled()) { + PERF_TRIE_INIT_LOG.debug(toString()); + } + + if (PERF_TRIE_INIT_LOG.isTraceEnabled()) { + StringBuilder sb = new StringBuilder(); + root.toString("", sb); + PERF_TRIE_INIT_LOG.trace("Trie Dump:\n{" + sb.toString() + "}"); + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerResourceTrie(" + resourceDef.getName() + ", evaluatorCount=" + evaluators.size() + ", isOptimizedForRetrieval=" + isOptimizedForRetrieval + "): " + toString()); + } + } + + public String getResourceName() { + return resourceName; + } + + public List getEvaluatorsForResource(Object resource) { + if (resource instanceof String) { + return getEvaluatorsForResource((String) resource); + } else if (resource instanceof Collection) { + if (CollectionUtils.isEmpty((Collection) resource)) { // treat empty collection same as empty-string + return getEvaluatorsForResource(""); + } else { + @SuppressWarnings("unchecked") + Collection resources = (Collection) resource; + + return getEvaluatorsForResources(resources); + } + } + + return null; + } - for(T evaluator : evaluators) { + private TrieNode buildTrie(RangerServiceDef.RangerResourceDef resourceDef, List evaluators, Comparator comparator, int builderThreadCount) { + if(LOG.isDebugEnabled()) { + LOG.debug("==> buildTrie(" + resourceDef.getName() + ", evaluatorCount=" + evaluators.size() + ", isMultiThreaded=" + (builderThreadCount > 1) + ")"); + } + + RangerPerfTracer perf = null; + + if(RangerPerfTracer.isPerfTraceEnabled(PERF_TRIE_INIT_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_TRIE_INIT_LOG, "RangerResourceTrie.init(resourceDef=" + resourceDef.getName() + ")"); + } + + TrieNode ret = new TrieNode<>(null); + final boolean isMultiThreaded = builderThreadCount > 1; + final List builderThreads; + final Map builderThreadMap; + int lastUsedThreadIndex = 0; + + if (isMultiThreaded) { + builderThreads = new ArrayList<>(); + for (int i = 0; i < builderThreadCount; i++) { + ResourceTrieBuilderThread t = new ResourceTrieBuilderThread(isOptimizedForRetrieval); + t.setDaemon(true); + builderThreads.add(t); + t.start(); + } + builderThreadMap = new HashMap<>(); + } else { + builderThreads = null; + builderThreadMap = null; + } + + for (T evaluator : evaluators) { Map policyResources = evaluator.getPolicyResource(); - RangerPolicyResource policyResource = policyResources != null ? policyResources.get(resourceName) : null; + RangerPolicyResource policyResource = policyResources != null ? policyResources.get(resourceName) : null; - if(policyResource == null) { - if(evaluator.getLeafResourceLevel() != null && resourceDef.getLevel() != null && evaluator.getLeafResourceLevel() < resourceDef.getLevel()) { - root.addWildcardEvaluator(evaluator); + if (policyResource == null) { + if (evaluator.getLeafResourceLevel() != null && resourceDef.getLevel() != null && evaluator.getLeafResourceLevel() < resourceDef.getLevel()) { + ret.addWildcardEvaluator(evaluator); } continue; } - if(policyResource.getIsExcludes()) { - root.addWildcardEvaluator(evaluator); + if (policyResource.getIsExcludes()) { + ret.addWildcardEvaluator(evaluator); } else { RangerResourceMatcher resourceMatcher = evaluator.getResourceMatcher(resourceName); - if(resourceMatcher != null && (resourceMatcher.isMatchAny())) { - root.addWildcardEvaluator(evaluator); + if (resourceMatcher != null && (resourceMatcher.isMatchAny())) { + ret.addWildcardEvaluator(evaluator); } else { - if(CollectionUtils.isNotEmpty(policyResource.getValues())) { + if (CollectionUtils.isNotEmpty(policyResource.getValues())) { for (String resource : policyResource.getValues()) { - insert(resource, policyResource.getIsRecursive(), evaluator); + if (!isMultiThreaded) { + insert(ret, resource, policyResource.getIsRecursive(), evaluator); + } else { + try { + lastUsedThreadIndex = insert(ret, resource, policyResource.getIsRecursive(), evaluator, builderThreadMap, builderThreads, lastUsedThreadIndex); + } catch (InterruptedException ex) { + LOG.error("Failed to dispatch " + resource + " to " + builderThreads.get(lastUsedThreadIndex)); + LOG.error("Failing and retrying with one thread"); + + ret = null; + + break; + } + } + } + if (ret == null) { + break; } } } } } + if (ret != null) { + if (isMultiThreaded) { + ret.setup(null, comparator); + + for (ResourceTrieBuilderThread t : builderThreads) { + t.setParentWildcardEvaluators(ret.wildcardEvaluators); + try { + // Send termination signal to each thread + t.add("", false, null); + // Wait for threads to finish work + t.join(); + ret.getChildren().putAll(t.getSubtrees()); + } catch (InterruptedException ex) { + LOG.error("BuilderThread " + t + " was interrupted:", ex); + LOG.error("Failing and retrying with one thread"); + + ret = null; + + break; + } + } + } else { + if (isOptimizedForRetrieval) { + RangerPerfTracer postSetupPerf = null; - root.postSetup(null); + if (RangerPerfTracer.isPerfTraceEnabled(PERF_TRIE_INIT_LOG)) { + postSetupPerf = RangerPerfTracer.getPerfTracer(PERF_TRIE_INIT_LOG, "RangerResourceTrie.init(name=" + resourceDef.getName() + "-postSetup)"); + } - LOG.info(toString()); + ret.postSetup(null, comparator); + + RangerPerfTracer.logAlways(postSetupPerf); + } else { + ret.setup(null, comparator); + } + } + } + + if (isMultiThreaded) { + cleanUpThreads(builderThreads); + } + + RangerPerfTracer.logAlways(perf); if(LOG.isDebugEnabled()) { - LOG.debug("<== RangerResourceTrie(" + resourceDef.getName() + ", evaluatorCount=" + evaluators.size() + "): " + toString()); + LOG.debug("<== buildTrie(" + resourceDef.getName() + ", evaluatorCount=" + evaluators.size() + ", isMultiThreaded=" + isMultiThreaded + ") :" + ret); } + + return ret; } - public String getResourceName() { - return resourceName; + private void cleanUpThreads(List builderThreads) { + if (CollectionUtils.isNotEmpty(builderThreads)) { + for (ResourceTrieBuilderThread t : builderThreads) { + try { + if (t.isAlive()) { + t.interrupt(); + t.join(); + } + } catch (InterruptedException ex) { + LOG.error("Could not terminate thread " + t); + } + } + } + } + + private TrieData getTrieData() { + TrieData ret = new TrieData(); + + root.populateTrieData(ret); + ret.maxDepth = getMaxDepth(); + + return ret; + } + + private int getMaxDepth() { + return root.getMaxDepth(); + } + + private Character getLookupChar(char ch) { + return optIgnoreCase ? Character.toLowerCase(ch) : ch; + } + + private Character getLookupChar(String str, int index) { + return getLookupChar(str.charAt(index)); + } + + private int insert(TrieNode currentRoot, String resource, boolean isRecursive, T evaluator, Map builderThreadMap, List builderThreads, int lastUsedThreadIndex) throws InterruptedException { + int ret = lastUsedThreadIndex; + final String prefix = getNonWildcardPrefix(resource); + + if (StringUtils.isNotEmpty(prefix)) { + char c = getLookupChar(prefix.charAt(0)); + Integer index = builderThreadMap.get(c); + + if (index == null) { + ret = index = (lastUsedThreadIndex + 1) % builderThreads.size(); + builderThreadMap.put(c, index); + } + + builderThreads.get(index).add(resource, isRecursive, evaluator); + } else { + currentRoot.addWildcardEvaluator(evaluator); + } + + return ret; + } + + private void insert(TrieNode currentRoot, String resource, boolean isRecursive, T evaluator) { + + TrieNode curr = currentRoot; + final String prefix = getNonWildcardPrefix(resource); + final boolean isWildcard = prefix.length() != resource.length(); + + if (StringUtils.isNotEmpty(prefix)) { + curr = curr.getOrCreateChild(prefix); + } + + if(isWildcard || isRecursive) { + curr.addWildcardEvaluator(evaluator); + } else { + curr.addEvaluator(evaluator); + } + + } + + private String getNonWildcardPrefix(String str) { + + int minIndex = str.length(); + + for (int i = 0; i < wildcardChars.length(); i++) { + int index = str.indexOf(wildcardChars.charAt(i)); + + if (index != -1 && index < minIndex) { + minIndex = index; + } + } + + return str.substring(0, minIndex); } public List getEvaluatorsForResource(String resource) { @@ -121,30 +370,47 @@ public List getEvaluatorsForResource(String resource) { LOG.debug("==> RangerResourceTrie.getEvaluatorsForResource(" + resource + ")"); } - List ret = null; + RangerPerfTracer perf = null; + + if(RangerPerfTracer.isPerfTraceEnabled(PERF_TRIE_OP_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_TRIE_OP_LOG, "RangerResourceTrie.getEvaluatorsForResource(resource=" + resource + ")"); + } + + TrieNode curr = root; + TrieNode parent = null; + final int len = resource.length(); + int i = 0; - TrieNode curr = root; + while (i < len) { + if (!isOptimizedForRetrieval) { + curr.setupIfNeeded(parent, comparator); + } - final int len = resource.length(); - for(int i = 0; i < len; i++) { - Character ch = getLookupChar(resource.charAt(i)); - TrieNode child = curr.getChild(ch); + final TrieNode child = curr.getChild(getLookupChar(resource, i)); - if(child == null) { - ret = curr.getWildcardEvaluators(); - curr = null; // so that curr.getEvaluators() will not be called below + if (child == null) { break; } + final String childStr = child.getStr(); + + if (!resource.regionMatches(optIgnoreCase, i, childStr, 0, childStr.length())) { + break; + } + + parent = curr; curr = child; + i += childStr.length(); } - if(ret == null) { - if(curr != null) { - ret = curr.getEvaluators(); - } + if (!isOptimizedForRetrieval) { + curr.setupIfNeeded(parent, comparator); } + List ret = i == len ? curr.getEvaluators() : curr.getWildcardEvaluators(); + + RangerPerfTracer.logAlways(perf); + if(LOG.isDebugEnabled()) { LOG.debug("<== RangerResourceTrie.getEvaluatorsForResource(" + resource + "): evaluatorCount=" + (ret == null ? 0 : ret.size())); } @@ -152,54 +418,55 @@ public List getEvaluatorsForResource(String resource) { return ret; } - public TrieData getTrieData() { - TrieData ret = new TrieData(); - - root.populateTrieData(ret); - ret.maxDepth = getMaxDepth(); - - return ret; - } + private List getEvaluatorsForResources(Collection resources) { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerResourceTrie.getEvaluatorsForResources(" + resources + ")"); + } - public int getMaxDepth() { - return root.getMaxDepth(); - } + List ret = null; + Map evaluatorsMap = null; - public void reorderEvaluators() { - root.reorderEvaluators(null); - } + for (String resource : resources) { + List resourceEvaluators = getEvaluatorsForResource(resource); - private final Character getLookupChar(char ch) { - if(optIgnoreCase) { - ch = Character.toLowerCase(ch); - } + if (CollectionUtils.isEmpty(resourceEvaluators)) { + continue; + } - return Character.valueOf(ch); - } + if (evaluatorsMap == null) { + if (ret == null) { // first resource: don't create map yet + ret = resourceEvaluators; + } else if (ret != resourceEvaluators) { // if evaluator list is same as earlier resources, retain the list, else create a map + evaluatorsMap = new HashMap<>(); - private void insert(String resource, boolean isRecursive, T evaluator) { - TrieNode curr = root; - boolean isWildcard = false; + for (T evaluator : ret) { + evaluatorsMap.put(evaluator.getId(), evaluator); + } - final int len = resource.length(); - for(int i = 0; i < len; i++) { - Character ch = getLookupChar(resource.charAt(i)); + ret = null; + } + } - if(optWildcard) { - if (wildcardChars.indexOf(ch) != -1) { - isWildcard = true; - break; + if (evaluatorsMap != null) { + for (T evaluator : resourceEvaluators) { + evaluatorsMap.put(evaluator.getId(), evaluator); } } + } - curr = curr.getOrCreateChild(ch); + if (ret == null && evaluatorsMap != null) { + ret = new ArrayList<>(evaluatorsMap.values()); + + if (comparator != null) { + ret.sort(comparator); + } } - if(isWildcard || isRecursive) { - curr.addWildcardEvaluator(evaluator); - } else { - curr.addEvaluator(evaluator); + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerResourceTrie.getEvaluatorsForResources(" + resources + "): evaluatorCount=" + (ret == null ? 0 : ret.size())); } + + return ret; } @Override @@ -224,256 +491,372 @@ public String toString() { return sb.toString(); } - public class TrieData { - int nodeCount = 0; - int leafNodeCount = 0; - int singleChildNodeCount = 0; - int maxDepth = 0; - int evaluatorListCount = 0; - int wildcardEvaluatorListCount = 0; - int evaluatorListRefCount = 0; - int wildcardEvaluatorListRefCount = 0; - } -} + class ResourceTrieBuilderThread extends Thread { -class TrieNode { - private final Character c; - private Map children = null; - private List evaluators = null; - private List wildcardEvaluators = null; - private boolean isSharingParentWildcardEvaluators = false; - - TrieNode(Character c) { - this.c = c; - } + class WorkItem { + final String resourceName; + final boolean isRecursive; + final T evaluator; - Character getChar() { - return c; - } - - Map getChildren() { - return children; - } + WorkItem(String resourceName, boolean isRecursive, T evaluator) { + this.resourceName = resourceName; + this.isRecursive = isRecursive; + this.evaluator = evaluator; + } + @Override + public String toString() { + return + "resourceName=" + resourceName + + "isRecursive=" + isRecursive + + "evaluator=" + (evaluator != null? evaluator.getId() : null); + } + } - List getEvaluators() { - return evaluators; - } + private final TrieNode thisRoot = new TrieNode<>(null); + private final BlockingQueue workQueue = new LinkedBlockingQueue<>(); + private final boolean isOptimizedForRetrieval; + private List parentWildcardEvaluators; - List getWildcardEvaluators() { - return wildcardEvaluators; - } + ResourceTrieBuilderThread(boolean isOptimizedForRetrieval) { + this.isOptimizedForRetrieval = isOptimizedForRetrieval; + } - TrieNode getChild(Character c) { - TrieNode ret = children == null ? null : children.get(c); + void add(String resourceName, boolean isRecursive, T evaluator) throws InterruptedException { + workQueue.put(new WorkItem(resourceName, isRecursive, evaluator)); + } - return ret; - } + void setParentWildcardEvaluators(List parentWildcardEvaluators) { + this.parentWildcardEvaluators = parentWildcardEvaluators; + } - void populateTrieData(RangerResourceTrie.TrieData trieData) { - trieData.nodeCount++; + Map> getSubtrees() { return thisRoot.getChildren(); } - if(wildcardEvaluators != null) { - if(isSharingParentWildcardEvaluators) { - trieData.wildcardEvaluatorListRefCount++; - } else { - trieData.wildcardEvaluatorListCount++; + @Override + public void run() { + if (LOG.isDebugEnabled()) { + LOG.debug("Running " + this); } - } - if(evaluators != null) { - if(evaluators == wildcardEvaluators) { - trieData.evaluatorListRefCount++; - } else { - trieData.evaluatorListCount++; - } - } + while (true) { + final WorkItem workItem; - if(children != null && children.size() > 0) { - if(children.size() == 1) { - trieData.singleChildNodeCount++; - } + try { + workItem = workQueue.take(); + } catch (InterruptedException exception) { + LOG.error("Thread=" + this + " is interrupted", exception); - for(Map.Entry entry : children.entrySet()) { - TrieNode child = entry.getValue(); + break; + } - child.populateTrieData(trieData); + if (workItem.evaluator != null) { + insert(thisRoot, workItem.resourceName, workItem.isRecursive, workItem.evaluator); + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("Received termination signal. " + workItem); + } + break; + } } - } else { - trieData.leafNodeCount++; - } - } - int getMaxDepth() { - int ret = 0; + if (!isInterrupted() && isOptimizedForRetrieval) { + RangerPerfTracer postSetupPerf = null; - if(children != null) { - for(Map.Entry entry : children.entrySet()) { - TrieNode child = entry.getValue(); + if (RangerPerfTracer.isPerfTraceEnabled(PERF_TRIE_INIT_LOG)) { + postSetupPerf = RangerPerfTracer.getPerfTracer(PERF_TRIE_INIT_LOG, "RangerResourceTrie.init(thread=" + this.getName() + "-postSetup)"); + } - int maxChildDepth = child.getMaxDepth(); + thisRoot.postSetup(parentWildcardEvaluators, comparator); - if(maxChildDepth > ret) { - ret = maxChildDepth; - } + RangerPerfTracer.logAlways(postSetupPerf); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("Exiting " + this); } } + } - return ret + 1; + class TrieData { + int nodeCount; + int leafNodeCount; + int singleChildNodeCount; + int maxDepth; + int evaluatorListCount; + int wildcardEvaluatorListCount; + int evaluatorListRefCount; + int wildcardEvaluatorListRefCount; } - TrieNode getOrCreateChild(Character c) { - if(children == null) { - children = new HashMap(); - } + class TrieNode { + private String str; + private final Map> children = new HashMap<>(); + private List evaluators; + private List wildcardEvaluators; + private boolean isSharingParentWildcardEvaluators; + private volatile boolean isSetup = false; - TrieNode child = children.get(c); + TrieNode(String str) { + this.str = str; + } - if(child == null) { - child = new TrieNode(c); - children.put(c, child); + String getStr() { + return str; } - return child; - } + void setStr(String str) { + this.str = str; + } - void addEvaluator(T evaluator) { - if(evaluators == null) { - evaluators = new ArrayList(); + Map> getChildren() { + return children; } - if(!evaluators.contains(evaluator)) { - evaluators.add(evaluator); + List getEvaluators() { + return evaluators; } - } - void addWildcardEvaluator(T evaluator) { - if(wildcardEvaluators == null) { - wildcardEvaluators = new ArrayList(); + List getWildcardEvaluators() { + return wildcardEvaluators; } - if(!wildcardEvaluators.contains(evaluator)) { - wildcardEvaluators.add(evaluator); + TrieNode getChild(Character ch) { + return children == null ? null : children.get(ch); } - } - void postSetup(List parentWildcardEvaluators) { - // finalize wildcard-evaluators list by including parent's wildcard evaluators - if(parentWildcardEvaluators != null) { - if(CollectionUtils.isEmpty(this.wildcardEvaluators)) { - this.wildcardEvaluators = parentWildcardEvaluators; + void populateTrieData(RangerResourceTrie.TrieData trieData) { + trieData.nodeCount++; + + if (wildcardEvaluators != null) { + if (isSharingParentWildcardEvaluators) { + trieData.wildcardEvaluatorListRefCount++; + } else { + trieData.wildcardEvaluatorListCount++; + } + } + + if (evaluators != null) { + if (evaluators == wildcardEvaluators) { + trieData.evaluatorListRefCount++; + } else { + trieData.evaluatorListCount++; + } + } + + if (children != null && !children.isEmpty()) { + if (children.size() == 1) { + trieData.singleChildNodeCount++; + } + + for (Map.Entry> entry : children.entrySet()) { + TrieNode child = entry.getValue(); + + child.populateTrieData(trieData); + } } else { - for (T evaluator : parentWildcardEvaluators) { - addWildcardEvaluator(evaluator); + trieData.leafNodeCount++; + } + } + + int getMaxDepth() { + int ret = 0; + + if (children != null) { + for (Map.Entry> entry : children.entrySet()) { + TrieNode child = entry.getValue(); + + int maxChildDepth = child.getMaxDepth(); + + if (maxChildDepth > ret) { + ret = maxChildDepth; + } } } + + return ret + 1; } - this.isSharingParentWildcardEvaluators = wildcardEvaluators == parentWildcardEvaluators; - // finalize evaluators list by including wildcard evaluators - if(wildcardEvaluators != null) { - if(CollectionUtils.isEmpty(this.evaluators)) { - this.evaluators = wildcardEvaluators; + TrieNode getOrCreateChild(String str) { + int len = str.length(); + + TrieNode child = children.get(getLookupChar(str, 0)); + + if (child == null) { + child = new TrieNode<>(str); + addChild(child); } else { - for (T evaluator : wildcardEvaluators) { - addEvaluator(evaluator); + final String childStr = child.getStr(); + final int childStrLen = childStr.length(); + + final boolean isExactMatch = optIgnoreCase ? StringUtils.equalsIgnoreCase(childStr, str) : StringUtils.equals(childStr, str); + + if (!isExactMatch) { + final int numOfCharactersToMatch = childStrLen < len ? childStrLen : len; + int index = 1; + for (; index < numOfCharactersToMatch; index++) { + if (getLookupChar(childStr, index) != getLookupChar(str, index)) { + break; + } + } + if (index == numOfCharactersToMatch) { + // Matched all + if (childStrLen > len) { + // Existing node has longer string, need to break up this node + TrieNode newChild = new TrieNode<>(str); + this.addChild(newChild); + child.setStr(childStr.substring(index)); + newChild.addChild(child); + child = newChild; + } else { + // This is a longer string, build a child with leftover string + child = child.getOrCreateChild(str.substring(index)); + } + } else { + // Partial match for both; both have leftovers + String matchedPart = str.substring(0, index); + TrieNode newChild = new TrieNode<>(matchedPart); + this.addChild(newChild); + child.setStr(childStr.substring(index)); + newChild.addChild(child); + child = newChild.getOrCreateChild(str.substring(index)); + } } } - } - RangerPolicyResourceEvaluator.IdComparator comparator = new RangerPolicyResourceEvaluator.IdComparator(); - if(!isSharingParentWildcardEvaluators && CollectionUtils.isNotEmpty(wildcardEvaluators)) { - Collections.sort(wildcardEvaluators, comparator); + return child; } - if(evaluators != wildcardEvaluators && CollectionUtils.isNotEmpty(evaluators)) { - Collections.sort(evaluators, comparator); + private void addChild(TrieNode child) { + children.put(getLookupChar(child.getStr(), 0), child); } - if(children != null) { - for(Map.Entry entry : children.entrySet()) { - TrieNode child = entry.getValue(); + void addEvaluator(U evaluator) { + if (evaluators == null) { + evaluators = new ArrayList<>(); + } - child.postSetup(wildcardEvaluators); + if (!evaluators.contains(evaluator)) { + evaluators.add(evaluator); } } - } - void reorderEvaluators(List parentWildcardEvaluators) { - boolean isEvaluatorsSameAsWildcardEvaluators = evaluators == wildcardEvaluators; + void addWildcardEvaluator(U evaluator) { + if (wildcardEvaluators == null) { + wildcardEvaluators = new ArrayList<>(); + } - if(isSharingParentWildcardEvaluators) { - wildcardEvaluators = parentWildcardEvaluators; - } else { - wildcardEvaluators = getSortedCopy(wildcardEvaluators); + if (!wildcardEvaluators.contains(evaluator)) { + wildcardEvaluators.add(evaluator); + } } - if(isEvaluatorsSameAsWildcardEvaluators) { - evaluators = wildcardEvaluators; - } else { - evaluators = getSortedCopy(evaluators); - } + void postSetup(List parentWildcardEvaluators, Comparator comparator) { - if(children != null) { - for(Map.Entry entry : children.entrySet()) { - TrieNode child = entry.getValue(); + setup(parentWildcardEvaluators, comparator); - child.reorderEvaluators(wildcardEvaluators); + if (children != null) { + for (Map.Entry> entry : children.entrySet()) { + TrieNode child = entry.getValue(); + + child.postSetup(wildcardEvaluators, comparator); + } } } - } - public void toString(String prefix, StringBuilder sb) { - String nodeValue = prefix; + void setupIfNeeded(TrieNode parent, Comparator comparator) { + if (parent == null) { + return; + } - if(c != 0) { - nodeValue += c; - } + boolean setupNeeded = !isSetup; - sb.append("nodeValue=").append(nodeValue); - sb.append("; childCount=").append(children == null ? 0 : children.size()); - sb.append("; evaluators=[ "); - if(evaluators != null) { - for(T evaluator : evaluators) { - sb.append(evaluator.getId()).append(" "); + if (setupNeeded) { + synchronized (this) { + setupNeeded = !isSetup; + + if (setupNeeded) { + setup(parent.getWildcardEvaluators(), comparator); + isSetup = true; + } + } } } - sb.append("]"); - sb.append("; wildcardEvaluators=[ "); - if(wildcardEvaluators != null) { - for(T evaluator : wildcardEvaluators) { - sb.append(evaluator.getId()).append(" "); + void setup(List parentWildcardEvaluators, Comparator comparator) { + // finalize wildcard-evaluators list by including parent's wildcard evaluators + if (parentWildcardEvaluators != null) { + if (CollectionUtils.isEmpty(this.wildcardEvaluators)) { + this.wildcardEvaluators = parentWildcardEvaluators; + } else { + for (U evaluator : parentWildcardEvaluators) { + addWildcardEvaluator(evaluator); + } + } + } + this.isSharingParentWildcardEvaluators = wildcardEvaluators == parentWildcardEvaluators; + + // finalize evaluators list by including wildcard evaluators + if (wildcardEvaluators != null) { + if (CollectionUtils.isEmpty(this.evaluators)) { + this.evaluators = wildcardEvaluators; + } else { + for (U evaluator : wildcardEvaluators) { + addEvaluator(evaluator); + } + } } - } - sb.append("]"); - sb.append(Character.LINE_SEPARATOR); - if(children != null) { - for(Map.Entry entry : children.entrySet()) { - TrieNode child = entry.getValue(); + if (comparator != null) { + if (!isSharingParentWildcardEvaluators && CollectionUtils.isNotEmpty(wildcardEvaluators)) { + wildcardEvaluators.sort(comparator); + } - child.toString(nodeValue, sb); + if (evaluators != wildcardEvaluators && CollectionUtils.isNotEmpty(evaluators)) { + evaluators.sort(comparator); + } } } - } - public void clear() { - children = null; - evaluators = null; - wildcardEvaluators = null; - } + public void toString(String prefix, StringBuilder sb) { + String nodeValue = prefix; - private List getSortedCopy(List evaluators) { - final List ret; + if (str != null) { + nodeValue += str; + } - if(CollectionUtils.isNotEmpty(evaluators)) { - ret = new ArrayList(evaluators); + sb.append("nodeValue=").append(nodeValue); + sb.append("; childCount=").append(children == null ? 0 : children.size()); + sb.append("; evaluators=[ "); + if (evaluators != null) { + for (U evaluator : evaluators) { + sb.append(evaluator.getId()).append(" "); + } + } + sb.append("]"); - Collections.sort(ret, new RangerPolicyResourceEvaluator.IdComparator()); - } else { - ret = evaluators; + sb.append("; wildcardEvaluators=[ "); + if (wildcardEvaluators != null) { + for (U evaluator : wildcardEvaluators) { + sb.append(evaluator.getId()).append(" "); + } + } + sb.append("]\n"); + + if (children != null) { + for (Map.Entry> entry : children.entrySet()) { + TrieNode child = entry.getValue(); + + child.toString(nodeValue, sb); + } + } } - return ret; + public void clear() { + if (children != null) { + children.clear(); + } + + evaluators = null; + wildcardEvaluators = null; + } } } diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/ServiceDefUtil.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/ServiceDefUtil.java index dbdc935015..b0090d4bb6 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/util/ServiceDefUtil.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/ServiceDefUtil.java @@ -22,6 +22,7 @@ import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.MapUtils; import org.apache.commons.lang.StringUtils; +import org.apache.ranger.authorization.hadoop.config.RangerConfiguration; import org.apache.ranger.plugin.model.RangerPolicy; import org.apache.ranger.plugin.model.RangerServiceDef; import org.apache.ranger.plugin.model.RangerServiceDef.RangerAccessTypeDef; @@ -40,7 +41,8 @@ public static boolean getOption_enableDenyAndExceptionsInPolicies(RangerServiceD boolean ret = false; if(serviceDef != null) { - boolean defaultValue = StringUtils.equalsIgnoreCase(serviceDef.getName(), EmbeddedServiceDefsUtil.EMBEDDED_SERVICEDEF_TAG_NAME); + boolean enableDenyAndExceptionsInPoliciesHiddenOption = RangerConfiguration.getInstance().getBoolean("ranger.servicedef.enableDenyAndExceptionsInPolicies", true); + boolean defaultValue = enableDenyAndExceptionsInPoliciesHiddenOption || StringUtils.equalsIgnoreCase(serviceDef.getName(), EmbeddedServiceDefsUtil.EMBEDDED_SERVICEDEF_TAG_NAME); ret = ServiceDefUtil.getBooleanValue(serviceDef.getOptions(), RangerServiceDef.OPTION_ENABLE_DENY_AND_EXCEPTIONS_IN_POLICIES, defaultValue); } diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/StringTokenReplacer.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/StringTokenReplacer.java index 4ec1595080..da2b866f57 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/util/StringTokenReplacer.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/StringTokenReplacer.java @@ -50,11 +50,25 @@ public String replaceTokens(String value, Map tokens) { i++; if(i < value.length()) { c = value.charAt(i); - if(token != null) { + if (token != null) { + // if next char is not the escape char or endChar, retain the escapeChar + if (c != escapeChar && c != endChar) { + token.append(escapeChar); + } token.append(c); } else { + // if next char is not the escape char or startChar, retain the escapeChar + if (c != escapeChar && c != startChar) { + ret.append(escapeChar); + } ret.append(c); } + } else { + if (token != null) { + token.append(escapeChar); + } else { + ret.append(escapeChar); + } } continue; } @@ -72,6 +86,10 @@ public String replaceTokens(String value, Map tokens) { Object replaced = RangerAccessRequestUtil.getTokenFromContext(tokens, rawToken.substring(tokenPrefix.length())); if (replaced != null) { ret.append(replaced.toString()); + } else { + ret = null; + token = null; + break; } } else { ret.append(startChar).append(token).append(endChar); @@ -87,6 +105,6 @@ public String replaceTokens(String value, Map tokens) { ret.append(startChar).append(token); } - return ret.toString(); + return ret != null ? ret.toString() : null; } } diff --git a/agents-common/src/main/resources/service-defs/ranger-servicedef-abfs.json b/agents-common/src/main/resources/service-defs/ranger-servicedef-abfs.json new file mode 100644 index 0000000000..18454defff --- /dev/null +++ b/agents-common/src/main/resources/service-defs/ranger-servicedef-abfs.json @@ -0,0 +1,123 @@ +{ + "id":103, + "name": "abfs", + "implClass": "", + "label": "Azure Blob File System", + "description": "Ranger plugin for ABFS", + "guid":"", + "options": { "enableDenyAndExceptionsInPolicies": "true" }, + "resources": [ + { + "itemId": 1, + "name": "storageaccount", + "type": "string", + "parent": "", + "level": 10, + "mandatory": true, + "lookupSupported": false, + "excludesSupported": false, + "matcher": "org.apache.ranger.plugin.resourcematcher.RangerDefaultResourceMatcher", + "matcherOptions": {"wildCard":true, "ignoreCase":false}, + "validationRegEx": "", + "validationMessage": "", + "uiHint": "", + "label": "Storage Account", + "description": "Storage Account for the Path" + }, + { + "itemId":2, + "name": "container", + "type": "string", + "parent": "storageaccount", + "level":20, + "mandatory": true, + "lookupSupported": false, + "excludesSupported": false, + "matcher": "org.apache.ranger.plugin.resourcematcher.RangerDefaultResourceMatcher", + "matcherOptions": {"wildCard":true, "ignoreCase":false}, + "validationRegEx":"", + "validationMessage": "", + "uiHint":"", + "label": "Storage Account Container", + "description": "Storage Account Container for the Path" + }, + { + "itemId":3, + "name": "relativepath", + "type": "path", + "parent": "container", + "level":30, + "mandatory": true, + "lookupSupported": true, + "recursiveSupported": true, + "excludesSupported": false, + "matcher": "org.apache.ranger.plugin.resourcematcher.RangerPathResourceMatcher", + "matcherOptions": {"wildCard":true, "ignoreCase":false}, + "validationRegEx":"^[/*]$|^\/.*?[^\/]$", + "validationMessage": "Relative Path must not end with a slash", + "uiHint":"", + "label": "Relative Path", + "description": "Relative Path inside Storage Account Container" + } + ], + "accessTypes": + [ + { + "itemId": 1, + "name": "read", + "label": "Read" + }, + { + "itemId": 2, + "name": "write", + "label": "Write" + } + ], + "configs": + [ + { + "itemId": 1, + "name": "username", + "type": "string", + "subType": "", + "mandatory": false, + "validationRegEx":"", + "validationMessage": "", + "uiHint":"", + "label": "Username" + }, + { + "itemId": 2, + "name": "password", + "type": "string", + "subType": "", + "mandatory": false, + "validationRegEx":"", + "validationMessage": "", + "uiHint":"", + "label": "Password" + }, + { + "itemId":3, + "name": "commonNameForCertificate", + "type": "string", + "subType": "", + "mandatory": false, + "validationRegEx":"", + "validationMessage": "", + "uiHint":"", + "label": "Common Name for Certificate" + } + ], + "contextEnrichers": [], + "policyConditions": + [ + { + "itemId": 1, + "name": "ip-range", + "evaluator": "org.apache.ranger.plugin.conditionevaluator.RangerIpMatcher", + "label": "IP Address Range", + "description": "IP Address Range" + } + ] +} \ No newline at end of file diff --git a/agents-common/src/main/resources/service-defs/ranger-servicedef-nifi.json b/agents-common/src/main/resources/service-defs/ranger-servicedef-nifi.json index b81785d9e3..1d1123225a 100644 --- a/agents-common/src/main/resources/service-defs/ranger-servicedef-nifi.json +++ b/agents-common/src/main/resources/service-defs/ranger-servicedef-nifi.json @@ -14,7 +14,7 @@ "mandatory":true, "lookupSupported":true, "recursiveSupported":false, - "excludesSupported":true, + "excludesSupported":false, "matcher":"org.apache.ranger.plugin.resourcematcher.RangerDefaultResourceMatcher", "matcherOptions":{ "wildCard":true, diff --git a/agents-common/src/main/resources/service-defs/ranger-servicedef-tag.json b/agents-common/src/main/resources/service-defs/ranger-servicedef-tag.json index 3bad2222f5..c17b750706 100644 --- a/agents-common/src/main/resources/service-defs/ranger-servicedef-tag.json +++ b/agents-common/src/main/resources/service-defs/ranger-servicedef-tag.json @@ -69,6 +69,14 @@ "uiHint": "{ \"singleValue\":true }", "label":"Accessed after expiry_date (yes/no)?", "description": "Accessed after expiry_date? (yes/no)" + }, + { + "itemId":2, + "name":"expression", + "evaluator": "org.apache.ranger.plugin.conditionevaluator.RangerScriptConditionEvaluator", + "evaluatorOptions" : {"engineName":"JavaScript", "ui.isMultiline":"true"}, + "label":"Enter boolean expression", + "description": "Boolean expression" } ] } diff --git a/agents-common/src/main/resources/service-defs/ranger-servicedef-wasb.json b/agents-common/src/main/resources/service-defs/ranger-servicedef-wasb.json index 9b3eafe338..946b662ad2 100644 --- a/agents-common/src/main/resources/service-defs/ranger-servicedef-wasb.json +++ b/agents-common/src/main/resources/service-defs/ranger-servicedef-wasb.json @@ -48,6 +48,7 @@ "level":30, "mandatory": true, "lookupSupported": true, + "recursiveSupported": true, "excludesSupported": false, "matcher": "org.apache.ranger.plugin.resourcematcher.RangerPathResourceMatcher", "matcherOptions": {"wildCard":true, "ignoreCase":false}, diff --git a/agents-common/src/test/java/org/apache/ranger/plugin/model/validation/TestRangerPolicyValidator.java b/agents-common/src/test/java/org/apache/ranger/plugin/model/validation/TestRangerPolicyValidator.java index caa8e35438..e55c6eb832 100644 --- a/agents-common/src/test/java/org/apache/ranger/plugin/model/validation/TestRangerPolicyValidator.java +++ b/agents-common/src/test/java/org/apache/ranger/plugin/model/validation/TestRangerPolicyValidator.java @@ -42,7 +42,6 @@ import org.apache.ranger.plugin.model.validation.RangerValidator.Action; import org.apache.ranger.plugin.store.ServiceStore; import org.apache.ranger.plugin.util.RangerObjectFactory; -import org.apache.ranger.plugin.util.SearchFilter; import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -124,13 +123,13 @@ public void setUp() throws Exception { private final Object[][] policyResourceMap_good = new Object[][] { // resource-name, values, excludes, recursive { "db", new String[] { "db1", "db2" }, null, null }, - { "TBL", new String[] { "tbl1", "tbl2" }, true, false } // case should not matter + { "tbl", new String[] { "tbl1", "tbl2" }, true, false } // case matters - use only lowercase characters }; private final Object[][] policyResourceMap_goodMultipleHierarchies = new Object[][] { // resource-name, values, excludes, recursive { "db", new String[] { "db1", "db2" }, null, null }, - { "UDF", new String[] { "udf1", "udf2" }, true, false } // case should not matter + { "udf", new String[] { "udf1", "udf2" }, true, false } // case matters - use only lowercase characters }; private final Object[][] policyResourceMap_bad = new Object[][] { @@ -229,6 +228,7 @@ public final void testIsValid_happyPath() throws Exception { // service name exists RangerService service = mock(RangerService.class); when(service.getType()).thenReturn("service-type"); + when(service.getId()).thenReturn(2L); when(_store.getServiceByName("service-name")).thenReturn(service); // service points to a valid service-def _serviceDef = _utils.createServiceDefWithAccessTypes(accessTypes); @@ -240,17 +240,7 @@ public final void testIsValid_happyPath() throws Exception { when(existingPolicy.getId()).thenReturn(8L); when(existingPolicy.getService()).thenReturn("service-name"); when(_store.getPolicy(8L)).thenReturn(existingPolicy); - SearchFilter createFilter = new SearchFilter(); - createFilter.setParam(SearchFilter.SERVICE_TYPE, "service-type"); - createFilter.setParam(SearchFilter.POLICY_NAME, "policy-name-1"); // this name would be used for create - when(_store.getPolicies(createFilter)).thenReturn(new ArrayList()); // a matching policy should not exist for update. - SearchFilter updateFilter = new SearchFilter(); - updateFilter.setParam(SearchFilter.SERVICE_TYPE, "service-type"); - updateFilter.setParam(SearchFilter.POLICY_NAME, "policy-name-2"); // this name would be used for update - List existingPolicies = new ArrayList(); - existingPolicies.add(existingPolicy); - when(_store.getPolicies(updateFilter)).thenReturn(existingPolicies); // valid policy can have empty set of policy items if audit is turned on // null value for audit is treated as audit on. // for now we want to turn any resource related checking off @@ -262,6 +252,7 @@ public final void testIsValid_happyPath() throws Exception { if (action == Action.CREATE) { when(_policy.getId()).thenReturn(7L); when(_policy.getName()).thenReturn("policy-name-1"); + when(_store.getPolicyId(service.getId(), _policy.getName())).thenReturn(null); Assert.assertTrue("" + action + ", " + auditEnabled, _validator.isValid(_policy, action, isAdmin, _failures)); Assert.assertTrue(_failures.isEmpty()); } else { @@ -272,6 +263,7 @@ public final void testIsValid_happyPath() throws Exception { Assert.assertTrue(_failures.isEmpty()); when(_policy.getName()).thenReturn("policy-name-2"); + when(_store.getPolicyId(service.getId(), _policy.getName())).thenReturn(null); Assert.assertTrue("" + action + ", " + auditEnabled, _validator.isValid(_policy, action, isAdmin, _failures)); Assert.assertTrue(_failures.isEmpty()); } @@ -370,20 +362,22 @@ public final void testIsValid_failures() throws Exception { checkFailure_isValid(action, "missing", "id"); } } + RangerService service = mock(RangerService.class); /* * Id is ignored for Create but name should not belong to an existing policy. For update, policy should exist for its id and should match its name. */ when(_policy.getName()).thenReturn("policy-name"); when(_policy.getService()).thenReturn("service-name"); + when(_store.getServiceByName("service-name")).thenReturn(service); + when(service.getId()).thenReturn(2L); + RangerPolicy existingPolicy = mock(RangerPolicy.class); when(existingPolicy.getId()).thenReturn(7L); - List existingPolicies = new ArrayList(); - existingPolicies.add(existingPolicy); - SearchFilter filter = new SearchFilter(); - filter.setParam(SearchFilter.SERVICE_NAME, "service-name"); - filter.setParam(SearchFilter.POLICY_NAME, "policy-name"); - when(_store.getPolicies(filter)).thenReturn(existingPolicies); + when(existingPolicy.getService()).thenReturn("service-name"); + List existingPolicies = new ArrayList<>(); + + when(_store.getPolicyId(service.getId(), "policy-name")).thenReturn(7L); checkFailure_isValid(Action.CREATE, "semantic", "policy name"); // update : does not exist for id @@ -395,21 +389,11 @@ public final void testIsValid_failures() throws Exception { when(_store.getPolicy(7L)).thenReturn(existingPolicy); RangerPolicy anotherExistingPolicy = mock(RangerPolicy.class); when(anotherExistingPolicy.getId()).thenReturn(8L); - existingPolicies.clear(); + when(anotherExistingPolicy.getService()).thenReturn("service-name"); + existingPolicies.add(anotherExistingPolicy); - when(_store.getPolicies(filter)).thenReturn(existingPolicies); + when(_store.getPolicyId(service.getId(), "policy-name")).thenReturn(8L); checkFailure_isValid(Action.UPDATE, "semantic", "id/name"); - - // more than one policies with same name is also an internal error - when(_policy.getName()).thenReturn("policy-name"); - when(_store.getPolicies(filter)).thenReturn(existingPolicies); - existingPolicies.add(existingPolicy); - existingPolicy = mock(RangerPolicy.class); - existingPolicies.add(existingPolicy); - for (boolean isAdmin : new boolean[] { true, false }) { - _failures.clear(); Assert.assertFalse(_validator.isValid(_policy, Action.UPDATE, isAdmin, _failures)); - _utils.checkFailureForInternalError(_failures); - } // policy must have service name on it and it should be valid when(_policy.getName()).thenReturn("policy-name"); @@ -449,10 +433,7 @@ public final void testIsValid_failures() throws Exception { } // policy must contain at least one policy item - List policyItems = new ArrayList(); - when(_policy.getService()).thenReturn("service-name"); - RangerService service = mock(RangerService.class); - when(_store.getServiceByName("service-name")).thenReturn(service); + List policyItems = new ArrayList<>(); for (Action action : cu) { for (boolean isAdmin : new boolean[] { true, false }) { // when it is null @@ -474,6 +455,8 @@ public final void testIsValid_failures() throws Exception { when(_store.getServiceDefByName("service-type")).thenReturn(null); for (Action action : cu) { for (boolean isAdmin : new boolean[] { true, false }) { + when(_policy.getService()).thenReturn("service-name"); + when(_store.getServiceByName("service-name")).thenReturn(service); _failures.clear(); Assert.assertFalse(_validator.isValid(_policy, action, isAdmin, _failures)); _utils.checkFailureForInternalError(_failures, "policy service def"); } @@ -491,7 +474,7 @@ public final void testIsValid_failures() throws Exception { // create the right service def with right resource defs - this is the same as in the happypath test above. _serviceDef = _utils.createServiceDefWithAccessTypes(accessTypes, "service-type"); - when(_store.getPolicies(filter)).thenReturn(null); + when(_store.getPolicyId(service.getId(), "policy-name")).thenReturn(null); List resourceDefs = _utils.createResourceDefs(resourceDefData); when(_serviceDef.getResources()).thenReturn(resourceDefs); when(_store.getServiceDefByName("service-type")).thenReturn(_serviceDef); diff --git a/agents-common/src/test/java/org/apache/ranger/plugin/model/validation/TestRangerServiceDefValidator.java b/agents-common/src/test/java/org/apache/ranger/plugin/model/validation/TestRangerServiceDefValidator.java index 33e6f4a946..370f549a9d 100644 --- a/agents-common/src/test/java/org/apache/ranger/plugin/model/validation/TestRangerServiceDefValidator.java +++ b/agents-common/src/test/java/org/apache/ranger/plugin/model/validation/TestRangerServiceDefValidator.java @@ -211,46 +211,50 @@ public final void test_isValidName_update() throws Exception { @Test public final void test_isValidAccessTypes_happyPath() { + long id = 7; + when(_serviceDef.getId()).thenReturn(id); List input = _utils.createAccessTypeDefs(accessTypes_good); - assertTrue(_validator.isValidAccessTypes(input, _failures)); + assertTrue(_validator.isValidAccessTypes(id, input, _failures, Action.CREATE)); assertTrue(_failures.isEmpty()); } @Test public final void test_isValidAccessTypes_failures() { + long id = 7; + when(_serviceDef.getId()).thenReturn(id); // null or empty access type defs List accessTypeDefs = null; - _failures.clear(); assertFalse(_validator.isValidAccessTypes(accessTypeDefs, _failures)); + _failures.clear(); assertFalse(_validator.isValidAccessTypes(id, accessTypeDefs, _failures, Action.CREATE)); _utils.checkFailureForMissingValue(_failures, "access types"); accessTypeDefs = new ArrayList(); - _failures.clear(); assertFalse(_validator.isValidAccessTypes(accessTypeDefs, _failures)); + _failures.clear(); assertFalse(_validator.isValidAccessTypes(id, accessTypeDefs, _failures, Action.CREATE)); _utils.checkFailureForMissingValue(_failures, "access types"); // null/empty access types accessTypeDefs = _utils.createAccessTypeDefs(new String[] { null, "", " " }); - _failures.clear(); assertFalse(_validator.isValidAccessTypes(accessTypeDefs, _failures)); + _failures.clear(); assertFalse(_validator.isValidAccessTypes(id, accessTypeDefs, _failures, Action.CREATE)); _utils.checkFailureForMissingValue(_failures, "access type name"); // duplicate access types accessTypeDefs = _utils.createAccessTypeDefs(new String[] { "read", "write", "execute", "read" } ); - _failures.clear(); assertFalse(_validator.isValidAccessTypes(accessTypeDefs, _failures)); + _failures.clear(); assertFalse(_validator.isValidAccessTypes(id, accessTypeDefs, _failures, Action.CREATE)); _utils.checkFailureForSemanticError(_failures, "access type name", "read"); // duplicate access types - case-insensitive accessTypeDefs = _utils.createAccessTypeDefs(new String[] { "read", "write", "execute", "READ" } ); - _failures.clear(); assertFalse(_validator.isValidAccessTypes(accessTypeDefs, _failures)); + _failures.clear(); assertFalse(_validator.isValidAccessTypes(id, accessTypeDefs, _failures, Action.CREATE)); _utils.checkFailureForSemanticError(_failures, "access type name", "READ"); // unknown access type in implied grants list accessTypeDefs = _utils.createAccessTypeDefs(accessTypes_bad_unknownType); - _failures.clear(); assertFalse(_validator.isValidAccessTypes(accessTypeDefs, _failures)); + _failures.clear(); assertFalse(_validator.isValidAccessTypes(id, accessTypeDefs, _failures, Action.CREATE)); _utils.checkFailureForSemanticError(_failures, "implied grants", "execute"); _utils.checkFailureForSemanticError(_failures, "access type itemId", "1"); // id 1 is duplicated // access type with implied grant referring to itself accessTypeDefs = _utils.createAccessTypeDefs(accessTypes_bad_selfReference); - _failures.clear(); assertFalse(_validator.isValidAccessTypes(accessTypeDefs, _failures)); + _failures.clear(); assertFalse(_validator.isValidAccessTypes(id, accessTypeDefs, _failures, Action.CREATE)); _utils.checkFailureForSemanticError(_failures, "implied grants", "admin"); } @@ -396,16 +400,16 @@ public final void test_isValidEnumElements_failures() { public final void test_isValidResources() { // null/empty resources are an error when(_serviceDef.getResources()).thenReturn(null); - _failures.clear(); assertFalse(_validator.isValidResources(_serviceDef, _failures)); + _failures.clear(); assertFalse(_validator.isValidResources(_serviceDef, _failures, Action.CREATE)); _utils.checkFailureForMissingValue(_failures, "resources"); List resources = new ArrayList(); when(_serviceDef.getResources()).thenReturn(resources); - _failures.clear(); assertFalse(_validator.isValidResources(_serviceDef, _failures)); + _failures.clear(); assertFalse(_validator.isValidResources(_serviceDef, _failures, Action.CREATE)); _utils.checkFailureForMissingValue(_failures, "resources"); resources.addAll(_utils.createResourceDefsWithIds(invalidResources)); - _failures.clear(); assertFalse(_validator.isValidResources(_serviceDef, _failures)); + _failures.clear(); assertFalse(_validator.isValidResources(_serviceDef, _failures, Action.CREATE)); _utils.checkFailureForMissingValue(_failures, "resource name"); _utils.checkFailureForMissingValue(_failures, "resource itemId"); _utils.checkFailureForSemanticError(_failures, "resource itemId", "1"); // id 1 is duplicate @@ -456,7 +460,7 @@ public final void test_isValidResources_happyPath() { }; List resources = _utils.createResourceDefsWithIds(data); when(_serviceDef.getResources()).thenReturn(resources); - assertTrue(_validator.isValidResources(_serviceDef, _failures)); + assertTrue(_validator.isValidResources(_serviceDef, _failures, Action.CREATE)); assertTrue(_failures.isEmpty()); } @@ -490,11 +494,13 @@ public final void test_isValidConfigs_failures() { @Test public final void test_isValidPolicyConditions() { + long id = 7; + when(_serviceDef.getId()).thenReturn(id); // null/empty policy conditions are ok - assertTrue(_validator.isValidPolicyConditions(null, _failures)); + assertTrue(_validator.isValidPolicyConditions(id,null, _failures, Action.CREATE)); assertTrue(_failures.isEmpty()); List conditionDefs = new ArrayList(); - assertTrue(_validator.isValidPolicyConditions(conditionDefs, _failures)); + assertTrue(_validator.isValidPolicyConditions(id, conditionDefs, _failures, Action.CREATE)); assertTrue(_failures.isEmpty()); Object[][] policyCondition_data = { @@ -506,7 +512,7 @@ public final void test_isValidPolicyConditions() { }; conditionDefs.addAll(_utils.createPolicyConditionDefs(policyCondition_data)); - _failures.clear(); assertFalse(_validator.isValidPolicyConditions(conditionDefs, _failures)); + _failures.clear(); assertFalse(_validator.isValidPolicyConditions(id, conditionDefs, _failures, Action.CREATE)); _utils.checkFailureForMissingValue(_failures, "policy condition def itemId"); _utils.checkFailureForMissingValue(_failures, "policy condition def name"); _utils.checkFailureForMissingValue(_failures, "policy condition def evaluator"); diff --git a/agents-common/src/test/java/org/apache/ranger/plugin/model/validation/TestRangerServiceValidator.java b/agents-common/src/test/java/org/apache/ranger/plugin/model/validation/TestRangerServiceValidator.java index 40af3ce31e..be6dc3dfe2 100644 --- a/agents-common/src/test/java/org/apache/ranger/plugin/model/validation/TestRangerServiceValidator.java +++ b/agents-common/src/test/java/org/apache/ranger/plugin/model/validation/TestRangerServiceValidator.java @@ -23,6 +23,7 @@ import static org.mockito.Mockito.when; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; @@ -40,6 +41,7 @@ public class TestRangerServiceValidator { final Action[] cud = new Action[] { Action.CREATE, Action.UPDATE, Action.DELETE }; final Action[] cu = new Action[] { Action.CREATE, Action.UPDATE }; final Action[] ud = new Action[] { Action.UPDATE, Action.DELETE }; + String serviceNameValidationErrorMessage = "Name should not start with space, it should be less than 256 characters and special characters are not allowed(except _ - and space). "; @Before public void before() { @@ -72,6 +74,216 @@ void checkFailure_isValid(RangerServiceValidator validator, RangerService servic } @Test + public void testIsValidServiceNameCreationWithOutSpecialCharacters() throws Exception{ + RangerService rangerService = new RangerService(); + rangerService.setName("c1_yarn"); + rangerService.setType("yarn"); + rangerService.setTagService(""); + + RangerServiceConfigDef configDef = new RangerServiceConfigDef(); + configDef.setMandatory(true); + + List listRangerServiceConfigDef = new ArrayList(); + listRangerServiceConfigDef.add(configDef); + + + configDef.setName("myconfig1"); + + Map testMap = new HashMap(); + testMap.put("myconfig1", "myconfig1"); + + rangerService.setConfigs(testMap); + + + RangerServiceDef rangerServiceDef = new RangerServiceDef(); + rangerServiceDef.setConfigs(listRangerServiceConfigDef); + + when(_store.getServiceDefByName("yarn")).thenReturn(rangerServiceDef); + boolean valid = _validator.isValid(rangerService, Action.CREATE, _failures); + Assert.assertEquals(0, _failures.size()); + Assert.assertTrue(valid); + + } + + @Test + public void testIsValidServiceNameUpdationWithOutSpecialCharacters() throws Exception{ + RangerService rangerService = new RangerService(); + rangerService.setId(1L); + rangerService.setName("c1_yarn"); + rangerService.setType("yarn"); + rangerService.setTagService(""); + + RangerServiceConfigDef configDef = new RangerServiceConfigDef(); + configDef.setMandatory(true); + + List listRangerServiceConfigDef = new ArrayList(); + listRangerServiceConfigDef.add(configDef); + + + configDef.setName("myconfig1"); + + Map testMap = new HashMap(); + testMap.put("myconfig1", "myconfig1"); + + rangerService.setConfigs(testMap); + + + RangerServiceDef rangerServiceDef = new RangerServiceDef(); + rangerServiceDef.setConfigs(listRangerServiceConfigDef); + + when(_store.getService(1L)).thenReturn(rangerService); + when(_store.getServiceDefByName("yarn")).thenReturn(rangerServiceDef); + boolean valid = _validator.isValid(rangerService, Action.UPDATE, _failures); + Assert.assertEquals(0, _failures.size()); + Assert.assertTrue(valid); + + } + + @Test + public void testIsValidServiceNameUpdationWithSpecialCharacters() throws Exception{ + RangerService rangerService = new RangerService(); + rangerService.setId(1L); + rangerService.setName("c1_yarn"); + rangerService.setType("yarn"); + rangerService.setTagService(""); + + RangerServiceConfigDef configDef = new RangerServiceConfigDef(); + configDef.setMandatory(true); + + List listRangerServiceConfigDef = new ArrayList(); + listRangerServiceConfigDef.add(configDef); + + + configDef.setName("myconfig1"); + + Map testMap = new HashMap(); + testMap.put("myconfig1", "myconfig1"); + + rangerService.setConfigs(testMap); + + + RangerServiceDef rangerServiceDef = new RangerServiceDef(); + rangerServiceDef.setConfigs(listRangerServiceConfigDef); + + when(_store.getService(1L)).thenReturn(rangerService); + when(_store.getServiceDefByName("yarn")).thenReturn(rangerServiceDef); + boolean valid = _validator.isValid(rangerService, Action.UPDATE, _failures); + ValidationFailureDetails failureMessage = _failures.get(0); + Assert.assertFalse(valid); + Assert.assertEquals("name",failureMessage.getFieldName()); + Assert.assertEquals(serviceNameValidationErrorMessage + ": name=[c1_yarn]",failureMessage._reason); + Assert.assertEquals(3031, failureMessage._errorCode); + + } + + @Test + public void testIsValidServiceNameCreationWithSpecialCharacters() throws Exception{ + RangerService rangerService = new RangerService(); + rangerService.setName(""); + rangerService.setType("yarn"); + rangerService.setTagService(""); + + RangerServiceConfigDef configDef = new RangerServiceConfigDef(); + configDef.setMandatory(true); + + List listRangerServiceConfigDef = new ArrayList(); + listRangerServiceConfigDef.add(configDef); + + + configDef.setName("myconfig1"); + + Map testMap = new HashMap(); + testMap.put("myconfig1", "myconfig1"); + + rangerService.setConfigs(testMap); + + + RangerServiceDef rangerServiceDef = new RangerServiceDef(); + rangerServiceDef.setConfigs(listRangerServiceConfigDef); + + when(_store.getServiceDefByName("yarn")).thenReturn(rangerServiceDef); + boolean valid = _validator.isValid(rangerService, _action, _failures); + ValidationFailureDetails failureMessage = _failures.get(0); + Assert.assertFalse(valid); + Assert.assertEquals("name",failureMessage.getFieldName()); + Assert.assertEquals(serviceNameValidationErrorMessage + ": name=[]",failureMessage._reason); + Assert.assertEquals(3031, failureMessage._errorCode); + + } + + @Test + public void testIsValidServiceNameCreationWithGreater255Characters() throws Exception{ + RangerService rangerService = new RangerService(); + rangerService.setName("c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1"); + rangerService.setType("yarn"); + rangerService.setTagService(""); + + RangerServiceConfigDef configDef = new RangerServiceConfigDef(); + configDef.setMandatory(true); + + List listRangerServiceConfigDef = new ArrayList(); + listRangerServiceConfigDef.add(configDef); + + + configDef.setName("myconfig1"); + + Map testMap = new HashMap(); + testMap.put("myconfig1", "myconfig1"); + + rangerService.setConfigs(testMap); + + + RangerServiceDef rangerServiceDef = new RangerServiceDef(); + rangerServiceDef.setConfigs(listRangerServiceConfigDef); + + when(_store.getServiceDefByName("yarn")).thenReturn(rangerServiceDef); + boolean valid = _validator.isValid(rangerService, _action, _failures); + ValidationFailureDetails failureMessage = _failures.get(0); + Assert.assertFalse(valid); + Assert.assertEquals("name",failureMessage.getFieldName()); + Assert.assertEquals(serviceNameValidationErrorMessage + ": name=[c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1]",failureMessage._reason); + Assert.assertEquals(3031, failureMessage._errorCode); + + } + + @Test + public void testIsValidServiceNameUpdationWithGreater255Characters() throws Exception{ + RangerService rangerService = new RangerService(); + rangerService.setId(1L); + rangerService.setName("c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1"); + rangerService.setType("yarn"); + rangerService.setTagService(""); + + RangerServiceConfigDef configDef = new RangerServiceConfigDef(); + configDef.setMandatory(true); + + List listRangerServiceConfigDef = new ArrayList(); + listRangerServiceConfigDef.add(configDef); + + + configDef.setName("myconfig1"); + + Map testMap = new HashMap(); + testMap.put("myconfig1", "myconfig1"); + + rangerService.setConfigs(testMap); + + + RangerServiceDef rangerServiceDef = new RangerServiceDef(); + rangerServiceDef.setConfigs(listRangerServiceConfigDef); + + when(_store.getService(1L)).thenReturn(rangerService); + when(_store.getServiceDefByName("yarn")).thenReturn(rangerServiceDef); + boolean valid = _validator.isValid(rangerService, Action.UPDATE, _failures); + ValidationFailureDetails failureMessage = _failures.get(0); + Assert.assertFalse(valid); + Assert.assertEquals("name",failureMessage.getFieldName()); + Assert.assertEquals(serviceNameValidationErrorMessage +": name=[c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1_yarn_c1]",failureMessage._reason); + Assert.assertEquals(3031, failureMessage._errorCode); + + } + + @Test public void testIsValid_failures() throws Exception { RangerService service = mock(RangerService.class); // passing in a null service to the check itself is an error diff --git a/agents-common/src/test/java/org/apache/ranger/plugin/model/validation/TestRangerValidator.java b/agents-common/src/test/java/org/apache/ranger/plugin/model/validation/TestRangerValidator.java index 5519a2c060..a6e31511d2 100644 --- a/agents-common/src/test/java/org/apache/ranger/plugin/model/validation/TestRangerValidator.java +++ b/agents-common/src/test/java/org/apache/ranger/plugin/model/validation/TestRangerValidator.java @@ -21,7 +21,6 @@ import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import java.util.ArrayList; @@ -34,7 +33,6 @@ import java.util.Set; import org.apache.ranger.plugin.model.RangerPolicy; -import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyResource; import org.apache.ranger.plugin.model.RangerService; import org.apache.ranger.plugin.model.RangerServiceDef; import org.apache.ranger.plugin.model.RangerServiceDef.RangerAccessTypeDef; @@ -43,13 +41,10 @@ import org.apache.ranger.plugin.model.RangerServiceDef.RangerServiceConfigDef; import org.apache.ranger.plugin.model.validation.RangerValidator.Action; import org.apache.ranger.plugin.store.ServiceStore; -import org.apache.ranger.plugin.util.SearchFilter; import org.junit.Assert; import org.junit.Before; import org.junit.Test; -import com.google.common.collect.Maps; - public class TestRangerValidator { static class RangerValidatorForTest extends RangerValidator { @@ -272,8 +267,8 @@ public void test_getAccessTypes() { Assert.assertEquals(4, accessTypes.size()); Assert.assertTrue(accessTypes.contains("a")); Assert.assertTrue(accessTypes.contains("b ")); - Assert.assertTrue(accessTypes.contains(" c")); - Assert.assertTrue(accessTypes.contains(" d ")); + Assert.assertTrue(accessTypes.contains(" C")); + Assert.assertTrue(accessTypes.contains(" D ")); } @Test @@ -360,36 +355,6 @@ public void test_getValidationRegExes() { Assert.assertEquals("regex3", regExMap.get("f")); } - @Test - public void test_getPolicyResources() { - - Set result; - RangerPolicy policy = null; - // null policy - result = _validator.getPolicyResources(null); - Assert.assertTrue(result != null); - Assert.assertTrue(result.isEmpty()); - // null resource map - policy = mock(RangerPolicy.class); - when(policy.getResources()).thenReturn(null); - result = _validator.getPolicyResources(null); - Assert.assertTrue(result != null); - Assert.assertTrue(result.isEmpty()); - // empty resource map - Map input = Maps.newHashMap(); - when(policy.getResources()).thenReturn(input); - result = _validator.getPolicyResources(policy); - Assert.assertTrue(result != null); - Assert.assertTrue(result.isEmpty()); - // known resource map - input.put("r1", mock(RangerPolicyResource.class)); - input.put("R2", mock(RangerPolicyResource.class)); - result = _validator.getPolicyResources(policy); - Assert.assertEquals(2, result.size()); - Assert.assertTrue("r1", result.contains("r1")); - Assert.assertTrue("R2", result.contains("r2")); // result should lowercase the resource-names - } - @Test public void test_getIsAuditEnabled() { // null policy @@ -410,42 +375,6 @@ public void test_getIsAuditEnabled() { result = _validator.getIsAuditEnabled(policy); Assert.assertTrue(result); } - - @Test - public void test_getPolicies() throws Exception { - - // returns null when store returns null - String policyName = "aPolicy"; - String serviceName = "aService"; - SearchFilter filter = new SearchFilter(); - filter.setParam(SearchFilter.POLICY_NAME, policyName); - filter.setParam(SearchFilter.SERVICE_NAME, serviceName); - - when(_store.getPolicies(filter)).thenReturn(null); - List result = _validator.getPolicies(serviceName, policyName); - // validate store is queried with both parameters - verify(_store).getPolicies(filter); - Assert.assertNull(result); - - // returns null if store throws an exception - when(_store.getPolicies(filter)).thenThrow(new Exception()); - result = _validator.getPolicies(serviceName, policyName); - Assert.assertNull(result); - - // does not shove policy into search filter if policy name passed in is "blank" - filter = new SearchFilter(); - filter.setParam(SearchFilter.SERVICE_NAME, serviceName); - - List policies = new ArrayList(); - RangerPolicy policy = mock(RangerPolicy.class); - policies.add(policy); - - when(_store.getPolicies(filter)).thenReturn(policies); - for (String aName : new String[]{ null, "", " "}) { - result = _validator.getPolicies(serviceName, aName); - Assert.assertTrue(result.iterator().next() == policy); - } - } @Test public void test_getServiceDef_byId() throws Exception { diff --git a/agents-common/src/test/java/org/apache/ranger/plugin/policyengine/TestPolicyEngine.java b/agents-common/src/test/java/org/apache/ranger/plugin/policyengine/TestPolicyEngine.java index d4c16c1164..45865618a2 100644 --- a/agents-common/src/test/java/org/apache/ranger/plugin/policyengine/TestPolicyEngine.java +++ b/agents-common/src/test/java/org/apache/ranger/plugin/policyengine/TestPolicyEngine.java @@ -152,7 +152,11 @@ public static void setUpBeforeClass() throws Exception { " ranger.plugin.tag.attr.additional.date.formats\n" + " abcd||xyz||yyyy/MM/dd'T'HH:mm:ss.SSS'Z'\n" + " \n" + - "\n"); + " \n" + + " ranger.policyengine.trie.builder.thread.count\n" + + " 3\n" + + " \n" + + "\n"); writer.close(); RangerConfiguration config = RangerConfiguration.getInstance(); @@ -289,6 +293,13 @@ public void testPolicyEngine_hiveMasking() { runTestsFromResourceFiles(resourceFiles); } + @Test + public void testPolicyEngine_hiveTagMasking() { + String[] resourceFiles = {"/policyengine/test_policyengine_tag_hive_mask.json"}; + + runTestsFromResourceFiles(resourceFiles); + } + @Test public void testPolicyEngine_owner() { String[] resourceFiles = {"/policyengine/test_policyengine_owner.json"}; @@ -334,6 +345,7 @@ private void runTests(InputStreamReader reader, String testName) { RangerPolicyEngineOptions policyEngineOptions = new RangerPolicyEngineOptions(); policyEngineOptions.disableTagPolicyEvaluation = false; + policyEngineOptions.optimizeTrieForRetrieval = false; boolean useForwardedIPAddress = RangerConfiguration.getInstance().getBoolean("ranger.plugin.hive.use.x-forwarded-for.ipaddress", false); String trustedProxyAddressString = RangerConfiguration.getInstance().get("ranger.plugin.hive.trusted.proxy.ipaddresses"); @@ -346,6 +358,7 @@ private void runTests(InputStreamReader reader, String testName) { RangerPolicyEngine policyEngine = new RangerPolicyEngineImpl(testName, servicePolicies, policyEngineOptions); policyEngine.setUseForwardedIPAddress(useForwardedIPAddress); policyEngine.setTrustedProxyAddresses(trustedProxyAddresses); + long requestCount = 0L; RangerAccessRequest request = null; diff --git a/agents-common/src/test/resources/log4j.xml b/agents-common/src/test/resources/log4j.xml index d863cf1d69..802c308bfb 100644 --- a/agents-common/src/test/resources/log4j.xml +++ b/agents-common/src/test/resources/log4j.xml @@ -35,6 +35,20 @@ + diff --git a/embeddedwebserver/pom.xml b/embeddedwebserver/pom.xml index 149ed1766e..852f04ba81 100644 --- a/embeddedwebserver/pom.xml +++ b/embeddedwebserver/pom.xml @@ -33,6 +33,11 @@ tomcat-embed-core ${tomcat.embed.version} + + org.apache.tomcat + tomcat-annotations-api + ${tomcat.embed.version} + org.apache.tomcat.embed tomcat-embed-el diff --git a/embeddedwebserver/scripts/ranger-admin-services.sh b/embeddedwebserver/scripts/ranger-admin-services.sh index f2d2bf5680..e5643a34b5 100755 --- a/embeddedwebserver/scripts/ranger-admin-services.sh +++ b/embeddedwebserver/scripts/ranger-admin-services.sh @@ -32,8 +32,8 @@ XAPOLICYMGR_DIR=`(cd $realScriptDir/..; pwd)` XAPOLICYMGR_EWS_DIR=${XAPOLICYMGR_DIR}/ews RANGER_JAAS_LIB_DIR="${XAPOLICYMGR_EWS_DIR}/ranger_jaas" RANGER_JAAS_CONF_DIR="${XAPOLICYMGR_EWS_DIR}/webapp/WEB-INF/classes/conf/ranger_jaas" -JAVA_OPTS=" ${JAVA_OPTS} -XX:MaxPermSize=256m -Xmx1024m -Xms1024m " -if [[ ${JAVA_OPTS} != *"-Duser.timezone"* ]] ;then export JAVA_OPTS=" ${JAVA_OPTS} -Duser.timezone=UTC" ;fi +ranger_admin_max_heap_size=1g + if [ -f ${XAPOLICYMGR_DIR}/ews/webapp/WEB-INF/classes/conf/java_home.sh ]; then . ${XAPOLICYMGR_DIR}/ews/webapp/WEB-INF/classes/conf/java_home.sh fi @@ -44,6 +44,9 @@ for custom_env_script in `find ${XAPOLICYMGR_DIR}/ews/webapp/WEB-INF/classes/con fi done +JAVA_OPTS=" ${JAVA_OPTS} -XX:MetaspaceSize=100m -XX:MaxMetaspaceSize=200m -Xmx${ranger_admin_max_heap_size} -Xms1g -Xloggc:${XAPOLICYMGR_EWS_DIR}/logs/gc-worker.log -verbose:gc -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=1m -XX:+PrintGCDetails -XX:+PrintHeapAtGC -XX:+PrintGCDateStamps" +if [[ ${JAVA_OPTS} != *"-Duser.timezone"* ]] ;then export JAVA_OPTS=" ${JAVA_OPTS} -Duser.timezone=UTC" ;fi + if [ "$JAVA_HOME" != "" ]; then export PATH=$JAVA_HOME/bin:$PATH fi @@ -137,7 +140,11 @@ stop(){ } metric(){ - java ${JAVA_OPTS} -Dlogdir=${RANGER_ADMIN_LOG_DIR} -cp "${XAPOLICYMGR_EWS_DIR}/webapp/WEB-INF/classes/conf:${XAPOLICYMGR_EWS_DIR}/webapp/WEB-INF/classes/:${XAPOLICYMGR_EWS_DIR}/webapp/WEB-INF/classes/lib/*:${XAPOLICYMGR_EWS_DIR}/webapp/WEB-INF/classes/META-INF:${XAPOLICYMGR_EWS_DIR}/webapp/WEB-INF/lib/*:${XAPOLICYMGR_EWS_DIR}/webapp/META-INF:${XAPOLICYMGR_EWS_DIR}/lib/*:${RANGER_JAAS_LIB_DIR}/*:${RANGER_JAAS_CONF_DIR}:${JAVA_HOME}/lib/*:${RANGER_HADOOP_CONF_DIR}/*:$CLASSPATH" org.apache.ranger.patch.cliutil.MetricUtil ${arg2} ${arg3} 2>/dev/null + if [ "$JAVA_HOME" == "" ]; then + echo "[E] JAVA_HOME environment variable not defined, aborting Apache Ranger Admin metric collection" + exit 1; + fi + java ${JAVA_OPTS} -Dlogdir=${RANGER_ADMIN_LOG_DIR} -cp "${XAPOLICYMGR_EWS_DIR}/webapp/WEB-INF/classes/conf:${XAPOLICYMGR_EWS_DIR}/webapp/WEB-INF/classes/:${XAPOLICYMGR_EWS_DIR}/webapp/WEB-INF/classes/lib/*:${XAPOLICYMGR_EWS_DIR}/webapp/WEB-INF/classes/META-INF:${XAPOLICYMGR_EWS_DIR}/webapp/WEB-INF/lib/*:${XAPOLICYMGR_EWS_DIR}/webapp/META-INF:${XAPOLICYMGR_EWS_DIR}/lib/*:${RANGER_JAAS_LIB_DIR}/*:${RANGER_JAAS_CONF_DIR}:${JAVA_HOME}/lib/*:${RANGER_HADOOP_CONF_DIR}/*:$CLASSPATH" org.apache.ranger.patch.cliutil.MetricUtil ${arg2} ${arg3} 2>/dev/null } if [ "${action}" == "START" ]; then diff --git a/enunciate.xml b/enunciate.xml new file mode 100644 index 0000000000..2bd313d257 --- /dev/null +++ b/enunciate.xml @@ -0,0 +1,40 @@ + + + Ranger REST API + Apache Ranger is a framework to enable, monitor and manage comprehensive data security across the Hadoop platform. Apache Ranger currently provides a centralized security adminstration, fine grain access control and detailed auditing for user access within Apache Hadoop, Apache Hive, Apache HBase and other Apache components + + + + + + + + + + + + + + + + + + + diff --git a/hbase-agent/src/main/java/org/apache/ranger/authorization/hbase/RangerAuthorizationCoprocessor.java b/hbase-agent/src/main/java/org/apache/ranger/authorization/hbase/RangerAuthorizationCoprocessor.java index cf2ffcf150..4c863c0509 100644 --- a/hbase-agent/src/main/java/org/apache/ranger/authorization/hbase/RangerAuthorizationCoprocessor.java +++ b/hbase-agent/src/main/java/org/apache/ranger/authorization/hbase/RangerAuthorizationCoprocessor.java @@ -111,9 +111,11 @@ import com.google.protobuf.RpcCallback; import com.google.protobuf.RpcController; import com.google.protobuf.Service; +import org.apache.ranger.plugin.util.RangerPerfTracer; public class RangerAuthorizationCoprocessor extends RangerAuthorizationCoprocessorBase implements AccessControlService.Interface, CoprocessorService { private static final Log LOG = LogFactory.getLog(RangerAuthorizationCoprocessor.class.getName()); + private static final Log PERF_HBASEAUTH_REQUEST_LOG = RangerPerfTracer.getPerfLogger("hbaseauth.request"); private static boolean UpdateRangerPoliciesOnGrantRevoke = RangerHadoopConstants.HBASE_UPDATE_RANGER_POLICIES_ON_GRANT_REVOKE_DEFAULT_VALUE; private static final String GROUP_PREFIX = "@"; @@ -334,7 +336,7 @@ ColumnFamilyAccessResult evaluateAccess(String operation, Action action, final R } return result; } - + // let's create a session that would be reused. Set things on it that won't change. HbaseAuditHandler auditHandler = _factory.getAuditHandler(); AuthorizationSession session = new AuthorizationSession(hbasePlugin) @@ -402,47 +404,33 @@ ColumnFamilyAccessResult evaluateAccess(String operation, Action action, final R if (columns == null || columns.isEmpty()) { LOG.debug("evaluateAccess: columns collection null or empty, ok. Family level access is desired."); session.column(null) // zap stale column from prior iteration of this loop, if any - .buildRequest() - .authorize(); + .buildRequest() + .authorize(); AuthzAuditEvent auditEvent = auditHandler.getAndDiscardMostRecentEvent(); // capture it only for success if (session.isAuthorized()) { - if (LOG.isDebugEnabled()) { - LOG.debug("evaluateAccess: has family level access [" + family + "]"); - } - // we need to do 3 things: housekeeping, decide about audit events, building the results cache for filter somethingIsAccessible = true; - familesAccessAllowed.add(family); - if (auditEvent != null) { - LOG.debug("evaluateAccess: adding to family-level-access-granted-event-set"); - familyLevelAccessEvents.add(auditEvent); - } - } else { - everythingIsAccessible = false; - if (auditEvent != null && deniedEvent == null) { // we need to capture just one denial event - LOG.debug("evaluateAccess: Setting denied access audit event with last auth failure audit event."); - deniedEvent = auditEvent; - } if (LOG.isDebugEnabled()) { - LOG.debug("evaluateAccess: no family level access [" + family + "]. Checking if has partial access (of any type)..."); + LOG.debug("evaluateAccess: has family level access [" + family + "]. Checking if [" + family + "] descendants have access."); } - session.resourceMatchingScope(RangerAccessRequest.ResourceMatchingScope.SELF_OR_DESCENDANTS) .buildRequest() .authorize(); auditEvent = auditHandler.getAndDiscardMostRecentEvent(); // capture it only for failure if (session.isAuthorized()) { if (LOG.isDebugEnabled()) { - LOG.debug("evaluateAccess: has partial access (of some type) in family [" + family + "]"); + LOG.debug("evaluateAccess: [" + family + "] descendants have access"); + } + familesAccessAllowed.add(family); + if (auditEvent != null) { + LOG.debug("evaluateAccess: adding to family-level-access-granted-event-set"); + familyLevelAccessEvents.add(auditEvent); } - // we need to do 3 things: housekeeping, decide about audit events, building the results cache for filter - somethingIsAccessible = true; - familesAccessIndeterminate.add(family); } else { if (LOG.isDebugEnabled()) { - LOG.debug("evaluateAccess: has no access of ["+ access + "] type in family [" + family + "]"); + LOG.debug("evaluateAccess: has partial access (of some type) in family [" + family + "]"); } - familesAccessDenied.add(family); - denialReason = String.format("Insufficient permissions for user ‘%s',action: %s, tableName:%s, family:%s.", user.getName(), operation, table, family); + everythingIsAccessible = false; + familesAccessIndeterminate.add(family); if (auditEvent != null && deniedEvent == null) { // we need to capture just one denial event LOG.debug("evaluateAccess: Setting denied access audit event with last auth failure audit event."); deniedEvent = auditEvent; @@ -450,6 +438,17 @@ ColumnFamilyAccessResult evaluateAccess(String operation, Action action, final R } // Restore the headMatch setting session.resourceMatchingScope(RangerAccessRequest.ResourceMatchingScope.SELF); + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("evaluateAccess: has no access of [" + access + "] type in family [" + family + "]"); + } + everythingIsAccessible = false; + familesAccessDenied.add(family); + denialReason = String.format("Insufficient permissions for user ‘%s',action: %s, tableName:%s, family:%s.", user.getName(), operation, table, family); + if (auditEvent != null && deniedEvent == null) { // we need to capture just one denial event + LOG.debug("evaluateAccess: Setting denied access audit event with last auth failure audit event."); + deniedEvent = auditEvent; + } } } else { LOG.debug("evaluateAccess: columns collection not empty. Skipping Family level check, will do finer level access check."); @@ -505,7 +504,11 @@ Filter authorizeAccess(String operation, Action action, final RegionCoprocessorE if (LOG.isDebugEnabled()) { LOG.debug("==> authorizeAccess"); } + RangerPerfTracer perf = null; + try { + perf = RangerPerfTracer.getPerfTracer(PERF_HBASEAUTH_REQUEST_LOG, "RangerAuthorizationCoprocessor.authorizeAccess(request=Operation[" + operation + "]"); + ColumnFamilyAccessResult accessResult = evaluateAccess(operation, action, env, familyMap); RangerDefaultAuditHandler auditHandler = new RangerDefaultAuditHandler(); if (accessResult._everythingIsAccessible) { @@ -525,6 +528,7 @@ Filter authorizeAccess(String operation, Action action, final RegionCoprocessorE throw new AccessDeniedException(accessResult._denialReason); } } finally { + RangerPerfTracer.log(perf); if (LOG.isDebugEnabled()) { LOG.debug("<== authorizeAccess"); } @@ -542,17 +546,26 @@ Filter combineFilters(Filter filter, Filter existingFilter) { void requirePermission(final String operation, final Action action, final RegionCoprocessorEnvironment regionServerEnv, final Map> familyMap) throws AccessDeniedException { - ColumnFamilyAccessResult accessResult = evaluateAccess(operation, action, regionServerEnv, familyMap); - RangerDefaultAuditHandler auditHandler = new RangerDefaultAuditHandler(); - if (accessResult._everythingIsAccessible) { - auditHandler.logAuthzAudits(accessResult._accessAllowedEvents); - auditHandler.logAuthzAudits(accessResult._familyLevelAccessEvents); - LOG.debug("requirePermission: exiting: all access was allowed"); - return; - } else { - auditHandler.logAuthzAudit(accessResult._accessDeniedEvent); - LOG.debug("requirePermission: exiting: throwing exception as everything wasn't accessible"); - throw new AccessDeniedException(accessResult._denialReason); + RangerPerfTracer perf = null; + + try { + if (RangerPerfTracer.isPerfTraceEnabled(PERF_HBASEAUTH_REQUEST_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_HBASEAUTH_REQUEST_LOG, "RangerAuthorizationCoprocessor.requirePermission(request=Operation[" + operation + "]"); + } + ColumnFamilyAccessResult accessResult = evaluateAccess(operation, action, regionServerEnv, familyMap); + RangerDefaultAuditHandler auditHandler = new RangerDefaultAuditHandler(); + if (accessResult._everythingIsAccessible) { + auditHandler.logAuthzAudits(accessResult._accessAllowedEvents); + auditHandler.logAuthzAudits(accessResult._familyLevelAccessEvents); + LOG.debug("requirePermission: exiting: all access was allowed"); + return; + } else { + auditHandler.logAuthzAudit(accessResult._accessDeniedEvent); + LOG.debug("requirePermission: exiting: throwing exception as everything wasn't accessible"); + throw new AccessDeniedException(accessResult._denialReason); + } + } finally { + RangerPerfTracer.log(perf); } } @@ -796,7 +809,7 @@ public void preDeleteColumn(ObserverContext c, Tab } @Override public void preDeleteSnapshot(ObserverContext ctx, SnapshotDescription snapshot) throws IOException { - requirePermission("deleteSnapshot", Permission.Action.ADMIN); + requirePermission("deleteSnapshot", snapshot.getTableBytes().toByteArray(), Permission.Action.ADMIN); } @Override public void preDeleteTable(ObserverContext c, TableName tableName) throws IOException { diff --git a/hbase-agent/src/test/java/org/apache/ranger/authorization/hbase/HBaseRangerAuthorizationTest.java b/hbase-agent/src/test/java/org/apache/ranger/authorization/hbase/HBaseRangerAuthorizationTest.java index 7aec352bec..90e09ad8fe 100644 --- a/hbase-agent/src/test/java/org/apache/ranger/authorization/hbase/HBaseRangerAuthorizationTest.java +++ b/hbase-agent/src/test/java/org/apache/ranger/authorization/hbase/HBaseRangerAuthorizationTest.java @@ -317,8 +317,9 @@ public Void run() throws Exception { // Read a row try { Get get = new Get(Bytes.toBytes("row1")); - table.get(get); - Assert.fail("Failure expected on an unauthorized user"); + Result result = table.get(get); + byte[] valResult = result.getValue(Bytes.toBytes("colfam1"), Bytes.toBytes("col1")); + Assert.assertNull("Failure expected on an unauthorized user", valResult); } catch (IOException ex) { // expected } @@ -526,8 +527,9 @@ public Void run() throws Exception { // Read a row try { Get get = new Get(Bytes.toBytes("row1")); - table.get(get); - Assert.fail("Failure expected on an unauthorized user"); + Result result = table.get(get); + byte[] valResult = result.getValue(Bytes.toBytes("colfam2"), Bytes.toBytes("col1")); + Assert.assertNull("Failure expected on an unauthorized user", valResult); } catch (IOException ex) { // expected } diff --git a/hdfs-agent/src/main/java/org/apache/ranger/authorization/hadoop/RangerHdfsAuthorizer.java b/hdfs-agent/src/main/java/org/apache/ranger/authorization/hadoop/RangerHdfsAuthorizer.java index d28685a481..22de0e8f63 100644 --- a/hdfs-agent/src/main/java/org/apache/ranger/authorization/hadoop/RangerHdfsAuthorizer.java +++ b/hdfs-agent/src/main/java/org/apache/ranger/authorization/hadoop/RangerHdfsAuthorizer.java @@ -20,10 +20,12 @@ package org.apache.ranger.authorization.hadoop; import static org.apache.ranger.authorization.hadoop.constants.RangerHadoopConstants.EXECUTE_ACCCESS_TYPE; +import static org.apache.ranger.authorization.hadoop.constants.RangerHadoopConstants.HDFS_ROOT_FOLDER_PATH; import static org.apache.ranger.authorization.hadoop.constants.RangerHadoopConstants.READ_ACCCESS_TYPE; import static org.apache.ranger.authorization.hadoop.constants.RangerHadoopConstants.WRITE_ACCCESS_TYPE; import java.net.InetAddress; +import java.security.SecureRandom; import java.util.Date; import java.util.HashMap; import java.util.HashSet; @@ -36,6 +38,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.permission.FsAction; +import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.server.namenode.INode; import org.apache.hadoop.hdfs.server.namenode.INodeAttributeProvider; import org.apache.hadoop.hdfs.server.namenode.INodeAttributes; @@ -54,7 +57,9 @@ import org.apache.ranger.plugin.policyengine.RangerAccessResource; import org.apache.ranger.plugin.policyengine.RangerAccessResourceImpl; import org.apache.ranger.plugin.policyengine.RangerAccessResult; +import org.apache.ranger.plugin.resourcematcher.RangerPathResourceMatcher; import org.apache.ranger.plugin.service.RangerBasePlugin; +import org.apache.ranger.plugin.util.RangerPerfTracer; import com.google.common.collect.Sets; @@ -70,6 +75,7 @@ public class RangerHdfsAuthorizer extends INodeAttributeProvider { public static final String RANGER_FILENAME_EXTENSION_SEPARATOR_PROP = "ranger.plugin.hdfs.filename.extension.separator"; private static final Log LOG = LogFactory.getLog(RangerHdfsAuthorizer.class); + private static final Log PERF_HDFSAUTH_REQUEST_LOG = RangerPerfTracer.getPerfLogger("hdfsauth.request"); private RangerHdfsPlugin rangerPlugin = null; private Map> access2ActionListMapper = new HashMap>(); @@ -92,6 +98,10 @@ public void start() { RangerHdfsPlugin plugin = new RangerHdfsPlugin(); plugin.init(); + if (plugin.isOptimizeSubAccessAuthEnabled()) { + LOG.info(RangerHadoopConstants.RANGER_OPTIMIZE_SUBACCESS_AUTHORIZATION_PROP + " is enabled"); + } + access2ActionListMapper.put(FsAction.NONE, new HashSet()); access2ActionListMapper.put(FsAction.ALL, Sets.newHashSet(READ_ACCCESS_TYPE, WRITE_ACCCESS_TYPE, EXECUTE_ACCCESS_TYPE)); access2ActionListMapper.put(FsAction.READ, Sets.newHashSet(READ_ACCCESS_TYPE)); @@ -188,6 +198,16 @@ public RangerAccessControlEnforcer(AccessControlEnforcer defaultEnforcer) { } } + class SubAccessData { + final INodeDirectory dir; + final String resourcePath; + + SubAccessData(INodeDirectory dir, String resourcePath) { + this.dir = dir; + this.resourcePath = resourcePath; + } + } + @Override public void checkPermission(String fsOwner, String superGroup, UserGroupInformation ugi, INodeAttributes[] inodeAttrs, INode[] inodes, byte[][] pathByNameArr, @@ -199,15 +219,22 @@ public void checkPermission(String fsOwner, String superGroup, UserGroupInformat RangerHdfsAuditHandler auditHandler = null; String user = ugi != null ? ugi.getShortUserName() : null; Set groups = ugi != null ? Sets.newHashSet(ugi.getGroupNames()) : null; + String resourcePath = path; if(LOG.isDebugEnabled()) { LOG.debug("==> RangerAccessControlEnforcer.checkPermission(" + "fsOwner=" + fsOwner + "; superGroup=" + superGroup + ", inodesCount=" + (inodes != null ? inodes.length : 0) - + ", snapshotId=" + snapshotId + ", user=" + user + ", path=" + path + ", ancestorIndex=" + ancestorIndex + + ", snapshotId=" + snapshotId + ", user=" + user + ", provided-path=" + path + ", ancestorIndex=" + ancestorIndex + ", doCheckOwner="+ doCheckOwner + ", ancestorAccess=" + ancestorAccess + ", parentAccess=" + parentAccess + ", access=" + access + ", subAccess=" + subAccess + ", ignoreEmptyDir=" + ignoreEmptyDir + ")"); } + RangerPerfTracer perf = null; + + if(RangerPerfTracer.isPerfTraceEnabled(PERF_HDFSAUTH_REQUEST_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_HDFSAUTH_REQUEST_LOG, "RangerHdfsAuthorizer.checkPermission(provided-path=" + path + ")"); + } + try { boolean isTraverseOnlyCheck = access == null && parentAccess == null && ancestorAccess == null && subAccess == null; INode ancestor = null; @@ -215,6 +242,29 @@ public void checkPermission(String fsOwner, String superGroup, UserGroupInformat INode inode = null; if(plugin != null && !ArrayUtils.isEmpty(inodes)) { + int sz = inodeAttrs.length; + if (LOG.isDebugEnabled()) { + LOG.debug("Size of INodeAttrs array:[" + sz + "]"); + } + byte[][] components = new byte[sz][]; + + int i = 0; + for (; i < sz; i++) { + if (inodeAttrs[i] != null) { + components[i] = inodeAttrs[i].getLocalNameBytes(); + } else { + break; + } + } + if (i != sz) { + if (LOG.isDebugEnabled()) { + LOG.debug("Input INodeAttributes array contains null at position " + i); + LOG.debug("Will use only first [" + i + "] components to build resourcePath"); + } + } + + resourcePath = DFSUtil.byteArray2PathString(components, 0, i); + if(ancestorIndex >= inodes.length) { ancestorIndex = inodes.length - 1; } @@ -227,25 +277,10 @@ public void checkPermission(String fsOwner, String superGroup, UserGroupInformat parent = inodes.length > 1 ? inodes[inodes.length - 2] : null; inode = inodes[inodes.length - 1]; // could be null while creating a new file - auditHandler = new RangerHdfsAuditHandler(path, isTraverseOnlyCheck); + auditHandler = new RangerHdfsAuditHandler(resourcePath, isTraverseOnlyCheck); if(isTraverseOnlyCheck) { - INode nodeToCheck = inode; - INodeAttributes nodeAttribs = inodeAttrs.length > 0 ? inodeAttrs[inodeAttrs.length - 1] : null; - - if(nodeToCheck == null || nodeToCheck.isFile()) { - if(parent != null) { - nodeToCheck = parent; - nodeAttribs = inodeAttrs.length > 1 ? inodeAttrs[inodeAttrs.length - 2] : null; - } else if(ancestor != null) { - nodeToCheck = ancestor; - nodeAttribs = inodeAttrs.length > ancestorIndex ? inodeAttrs[ancestorIndex] : null; - } - } - - if(nodeToCheck != null) { - authzStatus = isAccessAllowed(nodeToCheck, nodeAttribs, FsAction.EXECUTE, user, groups, plugin, auditHandler); - } + authzStatus = traverseOnlyCheck(inode, inodeAttrs, resourcePath, components, parent, ancestor, ancestorIndex, user, groups, plugin, auditHandler); } // checkStickyBit @@ -259,12 +294,13 @@ public void checkPermission(String fsOwner, String superGroup, UserGroupInformat // checkAncestorAccess if(authzStatus == AuthzStatus.ALLOW && ancestorAccess != null && ancestor != null) { INodeAttributes ancestorAttribs = inodeAttrs.length > ancestorIndex ? inodeAttrs[ancestorIndex] : null; + String ancestorPath = ancestorAttribs != null ? DFSUtil.byteArray2PathString(components, 0, ancestorIndex + 1) : null; - authzStatus = isAccessAllowed(ancestor, ancestorAttribs, ancestorAccess, user, groups, plugin, auditHandler); + authzStatus = isAccessAllowed(ancestor, ancestorAttribs, ancestorPath, ancestorAccess, user, groups, plugin, auditHandler); if (authzStatus == AuthzStatus.NOT_DETERMINED) { authzStatus = checkDefaultEnforcer(fsOwner, superGroup, ugi, inodeAttrs, inodes, pathByNameArr, snapshotId, path, ancestorIndex, doCheckOwner, - ancestorAccess, FsAction.NONE, FsAction.NONE, FsAction.NONE, ignoreEmptyDir, + ancestorAccess, null, null, null, ignoreEmptyDir, isTraverseOnlyCheck, ancestor, parent, inode, auditHandler); } } @@ -272,12 +308,13 @@ public void checkPermission(String fsOwner, String superGroup, UserGroupInformat // checkParentAccess if(authzStatus == AuthzStatus.ALLOW && parentAccess != null && parent != null) { INodeAttributes parentAttribs = inodeAttrs.length > 1 ? inodeAttrs[inodeAttrs.length - 2] : null; + String parentPath = parentAttribs != null ? DFSUtil.byteArray2PathString(components, 0, inodeAttrs.length - 1) : null; - authzStatus = isAccessAllowed(parent, parentAttribs, parentAccess, user, groups, plugin, auditHandler); + authzStatus = isAccessAllowed(parent, parentAttribs, parentPath, parentAccess, user, groups, plugin, auditHandler); if (authzStatus == AuthzStatus.NOT_DETERMINED) { authzStatus = checkDefaultEnforcer(fsOwner, superGroup, ugi, inodeAttrs, inodes, pathByNameArr, snapshotId, path, ancestorIndex, doCheckOwner, - FsAction.NONE, parentAccess, FsAction.NONE, FsAction.NONE, ignoreEmptyDir, + null, parentAccess, null, null, ignoreEmptyDir, isTraverseOnlyCheck, ancestor, parent, inode, auditHandler); } } @@ -286,44 +323,56 @@ public void checkPermission(String fsOwner, String superGroup, UserGroupInformat if(authzStatus == AuthzStatus.ALLOW && access != null && inode != null) { INodeAttributes inodeAttribs = inodeAttrs.length > 0 ? inodeAttrs[inodeAttrs.length - 1] : null; - authzStatus = isAccessAllowed(inode, inodeAttribs, access, user, groups, plugin, auditHandler); + authzStatus = isAccessAllowed(inode, inodeAttribs, resourcePath, access, user, groups, plugin, auditHandler); if (authzStatus == AuthzStatus.NOT_DETERMINED) { authzStatus = checkDefaultEnforcer(fsOwner, superGroup, ugi, inodeAttrs, inodes, pathByNameArr, snapshotId, path, ancestorIndex, doCheckOwner, - FsAction.NONE, FsAction.NONE, access, FsAction.NONE, ignoreEmptyDir, + null, null, access, null, ignoreEmptyDir, isTraverseOnlyCheck, ancestor, parent, inode, auditHandler); } } // checkSubAccess if(authzStatus == AuthzStatus.ALLOW && subAccess != null && inode != null && inode.isDirectory()) { - Stack directories = new Stack(); + Stack directories = new Stack<>(); - for(directories.push(inode.asDirectory()); !directories.isEmpty(); ) { - INodeDirectory dir = directories.pop(); - ReadOnlyList cList = dir.getChildrenList(snapshotId); + for(directories.push(new SubAccessData(inode.asDirectory(), resourcePath)); !directories.isEmpty(); ) { + SubAccessData data = directories.pop(); + ReadOnlyList cList = data.dir.getChildrenList(snapshotId); if (!(cList.isEmpty() && ignoreEmptyDir)) { - INodeAttributes dirAttribs = dir.getSnapshotINode(snapshotId); + INodeAttributes dirAttribs = data.dir.getSnapshotINode(snapshotId); - authzStatus = isAccessAllowed(dir, dirAttribs, subAccess, user, groups, plugin, auditHandler); + authzStatus = isAccessAllowed(data.dir, dirAttribs, data.resourcePath, subAccess, user, groups, plugin, auditHandler); if(authzStatus != AuthzStatus.ALLOW) { break; } - } - for(INode child : cList) { - if (child.isDirectory()) { - directories.push(child.asDirectory()); + AuthzStatus subDirAuthStatus = AuthzStatus.NOT_DETERMINED; + + boolean optimizeSubAccessAuthEnabled = RangerHdfsPlugin.isOptimizeSubAccessAuthEnabled(); + + if (optimizeSubAccessAuthEnabled) { + subDirAuthStatus = isAccessAllowedForHierarchy(data.dir, dirAttribs, data.resourcePath, subAccess, user, groups, plugin); + } + + if (subDirAuthStatus != AuthzStatus.ALLOW) { + for(INode child : cList) { + if (child.isDirectory()) { + directories.push(new SubAccessData(child.asDirectory(), resourcePath + org.apache.hadoop.fs.Path.SEPARATOR_CHAR + child.getLocalName())); + } + } } } } if (authzStatus == AuthzStatus.NOT_DETERMINED) { + authzStatus = checkDefaultEnforcer(fsOwner, superGroup, ugi, inodeAttrs, inodes, pathByNameArr, snapshotId, path, ancestorIndex, doCheckOwner, - FsAction.NONE, FsAction.NONE, FsAction.NONE, subAccess, ignoreEmptyDir, + null, null, null, subAccess, ignoreEmptyDir, isTraverseOnlyCheck, ancestor, parent, inode, auditHandler); + } } @@ -356,17 +405,107 @@ public void checkPermission(String fsOwner, String superGroup, UserGroupInformat } } - throw new RangerAccessControlException("Permission denied: user=" + user + ", access=" + action + ", inode=\"" + path + "\""); + throw new RangerAccessControlException("Permission denied: user=" + user + ", access=" + action + ", inode=\"" + resourcePath + "\""); } } finally { if(auditHandler != null) { auditHandler.flushAudit(); } + RangerPerfTracer.log(perf); + if(LOG.isDebugEnabled()) { - LOG.debug("<== RangerAccessControlEnforcer.checkPermission(" + path + ", " + access + ", user=" + user + ") : " + authzStatus); + LOG.debug("<== RangerAccessControlEnforcer.checkPermission(" + resourcePath + ", " + access + ", user=" + user + ") : " + authzStatus); + } + } + } + + /* + Check if parent or ancestor of the file being accessed is denied EXECUTE permission. If not, assume that Ranger-acls + allowed EXECUTE access. Do not audit this authorization check if resource is a file unless access is explicitly denied + */ + private AuthzStatus traverseOnlyCheck(INode inode, INodeAttributes[] inodeAttrs, String path, byte[][] components, INode parent, INode ancestor, int ancestorIndex, + String user, Set groups, RangerHdfsPlugin plugin, RangerHdfsAuditHandler auditHandler) { + + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerAccessControlEnforcer.traverseOnlyCheck(" + + "path=" + path + ", user=" + user + ", groups=" + groups + ")"); + } + final AuthzStatus ret; + + INode nodeToCheck = inode; + INodeAttributes nodeAttribs = inodeAttrs.length > 0 ? inodeAttrs[inodeAttrs.length - 1] : null; + boolean skipAuditOnAllow = false; + + String resourcePath = path; + if (nodeToCheck == null || nodeToCheck.isFile()) { + skipAuditOnAllow = true; + if (parent != null) { + nodeToCheck = parent; + nodeAttribs = inodeAttrs.length > 1 ? inodeAttrs[inodeAttrs.length - 2] : null; + resourcePath = inodeAttrs.length > 0 ? DFSUtil.byteArray2PathString(components, 0, inodeAttrs.length - 1) : HDFS_ROOT_FOLDER_PATH; + } else if (ancestor != null) { + nodeToCheck = ancestor; + nodeAttribs = inodeAttrs.length > ancestorIndex ? inodeAttrs[ancestorIndex] : null; + resourcePath = nodeAttribs != null ? DFSUtil.byteArray2PathString(components, 0, ancestorIndex+1) : HDFS_ROOT_FOLDER_PATH; + } + } + + if (nodeToCheck != null) { + if (resourcePath.length() > 1) { + if (resourcePath.endsWith(HDFS_ROOT_FOLDER_PATH)) { + resourcePath = resourcePath.substring(0, resourcePath.length()-1); + } } + ret = isAccessAllowedForTraversal(nodeToCheck, nodeAttribs, resourcePath, user, groups, plugin, auditHandler, skipAuditOnAllow); + } else { + ret = AuthzStatus.ALLOW; + } + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerAccessControlEnforcer.traverseOnlyCheck(" + + "path=" + path + ", resourcePath=" + resourcePath + ", user=" + user + ", groups=" + groups + ") : " + ret); } + return ret; + } + + private AuthzStatus isAccessAllowedForTraversal(INode inode, INodeAttributes inodeAttribs, String path, String user, Set groups, RangerHdfsPlugin plugin, RangerHdfsAuditHandler auditHandler, boolean skipAuditOnAllow) { + final AuthzStatus ret; + String pathOwner = inodeAttribs != null ? inodeAttribs.getUserName() : null; + String clusterName = plugin.getClusterName(); + FsAction access = FsAction.EXECUTE; + + + if (pathOwner == null) { + pathOwner = inode.getUserName(); + } + + if (RangerHadoopConstants.HDFS_ROOT_FOLDER_PATH_ALT.equals(path)) { + path = RangerHadoopConstants.HDFS_ROOT_FOLDER_PATH; + } + + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerAccessControlEnforcer.isAccessAllowedForTraversal(" + path + ", " + access + ", " + user + ", " + skipAuditOnAllow + ")"); + } + + RangerHdfsAccessRequest request = new RangerHdfsAccessRequest(inode, path, pathOwner, access, EXECUTE_ACCCESS_TYPE, user, groups, clusterName); + + RangerAccessResult result = plugin.isAccessAllowed(request, null); + + if (result != null && result.getIsAccessDetermined() && !result.getIsAllowed()) { + ret = AuthzStatus.DENY; + } else { + ret = AuthzStatus.ALLOW; + } + + if (ret == AuthzStatus.DENY || (!skipAuditOnAllow && result != null && result.getIsAccessDetermined())) { + auditHandler.processResult(result); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerAccessControlEnforcer.isAccessAllowedForTraversal(" + path + ", " + access + ", " + user + ", " + skipAuditOnAllow + "): " + ret); + } + + return ret; } private AuthzStatus checkDefaultEnforcer(String fsOwner, String superGroup, UserGroupInformation ugi, @@ -377,53 +516,79 @@ private AuthzStatus checkDefaultEnforcer(String fsOwner, String superGroup, User boolean isTraverseOnlyCheck, INode ancestor, INode parent, INode inode, RangerHdfsAuditHandler auditHandler ) throws AccessControlException { - AuthzStatus authzStatus = AuthzStatus.NOT_DETERMINED; - if(RangerHdfsPlugin.isHadoopAuthEnabled() && defaultEnforcer != null) { - try { - defaultEnforcer.checkPermission(fsOwner, superGroup, ugi, inodeAttrs, inodes, - pathByNameArr, snapshotId, path, ancestorIndex, doCheckOwner, - ancestorAccess, parentAccess, access, subAccess, ignoreEmptyDir); - - authzStatus = AuthzStatus.ALLOW; - } finally { - if(auditHandler != null) { - INode nodeChecked = inode; - FsAction action = access; - if(isTraverseOnlyCheck) { - if(nodeChecked == null || nodeChecked.isFile()) { - if(parent != null) { - nodeChecked = parent; - } else if(ancestor != null) { - nodeChecked = ancestor; - } - } + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerAccessControlEnforcer.checkDefaultEnforcer(" + + "fsOwner=" + fsOwner + "; superGroup=" + superGroup + ", inodesCount=" + (inodes != null ? inodes.length : 0) + + ", snapshotId=" + snapshotId + ", path=" + path + ", ancestorIndex=" + ancestorIndex + + ", doCheckOwner=" + doCheckOwner + ", ancestorAccess=" + ancestorAccess + ", parentAccess=" + parentAccess + + ", access=" + access + ", subAccess=" + subAccess + ", ignoreEmptyDir=" + ignoreEmptyDir + + ", isTraverseOnlyCheck=" + isTraverseOnlyCheck + ",ancestor=" + (ancestor == null ? null : ancestor.getFullPathName()) + + ", parent=" + (parent == null ? null : parent.getFullPathName()) + ", inode=" + (inode == null ? null : inode.getFullPathName()) + + ")"); + } + + AuthzStatus authzStatus = AuthzStatus.NOT_DETERMINED; + if(RangerHdfsPlugin.isHadoopAuthEnabled() && defaultEnforcer != null) { + + RangerPerfTracer hadoopAuthPerf = null; + + if(RangerPerfTracer.isPerfTraceEnabled(PERF_HDFSAUTH_REQUEST_LOG)) { + hadoopAuthPerf = RangerPerfTracer.getPerfTracer(PERF_HDFSAUTH_REQUEST_LOG, "RangerAccessControlEnforcer.checkDefaultEnforcer(path=" + path + ")"); + } + + try { + defaultEnforcer.checkPermission(fsOwner, superGroup, ugi, inodeAttrs, inodes, + pathByNameArr, snapshotId, path, ancestorIndex, doCheckOwner, + ancestorAccess, parentAccess, access, subAccess, ignoreEmptyDir); - action = FsAction.EXECUTE; - } else if(action == null || action == FsAction.NONE) { - if(parentAccess != null && parentAccess != FsAction.NONE ) { + authzStatus = AuthzStatus.ALLOW; + } finally { + if (auditHandler != null) { + INode nodeChecked = inode; + FsAction action = access; + if (isTraverseOnlyCheck) { + if (nodeChecked == null || nodeChecked.isFile()) { + if (parent != null) { nodeChecked = parent; - action = parentAccess; - } else if(ancestorAccess != null && ancestorAccess != FsAction.NONE ) { + } else if (ancestor != null) { nodeChecked = ancestor; - action = ancestorAccess; - } else if(subAccess != null && subAccess != FsAction.NONE ) { - action = subAccess; } } - String pathChecked = nodeChecked != null ? nodeChecked.getFullPathName() : path; - - auditHandler.logHadoopEvent(pathChecked, action, authzStatus == AuthzStatus.ALLOW); + action = FsAction.EXECUTE; + } else if (action == null || action == FsAction.NONE) { + if (parentAccess != null && parentAccess != FsAction.NONE) { + nodeChecked = parent; + action = parentAccess; + } else if (ancestorAccess != null && ancestorAccess != FsAction.NONE) { + nodeChecked = ancestor; + action = ancestorAccess; + } else if (subAccess != null && subAccess != FsAction.NONE) { + action = subAccess; + } } + + String pathChecked = nodeChecked != null ? nodeChecked.getFullPathName() : path; + + auditHandler.logHadoopEvent(pathChecked, action, authzStatus == AuthzStatus.ALLOW); } - return authzStatus; + RangerPerfTracer.log(hadoopAuthPerf); } - return authzStatus; - } + } + LOG.debug("<== RangerAccessControlEnforcer.checkDefaultEnforcer(" + + "fsOwner=" + fsOwner + "; superGroup=" + superGroup + ", inodesCount=" + (inodes != null ? inodes.length : 0) + + ", snapshotId=" + snapshotId + ", path=" + path + ", ancestorIndex=" + ancestorIndex + + ", doCheckOwner="+ doCheckOwner + ", ancestorAccess=" + ancestorAccess + ", parentAccess=" + parentAccess + + ", access=" + access + ", subAccess=" + subAccess + ", ignoreEmptyDir=" + ignoreEmptyDir + + ", isTraverseOnlyCheck=" + isTraverseOnlyCheck + ",ancestor=" + (ancestor == null ? null : ancestor.getFullPathName()) + + ", parent=" + (parent == null ? null : parent.getFullPathName()) + ", inode=" + (inode == null ? null : inode.getFullPathName()) + + ") : " + authzStatus ); + + return authzStatus; + } - private AuthzStatus isAccessAllowed(INode inode, INodeAttributes inodeAttribs, FsAction access, String user, Set groups, RangerHdfsPlugin plugin, RangerHdfsAuditHandler auditHandler) { + private AuthzStatus isAccessAllowed(INode inode, INodeAttributes inodeAttribs, String path, FsAction access, String user, Set groups, RangerHdfsPlugin plugin, RangerHdfsAuditHandler auditHandler) { AuthzStatus ret = null; - String path = inode != null ? inode.getFullPathName() : null; String pathOwner = inodeAttribs != null ? inodeAttribs.getUserName() : null; String clusterName = plugin.getClusterName(); @@ -475,6 +640,69 @@ private AuthzStatus isAccessAllowed(INode inode, INodeAttributes inodeAttribs, F return ret; } + + private AuthzStatus isAccessAllowedForHierarchy(INode inode, INodeAttributes inodeAttribs, String path, FsAction access, String user, Set groups, RangerHdfsPlugin plugin) { + AuthzStatus ret = null; + String pathOwner = inodeAttribs != null ? inodeAttribs.getUserName() : null; + String clusterName = plugin.getClusterName(); + + if (pathOwner == null && inode != null) { + pathOwner = inode.getUserName(); + } + + if (RangerHadoopConstants.HDFS_ROOT_FOLDER_PATH_ALT.equals(path)) { + path = RangerHadoopConstants.HDFS_ROOT_FOLDER_PATH; + } + + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerAccessControlEnforcer.isAccessAllowedForHierarchy(" + path + ", " + access + ", " + user + ")"); + } + + if (path != null) { + + Set accessTypes = access2ActionListMapper.get(access); + + if (accessTypes == null) { + LOG.warn("RangerAccessControlEnforcer.isAccessAllowedForHierarchy(" + path + ", " + access + ", " + user + "): no Ranger accessType found for " + access); + + accessTypes = access2ActionListMapper.get(FsAction.NONE); + } + + String subDirPath = path; + if (subDirPath.charAt(subDirPath.length() - 1) != org.apache.hadoop.fs.Path.SEPARATOR_CHAR) { + subDirPath = subDirPath + Character.toString(org.apache.hadoop.fs.Path.SEPARATOR_CHAR); + } + subDirPath = subDirPath + RangerHdfsPlugin.getRandomizedWildcardPathName(); + + for (String accessType : accessTypes) { + RangerHdfsAccessRequest request = new RangerHdfsAccessRequest(null, subDirPath, pathOwner, access, accessType, user, groups, clusterName); + + RangerAccessResult result = plugin.isAccessAllowed(request, null); + + if (result == null || !result.getIsAccessDetermined()) { + ret = AuthzStatus.NOT_DETERMINED; + // don't break yet; subsequent accessType could be denied + } else if(! result.getIsAllowed()) { // explicit deny + ret = AuthzStatus.DENY; + break; + } else { // allowed + if(!AuthzStatus.NOT_DETERMINED.equals(ret)) { // set to ALLOW only if there was no NOT_DETERMINED earlier + ret = AuthzStatus.ALLOW; + } + } + } + } + + if(ret == null) { + ret = AuthzStatus.NOT_DETERMINED; + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerAccessControlEnforcer.isAccessAllowedForHierarchy(" + path + ", " + access + ", " + user + "): " + ret); + } + + return ret; + } } } @@ -482,7 +710,8 @@ private AuthzStatus isAccessAllowed(INode inode, INodeAttributes inodeAttribs, F class RangerHdfsPlugin extends RangerBasePlugin { private static boolean hadoopAuthEnabled = RangerHadoopConstants.RANGER_ADD_HDFS_PERMISSION_DEFAULT; private static String fileNameExtensionSeparator; - + private static boolean optimizeSubAccessAuthEnabled = RangerHadoopConstants.RANGER_OPTIMIZE_SUBACCESS_AUTHORIZATION_DEFAULT; + private static String randomizedWildcardPathName; public RangerHdfsPlugin() { super("hdfs", "hdfs"); @@ -493,6 +722,30 @@ public void init() { RangerHdfsPlugin.hadoopAuthEnabled = RangerConfiguration.getInstance().getBoolean(RangerHadoopConstants.RANGER_ADD_HDFS_PERMISSION_PROP, RangerHadoopConstants.RANGER_ADD_HDFS_PERMISSION_DEFAULT); RangerHdfsPlugin.fileNameExtensionSeparator = RangerConfiguration.getInstance().get(RangerHdfsAuthorizer.RANGER_FILENAME_EXTENSION_SEPARATOR_PROP, RangerHdfsAuthorizer.DEFAULT_FILENAME_EXTENSION_SEPARATOR); + RangerHdfsPlugin.optimizeSubAccessAuthEnabled = RangerConfiguration.getInstance().getBoolean(RangerHadoopConstants.RANGER_OPTIMIZE_SUBACCESS_AUTHORIZATION_PROP, RangerHadoopConstants.RANGER_OPTIMIZE_SUBACCESS_AUTHORIZATION_DEFAULT); + + String random = generateString("^&#@!%()-_+=@:;'<>`~abcdefghijklmnopqrstuvwxyz01234567890"); + randomizedWildcardPathName = RangerPathResourceMatcher.WILDCARD_ASTERISK + random + RangerPathResourceMatcher.WILDCARD_ASTERISK; + } + + // Build random string of length between 56 and 112 characters + private static String generateString(String source) + { + SecureRandom rng = new SecureRandom(); + + byte[] bytes = new byte[1]; + rng.nextBytes(bytes); + int length = bytes[0]; + length = length < 56 ? 56 : length; + length = length > 112 ? 112 : length; + + char[] text = new char[length]; + + for (int i = 0; i < length; i++) + { + text[i] = source.charAt(rng.nextInt(source.length())); + } + return new String(text); } public static boolean isHadoopAuthEnabled() { @@ -501,6 +754,12 @@ public static boolean isHadoopAuthEnabled() { public static String getFileNameExtensionSeparator() { return RangerHdfsPlugin.fileNameExtensionSeparator; } + public static boolean isOptimizeSubAccessAuthEnabled() { + return RangerHdfsPlugin.optimizeSubAccessAuthEnabled; + } + public static String getRandomizedWildcardPathName() { + return RangerHdfsPlugin.randomizedWildcardPathName; + } } class RangerHdfsResource extends RangerAccessResourceImpl { @@ -589,17 +848,28 @@ public void processResult(RangerAccessResult result) { isAuditEnabled = true; } - auditEvent = super.getAuthzEvents(result); + if (auditEvent == null) { + auditEvent = super.getAuthzEvents(result); + } if (auditEvent != null) { RangerAccessRequest request = result.getAccessRequest(); RangerAccessResource resource = request.getResource(); String resourcePath = resource != null ? resource.getAsString() : null; + // Overwrite fields in original auditEvent auditEvent.setEventTime(request.getAccessTime()); auditEvent.setAccessType(request.getAction()); auditEvent.setResourcePath(this.pathToBeValidated); auditEvent.setResultReason(resourcePath); + + auditEvent.setAccessResult((short) (result.getIsAllowed() ? 1 : 0)); + auditEvent.setPolicyId(result.getPolicyId()); + + Set tags = getTags(request); + if (tags != null) { + auditEvent.setTags(tags); + } } if(LOG.isDebugEnabled()) { diff --git a/hdfs-agent/src/test/java/org/apache/ranger/services/hdfs/HDFSRangerTest.java b/hdfs-agent/src/test/java/org/apache/ranger/services/hdfs/HDFSRangerTest.java index f5e8cb7e0a..c5b7a75b7a 100644 --- a/hdfs-agent/src/test/java/org/apache/ranger/services/hdfs/HDFSRangerTest.java +++ b/hdfs-agent/src/test/java/org/apache/ranger/services/hdfs/HDFSRangerTest.java @@ -23,6 +23,7 @@ import org.apache.commons.io.IOUtils; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.ContentSummary; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; @@ -243,6 +244,11 @@ public void HDFSBaseFileNameTokenReadTest() throws Exception { HDFSReadFailTest("/tmp/tmpdir5/t/data-file.txt"); } + @org.junit.Test + public void HDFSContentSummaryTest() throws Exception { + HDFSGetContentSummary("/tmp/get-content-summary"); + } + void HDFSReadTest(String fileName) throws Exception { FileSystem fileSystem = hdfsCluster.getFileSystem(); @@ -413,4 +419,53 @@ public Void run() throws Exception { } }); } + + void HDFSGetContentSummary(final String dirName) throws Exception { + + String subdirName = dirName + "/tmpdir"; + + createFile(subdirName, 1); + createFile(subdirName, 2); + + UserGroupInformation ugi = UserGroupInformation.createUserForTesting("bob", new String[] {}); + ugi.doAs(new PrivilegedExceptionAction() { + + public Void run() throws Exception { + Configuration conf = new Configuration(); + conf.set("fs.defaultFS", defaultFs); + + FileSystem fs = FileSystem.get(conf); + + try { + // GetContentSummary on the directory dirName + ContentSummary contentSummary = fs.getContentSummary(new Path(dirName)); + + long directoryCount = contentSummary.getDirectoryCount(); + Assert.assertTrue("Found unexpected number of directories; expected-count=3, actual-count=" + directoryCount, directoryCount == 3); + } catch (Exception e) { + Assert.fail("Failed to getContentSummary, exception=" + e); + } + fs.close(); + return null; + } + }); + } + + void createFile(String baseDir, Integer index) throws Exception { + FileSystem fileSystem = hdfsCluster.getFileSystem(); + + // Write a file - the AccessControlEnforcer won't be invoked as we are the "superuser" + String dirName = baseDir + (index != null ? String.valueOf(index) : ""); + String fileName = dirName + "/dummy-data"; + final Path file = new Path(fileName); + FSDataOutputStream out = fileSystem.create(file); + for (int i = 0; i < 1024; ++i) { + out.write(("data" + i + "\n").getBytes("UTF-8")); + out.flush(); + } + out.close(); + + // Change permissions to read-only + fileSystem.setPermission(file, new FsPermission(FsAction.READ, FsAction.NONE, FsAction.NONE)); + } } diff --git a/hdfs-agent/src/test/resources/hdfs-policies.json b/hdfs-agent/src/test/resources/hdfs-policies.json index 4565b8b64b..52a2cc1608 100644 --- a/hdfs-agent/src/test/resources/hdfs-policies.json +++ b/hdfs-agent/src/test/resources/hdfs-policies.json @@ -233,6 +233,33 @@ "createdBy": "Admin", "updatedBy": "Admin", "version": 1 + }, + { + "service": "cl1_hadoop", + "name": "/tmp/get-content-summary", + "policyType": 0, + "description": "", + "isAuditEnabled": true, + "resources": { + "path": {"values": ["/tmp/get-content-summary", "/tmp/get-content-summary/tmpdir1", "/tmp/get-content-summary/tmpdir2"], "isExcludes": false, "isRecursive": false} + }, + "policyItems": [ + { + "accesses": [{"type": "read","isAllowed": true}, {"type": "execute","isAllowed": true}], + "users": ["bob"], + "groups": ["IT"], + "conditions": [], + "delegateAdmin": false + } + ], + "denyPolicyItems": [], + "allowExceptions": [], + "denyExceptions": [], + "dataMaskPolicyItems": [], + "rowFilterPolicyItems": [], + "id": 40, + "isEnabled": true, + "version": 1 } ], "serviceDef": { diff --git a/hdfs-agent/src/test/resources/log4j.properties b/hdfs-agent/src/test/resources/log4j.properties new file mode 100644 index 0000000000..f7ab2bad53 --- /dev/null +++ b/hdfs-agent/src/test/resources/log4j.properties @@ -0,0 +1,34 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +##-- To prevent junits from cluttering the build run by default all test runs send output to null appender +log4j.appender.devnull=org.apache.log4j.varia.NullAppender +ranger.root.logger=FATAL,devnull + +##-- uncomment the following line during during development/debugging so see debug messages during test run to be emitted to console +# ranger.root.logger=DEBUG,console +log4j.rootLogger=${ranger.root.logger} + +# Logging Threshold +log4j.threshold=ALL + +# +# console +# Add "console" to rootlogger above if you want to use this +# +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.target=System.err +log4j.appender.console.layout=org.apache.log4j.PatternLayout +log4j.appender.console.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2}: %m%n diff --git a/hdfs-agent/src/test/resources/ranger-hdfs-security.xml b/hdfs-agent/src/test/resources/ranger-hdfs-security.xml index 3062108d50..57de6dbaaf 100644 --- a/hdfs-agent/src/test/resources/ranger-hdfs-security.xml +++ b/hdfs-agent/src/test/resources/ranger-hdfs-security.xml @@ -42,4 +42,12 @@ + + ranger.optimize-subaccess-authorization + true + + Enable skipping subdirectories if proper policy exists for getContentSummary and delete commands + + + diff --git a/hive-agent/src/main/java/org/apache/ranger/authorization/hive/authorizer/RangerHiveAuditHandler.java b/hive-agent/src/main/java/org/apache/ranger/authorization/hive/authorizer/RangerHiveAuditHandler.java index 9dea37a138..24a71fa695 100644 --- a/hive-agent/src/main/java/org/apache/ranger/authorization/hive/authorizer/RangerHiveAuditHandler.java +++ b/hive-agent/src/main/java/org/apache/ranger/authorization/hive/authorizer/RangerHiveAuditHandler.java @@ -54,6 +54,16 @@ AuthzAuditEvent createAuditEvent(RangerAccessResult result, String accessType, S auditEvent.setResourcePath(resourcePath); auditEvent.setResourceType("@" + resourceType); // to be consistent with earlier release + if (request instanceof RangerHiveAccessRequest && resource instanceof RangerHiveResource) { + RangerHiveAccessRequest hiveAccessRequest = (RangerHiveAccessRequest) request; + RangerHiveResource hiveResource = (RangerHiveResource) resource; + + if (hiveAccessRequest.getHiveAccessType() == HiveAccessType.USE && hiveResource.getObjectType() == HiveObjectType.DATABASE && StringUtils.isBlank(hiveResource.getDatabase())) { + // this should happen only for SHOWDATABASES + auditEvent.setTags(null); + } + } + return auditEvent; } diff --git a/hive-agent/src/main/java/org/apache/ranger/authorization/hive/authorizer/RangerHiveAuthorizer.java b/hive-agent/src/main/java/org/apache/ranger/authorization/hive/authorizer/RangerHiveAuthorizer.java index 56ef1872e0..85a865ad58 100644 --- a/hive-agent/src/main/java/org/apache/ranger/authorization/hive/authorizer/RangerHiveAuthorizer.java +++ b/hive-agent/src/main/java/org/apache/ranger/authorization/hive/authorizer/RangerHiveAuthorizer.java @@ -66,11 +66,14 @@ import com.google.common.collect.Sets; +import org.apache.ranger.plugin.util.RangerPerfTracer; import org.apache.ranger.plugin.util.RangerRequestedResources; public class RangerHiveAuthorizer extends RangerHiveAuthorizerBase { private static final Log LOG = LogFactory.getLog(RangerHiveAuthorizer.class); + private static final Log PERF_HIVEAUTH_REQUEST_LOG = RangerPerfTracer.getPerfLogger("hiveauth.request"); + private static final char COLUMN_SEP = ','; private static final String HIVE_CONF_VAR_QUERY_STRING = "hive.query.string"; @@ -221,6 +224,8 @@ public void checkPrivileges(HiveOperationType hiveOpType, RangerHiveAuditHandler auditHandler = new RangerHiveAuditHandler(); + RangerPerfTracer perf = null; + try { HiveAuthzSessionContext sessionContext = getHiveAuthzSessionContext(); String user = ugi.getShortUserName(); @@ -237,6 +242,10 @@ public void checkPrivileges(HiveOperationType hiveOpType, return; } + if(RangerPerfTracer.isPerfTraceEnabled(PERF_HIVEAUTH_REQUEST_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_HIVEAUTH_REQUEST_LOG, "RangerHiveAuthorizer.checkPrivileges(hiveOpType=" + hiveOpType + ")"); + } + List requests = new ArrayList(); if(!CollectionUtils.isEmpty(inputHObjs)) { @@ -420,6 +429,7 @@ public void checkPrivileges(HiveOperationType hiveOpType, } } finally { auditHandler.flushAudit(); + RangerPerfTracer.log(perf); } } @@ -439,7 +449,13 @@ public List filterListCmdObjects(List if (LOG.isDebugEnabled()) { LOG.debug(String.format("==> filterListCmdObjects(%s, %s)", objs, context)); } - + + RangerPerfTracer perf = null; + + if(RangerPerfTracer.isPerfTraceEnabled(PERF_HIVEAUTH_REQUEST_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_HIVEAUTH_REQUEST_LOG, "RangerHiveAuthorizer.filterListCmdObjects()"); + } + List ret = null; // bail out early if nothing is there to validate! @@ -509,6 +525,8 @@ public List filterListCmdObjects(List } } + RangerPerfTracer.log(perf); + if (LOG.isDebugEnabled()) { int count = ret == null ? 0 : ret.size(); LOG.debug(String.format("<== filterListCmdObjects: count[%d], ret[%s]", count, ret)); @@ -524,6 +542,12 @@ public List applyRowFilterAndColumnMasking(HiveAuthzContext LOG.debug("==> applyRowFilterAndColumnMasking(" + queryContext + ", objCount=" + hiveObjs.size() + ")"); } + RangerPerfTracer perf = null; + + if(RangerPerfTracer.isPerfTraceEnabled(PERF_HIVEAUTH_REQUEST_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_HIVEAUTH_REQUEST_LOG, "RangerHiveAuthorizer.applyRowFilterAndColumnMasking()"); + } + if(CollectionUtils.isNotEmpty(hiveObjs)) { for (HivePrivilegeObject hiveObj : hiveObjs) { HivePrivilegeObjectType hiveObjType = hiveObj.getType(); @@ -576,6 +600,8 @@ public List applyRowFilterAndColumnMasking(HiveAuthzContext } } + RangerPerfTracer.log(perf); + if(LOG.isDebugEnabled()) { LOG.debug("<== applyRowFilterAndColumnMasking(" + queryContext + ", objCount=" + hiveObjs.size() + "): retCount=" + ret.size()); } diff --git a/hive-agent/src/main/java/org/apache/ranger/authorization/hive/authorizer/RangerHiveResource.java b/hive-agent/src/main/java/org/apache/ranger/authorization/hive/authorizer/RangerHiveResource.java index 3f1279faee..d04d3bface 100644 --- a/hive-agent/src/main/java/org/apache/ranger/authorization/hive/authorizer/RangerHiveResource.java +++ b/hive-agent/src/main/java/org/apache/ranger/authorization/hive/authorizer/RangerHiveResource.java @@ -50,6 +50,9 @@ public RangerHiveResource(HiveObjectType objectType, String databaseorUrl, Strin break; case FUNCTION: + if (databaseorUrl == null) { + databaseorUrl = ""; + } setValue(KEY_DATABASE, databaseorUrl); setValue(KEY_UDF, tableOrUdf); break; diff --git a/hive-agent/src/main/java/org/apache/ranger/services/hive/client/HiveConnectionMgr.java b/hive-agent/src/main/java/org/apache/ranger/services/hive/client/HiveConnectionMgr.java index b36d5da4d0..9376358725 100644 --- a/hive-agent/src/main/java/org/apache/ranger/services/hive/client/HiveConnectionMgr.java +++ b/hive-agent/src/main/java/org/apache/ranger/services/hive/client/HiveConnectionMgr.java @@ -64,10 +64,19 @@ public HiveClient call() throws Exception { LOG.error("Error connecting hive repository : "+ serviceName +" using config : "+ configs, e); } - HiveClient oldClient = hiveConnectionCache.putIfAbsent(serviceName, hiveClient); + + HiveClient oldClient = null; + if (hiveClient != null) { + oldClient = hiveConnectionCache.putIfAbsent(serviceName, hiveClient); + } else { + oldClient = hiveConnectionCache.get(serviceName); + } + if (oldClient != null) { // in the meantime someone else has put a valid client into the cache, let's use that instead. - hiveClient.close(); + if (hiveClient != null) { + hiveClient.close(); + } hiveClient = oldClient; } repoConnectStatusMap.put(serviceName, true); diff --git a/jisql/pom.xml b/jisql/pom.xml index 87113ce3f3..8deac5c465 100644 --- a/jisql/pom.xml +++ b/jisql/pom.xml @@ -28,11 +28,6 @@ .. - - net.sourceforge.javacsv - javacsv - 2.0 - net.sf.jopt-simple jopt-simple diff --git a/jisql/src/main/java/org/apache/util/outputformatter/CSVFormatter.java b/jisql/src/main/java/org/apache/util/outputformatter/CSVFormatter.java index 158e25ce06..d55ecbca17 100644 --- a/jisql/src/main/java/org/apache/util/outputformatter/CSVFormatter.java +++ b/jisql/src/main/java/org/apache/util/outputformatter/CSVFormatter.java @@ -22,15 +22,9 @@ import java.io.PrintStream; import java.sql.ResultSet; import java.sql.ResultSetMetaData; - -import java.nio.charset.Charset; - import joptsimple.OptionParser; import joptsimple.OptionSet; -import com.csvreader.CsvWriter; - - /** * This is the default formatter for Jisql. It outputs data in a "normal" * format that is similar to most other database command line formatters. @@ -117,8 +111,7 @@ public void formatHeader( PrintStream out, ResultSetMetaData metaData ) throws E /** - * Called to output the data. This class uses a third party library to output - * the CSV data. The library escapes the data as needed. + * Called to output the data. * * @param out the PrintStream to output data to. * @param resultSet the ResultSet for the row. @@ -126,25 +119,7 @@ public void formatHeader( PrintStream out, ResultSetMetaData metaData ) throws E * * */ - public void formatData( PrintStream out, ResultSet resultSet, ResultSetMetaData metaData ) throws Exception { - - CsvWriter csvWriter = new CsvWriter( out, delimiter, Charset.forName( "us-ascii" ) ); - - while( resultSet.next() ) { - int numColumns = metaData.getColumnCount(); - - for (int i = 1; i <= numColumns; i++) { - String result = resultSet.getString(i); - if( !resultSet.wasNull() ) - csvWriter.write( result ); - else - csvWriter.write( "" ); - } - - csvWriter.endRecord(); - } - - csvWriter.flush(); + public void formatData( PrintStream out, ResultSet resultSet, ResultSetMetaData metaData ) throws Exception{ } diff --git a/jisql/src/main/java/org/apache/util/sql/Jisql.java b/jisql/src/main/java/org/apache/util/sql/Jisql.java index cf5f2c44f8..7c63aff90c 100644 --- a/jisql/src/main/java/org/apache/util/sql/Jisql.java +++ b/jisql/src/main/java/org/apache/util/sql/Jisql.java @@ -62,9 +62,7 @@ * default is the term "go" on a single line like Sybase's isql or MS/SQL's * isql/osql. Note that there is a dependency on JOpt Simple in for the base - * configuration. Additionally, if you are using the CSVFormatter then it is - * dependent on Java CSV. - *

+ * configuration. * * * Options: @@ -431,6 +429,9 @@ public void doIsql() throws IOException, SQLException { if (trimmedLine.toUpperCase().startsWith("DECLARE")) { commandTerminator="/"; } + if ((trimmedLine.toUpperCase().startsWith("CREATE OR REPLACE PROCEDURE")) || (trimmedLine.toUpperCase().startsWith("CREATE OR REPLACE FUNCTION"))) { + commandTerminator="/"; + } } if(connectString.toLowerCase().startsWith("jdbc:postgresql") && inputFileName!=null){ if (trimmedLine.toLowerCase().startsWith("select 'delimiter start';")) { diff --git a/kms/config/kms-webapp/dbks-site.xml b/kms/config/kms-webapp/dbks-site.xml index a098db1106..22480322c1 100755 --- a/kms/config/kms-webapp/dbks-site.xml +++ b/kms/config/kms-webapp/dbks-site.xml @@ -123,6 +123,57 @@ ranger.ks.kerberos.keytab + + + + + ranger.kms.keysecure.enabled + false + + + + + ranger.kms.keysecure.UserPassword.Authentication + true + + + + ranger.kms.keysecure.masterkey.name + safenetmasterkey + Safenet key secure master key name + + + ranger.kms.keysecure.login.username + user1 + Safenet key secure username + + + ranger.kms.keysecure.login.password + t1e2s3t4 + Safenet key secure user password + + + ranger.kms.keysecure.login.password.alias + ranger.ks.login.password + Safenet key secure user password + + + ranger.kms.keysecure.hostname + SunPKCS11-keysecurehn + Safenet key secure hostname + + + ranger.kms.keysecure.masterkey.size + 256 + key size + + + ranger.kms.keysecure.sunpkcs11.cfg.filepath + /opt/safenetConf/64/8.3.1/sunpkcs11.cfg + Location of Safenet key secure library configuration file + + + @@ -167,4 +218,8 @@ ranger.ks.db.ssl.verifyServerCertificate false + + ranger.ks.db.ssl.auth.type + 2-way + diff --git a/kms/pom.xml b/kms/pom.xml index dd3f0c5f36..54fa01cd80 100644 --- a/kms/pom.xml +++ b/kms/pom.xml @@ -429,6 +429,10 @@ ${derby.version} test + + com.webcohesion.enunciate + enunciate-core-annotations + @@ -438,6 +442,17 @@ maven-war-plugin 2.6 + + com.webcohesion.enunciate + enunciate-maven-plugin + + enunciate.xml + + docs/target/kms/ + 1.8 + 1.8 + + diff --git a/kms/scripts/DBMKTOKEYSECURE.sh b/kms/scripts/DBMKTOKEYSECURE.sh new file mode 100644 index 0000000000..c0aa6e58c2 --- /dev/null +++ b/kms/scripts/DBMKTOKEYSECURE.sh @@ -0,0 +1,19 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ------------------------------------------------------------------------------------- +RANGER_KMS_HOME=`dirname $0` +cp="${RANGER_KMS_HOME}/cred/lib/*:${RANGER_KMS_HOME}/./ews/webapp/WEB-INF/classes/conf/:${RANGER_KMS_HOME}/ews/webapp/config:${RANGER_KMS_HOME}/ews/lib/*:${RANGER_KMS_HOME}/ews/webapp/lib/*:${RANGER_KMS_HOME}/ews/webapp/META-INF" +java -cp "${cp}" org.apache.hadoop.crypto.key.DBToKeySecure ${1} ${2} ${3} ${4} diff --git a/kms/scripts/KEYSECUREMKTOKMSDB.sh b/kms/scripts/KEYSECUREMKTOKMSDB.sh new file mode 100644 index 0000000000..ba0a8a98b2 --- /dev/null +++ b/kms/scripts/KEYSECUREMKTOKMSDB.sh @@ -0,0 +1,19 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ------------------------------------------------------------------------------------- +RANGER_KMS_HOME=`dirname $0` +cp="${RANGER_KMS_HOME}/cred/lib/*:${RANGER_KMS_HOME}/./ews/webapp/WEB-INF/classes/conf/:${RANGER_KMS_HOME}/ews/webapp/config:${RANGER_KMS_HOME}/ews/lib/*:${RANGER_KMS_HOME}/ews/webapp/lib/*:${RANGER_KMS_HOME}/ews/webapp/META-INF" +java -cp "${cp}" org.apache.hadoop.crypto.key.KeySecureToRangerDBMKUtil ${1} diff --git a/kms/scripts/db_setup.py b/kms/scripts/db_setup.py index d8b4b635ab..090e551407 100644 --- a/kms/scripts/db_setup.py +++ b/kms/scripts/db_setup.py @@ -102,13 +102,14 @@ def import_db_file(self, db_name, db_user, db_password, file_name): class MysqlConf(BaseDB): # Constructor - def __init__(self, host,SQL_CONNECTOR_JAR,JAVA_BIN,db_ssl_enabled,db_ssl_required,db_ssl_verifyServerCertificate,javax_net_ssl_keyStore,javax_net_ssl_keyStorePassword,javax_net_ssl_trustStore,javax_net_ssl_trustStorePassword): + def __init__(self, host,SQL_CONNECTOR_JAR,JAVA_BIN,db_ssl_enabled,db_ssl_required,db_ssl_verifyServerCertificate,javax_net_ssl_keyStore,javax_net_ssl_keyStorePassword,javax_net_ssl_trustStore,javax_net_ssl_trustStorePassword,db_ssl_auth_type): self.host = host self.SQL_CONNECTOR_JAR = SQL_CONNECTOR_JAR self.JAVA_BIN = JAVA_BIN self.db_ssl_enabled=db_ssl_enabled.lower() self.db_ssl_required=db_ssl_required.lower() self.db_ssl_verifyServerCertificate=db_ssl_verifyServerCertificate.lower() + self.db_ssl_auth_type=db_ssl_auth_type.lower() self.javax_net_ssl_keyStore=javax_net_ssl_keyStore self.javax_net_ssl_keyStorePassword=javax_net_ssl_keyStorePassword self.javax_net_ssl_trustStore=javax_net_ssl_trustStore @@ -121,7 +122,10 @@ def get_jisql_cmd(self, user, password ,db_name): if self.db_ssl_enabled == 'true': db_ssl_param="?useSSL=%s&requireSSL=%s&verifyServerCertificate=%s" %(self.db_ssl_enabled,self.db_ssl_required,self.db_ssl_verifyServerCertificate) if self.db_ssl_verifyServerCertificate == 'true': - db_ssl_cert_param=" -Djavax.net.ssl.keyStore=%s -Djavax.net.ssl.keyStorePassword=%s -Djavax.net.ssl.trustStore=%s -Djavax.net.ssl.trustStorePassword=%s " %(self.javax_net_ssl_keyStore,self.javax_net_ssl_keyStorePassword,self.javax_net_ssl_trustStore,self.javax_net_ssl_trustStorePassword) + if self.db_ssl_auth_type == '1-way': + db_ssl_cert_param=" -Djavax.net.ssl.trustStore=%s -Djavax.net.ssl.trustStorePassword=%s " %(self.javax_net_ssl_trustStore,self.javax_net_ssl_trustStorePassword) + else: + db_ssl_cert_param=" -Djavax.net.ssl.keyStore=%s -Djavax.net.ssl.keyStorePassword=%s -Djavax.net.ssl.trustStore=%s -Djavax.net.ssl.trustStorePassword=%s " %(self.javax_net_ssl_keyStore,self.javax_net_ssl_keyStorePassword,self.javax_net_ssl_trustStore,self.javax_net_ssl_trustStorePassword) self.JAVA_BIN = self.JAVA_BIN.strip("'") if is_unix: jisql_cmd = "%s %s -cp %s:%s/jisql/lib/* org.apache.util.sql.Jisql -driver mysqlconj -cstring jdbc:mysql://%s/%s%s -u '%s' -p '%s' -noheader -trim -c \;" %(self.JAVA_BIN,db_ssl_cert_param,self.SQL_CONNECTOR_JAR,path,self.host,db_name,db_ssl_param,user,password) @@ -572,6 +576,7 @@ def main(argv): db_ssl_enabled='false' db_ssl_required='false' db_ssl_verifyServerCertificate='false' + db_ssl_auth_type='2-way' javax_net_ssl_keyStore='' javax_net_ssl_keyStorePassword='' javax_net_ssl_trustStore='' @@ -585,30 +590,33 @@ def main(argv): db_ssl_required=globalDict['db_ssl_required'].lower() if 'db_ssl_verifyServerCertificate' in globalDict: db_ssl_verifyServerCertificate=globalDict['db_ssl_verifyServerCertificate'].lower() + if 'db_ssl_auth_type' in globalDict: + db_ssl_auth_type=globalDict['db_ssl_auth_type'].lower() if db_ssl_verifyServerCertificate == 'true': - if 'javax_net_ssl_keyStore' in globalDict: - javax_net_ssl_keyStore=globalDict['javax_net_ssl_keyStore'] - if 'javax_net_ssl_keyStorePassword' in globalDict: - javax_net_ssl_keyStorePassword=globalDict['javax_net_ssl_keyStorePassword'] if 'javax_net_ssl_trustStore' in globalDict: javax_net_ssl_trustStore=globalDict['javax_net_ssl_trustStore'] if 'javax_net_ssl_trustStorePassword' in globalDict: javax_net_ssl_trustStorePassword=globalDict['javax_net_ssl_trustStorePassword'] - if not os.path.exists(javax_net_ssl_keyStore): - log("[E] Invalid file Name! Unable to find keystore file:"+javax_net_ssl_keyStore,"error") - sys.exit(1) if not os.path.exists(javax_net_ssl_trustStore): log("[E] Invalid file Name! Unable to find truststore file:"+javax_net_ssl_trustStore,"error") sys.exit(1) - if javax_net_ssl_keyStorePassword is None or javax_net_ssl_keyStorePassword =="": - log("[E] Invalid ssl keystore password!","error") - sys.exit(1) if javax_net_ssl_trustStorePassword is None or javax_net_ssl_trustStorePassword =="": log("[E] Invalid ssl truststore password!","error") sys.exit(1) + if db_ssl_auth_type == '2-way': + if 'javax_net_ssl_keyStore' in globalDict: + javax_net_ssl_keyStore=globalDict['javax_net_ssl_keyStore'] + if 'javax_net_ssl_keyStorePassword' in globalDict: + javax_net_ssl_keyStorePassword=globalDict['javax_net_ssl_keyStorePassword'] + if not os.path.exists(javax_net_ssl_keyStore): + log("[E] Invalid file Name! Unable to find keystore file:"+javax_net_ssl_keyStore,"error") + sys.exit(1) + if javax_net_ssl_keyStorePassword is None or javax_net_ssl_keyStorePassword =="": + log("[E] Invalid ssl keystore password!","error") + sys.exit(1) MYSQL_CONNECTOR_JAR=globalDict['SQL_CONNECTOR_JAR'] - xa_sqlObj = MysqlConf(xa_db_host, MYSQL_CONNECTOR_JAR, JAVA_BIN,db_ssl_enabled,db_ssl_required,db_ssl_verifyServerCertificate,javax_net_ssl_keyStore,javax_net_ssl_keyStorePassword,javax_net_ssl_trustStore,javax_net_ssl_trustStorePassword) + xa_sqlObj = MysqlConf(xa_db_host, MYSQL_CONNECTOR_JAR, JAVA_BIN,db_ssl_enabled,db_ssl_required,db_ssl_verifyServerCertificate,javax_net_ssl_keyStore,javax_net_ssl_keyStorePassword,javax_net_ssl_trustStore,javax_net_ssl_trustStorePassword,db_ssl_auth_type) xa_db_core_file = os.path.join(RANGER_KMS_HOME , mysql_core_file) elif XA_DB_FLAVOR == "ORACLE": diff --git a/kms/scripts/dba_script.py b/kms/scripts/dba_script.py index 1e264cce80..6350d7da05 100755 --- a/kms/scripts/dba_script.py +++ b/kms/scripts/dba_script.py @@ -133,13 +133,14 @@ def create_db(self, root_user, db_root_password, db_name, db_user, db_password,d class MysqlConf(BaseDB): # Constructor - def __init__(self, host,SQL_CONNECTOR_JAR,JAVA_BIN,db_ssl_enabled,db_ssl_required,db_ssl_verifyServerCertificate,javax_net_ssl_keyStore,javax_net_ssl_keyStorePassword,javax_net_ssl_trustStore,javax_net_ssl_trustStorePassword): + def __init__(self, host,SQL_CONNECTOR_JAR,JAVA_BIN,db_ssl_enabled,db_ssl_required,db_ssl_verifyServerCertificate,javax_net_ssl_keyStore,javax_net_ssl_keyStorePassword,javax_net_ssl_trustStore,javax_net_ssl_trustStorePassword,db_ssl_auth_type): self.host = host self.SQL_CONNECTOR_JAR = SQL_CONNECTOR_JAR self.JAVA_BIN = JAVA_BIN self.db_ssl_enabled=db_ssl_enabled.lower() self.db_ssl_required=db_ssl_required.lower() self.db_ssl_verifyServerCertificate=db_ssl_verifyServerCertificate.lower() + self.db_ssl_auth_type=db_ssl_auth_type.lower() self.javax_net_ssl_keyStore=javax_net_ssl_keyStore self.javax_net_ssl_keyStorePassword=javax_net_ssl_keyStorePassword self.javax_net_ssl_trustStore=javax_net_ssl_trustStore @@ -153,7 +154,10 @@ def get_jisql_cmd(self, user, password ,db_name): if self.db_ssl_enabled == 'true': db_ssl_param="?useSSL=%s&requireSSL=%s&verifyServerCertificate=%s" %(self.db_ssl_enabled,self.db_ssl_required,self.db_ssl_verifyServerCertificate) if self.db_ssl_verifyServerCertificate == 'true': - db_ssl_cert_param=" -Djavax.net.ssl.keyStore=%s -Djavax.net.ssl.keyStorePassword=%s -Djavax.net.ssl.trustStore=%s -Djavax.net.ssl.trustStorePassword=%s " %(self.javax_net_ssl_keyStore,self.javax_net_ssl_keyStorePassword,self.javax_net_ssl_trustStore,self.javax_net_ssl_trustStorePassword) + if self.db_ssl_auth_type == '1-way': + db_ssl_cert_param=" -Djavax.net.ssl.trustStore=%s -Djavax.net.ssl.trustStorePassword=%s " %(self.javax_net_ssl_trustStore,self.javax_net_ssl_trustStorePassword) + else: + db_ssl_cert_param=" -Djavax.net.ssl.keyStore=%s -Djavax.net.ssl.keyStorePassword=%s -Djavax.net.ssl.trustStore=%s -Djavax.net.ssl.trustStorePassword=%s " %(self.javax_net_ssl_keyStore,self.javax_net_ssl_keyStorePassword,self.javax_net_ssl_trustStore,self.javax_net_ssl_trustStorePassword) if is_unix: jisql_cmd = "%s %s -cp %s:%s/jisql/lib/* org.apache.util.sql.Jisql -driver mysqlconj -cstring jdbc:mysql://%s/%s%s -u %s -p '%s' -noheader -trim -c \;" %(self.JAVA_BIN,db_ssl_cert_param,self.SQL_CONNECTOR_JAR,path,self.host,db_name,db_ssl_param,user,password) elif os_name == "WINDOWS": @@ -1363,6 +1367,7 @@ def main(argv): db_ssl_enabled='false' db_ssl_required='false' db_ssl_verifyServerCertificate='false' + db_ssl_auth_type='2-way' javax_net_ssl_keyStore='' javax_net_ssl_keyStorePassword='' javax_net_ssl_trustStore='' @@ -1375,30 +1380,34 @@ def main(argv): db_ssl_required=globalDict['db_ssl_required'].lower() if 'db_ssl_verifyServerCertificate' in globalDict: db_ssl_verifyServerCertificate=globalDict['db_ssl_verifyServerCertificate'].lower() + if 'db_ssl_auth_type' in globalDict: + db_ssl_auth_type=globalDict['db_ssl_auth_type'].lower() if db_ssl_verifyServerCertificate == 'true': - if 'javax_net_ssl_keyStore' in globalDict: - javax_net_ssl_keyStore=globalDict['javax_net_ssl_keyStore'] - if 'javax_net_ssl_keyStorePassword' in globalDict: - javax_net_ssl_keyStorePassword=globalDict['javax_net_ssl_keyStorePassword'] if 'javax_net_ssl_trustStore' in globalDict: javax_net_ssl_trustStore=globalDict['javax_net_ssl_trustStore'] if 'javax_net_ssl_trustStorePassword' in globalDict: javax_net_ssl_trustStorePassword=globalDict['javax_net_ssl_trustStorePassword'] - if not os.path.exists(javax_net_ssl_keyStore): - log("[E] Invalid file Name! Unable to find keystore file:"+javax_net_ssl_keyStore,"error") - sys.exit(1) if not os.path.exists(javax_net_ssl_trustStore): log("[E] Invalid file Name! Unable to find truststore file:"+javax_net_ssl_trustStore,"error") sys.exit(1) - if javax_net_ssl_keyStorePassword is None or javax_net_ssl_keyStorePassword =="": - log("[E] Invalid ssl keystore password!","error") - sys.exit(1) if javax_net_ssl_trustStorePassword is None or javax_net_ssl_trustStorePassword =="": log("[E] Invalid ssl truststore password!","error") sys.exit(1) + if db_ssl_auth_type == '2-way': + if 'javax_net_ssl_keyStore' in globalDict: + javax_net_ssl_keyStore=globalDict['javax_net_ssl_keyStore'] + if 'javax_net_ssl_keyStorePassword' in globalDict: + javax_net_ssl_keyStorePassword=globalDict['javax_net_ssl_keyStorePassword'] + if not os.path.exists(javax_net_ssl_keyStore): + log("[E] Invalid file Name! Unable to find keystore file:"+javax_net_ssl_keyStore,"error") + sys.exit(1) + if javax_net_ssl_keyStorePassword is None or javax_net_ssl_keyStorePassword =="": + log("[E] Invalid ssl keystore password!","error") + sys.exit(1) + MYSQL_CONNECTOR_JAR=CONNECTOR_JAR - xa_sqlObj = MysqlConf(xa_db_host, MYSQL_CONNECTOR_JAR, JAVA_BIN,db_ssl_enabled,db_ssl_required,db_ssl_verifyServerCertificate,javax_net_ssl_keyStore,javax_net_ssl_keyStorePassword,javax_net_ssl_trustStore,javax_net_ssl_trustStorePassword) + xa_sqlObj = MysqlConf(xa_db_host, MYSQL_CONNECTOR_JAR, JAVA_BIN,db_ssl_enabled,db_ssl_required,db_ssl_verifyServerCertificate,javax_net_ssl_keyStore,javax_net_ssl_keyStorePassword,javax_net_ssl_trustStore,javax_net_ssl_trustStorePassword,db_ssl_auth_type) xa_db_core_file = os.path.join(RANGER_KMS_HOME,mysql_core_file) elif XA_DB_FLAVOR == "ORACLE": diff --git a/kms/scripts/install.properties b/kms/scripts/install.properties index b173d13542..a4fda7e232 100755 --- a/kms/scripts/install.properties +++ b/kms/scripts/install.properties @@ -55,6 +55,8 @@ db_host=localhost db_ssl_enabled=false db_ssl_required=false db_ssl_verifyServerCertificate=false +#db_ssl_auth_type=1-way|2-way, where 1-way represents standard one way ssl authentication and 2-way represents mutual ssl authentication +db_ssl_auth_type=2-way javax_net_ssl_keyStore= javax_net_ssl_keyStorePassword= javax_net_ssl_trustStore= @@ -87,6 +89,16 @@ HSM_ENABLED=false HSM_PARTITION_NAME=par19 HSM_PARTITION_PASSWORD=S@fenet123 +#------------------------- Ranger SAFENET KEYSECURE CONFIG ------------------------------ +KEYSECURE_ENABLED=false +KEYSECURE_USER_PASSWORD_AUTHENTICATION=true +KEYSECURE_MASTERKEY_NAME=safenetkeysecure +KEYSECURE_USERNAME=user1 +KEYSECURE_PASSWORD=t1e2s3t4 +KEYSECURE_HOSTNAME=SunPKCS11-keysecurehn +KEYSECURE_MASTER_KEY_SIZE=256 +KEYSECURE_LIB_CONFIG_PATH=/opt/safenetConf/64/8.3.1/sunpkcs11.cfg + # # ------- UNIX User CONFIG ---------------- # diff --git a/kms/scripts/ranger-kms b/kms/scripts/ranger-kms index 11dc4ff0c3..36c7533b56 100755 --- a/kms/scripts/ranger-kms +++ b/kms/scripts/ranger-kms @@ -33,7 +33,7 @@ RANGER_KMS_EWS_DIR=${RANGER_KMS_DIR}/ews RANGER_KMS_EWS_CONF_DIR="${RANGER_KMS_EWS_DIR}/webapp/WEB-INF/classes/conf" RANGER_KMS_EWS_LIB_DIR="${RANGER_KMS_EWS_DIR}/webapp/WEB-INF/classes/lib" -JAVA_OPTS=" ${JAVA_OPTS} -XX:MaxPermSize=256m -Xmx1024m -Xms1024m " +ranger_kms_max_heap_size=1g if [ -f ${RANGER_KMS_DIR}/ews/webapp/WEB-INF/classes/conf/java_home.sh ]; then . ${RANGER_KMS_DIR}/ews/webapp/WEB-INF/classes/conf/java_home.sh @@ -45,6 +45,8 @@ for custom_env_script in `find ${RANGER_KMS_DIR}/ews/webapp/WEB-INF/classes/conf fi done +JAVA_OPTS=" ${JAVA_OPTS} -XX:MetaspaceSize=100m -XX:MaxMetaspaceSize=256m -Xmx${ranger_kms_max_heap_size} -Xms1g " + if [ "$JAVA_HOME" != "" ]; then export PATH=$JAVA_HOME/bin:$PATH fi @@ -89,7 +91,7 @@ fi KMS_CONF_DIR=${RANGER_KMS_EWS_DIR}/webapp/WEB-INF/classes/conf SERVER_NAME=rangerkms -JAVA_OPTS="${JAVA_OPTS} -Dservername=${SERVER_NAME} -Dcatalina.base=${RANGER_KMS_EWS_DIR} -Dkms.config.dir=${KMS_CONF_DIR} -Dkms.log.dir=${TOMCAT_LOG_DIR} -cp ${RANGER_KMS_EWS_CONF_DIR}:${RANGER_KMS_EWS_LIB_DIR}/*:${RANGER_KMS_EWS_DIR}/webapp/lib/*:${JAVA_HOME}/lib/*:${RANGER_HADOOP_CONF_DIR}/*:$CLASSPATH " +JAVA_OPTS="${JAVA_OPTS} ${DB_SSL_PARAM} -Dservername=${SERVER_NAME} -Dcatalina.base=${RANGER_KMS_EWS_DIR} -Dkms.config.dir=${KMS_CONF_DIR} -Dkms.log.dir=${TOMCAT_LOG_DIR} -cp ${RANGER_KMS_EWS_CONF_DIR}:${RANGER_KMS_EWS_LIB_DIR}/*:${RANGER_KMS_EWS_DIR}/webapp/lib/*:${JAVA_HOME}/lib/*:${RANGER_HADOOP_CONF_DIR}/*:$CLASSPATH " createRangerKMSPid () { SLEEP_TIME_AFTER_START=5 nohup java -D${PROC_NAME} ${JAVA_OPTS} ${START_CLASS_NAME} ${KMS_CONFIG_FILENAME} > ${TOMCAT_LOG_FILE} 2>&1 & @@ -142,6 +144,10 @@ killRangerKMSPid () { fi } metric(){ + if [ "$JAVA_HOME" == "" ]; then + echo "[E] JAVA_HOME environment variable not defined, aborting Apache Ranger KMS metric collection" 1>&2; + exit 1; + fi java ${JAVA_OPTS} org.apache.hadoop.crypto.key.kms.server.KMSMetricUtil ${arg2} ${arg3} 2>/dev/null } if [ "${action}" == "START" ]; then diff --git a/kms/scripts/setup.sh b/kms/scripts/setup.sh index c8d751926d..39afa5ae0c 100755 --- a/kms/scripts/setup.sh +++ b/kms/scripts/setup.sh @@ -66,6 +66,7 @@ db_password=$(get_prop 'db_password' $PROPFILE) db_ssl_enabled=$(get_prop 'db_ssl_enabled' $PROPFILE) db_ssl_required=$(get_prop 'db_ssl_required' $PROPFILE) db_ssl_verifyServerCertificate=$(get_prop 'db_ssl_verifyServerCertificate' $PROPFILE) +db_ssl_auth_type=$(get_prop 'db_ssl_auth_type' $PROPFILE) KMS_MASTER_KEY_PASSWD=$(get_prop 'KMS_MASTER_KEY_PASSWD' $PROPFILE) unix_user=$(get_prop 'unix_user' $PROPFILE) unix_group=$(get_prop 'unix_group' $PROPFILE) @@ -97,6 +98,15 @@ HSM_ENABLED=$(get_prop 'HSM_ENABLED' $PROPFILE) HSM_PARTITION_NAME=$(get_prop 'HSM_PARTITION_NAME' $PROPFILE) HSM_PARTITION_PASSWORD=$(get_prop 'HSM_PARTITION_PASSWORD' $PROPFILE) +KEYSECURE_ENABLED=$(get_prop 'KEYSECURE_ENABLED' $PROPFILE) +KEYSECURE_USER_PASSWORD_AUTHENTICATION=$(get_prop 'KEYSECURE_USER_PASSWORD_AUTHENTICATION' $PROPFILE) +KEYSECURE_MASTERKEY_NAME=$(get_prop 'KEYSECURE_MASTERKEY_NAME' $PROPFILE) +KEYSECURE_USERNAME=$(get_prop 'KEYSECURE_USERNAME' $PROPFILE) +KEYSECURE_PASSWORD=$(get_prop 'KEYSECURE_PASSWORD' $PROPFILE) +KEYSECURE_HOSTNAME=$(get_prop 'KEYSECURE_HOSTNAME' $PROPFILE) +KEYSECURE_MASTER_KEY_SIZE=$(get_prop 'KEYSECURE_MASTER_KEY_SIZE' $PROPFILE) +KEYSECURE_LIB_CONFIG_PATH=$(get_prop 'KEYSECURE_LIB_CONFIG_PATH' $PROPFILE) + kms_principal=$(get_prop 'kms_principal' $PROPFILE) kms_keytab=$(get_prop 'kms_keytab' $PROPFILE) hadoop_conf=$(get_prop 'hadoop_conf' $PROPFILE) @@ -209,6 +219,17 @@ password_validation() { fi fi } + +password_validation_safenet_keysecure(){ + if [ -z "$1" ] + then + log "[I] Blank password is not allowed for" $2". Please enter valid password." + exit 1 + else + log "[I]" $2 "password validated." + fi +} + init_variables(){ curDt=`date '+%Y%m%d%H%M%S'` @@ -270,11 +291,13 @@ init_variables(){ db_ssl_enabled="false" db_ssl_required="false" db_ssl_verifyServerCertificate="false" + db_ssl_auth_type="2-way" fi if [ "${db_ssl_enabled}" == "true" ] then db_ssl_required=`echo $db_ssl_required | tr '[:upper:]' '[:lower:]'` db_ssl_verifyServerCertificate=`echo $db_ssl_verifyServerCertificate | tr '[:upper:]' '[:lower:]'` + db_ssl_auth_type=`echo $db_ssl_auth_type | tr '[:upper:]' '[:lower:]'` if [ "${db_ssl_required}" != "true" ] then db_ssl_required="false" @@ -283,6 +306,10 @@ init_variables(){ then db_ssl_verifyServerCertificate="false" fi + if [ "${db_ssl_auth_type}" != "1-way" ] + then + db_ssl_auth_type="2-way" + fi fi } @@ -448,17 +475,21 @@ update_properties() { if [ "${db_ssl_enabled}" != "" ] then - propertyName=ranger.db.ssl.enabled + propertyName=ranger.ks.db.ssl.enabled newPropertyValue="${db_ssl_enabled}" updatePropertyToFilePy $propertyName $newPropertyValue $to_file - propertyName=ranger.db.ssl.required + propertyName=ranger.ks.db.ssl.required newPropertyValue="${db_ssl_required}" updatePropertyToFilePy $propertyName $newPropertyValue $to_file - propertyName=ranger.db.ssl.verifyServerCertificate + propertyName=ranger.ks.db.ssl.verifyServerCertificate newPropertyValue="${db_ssl_verifyServerCertificate}" updatePropertyToFilePy $propertyName $newPropertyValue $to_file + + propertyName=ranger.ks.db.ssl.auth.type + newPropertyValue="${db_ssl_auth_type}" + updatePropertyToFilePy $propertyName $newPropertyValue $to_file fi if [ "${DB_FLAVOR}" == "MYSQL" ] @@ -563,7 +594,11 @@ update_properties() { HSM_PARTITION_PASSWD="ranger.ks.hsm.partition.password" HSM_PARTITION_PASSWORD_ALIAS="ranger.kms.hsm.partition.password" + KEYSECURE_PASSWD="ranger.kms.keysecure.login.password" + KEYSECURE_PASSWORD_ALIAS="ranger.ks.login.password" + HSM_ENABLED=`echo $HSM_ENABLED | tr '[:lower:]' '[:upper:]'` + KEYSECURE_ENABLED=`echo $KEYSECURE_ENABLED | tr '[:lower:]' '[:upper:]'` if [ "${keystore}" != "" ] then @@ -589,6 +624,20 @@ update_properties() { updatePropertyToFilePy $propertyName $newPropertyValue $to_file fi + if [ "${KEYSECURE_ENABLED}" == "TRUE" ] + then + password_validation_safenet_keysecure "$KEYSECURE_PASSWORD" "KEYSECURE User Password" + $PYTHON_COMMAND_INVOKER ranger_credential_helper.py -l "cred/lib/*" -f "$keystore" -k "${KEYSECURE_PASSWORD_ALIAS}" -v "${KEYSECURE_PASSWORD}" -c 1 + + propertyName=ranger.kms.keysecure.login.password.alias + newPropertyValue="${KEYSECURE_PASSWORD_ALIAS}" + updatePropertyToFilePy $propertyName $newPropertyValue $to_file + + propertyName=ranger.kms.keysecure.login.password + newPropertyValue="_" + updatePropertyToFilePy $propertyName $newPropertyValue $to_file + fi + propertyName=ranger.ks.jpa.jdbc.credential.alias newPropertyValue="${DB_CREDENTIAL_ALIAS}" updatePropertyToFilePy $propertyName $newPropertyValue $to_file @@ -620,6 +669,10 @@ update_properties() { propertyName="${HSM_PARTITION_PASSWD}" newPropertyValue="${HSM_PARTITION_PASSWORD}" updatePropertyToFilePy $propertyName $newPropertyValue $to_file + + propertyName="${KEYSECURE_PASSWD}" + newPropertyValue="${KEYSECURE_PASSWORD}" + updatePropertyToFilePy $propertyName $newPropertyValue $to_file fi if test -f $keystore; then @@ -684,6 +737,45 @@ update_properties() { updatePropertyToFilePy $propertyName $newPropertyValue $to_file fi + ########### SAFENET KEYSECURE CONFIG ################# + + + if [ "${KEYSECURE_ENABLED}" != "TRUE" ] + then + propertyName=ranger.kms.keysecure.enabled + newPropertyValue="false" + updatePropertyToFilePy $propertyName $newPropertyValue $to_file + else + propertyName=ranger.kms.keysecure.enabled + newPropertyValue="true" + updatePropertyToFilePy $propertyName $newPropertyValue $to_file + + propertyName=ranger.kms.keysecure.UserPassword.Authentication + newPropertyValue="${KEYSECURE_USER_PASSWORD_AUTHENTICATION}" + updatePropertyToFilePy $propertyName $newPropertyValue $to_file + + propertyName=ranger.kms.keysecure.masterkey.name + newPropertyValue="${KEYSECURE_MASTERKEY_NAME}" + updatePropertyToFilePy $propertyName $newPropertyValue $to_file + + propertyName=ranger.kms.keysecure.login.username + newPropertyValue="${KEYSECURE_USERNAME}" + updatePropertyToFilePy $propertyName $newPropertyValue $to_file + + propertyName=ranger.kms.keysecure.hostname + newPropertyValue="${KEYSECURE_HOSTNAME}" + updatePropertyToFilePy $propertyName $newPropertyValue $to_file + + propertyName=ranger.kms.keysecure.masterkey.size + newPropertyValue="${KEYSECURE_MASTER_KEY_SIZE}" + updatePropertyToFilePy $propertyName $newPropertyValue $to_file + + propertyName=ranger.kms.keysecure.sunpkcs11.cfg.filepath + newPropertyValue="${KEYSECURE_LIB_CONFIG_PATH}" + updatePropertyToFilePy $propertyName $newPropertyValue $to_file + + fi + to_file_kms_site=$PWD/ews/webapp/WEB-INF/classes/conf/ranger-kms-site.xml if test -f $to_file_kms_site; then log "[I] $to_file_kms_site file found" @@ -891,7 +983,12 @@ setup_install_files(){ if [ "${db_ssl_verifyServerCertificate}" == "true" ] then - DB_SSL_PARAM="' -Djavax.net.ssl.keyStore=${javax_net_ssl_keyStore} -Djavax.net.ssl.keyStorePassword=${javax_net_ssl_keyStorePassword} -Djavax.net.ssl.trustStore=${javax_net_ssl_trustStore} -Djavax.net.ssl.trustStorePassword=${javax_net_ssl_trustStorePassword} '" + if [ "${db_ssl_auth_type}" == "1-way" ] + then + DB_SSL_PARAM="' -Djavax.net.ssl.trustStore=${javax_net_ssl_trustStore} -Djavax.net.ssl.trustStorePassword=${javax_net_ssl_trustStorePassword} '" + else + DB_SSL_PARAM="' -Djavax.net.ssl.keyStore=${javax_net_ssl_keyStore} -Djavax.net.ssl.keyStorePassword=${javax_net_ssl_keyStorePassword} -Djavax.net.ssl.trustStore=${javax_net_ssl_trustStore} -Djavax.net.ssl.trustStorePassword=${javax_net_ssl_trustStorePassword} '" + fi echo "export DB_SSL_PARAM=${DB_SSL_PARAM}" > ${WEBAPP_ROOT}/WEB-INF/classes/conf/ranger-kms-env-dbsslparam.sh chmod a+rx ${WEBAPP_ROOT}/WEB-INF/classes/conf/ranger-kms-env-dbsslparam.sh else diff --git a/kms/src/main/java/org/apache/hadoop/crypto/key/DBToKeySecure.java b/kms/src/main/java/org/apache/hadoop/crypto/key/DBToKeySecure.java new file mode 100644 index 0000000000..ab0baa1dd4 --- /dev/null +++ b/kms/src/main/java/org/apache/hadoop/crypto/key/DBToKeySecure.java @@ -0,0 +1,127 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.crypto.key; + +import java.io.IOException; +import org.apache.hadoop.conf.Configuration; +import org.apache.ranger.kms.dao.DaoManager; + +import com.sun.org.apache.xml.internal.security.utils.Base64; + +public class DBToKeySecure { + + private static final String ENCRYPTION_KEY = "ranger.db.encrypt.key.password"; + private static final String KEYSECURE_MASTERKEY_NAME = "ranger.kms.keysecure.masterkey.name"; + private static final String KEYSECURE_LOGIN = "ranger.kms.keysecure.login"; + private static final String CFGFILEPATH = "ranger.kms.keysecure.sunpkcs11.cfg.filepath"; + + public static void showUsage() { + System.err + .println("USAGE: java " + + DBToKeySecure.class.getName() + + " "); + } + + public static void main(String[] args) { + + if (args.length < 4) { + System.err.println("Invalid number of parameters found."); + showUsage(); + System.exit(1); + } else { + + Configuration conf = RangerKeyStoreProvider.getDBKSConf(); + + String keyName = args[0]; + if (keyName == null || keyName.trim().isEmpty()) { + System.err.println("Key Secure master key name not provided."); + showUsage(); + System.exit(1); + } + + String username = args[1]; + if (username == null || username.trim().isEmpty()) { + System.err.println("Key Secure username not provided."); + showUsage(); + System.exit(1); + } + String password = args[2]; + if (password == null || password.trim().isEmpty()) { + System.err.println("Key Secure password not provided."); + showUsage(); + System.exit(1); + } + + String cfgFilePath = args[3]; + if (cfgFilePath == null || cfgFilePath.trim().isEmpty()) { + System.err.println("sunpkcs11 Configuration File Path not provided"); + showUsage(); + System.exit(1); + } + + boolean result = new DBToKeySecure().doExportMKToKeySecure(keyName, username, password, cfgFilePath, conf); + if (result) { + System.out + .println("Master Key from Ranger KMS DB has been successfully imported into Key Secure."); + } else { + System.out + .println("Import of Master Key from DB has been unsuccessful."); + System.exit(1); + } + System.exit(0); + } + } + + private boolean doExportMKToKeySecure(String keyName, String username, String password, String cfgFilePath, Configuration conf) { + try { + String keySecureMKPassword = conf.get(ENCRYPTION_KEY); + if (keySecureMKPassword == null + || keySecureMKPassword.trim().equals("") + || keySecureMKPassword.trim().equals("_") + || keySecureMKPassword.trim().equals("crypted")) { + throw new IOException("Master Key Jceks does not exists"); + } + + conf.set(CFGFILEPATH, cfgFilePath); + conf.set(KEYSECURE_MASTERKEY_NAME, keyName); + conf.set(KEYSECURE_LOGIN,username + ":" + password); + + RangerKMSDB rangerkmsDb = new RangerKMSDB(conf); + DaoManager daoManager = rangerkmsDb.getDaoManager(); + String mkPassword = conf.get(ENCRYPTION_KEY); + + // Get Master Key from Ranger DB + RangerMasterKey rangerMasterKey = new RangerMasterKey(daoManager); + String mkey = rangerMasterKey.getMasterKey(mkPassword); + byte[] key = Base64.decode(mkey); + + if (conf != null) { + RangerSafenetKeySecure rangerSafenetKeySecure = new RangerSafenetKeySecure( + conf); + return rangerSafenetKeySecure.setMasterKey(password, key,conf); + } + + return false; + } catch (Throwable t) { + throw new RuntimeException( + "Unable to import Master key from Ranger DB to KeySecure ", + t); + } + + } + +} diff --git a/kms/src/main/java/org/apache/hadoop/crypto/key/JKS2RangerUtil.java b/kms/src/main/java/org/apache/hadoop/crypto/key/JKS2RangerUtil.java index 22dce0f5fa..ee0913f8c4 100644 --- a/kms/src/main/java/org/apache/hadoop/crypto/key/JKS2RangerUtil.java +++ b/kms/src/main/java/org/apache/hadoop/crypto/key/JKS2RangerUtil.java @@ -26,13 +26,22 @@ import java.security.KeyStore; import java.security.KeyStoreException; +import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; +import org.apache.ranger.credentialapi.CredentialReader; import org.apache.ranger.kms.dao.DaoManager; public class JKS2RangerUtil { private static final String DEFAULT_KEYSTORE_TYPE = "jceks"; private static final String ENCRYPTION_KEY = "ranger.db.encrypt.key.password"; + private static final String KEYSECURE_ENABLED = "ranger.kms.keysecure.enabled"; + private static final String KEYSECURE_USERNAME = "ranger.kms.keysecure.login.username"; + private static final String KEYSECURE_PASSWORD = "ranger.kms.keysecure.login.password"; + private static final String KEYSECURE_PASSWORD_ALIAS = "ranger.kms.keysecure.login.password.alias"; + private static final String KEYSECURE_LOGIN = "ranger.kms.keysecure.login"; + private static final String CREDENTIAL_PATH = "ranger.ks.jpa.jdbc.credential.provider.path"; + public static void showUsage() { System.err.println("USAGE: java " + JKS2RangerUtil.class.getName() + " [KeyStoreType]"); @@ -80,12 +89,31 @@ private void doImportKeysFromJKS(String keyStoreFileName, String keyStoreType) { Configuration conf = RangerKeyStoreProvider.getDBKSConf(); RangerKMSDB rangerkmsDb = new RangerKMSDB(conf); DaoManager daoManager = rangerkmsDb.getDaoManager(); - RangerKeyStore dbStore = new RangerKeyStore(daoManager); + RangerKeyStore dbStore= new RangerKeyStore(daoManager); + char[] masterKey; String password = conf.get(ENCRYPTION_KEY); - RangerMasterKey rangerMasterKey = new RangerMasterKey(daoManager); - rangerMasterKey.generateMasterKey(password); - char[] masterKey = rangerMasterKey.getMasterKey(password).toCharArray(); InputStream in = null; + + if (conf != null + && StringUtils.isNotEmpty(conf.get(KEYSECURE_ENABLED)) + && conf.get(KEYSECURE_ENABLED).equalsIgnoreCase("true")) { + + getFromJceks(conf, CREDENTIAL_PATH, KEYSECURE_PASSWORD_ALIAS, KEYSECURE_PASSWORD); + String keySecureLoginCred = conf.get(KEYSECURE_USERNAME).trim() + ":" + conf.get(KEYSECURE_PASSWORD); + conf.set(KEYSECURE_LOGIN, keySecureLoginCred); + + RangerSafenetKeySecure rangerSafenetKeySecure = new RangerSafenetKeySecure( + conf); + rangerSafenetKeySecure.generateMasterKey(password); + masterKey = rangerSafenetKeySecure.getMasterKey(password).toCharArray(); + } else { + RangerMasterKey rangerMasterKey = new RangerMasterKey( + daoManager); + rangerMasterKey.generateMasterKey(password); + masterKey = rangerMasterKey.getMasterKey(password) + .toCharArray(); + } + try { in = new FileInputStream(new File(keyStoreFileName)); dbStore.engineLoadKeyStoreFile(in, keyStorePassword, keyPassword, masterKey, keyStoreType); @@ -105,6 +133,21 @@ private void doImportKeysFromJKS(String keyStoreFileName, String keyStoreType) { throw new RuntimeException("Unable to import keys from [" + keyStoreFileName + "] due to exception.", t); } } + private static void getFromJceks(Configuration conf, String path, String alias, String key) { + + //update credential from keystore + if (conf != null) { + String pathValue = conf.get(path); + String aliasValue = conf.get(alias); + if (pathValue != null && aliasValue != null) { + String xaDBPassword = CredentialReader.getDecryptedString(pathValue.trim(), aliasValue.trim()); + if (xaDBPassword != null && !xaDBPassword.trim().isEmpty() && + !xaDBPassword.trim().equalsIgnoreCase("none")) { + conf.set(key, xaDBPassword); + } + } + } + } private char[] getPasswordFromConsole(String prompt) throws IOException { diff --git a/kms/src/main/java/org/apache/hadoop/crypto/key/KeySecureToRangerDBMKUtil.java b/kms/src/main/java/org/apache/hadoop/crypto/key/KeySecureToRangerDBMKUtil.java new file mode 100644 index 0000000000..538fde95e6 --- /dev/null +++ b/kms/src/main/java/org/apache/hadoop/crypto/key/KeySecureToRangerDBMKUtil.java @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.crypto.key; + +import org.apache.hadoop.conf.Configuration; +import org.apache.ranger.credentialapi.CredentialReader; +import org.apache.ranger.kms.dao.DaoManager; + +import com.sun.org.apache.xml.internal.security.utils.Base64; + +public class KeySecureToRangerDBMKUtil { + private static final String ENCRYPTION_KEY = "ranger.db.encrypt.key.password"; + private static final String KEYSECURE_USERNAME = "ranger.kms.keysecure.login.username"; + private static final String KEYSECURE_PASSWORD = "ranger.kms.keysecure.login.password"; + private static final String KEYSECURE_PASSWORD_ALIAS = "ranger.kms.keysecure.login.password.alias"; + private static final String KEYSECURE_LOGIN = "ranger.kms.keysecure.login"; + private static final String CREDENTIAL_PATH = "ranger.ks.jpa.jdbc.credential.provider.path"; + + public static void showUsage() { + System.err.println("USAGE: java " + KeySecureToRangerDBMKUtil.class.getName() + " "); + } + + public static void main(String[] args) { + + if (args.length != 1) { + System.err.println("Invalid number of parameters found."); + showUsage(); + System.exit(1); + } + else { + String kmsMKPassword = args[0]; + if (kmsMKPassword == null || kmsMKPassword.trim().isEmpty()) { + System.err.println("KMS master key password not provided"); + showUsage(); + System.exit(1); + } + + new KeySecureToRangerDBMKUtil().doImportMKFromKeySecure(kmsMKPassword); + System.out.println("Master Key from Key Secure has been successfully imported into Ranger KMS DB."); + } + } + + private void doImportMKFromKeySecure(String kmsMKPassword) { + try { + Configuration conf = RangerKeyStoreProvider.getDBKSConf(); + conf.set(ENCRYPTION_KEY, kmsMKPassword); + getFromJceks(conf, CREDENTIAL_PATH, KEYSECURE_PASSWORD_ALIAS, KEYSECURE_PASSWORD); + String keySecureLoginCred = conf.get(KEYSECURE_USERNAME).trim() + ":" + conf.get(KEYSECURE_PASSWORD); + conf.set(KEYSECURE_LOGIN, keySecureLoginCred); + + RangerKMSDB rangerkmsDb = new RangerKMSDB(conf); + DaoManager daoManager = rangerkmsDb.getDaoManager(); + String password = conf.get(ENCRYPTION_KEY); + + RangerSafenetKeySecure rangerSafenetKeySecure = new RangerSafenetKeySecure( + conf); + String mKey = rangerSafenetKeySecure.getMasterKey(password); + + byte[] key = Base64.decode(mKey); + + // Put Master Key in Ranger DB + RangerMasterKey rangerMasterKey = new RangerMasterKey(daoManager); + rangerMasterKey.generateMKFromKeySecureMK(password, key); + + } catch (Throwable t) { + throw new RuntimeException( + "Unable to migrate Master key from KeySecure to Ranger DB", + t); + + } + } + + private static void getFromJceks(Configuration conf, String path, String alias, String key) { + + //update credential from keystore + if (conf != null) { + String pathValue = conf.get(path); + String aliasValue = conf.get(alias); + if (pathValue != null && aliasValue != null) { + String xaDBPassword = CredentialReader.getDecryptedString(pathValue.trim(), aliasValue.trim()); + if (xaDBPassword != null && !xaDBPassword.trim().isEmpty() && + !xaDBPassword.trim().equalsIgnoreCase("none")) { + conf.set(key, xaDBPassword); + } + } + } + } + + +} diff --git a/kms/src/main/java/org/apache/hadoop/crypto/key/Ranger2JKSUtil.java b/kms/src/main/java/org/apache/hadoop/crypto/key/Ranger2JKSUtil.java index 1abbf8e3cf..3a5404111f 100644 --- a/kms/src/main/java/org/apache/hadoop/crypto/key/Ranger2JKSUtil.java +++ b/kms/src/main/java/org/apache/hadoop/crypto/key/Ranger2JKSUtil.java @@ -26,13 +26,21 @@ import java.security.KeyStore; import java.security.KeyStoreException; +import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; +import org.apache.ranger.credentialapi.CredentialReader; import org.apache.ranger.kms.dao.DaoManager; public class Ranger2JKSUtil { private static final String DEFAULT_KEYSTORE_TYPE = "jceks"; private static final String ENCRYPTION_KEY = "ranger.db.encrypt.key.password"; + private static final String KEYSECURE_ENABLED = "ranger.kms.keysecure.enabled"; + private static final String KEYSECURE_USERNAME = "ranger.kms.keysecure.login.username"; + private static final String KEYSECURE_PASSWORD = "ranger.kms.keysecure.login.password"; + private static final String KEYSECURE_PASSWORD_ALIAS = "ranger.kms.keysecure.login.password.alias"; + private static final String KEYSECURE_LOGIN = "ranger.kms.keysecure.login"; + private static final String CREDENTIAL_PATH = "ranger.ks.jpa.jdbc.credential.provider.path"; public static void showUsage() { System.err.println("USAGE: java " + Ranger2JKSUtil.class.getName() + " [KeyStoreType]"); @@ -79,23 +87,43 @@ private void doExportKeysFromJKS(String keyStoreFileName, String keyStoreType) { char[] keyStorePassword = getPasswordFromConsole("Enter Password for the keystore FILE :"); char[] keyPassword = getPasswordFromConsole("Enter Password for the KEY(s) stored in the keystore:"); Configuration conf = RangerKeyStoreProvider.getDBKSConf(); - RangerKMSDB rangerkmsDb = new RangerKMSDB(conf); + RangerKMSDB rangerkmsDb = new RangerKMSDB(conf); DaoManager daoManager = rangerkmsDb.getDaoManager(); - RangerKeyStore dbStore = new RangerKeyStore(daoManager); + RangerKeyStore dbStore= new RangerKeyStore(daoManager); + + char[] masterKey; String password = conf.get(ENCRYPTION_KEY); - RangerMasterKey rangerMasterKey = new RangerMasterKey(daoManager); - char[] masterKey = rangerMasterKey.getMasterKey(password).toCharArray(); + if (conf != null + && StringUtils.isNotEmpty(conf.get(KEYSECURE_ENABLED)) + && conf.get(KEYSECURE_ENABLED).equalsIgnoreCase("true")) { + + getFromJceks(conf, CREDENTIAL_PATH, KEYSECURE_PASSWORD_ALIAS, KEYSECURE_PASSWORD); + String keySecureLoginCred = conf.get(KEYSECURE_USERNAME).trim() + ":" + conf.get(KEYSECURE_PASSWORD); + conf.set(KEYSECURE_LOGIN, keySecureLoginCred); + + RangerSafenetKeySecure rangerSafenetKeySecure = new RangerSafenetKeySecure( + conf); + masterKey = rangerSafenetKeySecure.getMasterKey(password).toCharArray(); + + } else { + RangerMasterKey rangerMasterKey = new RangerMasterKey( + daoManager); + masterKey = rangerMasterKey.getMasterKey(password) + .toCharArray(); + } OutputStream out = null; try { out = new FileOutputStream(new File(keyStoreFileName)); - dbStore.engineLoadToKeyStoreFile(out, keyStorePassword, keyPassword, masterKey, keyStoreType); - } - finally { + dbStore.engineLoadToKeyStoreFile(out, keyStorePassword, + keyPassword, masterKey, keyStoreType); + } finally { if (out != null) { try { out.close(); } catch (Exception e) { - throw new RuntimeException("ERROR: Unable to close file stream for [" + keyStoreFileName + "]", e); + throw new RuntimeException( + "ERROR: Unable to close file stream for [" + + keyStoreFileName + "]", e); } } } @@ -134,4 +162,20 @@ private char[] getPasswordFromConsole(String prompt) throws IOException { } return ret.toCharArray(); } + + private static void getFromJceks(Configuration conf, String path, String alias, String key) { + + //update credential from keystore + if (conf != null) { + String pathValue = conf.get(path); + String aliasValue = conf.get(alias); + if (pathValue != null && aliasValue != null) { + String xaDBPassword = CredentialReader.getDecryptedString(pathValue.trim(), aliasValue.trim()); + if (xaDBPassword != null && !xaDBPassword.trim().isEmpty() && + !xaDBPassword.trim().equalsIgnoreCase("none")) { + conf.set(key, xaDBPassword); + } + } + } + } } diff --git a/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKMSDB.java b/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKMSDB.java index 649da30c59..c745438851 100755 --- a/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKMSDB.java +++ b/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKMSDB.java @@ -57,6 +57,7 @@ public class RangerKMSDB { private static final String DB_SSL_ENABLED="db.ssl.enabled"; private static final String DB_SSL_REQUIRED="db.ssl.required"; private static final String DB_SSL_VerifyServerCertificate="db.ssl.verifyServerCertificate"; + private static final String DB_SSL_AUTH_TYPE="db.ssl.auth.type"; private static final String DB_SSL_KEYSTORE="keystore.file"; private static final String DB_SSL_KEYSTORE_PASSWORD="keystore.password"; private static final String DB_SSL_TRUSTSTORE="truststore.file"; @@ -190,9 +191,11 @@ private void updateDBSSLURL(){ db_ssl_verifyServerCertificate="false"; } db_ssl_verifyServerCertificate=db_ssl_verifyServerCertificate.toLowerCase(); + String db_ssl_auth_type=conf.get(PROPERTY_PREFIX+DB_SSL_AUTH_TYPE,"2-way"); conf.set(PROPERTY_PREFIX+DB_SSL_ENABLED, db_ssl_enabled); conf.set(PROPERTY_PREFIX+DB_SSL_REQUIRED, db_ssl_required); conf.set(PROPERTY_PREFIX+DB_SSL_VerifyServerCertificate, db_ssl_verifyServerCertificate); + conf.set(PROPERTY_PREFIX+DB_SSL_AUTH_TYPE, db_ssl_auth_type); String ranger_jpa_jdbc_url=conf.get(PROPERTY_PREFIX+DB_URL); if(!StringUtils.isEmpty(ranger_jpa_jdbc_url)){ StringBuffer ranger_jpa_jdbc_url_ssl=new StringBuffer(ranger_jpa_jdbc_url); @@ -204,19 +207,21 @@ private void updateDBSSLURL(){ if("true".equalsIgnoreCase(db_ssl_verifyServerCertificate)){ if (conf!=null) { - // update system key store path with custom key store. - String keystore=conf.get(PROPERTY_PREFIX+DB_SSL_KEYSTORE); - if(!StringUtils.isEmpty(keystore)){ - Path path = Paths.get(keystore); - if (Files.exists(path) && Files.isReadable(path)) { - System.setProperty("javax.net.ssl.keyStore", conf.get(PROPERTY_PREFIX+DB_SSL_KEYSTORE)); - System.setProperty("javax.net.ssl.keyStorePassword", conf.get(PROPERTY_PREFIX+DB_SSL_KEYSTORE_PASSWORD)); - System.setProperty("javax.net.ssl.keyStoreType", KeyStore.getDefaultType()); + if(!"1-way".equalsIgnoreCase((db_ssl_auth_type))){ + // update system key store path with custom key store. + String keystore=conf.get(PROPERTY_PREFIX+DB_SSL_KEYSTORE); + if(!StringUtils.isEmpty(keystore)){ + Path path = Paths.get(keystore); + if (Files.exists(path) && Files.isReadable(path)) { + System.setProperty("javax.net.ssl.keyStore", conf.get(PROPERTY_PREFIX+DB_SSL_KEYSTORE)); + System.setProperty("javax.net.ssl.keyStorePassword", conf.get(PROPERTY_PREFIX+DB_SSL_KEYSTORE_PASSWORD)); + System.setProperty("javax.net.ssl.keyStoreType", KeyStore.getDefaultType()); + }else{ + logger.debug("Could not find or read keystore file '"+keystore+"'"); + } }else{ - logger.debug("Could not find or read keystore file '"+keystore+"'"); + logger.debug("keystore property '"+PROPERTY_PREFIX+DB_SSL_KEYSTORE+"' value not found!"); } - }else{ - logger.debug("keystore property '"+PROPERTY_PREFIX+DB_SSL_KEYSTORE+"' value not found!"); } // update system trust store path with custom trust store. String truststore=conf.get(PROPERTY_PREFIX+DB_SSL_TRUSTSTORE); diff --git a/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKeyStore.java b/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKeyStore.java index a001c0876f..018ead5945 100644 --- a/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKeyStore.java +++ b/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKeyStore.java @@ -26,11 +26,13 @@ import java.io.ObjectInputStream; import java.io.ObjectOutputStream; import java.io.OutputStream; +import java.io.Serializable; import java.io.UnsupportedEncodingException; import java.lang.reflect.Constructor; import java.lang.reflect.Field; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; +import java.security.AlgorithmParameters; import java.security.DigestInputStream; import java.security.DigestOutputStream; import java.security.Key; @@ -39,16 +41,24 @@ import java.security.KeyStoreSpi; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; +import java.security.SecureRandom; import java.security.UnrecoverableKeyException; import java.security.cert.Certificate; import java.security.cert.CertificateException; -import java.util.ArrayList; import java.util.Date; import java.util.Enumeration; import java.util.Hashtable; import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import javax.crypto.Cipher; +import javax.crypto.IllegalBlockSizeException; import javax.crypto.SealedObject; +import javax.crypto.SecretKey; +import javax.crypto.SecretKeyFactory; +import javax.crypto.spec.PBEKeySpec; +import javax.crypto.spec.PBEParameterSpec; import javax.xml.bind.DatatypeConverter; import org.apache.hadoop.crypto.key.KeyProvider.Metadata; @@ -64,6 +74,8 @@ public class RangerKeyStore extends KeyStoreSpi { static final Logger logger = Logger.getLogger(RangerKeyStore.class); + private static final String KEY_NAME_VALIDATION = "[a-z,A-Z,0-9](?!.*--)(?!.*__)(?!.*-_)(?!.*_-)[\\w\\-\\_]*"; + private static final Pattern pattern = Pattern.compile(KEY_NAME_VALIDATION); private DaoManager daoManager; @@ -89,7 +101,7 @@ private static final class SecretKeyEntry { RangerKeyStore() { } - RangerKeyStore(DaoManager daoManager) { + public RangerKeyStore(DaoManager daoManager) { this.daoManager = daoManager; } @@ -108,17 +120,9 @@ public Key engineGetKey(String alias, char[] password)throws NoSuchAlgorithmExce return null; } - Class c = null; - Object o = null; try { - c = Class.forName("com.sun.crypto.provider.KeyProtector"); - Constructor constructor = c.getDeclaredConstructor(char[].class); - constructor.setAccessible(true); - o = constructor.newInstance(password); - Method m = c.getDeclaredMethod("unseal", SealedObject.class); - m.setAccessible(true); - key = (Key) m.invoke(o, ((SecretKeyEntry)entry).sealedKey); - } catch (ClassNotFoundException | NoSuchMethodException | SecurityException | InstantiationException | IllegalAccessException | IllegalArgumentException | InvocationTargetException e) { + key = unsealKey(((SecretKeyEntry)entry).sealedKey, password); + } catch (Exception e) { logger.error(e.getMessage()); } return key; @@ -144,22 +148,9 @@ public void addKeyEntry(String alias, Key key, char[] password, String cipher, i SecretKeyEntry entry = new SecretKeyEntry(); synchronized(deltaEntries) { try { - Class c = null; - Object o = null; - try { - c = Class.forName("com.sun.crypto.provider.KeyProtector"); - Constructor constructor = c.getDeclaredConstructor(char[].class); - constructor.setAccessible(true); - o = constructor.newInstance(password); - } catch (ClassNotFoundException | NoSuchMethodException | SecurityException | InstantiationException | IllegalAccessException | IllegalArgumentException | InvocationTargetException e) { - logger.error(e.getMessage()); - throw new KeyStoreException(e.getMessage()); - } entry.date = new Date(); // seal and store the key - Method m = c.getDeclaredMethod("seal", Key.class); - m.setAccessible(true); - entry.sealedKey = (SealedObject) m.invoke(o, key); + entry.sealedKey = sealKey(key, password); entry.cipher_field = cipher; entry.bit_length = bitLength; @@ -182,6 +173,47 @@ public void addKeyEntry(String alias, Key key, char[] password, String cipher, i } } + private SealedObject sealKey(Key key, char[] password) throws Exception { + // Create SecretKey + SecretKeyFactory secretKeyFactory = SecretKeyFactory.getInstance("PBEWithMD5AndTripleDES"); + PBEKeySpec pbeKeySpec = new PBEKeySpec(password); + SecretKey secretKey = secretKeyFactory.generateSecret(pbeKeySpec); + pbeKeySpec.clearPassword(); + + // Generate random bytes + set up the PBEParameterSpec + SecureRandom random = new SecureRandom(); + byte[] salt = new byte[8]; + random.nextBytes(salt); + PBEParameterSpec pbeSpec = new PBEParameterSpec(salt, 20); + + // Seal the Key + Cipher cipher = Cipher.getInstance("PBEWithMD5AndTripleDES"); + cipher.init(Cipher.ENCRYPT_MODE, secretKey, pbeSpec); + return new RangerSealedObject(key, cipher); + } + + private Key unsealKey(SealedObject sealedKey, char[] password) throws Exception { + // Create SecretKey + SecretKeyFactory secretKeyFactory = SecretKeyFactory.getInstance("PBEWithMD5AndTripleDES"); + PBEKeySpec pbeKeySpec = new PBEKeySpec(password); + SecretKey secretKey = secretKeyFactory.generateSecret(pbeKeySpec); + pbeKeySpec.clearPassword(); + + // Get the AlgorithmParameters from RangerSealedObject + AlgorithmParameters algorithmParameters = null; + if (sealedKey instanceof RangerSealedObject) { + algorithmParameters = ((RangerSealedObject)sealedKey).getParameters(); + } else { + algorithmParameters = new RangerSealedObject(sealedKey).getParameters(); + } + + // Unseal the Key + Cipher cipher = Cipher.getInstance("PBEWithMD5AndTripleDES"); + cipher.init(Cipher.DECRYPT_MODE, secretKey, algorithmParameters); + + return (Key)sealedKey.getObject(cipher); + } + @Override public void engineDeleteEntry(String alias) throws KeyStoreException @@ -526,6 +558,7 @@ public void engineLoadKeyStoreFile(InputStream stream, char[] storePass, char[] entry.cipher_field = k.getAlgorithm(); } String keyName = alias.split("@")[0]; + validateKeyName(keyName); entry.attributes = "{\"key.acl.name\":\"" + keyName + "\"}"; Class c = null; Object o = null; @@ -581,9 +614,44 @@ public void engineLoadToKeyStoreFile(OutputStream stream, char[] storePass, char } } } - + + private void validateKeyName(String name) { + Matcher matcher = pattern.matcher(name); + if (!matcher.matches()) { + throw new IllegalArgumentException( + "Key Name : " + + name + + ", should start with alpha/numeric letters and can have special characters - (hypen) or _ (underscore)"); + } + } + public void clearDeltaEntires(){ deltaEntries.clear(); } - + + /** + * Encapsulate the encrypted key, so that we can retrieve the AlgorithmParameters object on the decryption side + */ + private static class RangerSealedObject extends SealedObject { + + /** + * + */ + private static final long serialVersionUID = -7551578543434362070L; + + protected RangerSealedObject(SealedObject so) { + super(so); + } + + protected RangerSealedObject(Serializable object, Cipher cipher) throws IllegalBlockSizeException, IOException { + super(object, cipher); + } + + public AlgorithmParameters getParameters() throws NoSuchAlgorithmException, IOException { + AlgorithmParameters algorithmParameters = AlgorithmParameters.getInstance("PBEWithMD5AndTripleDES"); + algorithmParameters.init(super.encodedParams); + return algorithmParameters; + } + + } } diff --git a/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKeyStoreProvider.java b/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKeyStoreProvider.java index 267fcf08f1..448469b99c 100755 --- a/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKeyStoreProvider.java +++ b/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKeyStoreProvider.java @@ -68,6 +68,13 @@ public class RangerKeyStoreProvider extends KeyProvider{ private static final String HSM_ENABLED = "ranger.ks.hsm.enabled"; private static final String HSM_PARTITION_PASSWORD_ALIAS = "ranger.ks.hsm.partition.password.alias"; private static final String HSM_PARTITION_PASSWORD = "ranger.ks.hsm.partition.password"; + private static final String KEYSECURE_ENABLED = "ranger.kms.keysecure.enabled"; + + private static final String KEYSECURE_USERNAME = "ranger.kms.keysecure.login.username"; + private static final String KEYSECURE_PASSWORD_ALIAS = "ranger.kms.keysecure.login.password.alias"; + private static final String KEYSECURE_PASSWORD = "ranger.kms.keysecure.login.password"; + private static final String KEYSECURE_LOGIN = "ranger.kms.keysecure.login"; + private final RangerKeyStore dbStore; private char[] masterKey; @@ -80,37 +87,68 @@ public class RangerKeyStoreProvider extends KeyProvider{ public RangerKeyStoreProvider(Configuration conf) throws Throwable { super(conf); conf = getDBKSConf(); - getFromJceks(conf,CREDENTIAL_PATH, MK_CREDENTIAL_ALIAS, ENCRYPTION_KEY); - getFromJceks(conf,CREDENTIAL_PATH, DB_CREDENTIAL_ALIAS, DB_PASSWORD); - getFromJceks(conf,CREDENTIAL_PATH, HSM_PARTITION_PASSWORD_ALIAS, HSM_PARTITION_PASSWORD); + getFromJceks(conf, CREDENTIAL_PATH, MK_CREDENTIAL_ALIAS, ENCRYPTION_KEY); + getFromJceks(conf, CREDENTIAL_PATH, DB_CREDENTIAL_ALIAS, DB_PASSWORD); + getFromJceks(conf, CREDENTIAL_PATH, HSM_PARTITION_PASSWORD_ALIAS, + HSM_PARTITION_PASSWORD); RangerKMSDB rangerKMSDB = new RangerKMSDB(conf); daoManager = rangerKMSDB.getDaoManager(); - + RangerKMSMKI rangerMasterKey = null; String password = conf.get(ENCRYPTION_KEY); - if(password == null || password.trim().equals("") || password.trim().equals("_") || password.trim().equals("crypted")){ + if (password == null || password.trim().equals("") + || password.trim().equals("_") + || password.trim().equals("crypted")) { throw new IOException("Master Key Jceks does not exists"); } - if(StringUtils.isEmpty(conf.get(HSM_ENABLED)) || conf.get(HSM_ENABLED).equalsIgnoreCase("false")){ + if (StringUtils.isEmpty(conf.get(HSM_ENABLED)) + || conf.get(HSM_ENABLED).equalsIgnoreCase("false")) { rangerMasterKey = new RangerMasterKey(daoManager); - }else{ + } else { rangerMasterKey = new RangerHSM(conf); String partitionPasswd = conf.get(HSM_PARTITION_PASSWORD); - if(partitionPasswd == null || partitionPasswd.trim().equals("") || partitionPasswd.trim().equals("_") || partitionPasswd.trim().equals("crypted")){ + if (partitionPasswd == null || partitionPasswd.trim().equals("") + || partitionPasswd.trim().equals("_") + || partitionPasswd.trim().equals("crypted")) { throw new IOException("Partition Password doesn't exists"); } } - dbStore = new RangerKeyStore(daoManager); - rangerMasterKey.generateMasterKey(password); - //code to retrieve rangerMasterKey password - masterKey = rangerMasterKey.getMasterKey(password).toCharArray(); - if(masterKey == null){ - // Master Key does not exists - throw new IOException("Ranger MasterKey does not exists"); + + + if (conf != null && StringUtils.isNotEmpty(conf.get(KEYSECURE_ENABLED)) + && conf.get(KEYSECURE_ENABLED).equalsIgnoreCase("true")) { + getFromJceks(conf, CREDENTIAL_PATH, KEYSECURE_PASSWORD_ALIAS, KEYSECURE_PASSWORD); + String keySecureLoginCred = conf.get(KEYSECURE_USERNAME).trim() + ":" + conf.get(KEYSECURE_PASSWORD); + conf.set(KEYSECURE_LOGIN, keySecureLoginCred); + rangerMasterKey = new RangerSafenetKeySecure(conf); + + dbStore = new RangerKeyStore(daoManager); + // generate master key on key secure server + rangerMasterKey.generateMasterKey(password); + try { + masterKey = rangerMasterKey.getMasterKey(password) + .toCharArray(); + } catch (Exception ex) { + throw new Exception("Error while getting Safenet KeySecure master key " + ex); + } + + } else { + dbStore = new RangerKeyStore(daoManager); + rangerMasterKey.generateMasterKey(password); + // code to retrieve rangerMasterKey password + try { + masterKey = rangerMasterKey.getMasterKey(password) + .toCharArray(); + } catch (Exception ex) { + throw new Exception("Error while getting Ranger Master key " + ex); + } + } - reloadKeys(); + + reloadKeys(); ReadWriteLock lock = new ReentrantReadWriteLock(true); - readLock = lock.readLock(); + readLock = lock.readLock(); + } public static Configuration getDBKSConf() { @@ -164,7 +202,7 @@ public KeyVersion createKey(String name, byte[] material, Options options) throw new IOException("Wrong key length. Required " + options.getBitLength() + ", but got " + (8 * material.length)); } - cache.put(name, meta); + cache.put(name, meta); String versionName = buildVersionName(name, 0); return innerSetKeyVersion(name, versionName, material, meta.getCipher(), meta.getBitLength(), meta.getDescription(), meta.getVersions(), meta.getAttributes()); } @@ -205,7 +243,8 @@ public void deleteKey(String name) throws IOException { } catch (KeyStoreException e) { throw new IOException("Problem removing " + name + " from " + this, e); } - cache.remove(name); + cache.remove(name); + changed = true; } @@ -236,7 +275,8 @@ public void flush() throws IOException { } changed = false; }catch (IOException ioe) { - cache.clear(); + cache.clear(); + reloadKeys(); throw ioe; } @@ -245,37 +285,40 @@ public void flush() throws IOException { @Override public KeyVersion getKeyVersion(String versionName) throws IOException { readLock.lock(); - try { - SecretKeySpec key = null; - try { - if (!dbStore.engineContainsAlias(versionName)) { - dbStore.engineLoad(null, masterKey); - if (!dbStore.engineContainsAlias(versionName)) { - return null; - } + try { + SecretKeySpec key = null; + try { + if (!dbStore.engineContainsAlias(versionName)) { + dbStore.engineLoad(null, masterKey); + if (!dbStore.engineContainsAlias(versionName)) { + return null; + } + } + key = (SecretKeySpec) dbStore.engineGetKey(versionName, + masterKey); + } catch (NoSuchAlgorithmException e) { + + throw new IOException("Can't get algorithm for key " + key, e); + } catch (UnrecoverableKeyException e) { + throw new IOException("Can't recover key " + key, e); + } catch (CertificateException e) { + throw new IOException("Certificate exception storing key", e); } - key = (SecretKeySpec) dbStore.engineGetKey(versionName, masterKey); - } catch (NoSuchAlgorithmException e) { - throw new IOException("Can't get algorithm for key " + key, e); - } catch (UnrecoverableKeyException e) { - throw new IOException("Can't recover key " + key, e); - } - catch (CertificateException e) { - throw new IOException("Certificate exception storing key", e); + if (key == null) { + return null; + } else { + return new KeyVersion(getBaseName(versionName), versionName, + key.getEncoded()); + } + } finally { + readLock.unlock(); } - if (key == null) { - return null; - } else { - return new KeyVersion(getBaseName(versionName), versionName, key.getEncoded()); - } - } finally { - readLock.unlock(); - } } @Override public List getKeyVersions(String name) throws IOException { List list = new ArrayList(); + Metadata km = getMetadata(name); if (km != null) { int latestVersion = km.getVersions(); @@ -294,18 +337,18 @@ public List getKeyVersions(String name) throws IOException { @Override public List getKeys() throws IOException { - ArrayList list = new ArrayList(); - String alias = null; - reloadKeys(); - Enumeration e = dbStore.engineAliases(); - while (e.hasMoreElements()) { - alias = e.nextElement(); - // only include the metadata key names in the list of names - if (!alias.contains("@")) { - list.add(alias); - } - } - return list; + ArrayList list = new ArrayList(); + String alias = null; + reloadKeys(); + Enumeration e = dbStore.engineAliases(); + while (e.hasMoreElements()) { + alias = e.nextElement(); + // only include the metadata key names in the list of names + if (!alias.contains("@")) { + list.add(alias); + } + } + return list; } @Override @@ -326,7 +369,7 @@ public Metadata getMetadata(String name) throws IOException { Key key = dbStore.engineGetKey(name, masterKey); if(key != null){ Metadata meta = ((KeyMetadata) key).metadata; - cache.put(name, meta); + cache.put(name, meta); return meta; } } catch (NoSuchAlgorithmException e) { @@ -347,6 +390,7 @@ public Metadata getMetadata(String name) throws IOException { @Override public KeyVersion rollNewVersion(String name, byte[] material)throws IOException { reloadKeys(); + Metadata meta = getMetadata(name); if (meta == null) { throw new IOException("Key " + name + " not found"); @@ -378,8 +422,8 @@ private static void getFromJceks(Configuration conf, String path, String alias, private void reloadKeys() throws IOException { try { - cache.clear(); - loadKeys(masterKey); + cache.clear(); + loadKeys(masterKey); } catch (NoSuchAlgorithmException e) { throw new IOException("Can't load Keys"); }catch(CertificateException e){ diff --git a/kms/src/main/java/org/apache/hadoop/crypto/key/RangerMasterKey.java b/kms/src/main/java/org/apache/hadoop/crypto/key/RangerMasterKey.java index 5614c1640b..c0910a45cf 100755 --- a/kms/src/main/java/org/apache/hadoop/crypto/key/RangerMasterKey.java +++ b/kms/src/main/java/org/apache/hadoop/crypto/key/RangerMasterKey.java @@ -131,6 +131,17 @@ private SecretKey decryptMasterKeySK(byte masterKey[], String password) throws T return getMasterKeyFromBytes(masterKeyFromDBDecrypted); } + public boolean generateMKFromKeySecureMK(String password, byte[] key) throws Throwable{ + logger.info("Generating Master Key"); + String encryptedMasterKey = encryptMasterKey(password, key); + String savedKey = saveEncryptedMK(encryptedMasterKey, daoManager); + if(savedKey != null && !savedKey.trim().equals("")){ + logger.debug("Master Key Created with id = "+savedKey); + return true; + } + return false; + } + private byte[] getEncryptedMK() throws Base64DecodingException { logger.debug("Retrieving Encrypted Master Key from database"); try{ diff --git a/kms/src/main/java/org/apache/hadoop/crypto/key/RangerSafenetKeySecure.java b/kms/src/main/java/org/apache/hadoop/crypto/key/RangerSafenetKeySecure.java new file mode 100644 index 0000000000..70ec504493 --- /dev/null +++ b/kms/src/main/java/org/apache/hadoop/crypto/key/RangerSafenetKeySecure.java @@ -0,0 +1,156 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.crypto.key; + +import javax.crypto.KeyGenerator; +import javax.crypto.SecretKey; +import javax.crypto.spec.SecretKeySpec; + +import org.apache.hadoop.conf.Configuration; +import org.apache.log4j.Logger; + +import com.sun.org.apache.xml.internal.security.utils.Base64; + +import java.io.IOException; +import java.security.Key; +import java.security.KeyStore; +import java.security.KeyStoreException; +import java.security.NoSuchAlgorithmException; +import java.security.Provider; +import java.security.Security; +import java.security.cert.CertificateException; + +/** + * This Class is for HSM Keystore + */ +public class RangerSafenetKeySecure implements RangerKMSMKI { + + static final Logger logger = Logger.getLogger(RangerSafenetKeySecure.class); + + private final String alias; + private final KeyStore myStore; + private final String adp; + private final Provider provider; + private static final String MK_ALGO = "AES"; + private final int mkSize; + private static final int MK_KeySize = 256; + private String pkcs11CfgFilePath = null; + private static final String CFGFILEPATH = "ranger.kms.keysecure.sunpkcs11.cfg.filepath"; + private static final String MK_KEYSIZE = "ranger.kms.keysecure.masterkey.size"; + private static final String ALIAS = "ranger.kms.keysecure.masterkey.name"; + + private static final String KEYSECURE_LOGIN = "ranger.kms.keysecure.login"; + + public RangerSafenetKeySecure(Configuration conf) throws KeyStoreException, NoSuchAlgorithmException, CertificateException, IOException { + mkSize = conf.getInt(MK_KEYSIZE, MK_KeySize); + alias = conf.get(ALIAS, "RANGERMK"); + adp = conf.get(KEYSECURE_LOGIN); + pkcs11CfgFilePath = conf.get(CFGFILEPATH); + + try { + // Create a PKCS#11 session and initialize it + // using the sunPKCS11 config file + provider = new sun.security.pkcs11.SunPKCS11(pkcs11CfgFilePath); + Security.addProvider(provider); + myStore = KeyStore.getInstance("PKCS11", provider); + if(myStore != null){ + myStore.load(null, adp.toCharArray()); + }else{ + logger.error("Safenet Keysecure not found. Please verify the Ranger KMS Safenet Keysecure configuration setup."); + } + + } catch (NoSuchAlgorithmException nsae) { + throw new NoSuchAlgorithmException("Unexpected NoSuchAlgorithmException while loading keystore : " + + nsae.getMessage()); + } catch (CertificateException e) { + throw new CertificateException("Unexpected CertificateException while loading keystore : " + + e.getMessage()); + } catch (IOException e) { + throw new IOException("Unexpected IOException while loading keystore : " + + e.getMessage()); + } + } + + @Override + public boolean generateMasterKey(String password){ + if (myStore != null) { + KeyGenerator keyGen = null; + SecretKey aesKey = null; + try { + boolean result = myStore.containsAlias(alias); + + if (!result) { + keyGen = KeyGenerator.getInstance(MK_ALGO, provider); + + + keyGen.init(mkSize); + + aesKey = keyGen.generateKey(); + myStore.setKeyEntry(alias, aesKey, password.toCharArray(), + (java.security.cert.Certificate[]) null); + return true; + } else { + return true; + } + + } catch (Exception e) { + logger.error("generateMasterKey : Exception during Ranger Master Key Generation - " + + e); + return false; + } + } + return false; + } + + @Override + public String getMasterKey(String password) { + if (myStore != null) { + try { + boolean result = myStore.containsAlias(alias); + if (result) { + SecretKey key = (SecretKey) myStore.getKey(alias, + password.toCharArray()); + if (key != null) { + return Base64.encode(key.getEncoded()); + } + + } + } catch (Exception e) { + logger.error("getMasterKey : Exception searching for Ranger Master Key - " + + e.getMessage()); + } + } + return null; + } + + public boolean setMasterKey(String password, byte[] key, Configuration conf) { + if (myStore != null) { + try { + Key aesKey = new SecretKeySpec(key, MK_ALGO); + myStore.setKeyEntry(alias, aesKey, password.toCharArray(), + (java.security.cert.Certificate[]) null); + return true; + } catch (Exception e) { + logger.error("setMasterKey : Exception while setting Master Key - " + + e.getMessage()); + } + } + return false; + } + +} \ No newline at end of file diff --git a/kms/src/test/java/org/apache/hadoop/crypto/key/kms/TestRangerKeyStore.java b/kms/src/test/java/org/apache/hadoop/crypto/key/kms/TestRangerKeyStore.java new file mode 100644 index 0000000000..9c5898c462 --- /dev/null +++ b/kms/src/test/java/org/apache/hadoop/crypto/key/kms/TestRangerKeyStore.java @@ -0,0 +1,168 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.crypto.key.kms; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.security.Key; +import java.security.KeyStore; +import java.security.KeyStoreException; +import java.security.NoSuchAlgorithmException; +import java.security.cert.CertificateException; + +import javax.crypto.KeyGenerator; + +import org.apache.hadoop.crypto.key.RangerKeyStore; +import org.apache.ranger.kms.dao.DaoManager; +import org.junit.After; +import org.junit.Before; +import org.junit.FixMethodOrder; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.MethodSorters; +import org.mockito.Mockito; +import org.mockito.runners.MockitoJUnitRunner; + +@RunWith(MockitoJUnitRunner.class) +@FixMethodOrder(MethodSorters.NAME_ASCENDING) +public class TestRangerKeyStore { + + String fileFormat = "jceks"; + String keyStoreFileName = "KmsKeyStoreFile"; + char[] storePass = "none".toCharArray(); + char[] keyPass = "none".toCharArray(); + char[] masterKey = "MasterPassword".toCharArray(); + + @Before + public void checkFileIfExists() { + deleteKeyStoreFile(); + } + + @After + public void cleanKeystoreFile() { + deleteKeyStoreFile(); + } + + @Test(expected=IOException.class) + public void testInvalidKey1() throws NoSuchAlgorithmException, + CertificateException, IOException, KeyStoreException { + + DaoManager daoManager = Mockito.mock(DaoManager.class); + RangerKeyStore rangerKeyStore = new RangerKeyStore(daoManager); + String keyValue = "enckey:1"; + InputStream inputStream = generateKeyStoreFile(keyValue); + rangerKeyStore.engineLoadKeyStoreFile(inputStream, storePass, keyPass, masterKey, fileFormat); + inputStream.close(); + } + + @Test(expected=IOException.class) + public void testInvalidKey2() throws NoSuchAlgorithmException, + CertificateException, IOException, KeyStoreException { + + DaoManager daoManager = Mockito.mock(DaoManager.class); + RangerKeyStore rangerKeyStore = new RangerKeyStore(daoManager); + String keyValue = "1%enckey"; + InputStream inputStream = generateKeyStoreFile(keyValue); + rangerKeyStore.engineLoadKeyStoreFile(inputStream, storePass, keyPass, masterKey, fileFormat); + inputStream.close(); + } + + @Test(expected=IOException.class) + public void testInvalidKey3() throws NoSuchAlgorithmException, + CertificateException, IOException, KeyStoreException { + + DaoManager daoManager = Mockito.mock(DaoManager.class); + RangerKeyStore rangerKeyStore = new RangerKeyStore(daoManager); + String keyValue = "1 enckey"; + InputStream inputStream = generateKeyStoreFile(keyValue); + rangerKeyStore.engineLoadKeyStoreFile(inputStream, storePass, keyPass, masterKey, fileFormat); + inputStream.close(); + } + + @Test(expected=IOException.class) + public void testInvalidKey4() throws NoSuchAlgorithmException, + CertificateException, IOException, KeyStoreException { + + DaoManager daoManager = Mockito.mock(DaoManager.class); + RangerKeyStore rangerKeyStore = new RangerKeyStore(daoManager); + String keyValue = "_1-enckey"; + InputStream inputStream = generateKeyStoreFile(keyValue); + rangerKeyStore.engineLoadKeyStoreFile(inputStream, storePass, keyPass, masterKey, fileFormat); + inputStream.close(); + } + + @Test + public void testValidKey1() throws NoSuchAlgorithmException, + CertificateException, IOException, KeyStoreException { + + DaoManager daoManager = Mockito.mock(DaoManager.class); + RangerKeyStore rangerKeyStore = new RangerKeyStore(daoManager); + String keyValue = "enckey_1-test"; + InputStream inputStream = generateKeyStoreFile(keyValue); + rangerKeyStore.engineLoadKeyStoreFile(inputStream, storePass, keyPass, masterKey, fileFormat); + inputStream.close(); + } + + @Test + public void testValidKey2() throws NoSuchAlgorithmException, + CertificateException, IOException, KeyStoreException { + + DaoManager daoManager = Mockito.mock(DaoManager.class); + RangerKeyStore rangerKeyStore = new RangerKeyStore(daoManager); + String keyValue = "1-enckey_test"; + InputStream inputStream = generateKeyStoreFile(keyValue); + rangerKeyStore.engineLoadKeyStoreFile(inputStream, storePass, keyPass, masterKey, fileFormat); + inputStream.close(); + } + + private InputStream generateKeyStoreFile(String keyValue) throws KeyStoreException, NoSuchAlgorithmException, CertificateException, IOException { + FileOutputStream stream = new FileOutputStream(new File(keyStoreFileName)); + KeyStore ks; + try { + ks = KeyStore.getInstance(fileFormat); + if (ks != null) { + ks.load(null, storePass); + String alias = keyValue; + + KeyGenerator kg = KeyGenerator.getInstance("AES"); + kg.init(256); + Key key = kg.generateKey(); + ks.setKeyEntry(alias, key, keyPass, null); + ks.store(stream, storePass); + } + return new FileInputStream(new File(keyStoreFileName)); + } catch (Throwable t) { + throw new IOException(t); + } finally { + stream.close(); + } + } + + private void deleteKeyStoreFile() { + File f = new File(keyStoreFileName); + if (f.exists()) { + boolean bol = f.delete(); + if(!bol){ + System.out.println("Keystore File was not deleted successfully."); + } + } + } +} diff --git a/knox-agent/src/main/java/org/apache/ranger/authorization/knox/RangerPDPKnoxFilter.java b/knox-agent/src/main/java/org/apache/ranger/authorization/knox/RangerPDPKnoxFilter.java index 1d58b210e3..24e8702e14 100644 --- a/knox-agent/src/main/java/org/apache/ranger/authorization/knox/RangerPDPKnoxFilter.java +++ b/knox-agent/src/main/java/org/apache/ranger/authorization/knox/RangerPDPKnoxFilter.java @@ -42,11 +42,14 @@ import org.apache.ranger.audit.provider.MiscUtil; import org.apache.ranger.plugin.policyengine.RangerAccessRequest; import org.apache.ranger.plugin.policyengine.RangerAccessResult; +import org.apache.ranger.plugin.util.RangerPerfTracer; public class RangerPDPKnoxFilter implements Filter { private static final Log LOG = LogFactory.getLog(RangerPDPKnoxFilter.class); + private static final Log PERF_KNOXAUTH_REQUEST_LOG = RangerPerfTracer.getPerfLogger("knoxauth.request"); + private static final String KNOX_GATEWAY_JASS_CONFIG_SECTION = "com.sun.security.jgss.initiate"; private String resourceRole = null; @@ -93,6 +96,12 @@ public void doFilter(ServletRequest request, ServletResponse response, String topologyName = getTopologyName(sourceUrl); String serviceName = getServiceName(); + RangerPerfTracer perf = null; + + if(RangerPerfTracer.isPerfTraceEnabled(PERF_KNOXAUTH_REQUEST_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_KNOXAUTH_REQUEST_LOG, "RangerPDPKnoxFilter.doFilter(url=" + sourceUrl + ", topologyName=" + topologyName + ")"); + } + Subject subject = Subject.getSubject(AccessController.getContext()); Principal primaryPrincipal = (Principal) subject.getPrincipals( @@ -151,6 +160,8 @@ public void doFilter(ServletRequest request, ServletResponse response, LOG.debug("Access allowed: " + accessAllowed); } + RangerPerfTracer.log(perf); + if (accessAllowed) { chain.doFilter(request, response); } else { diff --git a/plugin-atlas/src/main/java/org/apache/ranger/authorization/atlas/authorizer/RangerAtlasAuthorizer.java b/plugin-atlas/src/main/java/org/apache/ranger/authorization/atlas/authorizer/RangerAtlasAuthorizer.java index 9712f95c71..acd11115ce 100644 --- a/plugin-atlas/src/main/java/org/apache/ranger/authorization/atlas/authorizer/RangerAtlasAuthorizer.java +++ b/plugin-atlas/src/main/java/org/apache/ranger/authorization/atlas/authorizer/RangerAtlasAuthorizer.java @@ -26,15 +26,18 @@ import org.apache.atlas.authorize.AtlasAuthorizationException; import org.apache.atlas.authorize.AtlasAuthorizer; import org.apache.atlas.authorize.AtlasResourceTypes; +import org.apache.commons.logging.Log; import org.apache.ranger.plugin.audit.RangerDefaultAuditHandler; import org.apache.ranger.plugin.policyengine.RangerAccessRequestImpl; import org.apache.ranger.plugin.policyengine.RangerAccessResult; import org.apache.ranger.plugin.service.RangerBasePlugin; +import org.apache.ranger.plugin.util.RangerPerfTracer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; public class RangerAtlasAuthorizer implements AtlasAuthorizer { private static final Logger LOG = LoggerFactory.getLogger(RangerAtlasAuthorizer.class); + private static final Log PERF_ATLASAUTH_REQUEST_LOG = RangerPerfTracer.getPerfLogger("atlasauth.request"); private static boolean isDebugEnabled = LOG.isDebugEnabled(); private static volatile RangerBasePlugin atlasPlugin = null; @@ -70,6 +73,11 @@ public boolean isAccessAllowed(AtlasAccessRequest request) throws AtlasAuthoriza if (isDebugEnabled) { LOG.debug("==> isAccessAllowed( " + request + " )"); } + RangerPerfTracer perf = null; + + if(RangerPerfTracer.isPerfTraceEnabled(PERF_ATLASAUTH_REQUEST_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_ATLASAUTH_REQUEST_LOG, "RangerAtlasAuthorizer.isAccessAllowed(request=" + request + ")"); + } String resource = request.getResource(); String user = request.getUser(); @@ -93,6 +101,8 @@ public boolean isAccessAllowed(AtlasAccessRequest request) throws AtlasAuthoriza } } + RangerPerfTracer.log(perf); + if (isDebugEnabled) { LOG.debug("<== isAccessAllowed Returning value :: " + isAccessAllowed); } diff --git a/plugin-kafka/pom.xml b/plugin-kafka/pom.xml index fb0be1819c..61bdfd510d 100644 --- a/plugin-kafka/pom.xml +++ b/plugin-kafka/pom.xml @@ -84,7 +84,12 @@ ${bouncycastle.version} test - + + junit + junit + ${junit.version} + test + diff --git a/plugin-kafka/src/main/java/org/apache/ranger/authorization/kafka/authorizer/RangerKafkaAuthorizer.java b/plugin-kafka/src/main/java/org/apache/ranger/authorization/kafka/authorizer/RangerKafkaAuthorizer.java index ec7f88774d..b3d5a74d59 100644 --- a/plugin-kafka/src/main/java/org/apache/ranger/authorization/kafka/authorizer/RangerKafkaAuthorizer.java +++ b/plugin-kafka/src/main/java/org/apache/ranger/authorization/kafka/authorizer/RangerKafkaAuthorizer.java @@ -42,12 +42,14 @@ import org.apache.ranger.plugin.policyengine.RangerAccessResult; import org.apache.ranger.plugin.service.RangerBasePlugin; +import org.apache.ranger.plugin.util.RangerPerfTracer; import scala.collection.immutable.HashSet; import scala.collection.immutable.Set; public class RangerKafkaAuthorizer implements Authorizer { private static final Log logger = LogFactory .getLog(RangerKafkaAuthorizer.class); + private static final Log PERF_KAFKAAUTH_REQUEST_LOG = RangerPerfTracer.getPerfLogger("kafkaauth.request"); public static final String KEY_TOPIC = "topic"; public static final String KEY_CLUSTER = "cluster"; @@ -130,11 +132,14 @@ public boolean authorize(Session session, Operation operation, return true; } + RangerPerfTracer perf = null; + + if(RangerPerfTracer.isPerfTraceEnabled(PERF_KAFKAAUTH_REQUEST_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_KAFKAAUTH_REQUEST_LOG, "RangerKafkaAuthorizer.authorize(resource=" + resource + ")"); + } String userName = null; if (session.principal() != null) { userName = session.principal().getName(); - userName = StringUtils.substringBefore(userName, "/"); - userName = StringUtils.substringBefore(userName, "@"); } java.util.Set userGroups = MiscUtil .getGroupsForRequestUser(userName); @@ -206,6 +211,8 @@ public boolean authorize(Session session, Operation operation, + rangerRequest, t); } } + RangerPerfTracer.log(perf); + if (logger.isDebugEnabled()) { logger.debug("rangerRequest=" + rangerRequest + ", return=" + returnValue); diff --git a/plugin-kafka/src/test/java/org/apache/ranger/authorization/kafka/authorizer/KafkaRangerAuthorizerSASLSSLTest.java b/plugin-kafka/src/test/java/org/apache/ranger/authorization/kafka/authorizer/KafkaRangerAuthorizerSASLSSLTest.java index cef9f124ac..22dce3a993 100644 --- a/plugin-kafka/src/test/java/org/apache/ranger/authorization/kafka/authorizer/KafkaRangerAuthorizerSASLSSLTest.java +++ b/plugin-kafka/src/test/java/org/apache/ranger/authorization/kafka/authorizer/KafkaRangerAuthorizerSASLSSLTest.java @@ -56,7 +56,8 @@ * CustomAuthorizer that enforces some authorization rules: * * - The "IT" group can do anything - * - The "public" group can only "read/describe" on the "test" topic, not "write". + * - The "public" group can "read/describe/write" on the "test" topic. + * - The "public" group can only "read/describe" on the "dev" topic, but not write. * * Policies available from admin via: * @@ -64,6 +65,7 @@ * * Clients and services authenticate to Kafka using the SASL SSL protocol as part of this test. */ +@org.junit.Ignore("This is failing on some VMs") public class KafkaRangerAuthorizerSASLSSLTest { private static KafkaServerStartable kafkaServer; diff --git a/plugin-kafka/src/test/java/org/apache/ranger/authorization/kafka/authorizer/KafkaRangerAuthorizerTest.java b/plugin-kafka/src/test/java/org/apache/ranger/authorization/kafka/authorizer/KafkaRangerAuthorizerTest.java index 898c785bb3..abc03fa526 100644 --- a/plugin-kafka/src/test/java/org/apache/ranger/authorization/kafka/authorizer/KafkaRangerAuthorizerTest.java +++ b/plugin-kafka/src/test/java/org/apache/ranger/authorization/kafka/authorizer/KafkaRangerAuthorizerTest.java @@ -58,7 +58,8 @@ * CustomAuthorizer that enforces some authorization rules: * * - The "IT" group can do anything - * - The "public" group can only "read/describe" on the "test" topic, not "write". + * - The "public" group can "read/describe/write" on the "test" topic. + * - The "public" group can only "read/describe" on the "dev" topic, but not write. * * Policies available from admin via: * @@ -259,8 +260,8 @@ public void testAuthorizedWrite() throws Exception { producer.close(); } - - // The "public" group can't write to "test" or "dev" + + // The "public" group can write to "test" but not "dev" @Test public void testUnauthorizedWrite() throws Exception { // Create the Producer @@ -280,18 +281,13 @@ public void testUnauthorizedWrite() throws Exception { final Producer producer = new KafkaProducer<>(producerProps); // Send a message - try { - Future record = - producer.send(new ProducerRecord("test", "somekey", "somevalue")); - producer.flush(); - record.get(); - Assert.fail("Authorization failure expected"); - } catch (Exception ex) { - Assert.assertTrue(ex.getMessage().contains("Not authorized to access topics")); - } + Future record = + producer.send(new ProducerRecord("test", "somekey", "somevalue")); + producer.flush(); + record.get(); try { - Future record = + record = producer.send(new ProducerRecord("dev", "somekey", "somevalue")); producer.flush(); record.get(); diff --git a/plugin-kafka/src/test/resources/kafka-policies.json b/plugin-kafka/src/test/resources/kafka-policies.json index cc1c927ef8..dd9b802499 100644 --- a/plugin-kafka/src/test/resources/kafka-policies.json +++ b/plugin-kafka/src/test/resources/kafka-policies.json @@ -124,11 +124,37 @@ ], "users": [], "groups": [ - "IT" + "public" ], "conditions": [], "delegateAdmin": false - }, + } + ], + "denyPolicyItems": [], + "allowExceptions": [], + "denyExceptions": [], + "dataMaskPolicyItems": [], + "rowFilterPolicyItems": [], + "id": 19, + "isEnabled": true, + "version": 1 + }, + { + "service": "cl1_kafka", + "name": "DevPolicy", + "policyType": 0, + "description": "", + "isAuditEnabled": true, + "resources": { + "topic": { + "values": [ + "dev" + ], + "isExcludes": false, + "isRecursive": false + } + }, + "policyItems": [ { "accesses": [ { diff --git a/plugin-kms/src/main/java/org/apache/ranger/authorization/kms/authorizer/RangerKmsAuthorizer.java b/plugin-kms/src/main/java/org/apache/ranger/authorization/kms/authorizer/RangerKmsAuthorizer.java index 4cda8fa686..c3d75a14d1 100755 --- a/plugin-kms/src/main/java/org/apache/ranger/authorization/kms/authorizer/RangerKmsAuthorizer.java +++ b/plugin-kms/src/main/java/org/apache/ranger/authorization/kms/authorizer/RangerKmsAuthorizer.java @@ -27,6 +27,8 @@ import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; + +import org.apache.commons.logging.Log; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.crypto.key.kms.server.KMSACLsType; import org.apache.hadoop.crypto.key.kms.server.KMSConfiguration; @@ -46,6 +48,7 @@ import org.apache.ranger.plugin.policyengine.RangerAccessResourceImpl; import org.apache.ranger.plugin.policyengine.RangerAccessResult; import org.apache.ranger.plugin.service.RangerBasePlugin; +import org.apache.ranger.plugin.util.RangerPerfTracer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -53,6 +56,7 @@ public class RangerKmsAuthorizer implements Runnable, KeyACLs { private static final Logger LOG = LoggerFactory.getLogger(RangerKmsAuthorizer.class); + private static final Log PERF_KMSAUTH_REQUEST_LOG = RangerPerfTracer.getPerfLogger("kmsauth.request"); private static final String KMS_USER_PRINCIPAL = "ranger.ks.kerberos.principal"; private static final String KMS_USER_KEYTAB = "ranger.ks.kerberos.keytab"; @@ -200,6 +204,11 @@ public boolean hasAccess(Type type, UserGroupInformation ugi, String clientIp) { if(LOG.isDebugEnabled()) { LOG.debug("==> RangerKmsAuthorizer.hasAccess(" + type + ", " + ugi + ")"); } + RangerPerfTracer perf = null; + + if(RangerPerfTracer.isPerfTraceEnabled(PERF_KMSAUTH_REQUEST_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_KMSAUTH_REQUEST_LOG, "RangerKmsAuthorizer.hasAccess(type=" + type + ")"); + } boolean ret = false; RangerKMSPlugin plugin = kmsPlugin; String rangerAccessType = getRangerAccessType(type); @@ -215,7 +224,7 @@ public boolean hasAccess(Type type, UserGroupInformation ugi, String clientIp) { RangerAccessResult result = plugin.isAccessAllowed(request); ret = result == null ? false : result.getIsAllowed(); } - + RangerPerfTracer.log(perf); if(LOG.isDebugEnabled()) { LOG.debug("<== RangerkmsAuthorizer.hasAccess(" + type + ", " + ugi + "): " + ret); } diff --git a/plugin-solr/src/main/java/org/apache/ranger/authorization/solr/authorizer/RangerSolrAuthorizer.java b/plugin-solr/src/main/java/org/apache/ranger/authorization/solr/authorizer/RangerSolrAuthorizer.java index 6ac0a1f038..068ac27b3b 100644 --- a/plugin-solr/src/main/java/org/apache/ranger/authorization/solr/authorizer/RangerSolrAuthorizer.java +++ b/plugin-solr/src/main/java/org/apache/ranger/authorization/solr/authorizer/RangerSolrAuthorizer.java @@ -29,9 +29,12 @@ import java.util.Set; import javax.security.auth.login.Configuration; + +import org.apache.commons.collections.CollectionUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.ranger.audit.provider.AuditProviderFactory; import org.apache.ranger.audit.provider.MiscUtil; import org.apache.ranger.authorization.hadoop.config.RangerConfiguration; import org.apache.ranger.plugin.audit.RangerMultiResourceAuditHandler; @@ -39,6 +42,7 @@ import org.apache.ranger.plugin.policyengine.RangerAccessResourceImpl; import org.apache.ranger.plugin.policyengine.RangerAccessResult; import org.apache.ranger.plugin.service.RangerBasePlugin; +import org.apache.ranger.plugin.util.RangerPerfTracer; import org.apache.solr.security.AuthorizationContext.RequestType; import org.apache.solr.security.AuthorizationPlugin; import org.apache.solr.security.AuthorizationResponse; @@ -48,6 +52,7 @@ public class RangerSolrAuthorizer implements AuthorizationPlugin { private static final Log logger = LogFactory .getLog(RangerSolrAuthorizer.class); + private static final Log PERF_SOLRAUTH_REQUEST_LOG = RangerPerfTracer.getPerfLogger("solrauth.request"); public static final String PROP_USE_PROXY_IP = "xasecure.solr.use_proxy_ip"; public static final String PROP_PROXY_IP_HEADER = "xasecure.solr.proxy_ip_header"; @@ -144,6 +149,11 @@ public void close() throws IOException { logger.info("close() called"); try { solrPlugin.cleanup(); + /* Solr shutdown is not graceful so that JVM shutdown hooks + * are not always invoked and the audit store are not flushed. So + * we are forcing a cleanup here. + */ + AuditProviderFactory.getInstance().shutdown(); } catch (Throwable t) { logger.error("Error cleaning up Ranger plugin. Ignoring error", t); } @@ -167,6 +177,12 @@ public AuthorizationResponse authorize(AuthorizationContext context) { RangerMultiResourceAuditHandler auditHandler = new RangerMultiResourceAuditHandler(); + RangerPerfTracer perf = null; + + if(RangerPerfTracer.isPerfTraceEnabled(PERF_SOLRAUTH_REQUEST_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_SOLRAUTH_REQUEST_LOG, "RangerSolrAuthorizer.authorize()"); + } + String userName = getUserName(context); Set userGroups = getGroupsForUser(userName); String ip = null; @@ -180,23 +196,38 @@ public AuthorizationResponse authorize(AuthorizationContext context) { ip = context.getHttpHeader("REMOTE_ADDR"); } - // Create the list of requests for access check. Each field is - // broken - // into a request List rangerRequests = new ArrayList(); - for (CollectionRequest collectionRequest : context - .getCollectionRequests()) { + List collectionRequests = context.getCollectionRequests(); + if (CollectionUtils.isEmpty(collectionRequests)) { + // if Collection is empty we set the collection to *. This happens when LIST is done. RangerAccessRequestImpl requestForCollection = createRequest( userName, userGroups, ip, eventTime, context, - collectionRequest); + null); if (requestForCollection != null) { rangerRequests.add(requestForCollection); } + } else { + // Create the list of requests for access check. Each field is + // broken + // into a request + for (CollectionRequest collectionRequest : context + .getCollectionRequests()) { + + RangerAccessRequestImpl requestForCollection = createRequest( + userName, userGroups, ip, eventTime, context, + collectionRequest); + if (requestForCollection != null) { + rangerRequests.add(requestForCollection); + } + } + } + if (logger.isDebugEnabled()) { logger.debug("rangerRequests.size()=" + rangerRequests.size()); } + try { // Let's check the access for each request/resource for (RangerAccessRequestImpl rangerRequest : rangerRequests) { @@ -213,6 +244,7 @@ public AuthorizationResponse authorize(AuthorizationContext context) { } } finally { auditHandler.flushAudit(); + RangerPerfTracer.log(perf); } } catch (Throwable t) { isDenied = true; @@ -304,25 +336,19 @@ private RangerAccessRequestImpl createRequest(String userName, String accessType = mapToRangerAccessType(context); String action = accessType; - - if (collectionRequest.collectionName != null) { - RangerAccessRequestImpl rangerRequest = createBaseRequest(userName, - userGroups, ip, eventTime); - RangerAccessResourceImpl rangerResource = new RangerAccessResourceImpl(); - rangerResource.setValue(KEY_COLLECTION, - collectionRequest.collectionName); - rangerRequest.setResource(rangerResource); - rangerRequest.setAccessType(accessType); - rangerRequest.setAction(action); - - return rangerRequest; + RangerAccessRequestImpl rangerRequest = createBaseRequest(userName, + userGroups, ip, eventTime); + RangerAccessResourceImpl rangerResource = new RangerAccessResourceImpl(); + if (collectionRequest == null) { + rangerResource.setValue(KEY_COLLECTION, "*"); + } else { + rangerResource.setValue(KEY_COLLECTION, collectionRequest.collectionName); } - - logger.fatal("Can't create RangerRequest oject. userName=" - + userName + ", accessType=" + accessType + ", ip=" + ip - + ", collectionRequest=" + collectionRequest); + rangerRequest.setResource(rangerResource); + rangerRequest.setAccessType(accessType); + rangerRequest.setAction(action); - return null; + return rangerRequest; } private RangerAccessRequestImpl createBaseRequest(String userName, diff --git a/plugin-yarn/src/main/java/org/apache/ranger/authorization/yarn/authorizer/RangerYarnAuthorizer.java b/plugin-yarn/src/main/java/org/apache/ranger/authorization/yarn/authorizer/RangerYarnAuthorizer.java index c589060ebf..b4488304f7 100644 --- a/plugin-yarn/src/main/java/org/apache/ranger/authorization/yarn/authorizer/RangerYarnAuthorizer.java +++ b/plugin-yarn/src/main/java/org/apache/ranger/authorization/yarn/authorizer/RangerYarnAuthorizer.java @@ -44,6 +44,7 @@ import org.apache.ranger.plugin.service.RangerBasePlugin; import com.google.common.collect.Sets; +import org.apache.ranger.plugin.util.RangerPerfTracer; public class RangerYarnAuthorizer extends YarnAuthorizationProvider { public static final String ACCESS_TYPE_ADMIN_QUEUE = "admin-queue"; @@ -56,6 +57,8 @@ public class RangerYarnAuthorizer extends YarnAuthorizationProvider { private static final Log LOG = LogFactory.getLog(RangerYarnAuthorizer.class); + private static final Log PERF_YARNAUTH_REQUEST_LOG = RangerPerfTracer.getPerfLogger("yarnauth.request"); + private static volatile RangerYarnPlugin yarnPlugin = null; private AccessControlList admins = null; @@ -101,7 +104,15 @@ public boolean checkPermission(AccessType accessType, PrivilegedEntity entity, U RangerAccessResult result = null; String clusterName = yarnPlugin.getClusterName(); + RangerPerfTracer perf = null; + RangerPerfTracer yarnAclPerf = null; + if(plugin != null) { + + if(RangerPerfTracer.isPerfTraceEnabled(PERF_YARNAUTH_REQUEST_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_YARNAUTH_REQUEST_LOG, "RangerYarnAuthorizer.checkPermission(entity=" + entity + ")"); + } + RangerYarnAccessRequest request = new RangerYarnAccessRequest(entity, getRangerAccessType(accessType), accessType.name(), ugi, clusterName); auditHandler = new RangerYarnAuditHandler(); @@ -110,6 +121,11 @@ public boolean checkPermission(AccessType accessType, PrivilegedEntity entity, U } if(RangerYarnAuthorizer.yarnAuthEnabled && (result == null || !result.getIsAccessDetermined())) { + + if(RangerPerfTracer.isPerfTraceEnabled(PERF_YARNAUTH_REQUEST_LOG)) { + yarnAclPerf = RangerPerfTracer.getPerfTracer(PERF_YARNAUTH_REQUEST_LOG, "RangerYarnNativeAuthorizer.isAllowedByYarnAcl(entity=" + entity + ")"); + } + ret = isAllowedByYarnAcl(accessType, entity, ugi, auditHandler); } else { ret = result == null ? false : result.getIsAllowed(); @@ -119,6 +135,10 @@ public boolean checkPermission(AccessType accessType, PrivilegedEntity entity, U auditHandler.flushAudit(); } + RangerPerfTracer.log(yarnAclPerf); + + RangerPerfTracer.log(perf); + if(LOG.isDebugEnabled()) { LOG.debug("<== RangerYarnAuthorizer.checkPermission(" + accessType + ", " + toString(entity) + ", " + ugi + "): " + ret); } diff --git a/pom.xml b/pom.xml index 91e4434ed8..787e7d1a78 100644 --- a/pom.xml +++ b/pom.xml @@ -125,18 +125,18 @@ 3.2 3.1 1.8.2 - 0.7-incubating + 0.8.2 14.0 2.5 1.3.7 + 1.4.1 1.1.3 1.7.1 1.55 - 0.9.1.2 + 0.9.5.2 2.2.0-b23 1.9.13 - 1.8.3 - 1.8.3 + 1.9.3 1.2 1.9 3.2.2 @@ -194,11 +194,11 @@ 3.0.2 1.8.4 5.1.31 - 3.6.2.Final + 3.10.5.Final 0.6 r239 2.3 - 3.12 + 3.17 UTF-8 2.5.0 2.11.8 @@ -209,7 +209,7 @@ 1.7.5 1.0.4.1 5.5.1 - 1.3.1.RELEASE + 2.3.2.RELEASE 3.2.10.RELEASE 3.2.10.RELEASE 3.2.10.RELEASE @@ -219,7 +219,7 @@ 1.4 1.4 0.5.2 - 7.0.77 + 7.0.82 1.7 0.52 1.0 @@ -353,6 +353,21 @@ gson ${gson.version} + + com.webcohesion.enunciate + enunciate-core-annotations + 2.8.0 + + + commons-beanutils + commons-beanutils + ${commons.beanutils.version} + + + com.sun.jersey.contribs + jersey-multipart + ${jersey-bundle.version} + @@ -475,6 +490,26 @@ maven-release-plugin 2.5.2 + + com.webcohesion.enunciate + enunciate-maven-plugin + 2.8.0 + + enunciate.xml + + docs/src/site/ + 1.8 + 1.8 + + + + + docs + + package + + + @@ -576,6 +611,8 @@ **/.pydevproject **/derby.log **/*.jaas + **/target/apidocs/** + **/target/kms/apidocs/** diff --git a/ranger-tools/conf/log4j.properties b/ranger-tools/conf/log4j.properties index 4ead8023d5..e95a6c8ea3 100644 --- a/ranger-tools/conf/log4j.properties +++ b/ranger-tools/conf/log4j.properties @@ -31,6 +31,7 @@ log4j.appender.console.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2}: %m% ranger.perf.logger=DEBUG,PERF ranger.perf.log.file=ranger-perf-test.log +log4j.logger.org.apache.ranger.plugin.util.PerfDataRecorder=${ranger.perf.logger} log4j.logger.org.apache.ranger.perf=${ranger.perf.logger} log4j.additivity.org.apache.ranger.perf=false diff --git a/ranger-tools/scripts/README.txt b/ranger-tools/scripts/README.txt index 55170d55b4..81b5b66781 100644 --- a/ranger-tools/scripts/README.txt +++ b/ranger-tools/scripts/README.txt @@ -51,6 +51,8 @@ This file describes how to build, setup, configure and run the performance testi tag-definitions, and service-resources with their associated tags; testdata/test_requests_hive.json - Contains access requests to be made to the policy-engine; + + testdata/ranger-config.xml - Contains any required Ranger configuration variables Please review the contents of these files and modify to suit your profiling needs. @@ -58,10 +60,13 @@ This file describes how to build, setup, configure and run the performance testi 6. Run the tool with the following command - % ./ranger-perftester.sh -s -r -c -n + % ./ranger-perftester.sh -s -r -c -n -t -d -f -p + + where, -t indicates enabling Trie, + -d indicates enabling lazy post-setup of Trie structure, Example: - % ./ranger-perftester.sh -s testdata/test_servicepolicies_hive.json -r testdata/test_requests_hive.json -c 2 -n 1 + % ./ranger-perftester.sh -s testdata/test_servicepolicies_hive.json -r testdata/test_requests_hive.json -c 2 -n 1 -t -d -f testdata/ranger-config.xml -p testdata/test_modules.txt 7. At the end of the run, the performance-statistics are printed on the console and in the log specified file in conf/log4j.properties file as shown below. This is for time spent in evaluating access by Ranger Policy Engine during the course of a test run. The time values shown are in milliseconds. diff --git a/ranger-tools/src/main/java/org/apache/ranger/policyengine/CommandLineParser.java b/ranger-tools/src/main/java/org/apache/ranger/policyengine/CommandLineParser.java index e8edd9eeb8..fe9ec179b3 100644 --- a/ranger-tools/src/main/java/org/apache/ranger/policyengine/CommandLineParser.java +++ b/ranger-tools/src/main/java/org/apache/ranger/policyengine/CommandLineParser.java @@ -55,6 +55,12 @@ public class CommandLineParser private boolean isDynamicReorderingDisabled = true; private boolean isTrieLookupPrefixDisabled = true; + private boolean isLazyTriePostSetupDisabled = true; + + private String configurationFileName; + private URL configurationFileURL; + + private Options options = new Options(); CommandLineParser() {} @@ -63,7 +69,7 @@ final PerfTestOptions parse(final String[] args) { PerfTestOptions ret = null; if (parseArguments(args) && validateInputFiles()) { // Instantiate a data-object and return - ret = new PerfTestOptions(servicePoliciesFileURL, requestFileURLs, statCollectionFileURL, concurrentClientCount, iterationsCount, isDynamicReorderingDisabled, isTrieLookupPrefixDisabled); + ret = new PerfTestOptions(servicePoliciesFileURL, requestFileURLs, statCollectionFileURL, concurrentClientCount, iterationsCount, isDynamicReorderingDisabled, isTrieLookupPrefixDisabled, isLazyTriePostSetupDisabled, configurationFileURL); } else { showUsage(); } @@ -78,7 +84,10 @@ final PerfTestOptions parse(final String[] args) { -r request-file-name-list -n number-of-iterations -p modules-to-collect-stats + -f configuration-file-name -o + -t + -d If the concurrent-client-count is more than the number of files in the request-file-name-list, then reuse the request-file-names in a round-robin way @@ -98,8 +107,11 @@ final boolean parseArguments(final String[] args) { options.addOption("p", "statistics", true, "Modules for stat collection File Name"); options.addOption("c", "clients", true, "Number of concurrent clients"); options.addOption("n", "cycles", true, "Number of iterations"); + options.addOption("f", "configurations", true, "Configuration File Name"); options.addOption("o", "optimize", false, "Enable usage-based policy reordering"); options.addOption("t", "trie-prefilter", false, "Enable trie-prefilter"); + options.addOption("d", "trie-lazy-setup", false, "Enable lazy trie-setup"); + org.apache.commons.cli.CommandLineParser commandLineParser = new DefaultParser(); @@ -133,11 +145,20 @@ final boolean parseArguments(final String[] args) { isTrieLookupPrefixDisabled = false; } + if (commandLine.hasOption("d")) { + isLazyTriePostSetupDisabled = false; + } + + configurationFileName = commandLine.getOptionValue("f"); + if (LOG.isDebugEnabled()) { LOG.debug("servicePoliciesFileName=" + servicePoliciesFileName + ", requestFileName=" + Arrays.toString(requestFileNames)); LOG.debug("concurrentClientCount=" + concurrentClientCount + ", iterationsCount=" + iterationsCount); LOG.debug("isDynamicReorderingDisabled=" + isDynamicReorderingDisabled); LOG.debug("isTrieLookupPrefixDisabled=" + isTrieLookupPrefixDisabled); + LOG.debug("isLazyTriePostSetupDisabled=" + isLazyTriePostSetupDisabled); + LOG.debug("configurationFileName=" + configurationFileName); + } ret = true; @@ -164,11 +185,14 @@ final boolean validateInputFiles() { if (servicePoliciesFileURL != null) { if (requestFileNames != null) { if (validateRequestFiles()) { + ret = true; if (statCollectionFileName != null) { statCollectionFileURL = getInputFileURL(statCollectionFileName); ret = statCollectionFileURL != null; - } else { - ret = true; + } + if (ret && configurationFileName != null) { + configurationFileURL = getInputFileURL(configurationFileName); + ret = configurationFileURL != null; } } } else { diff --git a/ranger-tools/src/main/java/org/apache/ranger/policyengine/PerfTestOptions.java b/ranger-tools/src/main/java/org/apache/ranger/policyengine/PerfTestOptions.java index d6e04eaadf..1486f52d8d 100644 --- a/ranger-tools/src/main/java/org/apache/ranger/policyengine/PerfTestOptions.java +++ b/ranger-tools/src/main/java/org/apache/ranger/policyengine/PerfTestOptions.java @@ -28,12 +28,15 @@ public class PerfTestOptions { private final URL statCollectionFileURL; private final boolean isDynamicReorderingDisabled; private final boolean isTrieLookupPrefixDisabled; + private final boolean isOnDemandTriePostSetupDisabled; private final int concurrentClientCount; private final int iterationsCount; + private final URL perfConfigurationFileURL; - PerfTestOptions(URL servicePoliciesFileURL, URL[] requestFileURLs, URL statCollectionFileURL, int concurrentClientCount, int iterationsCount, boolean isDynamicReorderingDisabled, boolean isTrieLookupPrefixDisabled) { + + PerfTestOptions(URL servicePoliciesFileURL, URL[] requestFileURLs, URL statCollectionFileURL, int concurrentClientCount, int iterationsCount, boolean isDynamicReorderingDisabled, boolean isTrieLookupPrefixDisabled, boolean isOnDemandTriePostSetupDisabled, URL perfConfigurationFileURL) { this.servicePoliciesFileURL = servicePoliciesFileURL; this.requestFileURLs = requestFileURLs; this.statCollectionFileURL = statCollectionFileURL; @@ -41,6 +44,8 @@ public class PerfTestOptions { this.concurrentClientCount = concurrentClientCount; this.isDynamicReorderingDisabled = isDynamicReorderingDisabled; this.isTrieLookupPrefixDisabled = isTrieLookupPrefixDisabled; + this.isOnDemandTriePostSetupDisabled = isOnDemandTriePostSetupDisabled; + this.perfConfigurationFileURL = perfConfigurationFileURL; } public URL getServicePoliciesFileURL() { @@ -66,4 +71,11 @@ public int getIterationsCount() { public boolean getIsDynamicReorderingDisabled() { return isDynamicReorderingDisabled; } public boolean getIsTrieLookupPrefixDisabled() { return isTrieLookupPrefixDisabled; } + + public boolean getIsOnDemandTriePostSetupDisabled() { return isOnDemandTriePostSetupDisabled; } + + public URL getPerfConfigurationFileURL() { + return this.perfConfigurationFileURL; + } + } diff --git a/ranger-tools/src/main/java/org/apache/ranger/policyengine/RangerPolicyenginePerfTester.java b/ranger-tools/src/main/java/org/apache/ranger/policyengine/RangerPolicyenginePerfTester.java index 056c548703..53e5cf023d 100644 --- a/ranger-tools/src/main/java/org/apache/ranger/policyengine/RangerPolicyenginePerfTester.java +++ b/ranger-tools/src/main/java/org/apache/ranger/policyengine/RangerPolicyenginePerfTester.java @@ -21,6 +21,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.ranger.authorization.hadoop.config.RangerConfiguration; import org.apache.ranger.plugin.policyengine.RangerPolicyEngineOptions; import org.apache.ranger.plugin.policyevaluator.RangerPolicyEvaluator; import org.apache.ranger.plugin.util.PerfDataRecorder; @@ -59,7 +60,16 @@ public static void main(String[] args) { RangerPolicyEngineOptions policyEngineOptions = new RangerPolicyEngineOptions(); policyEngineOptions.disableTagPolicyEvaluation = false; policyEngineOptions.evaluatorType = RangerPolicyEvaluator.EVALUATOR_TYPE_OPTIMIZED; + policyEngineOptions.cacheAuditResults = false; policyEngineOptions.disableTrieLookupPrefilter = perfTestOptions.getIsTrieLookupPrefixDisabled(); + policyEngineOptions.optimizeTrieForRetrieval = perfTestOptions.getIsOnDemandTriePostSetupDisabled(); + + URL configurationFileURL = perfTestOptions.getPerfConfigurationFileURL(); + + if (configurationFileURL != null) { + RangerConfiguration config = RangerConfiguration.getInstance(); + config.addResource(configurationFileURL); + } PerfTestEngine perfTestEngine = new PerfTestEngine(servicePoliciesFileURL, policyEngineOptions, perfTestOptions.getIsDynamicReorderingDisabled()); if (!perfTestEngine.init()) { diff --git a/ranger-tools/src/test/resources/commandline b/ranger-tools/src/test/resources/commandline index 9ea690e87f..73338d5e03 100644 --- a/ranger-tools/src/test/resources/commandline +++ b/ranger-tools/src/test/resources/commandline @@ -17,4 +17,4 @@ # under the License. # --s /testdata/test_servicepolicies_hive.json -r /testdata/test_requests_hive.json -p /testdata/test_modules.txt -c 3 -n 1 +-s /testdata/test_servicepolicies_hive.json -r /testdata/test_requests_hive.json -p /testdata/test_modules.txt -c 3 -n 1 -t -d -f /testdata/ranger-config.xml diff --git a/ranger-tools/src/test/resources/testdata/ranger-config.xml b/ranger-tools/src/test/resources/testdata/ranger-config.xml new file mode 100644 index 0000000000..933d6705a8 --- /dev/null +++ b/ranger-tools/src/test/resources/testdata/ranger-config.xml @@ -0,0 +1,18 @@ + + + + + + ranger.policyengine.trie.builder.thread.count + 1 + + \ No newline at end of file diff --git a/ranger-tools/testdata/ranger-config.xml b/ranger-tools/testdata/ranger-config.xml new file mode 100644 index 0000000000..933d6705a8 --- /dev/null +++ b/ranger-tools/testdata/ranger-config.xml @@ -0,0 +1,18 @@ + + + + + + ranger.policyengine.trie.builder.thread.count + 1 + + \ No newline at end of file diff --git a/security-admin/contrib/solr_for_audit_setup/conf/managed-schema b/security-admin/contrib/solr_for_audit_setup/conf/managed-schema index ee1d894059..6c87af7cf9 100644 --- a/security-admin/contrib/solr_for_audit_setup/conf/managed-schema +++ b/security-admin/contrib/solr_for_audit_setup/conf/managed-schema @@ -15,29 +15,30 @@ See the License for the specific language governing permissions and limitations under the License. --> - + id - - - + + + - + + - + - - - - + + + + @@ -49,15 +50,15 @@ - - - - - - + + + + + + - + @@ -89,4 +90,4 @@ - + \ No newline at end of file diff --git a/security-admin/db/mysql/patches/028-delete-xgroup-duplicate-references.sql b/security-admin/db/mysql/patches/028-delete-xgroup-duplicate-references.sql new file mode 100644 index 0000000000..811c5bb2fb --- /dev/null +++ b/security-admin/db/mysql/patches/028-delete-xgroup-duplicate-references.sql @@ -0,0 +1,96 @@ +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +-- -------------------------------------------------------------------------------- +-- Procedure which shall remove duplicate entries from x_group table +-- Duplicate entries were previously created due to unavailablity of unique index +-- -------------------------------------------------------------------------------- +DELIMITER $$ + +DROP PROCEDURE if exists deleteXGroupDuplicateReferences $$ +CREATE PROCEDURE `deleteXGroupDuplicateReferences`() +BEGIN +Block1: BEGIN + +DECLARE donecursor1 INT; +DECLARE group_name1 varchar(1024); +DECLARE mingroupid1 bigint; +DECLARE id2 bigint; + +DECLARE cursor1 CURSOR FOR + SELECT group_name,min(id) FROM x_group GROUP BY group_name HAVING count(group_name)>1; + +DECLARE CONTINUE HANDLER FOR NOT FOUND SET donecursor1 = 1; + OPEN cursor1; + REPEAT + FETCH cursor1 into group_name1, mingroupid1; + Block2: BEGIN + DECLARE donecursor2 INT DEFAULT 0; + DECLARE cursor2 CURSOR FOR SELECT id FROM x_group WHERE group_name= group_name1 AND id > mingroupid1; + DECLARE CONTINUE HANDLER FOR NOT FOUND SET donecursor2 = 1; + OPEN cursor2; + REPEAT + FETCH cursor2 INTO id2; + UPDATE x_group_users SET p_group_id=mingroupid1 where p_group_id=id2; + UNTIL donecursor2 END REPEAT; + CLOSE cursor2; + END Block2; + UNTIL donecursor1 END REPEAT; + CLOSE cursor1; +END Block1; + +Block3: BEGIN + +DECLARE donecursor3 INT; +DECLARE group_name3 varchar(1024); +DECLARE user_id3 bigint; +DECLARE minrowid3 bigint; + +DECLARE cursor3 CURSOR FOR + SELECT group_name,user_id,min(id) FROM x_group_users GROUP BY group_name,user_id HAVING count(1)>1; + +DECLARE CONTINUE HANDLER FOR NOT FOUND SET donecursor3 = 1; + OPEN cursor3; + REPEAT + FETCH cursor3 into group_name3, user_id3, minrowid3; + DELETE FROM x_group_users WHERE group_name=group_name3 AND user_id=user_id3 AND id > minrowid3; + UNTIL donecursor3 END REPEAT; + CLOSE cursor3; +END Block3; + +Block4: BEGIN + +DECLARE donecursor4 INT; +DECLARE group_name4 varchar(1024); +DECLARE group_id4 bigint; +DECLARE minrowid4 bigint; + +DECLARE cursor4 CURSOR FOR + SELECT group_name,min(id) FROM x_group GROUP BY group_name HAVING count(1)>1; + +DECLARE CONTINUE HANDLER FOR NOT FOUND SET donecursor4 = 1; + OPEN cursor4; + REPEAT + FETCH cursor4 into group_name4, minrowid4; + DELETE FROM x_group WHERE group_name=group_name4 AND id > minrowid4; + UNTIL donecursor4 END REPEAT; + CLOSE cursor4; +END Block4; + +END $$ +DELIMITER ; +call deleteXGroupDuplicateReferences(); + +DROP PROCEDURE if exists deleteXGroupDuplicateReferences; diff --git a/security-admin/db/mysql/patches/029-add-unique-constraint-on-table-x_group.sql b/security-admin/db/mysql/patches/029-add-unique-constraint-on-table-x_group.sql new file mode 100644 index 0000000000..078fb9900f --- /dev/null +++ b/security-admin/db/mysql/patches/029-add-unique-constraint-on-table-x_group.sql @@ -0,0 +1,45 @@ +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +drop procedure if exists create_unique_constraint_on_groupname; + +delimiter ;; +create procedure create_unique_constraint_on_groupname() begin +DECLARE loginID bigint(20); + /* check tables exist or not */ + if exists (select * from information_schema.columns where table_schema=database() and table_name = 'x_group' and column_name='group_name') then + /* check unique constraint exist on group_name column or not */ + if not exists (select * from information_schema.columns where table_schema=database() and table_name = 'x_group' and column_name='group_name' and column_key='UNI') then + if not exists (select * from information_schema.table_constraints where table_schema=database() and table_name = 'x_group' and constraint_name='x_group_UK_group_name') then + ALTER TABLE x_group ADD UNIQUE INDEX x_group_UK_group_name(group_name(767)); +-- ALTER TABLE x_group MODIFY COLUMN group_name varchar(767) NOT NULL, ADD CONSTRAINT x_group_UK_group_name UNIQUE(group_name(767)); + end if; + end if; + end if; + if exists (select * from information_schema.columns where table_schema=database() and table_name = 'x_group_users' and column_name='group_name') then + if exists (select * from information_schema.columns where table_schema=database() and table_name = 'x_group_users' and column_name='user_id') then + /* check unique constraint exist on group_name column or not */ + if not exists (select * from information_schema.table_constraints where table_schema=database() and table_name = 'x_group_users' and constraint_name='x_group_users_UK_uid_gname') then + ALTER TABLE x_group_users ADD UNIQUE INDEX x_group_users_UK_uid_gname(user_id,group_name(740)); +-- ALTER TABLE x_group_users MODIFY COLUMN group_name varchar(767), ADD CONSTRAINT x_group_users_UK_uid_gname UNIQUE(user_id,group_name(767)); + end if; + end if; + end if; +end;; + +delimiter ; +call create_unique_constraint_on_groupname(); + +drop procedure if exists create_unique_constraint_on_groupname; diff --git a/security-admin/db/mysql/patches/033-add-unique-constraint-on-table-x_policy.sql b/security-admin/db/mysql/patches/033-add-unique-constraint-on-table-x_policy.sql new file mode 100644 index 0000000000..8deb285f7d --- /dev/null +++ b/security-admin/db/mysql/patches/033-add-unique-constraint-on-table-x_policy.sql @@ -0,0 +1,35 @@ +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +drop procedure if exists create_unique_constraint_on_name_service; + +delimiter ;; +create procedure create_unique_constraint_on_name_service() begin + /* check tables exist or not */ + if exists (select * from information_schema.columns where table_schema=database() and table_name = 'x_policy' and column_name in('service','name')) then + /* check unique constraint exist on service and name column or not */ + if not exists (select * from information_schema.columns where table_schema=database() and table_name = 'x_policy' and column_name in('name') and column_key in('UNI','MUL')) then + if not exists (select * from information_schema.table_constraints where table_schema=database() and table_name = 'x_policy' and constraint_name='x_policy_UK_name_service') then + UPDATE x_policy set name=concat(name,'-duplicate-',id) where id in (select id from (select id from x_policy where concat(service,name) in (select concat(service,name) from x_policy group by service,name having count(*) >1)) as tmp); + ALTER TABLE x_policy ADD UNIQUE INDEX x_policy_UK_name_service(name(180),service); + end if; + end if; + end if; +end;; + +delimiter ; +call create_unique_constraint_on_name_service(); + +drop procedure if exists create_unique_constraint_on_name_service; diff --git a/security-admin/db/mysql/patches/035-update-schema-for-x-policy.sql b/security-admin/db/mysql/patches/035-update-schema-for-x-policy.sql new file mode 100644 index 0000000000..05bd850f2d --- /dev/null +++ b/security-admin/db/mysql/patches/035-update-schema-for-x-policy.sql @@ -0,0 +1,199 @@ +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +drop procedure if exists alter_table_x_policy; + +delimiter ;; +create procedure alter_table_x_policy() begin + +if exists (select * from information_schema.columns where table_schema=database() and table_name = 'x_policy') then + if not exists (select * from information_schema.columns where table_schema=database() and table_name = 'x_policy' and column_name = 'policy_text') then + ALTER TABLE `x_policy` ADD `policy_text` MEDIUMTEXT DEFAULT NULL; + end if; + end if; +end;; + +delimiter ; +call alter_table_x_policy(); + +drop procedure if exists alter_table_x_policy; + +DROP PROCEDURE IF EXISTS removeConstraints; +DELIMITER ;; +CREATE PROCEDURE removeConstraints(vTableName varchar(128)) +BEGIN + DECLARE done INT DEFAULT FALSE; + DECLARE cName VARCHAR(64); + DECLARE cur CURSOR FOR + SELECT DISTINCT CONSTRAINT_NAME + FROM INFORMATION_SCHEMA.Key_COLUMN_USAGE + WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME = vTableName + AND REFERENCED_TABLE_NAME IS NOT NULL; + DECLARE CONTINUE HANDLER FOR NOT FOUND SET done = TRUE; + + SET AUTOCOMMIT=0; + SET FOREIGN_KEY_CHECKS=0; + + OPEN cur; + + read_loop: LOOP + FETCH cur INTO cName; + IF done THEN + LEAVE read_loop; + END IF; + SET @sql = CONCAT('ALTER TABLE ',vTableName,' DROP FOREIGN KEY ',cName,';'); + PREPARE stmt FROM @sql; + EXECUTE stmt; + DEALLOCATE PREPARE stmt; + END LOOP; + + CLOSE cur; + + SET FOREIGN_KEY_CHECKS=1; + COMMIT; + SET AUTOCOMMIT=1; +END ;; +DELIMITER ; + +call removeConstraints('x_policy_item'); +call removeConstraints('x_policy_item_access'); +call removeConstraints('x_policy_item_condition'); +call removeConstraints('x_policy_item_datamask'); +call removeConstraints('x_policy_item_group_perm'); +call removeConstraints('x_policy_item_user_perm'); +call removeConstraints('x_policy_item_rowfilter'); +call removeConstraints('x_policy_resource'); +call removeConstraints('x_policy_resource_map'); + +DROP PROCEDURE removeConstraints; + +DROP TABLE IF EXISTS `x_policy_ref_resource`; +CREATE TABLE IF NOT EXISTS `x_policy_ref_resource` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `guid` varchar(1024) DEFAULT NULL, + `create_time` datetime DEFAULT NULL, + `update_time` datetime DEFAULT NULL, + `added_by_id` bigint(20) DEFAULT NULL, + `upd_by_id` bigint(20) DEFAULT NULL, + `policy_id` bigint(20) NOT NULL, + `resource_def_id` bigint(20) NOT NULL, + `resource_name` varchar(4000) DEFAULT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `x_policy_ref_res_UK_polId_resDefId`(`policy_id`, `resource_def_id`), + CONSTRAINT `x_policy_ref_res_FK_added_by_id` FOREIGN KEY (`added_by_id`) REFERENCES `x_portal_user` (`id`), + CONSTRAINT `x_policy_ref_res_FK_upd_by_id` FOREIGN KEY (`upd_by_id`) REFERENCES `x_portal_user` (`id`), + CONSTRAINT `x_policy_ref_res_FK_policy_id` FOREIGN KEY (`policy_id`) REFERENCES `x_policy` (`id`), + CONSTRAINT `x_policy_ref_res_FK_resource_def_id` FOREIGN KEY (`resource_def_id`) REFERENCES `x_resource_def` (`id`) + +) ROW_FORMAT=DYNAMIC; + + +DROP TABLE IF EXISTS `x_policy_ref_access_type`; +CREATE TABLE IF NOT EXISTS `x_policy_ref_access_type` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `guid` varchar(1024) DEFAULT NULL, + `create_time` datetime DEFAULT NULL, + `update_time` datetime DEFAULT NULL, + `added_by_id` bigint(20) DEFAULT NULL, + `upd_by_id` bigint(20) DEFAULT NULL, + `policy_id` bigint(20) NOT NULL, + `access_def_id` bigint(20) NOT NULL, + `access_type_name` varchar(4000) DEFAULT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `x_policy_ref_access_UK_polId_accessDefId`(`policy_id`, `access_def_id`), + CONSTRAINT `x_policy_ref_access_FK_added_by_id` FOREIGN KEY (`added_by_id`) REFERENCES `x_portal_user` (`id`), + CONSTRAINT `x_policy_ref_access_FK_upd_by_id` FOREIGN KEY (`upd_by_id`) REFERENCES `x_portal_user` (`id`), + CONSTRAINT `x_policy_ref_access_FK_policy_id` FOREIGN KEY (`policy_id`) REFERENCES `x_policy` (`id`), + CONSTRAINT `x_policy_ref_access_FK_access_def_id` FOREIGN KEY (`access_def_id`) REFERENCES `x_access_type_def` (`id`) +) ROW_FORMAT=DYNAMIC; + + +DROP TABLE IF EXISTS `x_policy_ref_condition`; +CREATE TABLE IF NOT EXISTS `x_policy_ref_condition` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `guid` varchar(1024) DEFAULT NULL, + `create_time` datetime DEFAULT NULL, + `update_time` datetime DEFAULT NULL, + `added_by_id` bigint(20) DEFAULT NULL, + `upd_by_id` bigint(20) DEFAULT NULL, + `policy_id` bigint(20) NOT NULL, + `condition_def_id` bigint(20) NOT NULL, + `condition_name` varchar(4000) DEFAULT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `x_policy_ref_condition_UK_polId_condDefId`(`policy_id`, `condition_def_id`), + CONSTRAINT `x_policy_ref_condition_FK_added_by_id` FOREIGN KEY (`added_by_id`) REFERENCES `x_portal_user` (`id`), + CONSTRAINT `x_policy_ref_condition_FK_upd_by_id` FOREIGN KEY (`upd_by_id`) REFERENCES `x_portal_user` (`id`), + CONSTRAINT `x_policy_ref_condition_FK_policy_id` FOREIGN KEY (`policy_id`) REFERENCES `x_policy` (`id`), + CONSTRAINT `x_policy_ref_condition_FK_condition_def_id` FOREIGN KEY (`condition_def_id`) REFERENCES `x_policy_condition_def` (`id`) + +) ROW_FORMAT=DYNAMIC; + +DROP TABLE IF EXISTS `x_policy_ref_datamask_type`; +CREATE TABLE IF NOT EXISTS `x_policy_ref_datamask_type` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `guid` varchar(1024) DEFAULT NULL, + `create_time` datetime DEFAULT NULL, + `update_time` datetime DEFAULT NULL, + `added_by_id` bigint(20) DEFAULT NULL, + `upd_by_id` bigint(20) DEFAULT NULL, + `policy_id` bigint(20) NOT NULL, + `datamask_def_id` bigint(20) NOT NULL, + `datamask_type_name` varchar(4000) DEFAULT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `x_policy_ref_datamask_UK_polId_dmaskDefId`(`policy_id`, `datamask_def_id`), + CONSTRAINT `x_policy_ref_datamask_FK_added_by_id` FOREIGN KEY (`added_by_id`) REFERENCES `x_portal_user` (`id`), + CONSTRAINT `x_policy_ref_datamask_FK_upd_by_id` FOREIGN KEY (`upd_by_id`) REFERENCES `x_portal_user` (`id`), + CONSTRAINT `x_policy_ref_datamask_FK_policy_id` FOREIGN KEY (`policy_id`) REFERENCES `x_policy` (`id`), + CONSTRAINT `x_policy_ref_datamask_FK_datamask_def_id` FOREIGN KEY (`datamask_def_id`) REFERENCES `x_datamask_type_def` (`id`) +) ROW_FORMAT=DYNAMIC; + +DROP TABLE IF EXISTS `x_policy_ref_user`; +CREATE TABLE IF NOT EXISTS `x_policy_ref_user` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `guid` varchar(1024) DEFAULT NULL, + `create_time` datetime DEFAULT NULL, + `update_time` datetime DEFAULT NULL, + `added_by_id` bigint(20) DEFAULT NULL, + `upd_by_id` bigint(20) DEFAULT NULL, + `policy_id` bigint(20) NOT NULL, + `user_id` bigint(20) NOT NULL, + `user_name` varchar(4000) DEFAULT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `x_policy_ref_user_UK_polId_userId`(`policy_id`, `user_id`), + CONSTRAINT `x_policy_ref_user_FK_added_by_id` FOREIGN KEY (`added_by_id`) REFERENCES `x_portal_user` (`id`), + CONSTRAINT `x_policy_ref_user_FK_upd_by_id` FOREIGN KEY (`upd_by_id`) REFERENCES `x_portal_user` (`id`), + CONSTRAINT `x_policy_ref_user_FK_policy_id` FOREIGN KEY (`policy_id`) REFERENCES `x_policy` (`id`), + CONSTRAINT `x_policy_ref_user_FK_user_id` FOREIGN KEY (`user_id`) REFERENCES `x_user` (`id`) +) ROW_FORMAT=DYNAMIC; + + +DROP TABLE IF EXISTS `x_policy_ref_group`; +CREATE TABLE IF NOT EXISTS `x_policy_ref_group` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `guid` varchar(1024) DEFAULT NULL, + `create_time` datetime DEFAULT NULL, + `update_time` datetime DEFAULT NULL, + `added_by_id` bigint(20) DEFAULT NULL, + `upd_by_id` bigint(20) DEFAULT NULL, + `policy_id` bigint(20) NOT NULL, + `group_id` bigint(20) NOT NULL, + `group_name` varchar(4000) DEFAULT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `x_policy_ref_group_UK_polId_groupId`(`policy_id`, `group_id`), + CONSTRAINT `x_policy_ref_group_FK_added_by_id` FOREIGN KEY (`added_by_id`) REFERENCES `x_portal_user` (`id`), + CONSTRAINT `x_policy_ref_group_FK_upd_by_id` FOREIGN KEY (`upd_by_id`) REFERENCES `x_portal_user` (`id`), + CONSTRAINT `x_policy_ref_group_FK_policy_id` FOREIGN KEY (`policy_id`) REFERENCES `x_policy` (`id`), + CONSTRAINT `x_policy_ref_group_FK_group_id` FOREIGN KEY (`group_id`) REFERENCES `x_group` (`id`) +) ROW_FORMAT=DYNAMIC; diff --git a/security-admin/db/mysql/patches/036-denormalize-tag-tables.sql b/security-admin/db/mysql/patches/036-denormalize-tag-tables.sql new file mode 100644 index 0000000000..63035bc407 --- /dev/null +++ b/security-admin/db/mysql/patches/036-denormalize-tag-tables.sql @@ -0,0 +1,82 @@ +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +drop procedure if exists denormalize_tag_tables; + +delimiter ;; +create procedure denormalize_tag_tables() begin + +if not exists (select * from information_schema.columns where table_schema=database() and table_name = 'x_tag_def' and column_name='tag_attrs_def_text') then + ALTER TABLE x_tag_def ADD tag_attrs_def_text MEDIUMTEXT NULL DEFAULT NULL; +end if; +if not exists (select * from information_schema.columns where table_schema=database() and table_name = 'x_tag' and column_name='tag_attrs_text') then + ALTER TABLE x_tag ADD tag_attrs_text MEDIUMTEXT NULL DEFAULT NULL; +end if; +if not exists (select * from information_schema.columns where table_schema=database() and table_name = 'x_service_resource' and column_name='service_resource_elements_text') then + ALTER TABLE x_service_resource ADD service_resource_elements_text MEDIUMTEXT NULL DEFAULT NULL; +end if; +if not exists (select * from information_schema.columns where table_schema=database() and table_name = 'x_service_resource' and column_name='tags_text') then + ALTER TABLE x_service_resource ADD tags_text MEDIUMTEXT NULL DEFAULT NULL; +end if; +end;; + +delimiter ; +call denormalize_tag_tables(); + +drop procedure if exists denormalize_tag_tables; + +DROP PROCEDURE IF EXISTS removeConstraints; +DELIMITER ;; +CREATE PROCEDURE removeConstraints(vTableName varchar(128)) +BEGIN + DECLARE done INT DEFAULT FALSE; + DECLARE cName VARCHAR(64); + DECLARE cur CURSOR FOR + SELECT DISTINCT CONSTRAINT_NAME + FROM INFORMATION_SCHEMA.Key_COLUMN_USAGE + WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME = vTableName + AND REFERENCED_TABLE_NAME IS NOT NULL; + DECLARE CONTINUE HANDLER FOR NOT FOUND SET done = TRUE; + + SET AUTOCOMMIT=0; + SET FOREIGN_KEY_CHECKS=0; + + OPEN cur; + + read_loop: LOOP + FETCH cur INTO cName; + IF done THEN + LEAVE read_loop; + END IF; + SET @sql = CONCAT('ALTER TABLE ',vTableName,' DROP FOREIGN KEY ',cName,';'); + PREPARE stmt FROM @sql; + EXECUTE stmt; + DEALLOCATE PREPARE stmt; + END LOOP; + + CLOSE cur; + + SET FOREIGN_KEY_CHECKS=1; + COMMIT; + SET AUTOCOMMIT=1; +END ;; +DELIMITER ; + +call removeConstraints('x_tag_attr_def'); +call removeConstraints('x_tag_attr'); +call removeConstraints('x_service_resource_element'); +call removeConstraints('x_service_resource_element_val'); + +DROP PROCEDURE removeConstraints; diff --git a/security-admin/db/oracle/patches/028-delete-xgroup-duplicate-references.sql b/security-admin/db/oracle/patches/028-delete-xgroup-duplicate-references.sql new file mode 100644 index 0000000000..7c017f9e7e --- /dev/null +++ b/security-admin/db/oracle/patches/028-delete-xgroup-duplicate-references.sql @@ -0,0 +1,65 @@ +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +DECLARE + donecursor1 number:=0; + group_name1 VARCHAR2(1024); + mingroupid1 number:=0; + id2 number:=0; + group_name3 VARCHAR2(1024); + user_id3 number:=0; + minrowid3 number:=0; + group_name4 VARCHAR2(1024); + group_id4 number:=0; + minrowid4 number:=0; + + CURSOR cursor1 IS + SELECT group_name, min(id) FROM x_group GROUP BY group_name HAVING count(group_name)>1; + CURSOR cursor2 IS + SELECT id FROM x_group WHERE group_name = group_name1 AND id > mingroupid1; + CURSOR cursor3 IS + SELECT group_name,user_id,min(id) FROM x_group_users GROUP BY group_name,user_id HAVING count(1)>1; + CURSOR cursor4 IS + SELECT group_name,min(id) FROM x_group GROUP BY group_name HAVING count(1)>1; + +BEGIN + OPEN cursor1; + LOOP + FETCH cursor1 into group_name1, mingroupid1; + EXIT WHEN cursor1%notfound; + OPEN cursor2; + LOOP + FETCH cursor2 INTO id2; + EXIT WHEN cursor2%notfound; + UPDATE x_group_users SET p_group_id=mingroupid1 where p_group_id=id2; + END LOOP; + CLOSE cursor2; + END LOOP; + CLOSE cursor1; + OPEN cursor3; + LOOP + FETCH cursor3 into group_name3, user_id3, minrowid3; + EXIT WHEN cursor3%notfound; + DELETE FROM x_group_users WHERE group_name=group_name3 AND user_id=user_id3 AND id > minrowid3; + END LOOP; + CLOSE cursor3; + OPEN cursor4; + LOOP + FETCH cursor4 into group_name4, minrowid4; + EXIT WHEN cursor4%notfound; + DELETE FROM x_group WHERE group_name=group_name4 AND id > minrowid4; + END LOOP; + CLOSE cursor4; +END;/ \ No newline at end of file diff --git a/security-admin/db/oracle/patches/029-add-unique-constraint-on-table-x_group.sql b/security-admin/db/oracle/patches/029-add-unique-constraint-on-table-x_group.sql new file mode 100644 index 0000000000..d512465a99 --- /dev/null +++ b/security-admin/db/oracle/patches/029-add-unique-constraint-on-table-x_group.sql @@ -0,0 +1,46 @@ +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +DECLARE + v_count number:=0; + gu_count number:=0; +BEGIN + select count(*) into v_count from user_tab_cols where table_name='X_GROUP' and column_name='GROUP_NAME'; + if (v_count = 1) then + v_count:=0; + select count(*) into v_count from user_constraints where table_name='X_GROUP' and constraint_name='X_GROUP_UK_GROUP_NAME' and constraint_type='U'; + if (v_count = 0) then + v_count:=0; + select count(*) into v_count from user_ind_columns WHERE table_name='X_GROUP' and column_name='GROUP_NAME' and index_name='X_GROUP_UK_GROUP_NAME'; + if (v_count = 0) then + execute immediate 'ALTER TABLE x_group MODIFY(group_name VARCHAR(767)) ADD CONSTRAINT x_group_UK_group_name UNIQUE (group_name)'; + end if; + commit; + end if; + end if; + + select count(*) into gu_count from user_tab_cols where table_name='X_GROUP_USERS' and column_name='GROUP_NAME'; + if (gu_count = 1) then + gu_count:=0; + select count(*) into gu_count from user_constraints where table_name='X_GROUP_USERS' and constraint_name='X_GROUP_USERS_UK_UID_GNAME' and constraint_type='U'; + if (gu_count = 0) then + gu_count:=0; + select count(*) into gu_count from user_ind_columns WHERE table_name='X_GROUP_USERS' and column_name='GROUP_NAME' and index_name='X_GROUP_USERS_UK_UID_GNAME'; + if (gu_count = 0) then + execute immediate 'ALTER TABLE x_group_users MODIFY(group_name VARCHAR(767)) ADD CONSTRAINT x_group_users_uk_uid_gname UNIQUE (user_id,group_name)'; + end if; + commit; + end if; + end if; +end;/ diff --git a/security-admin/db/oracle/patches/033-add-unique-constraint-on-table-x_policy.sql b/security-admin/db/oracle/patches/033-add-unique-constraint-on-table-x_policy.sql new file mode 100644 index 0000000000..dc97b3700f --- /dev/null +++ b/security-admin/db/oracle/patches/033-add-unique-constraint-on-table-x_policy.sql @@ -0,0 +1,35 @@ +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +DECLARE + v_count number:=0; + sql_stmt VARCHAR2(1000); + duplicate VARCHAR2(11):='-duplicate-'; +BEGIN + select count(*) into v_count from user_tab_cols where table_name='X_POLICY' and column_name IN('NAME','SERVICE'); + if (v_count = 2) then + v_count:=0; + select count(*) into v_count from user_constraints where table_name='X_POLICY' and constraint_name='X_POLICY_UK_NAME_SERVICE' and constraint_type='U'; + if (v_count = 0) then + v_count:=0; + select count(*) into v_count from user_ind_columns WHERE table_name='X_POLICY' and column_name IN('NAME','SERVICE') and index_name='X_POLICY_UK_NAME_SERVICE'; + if (v_count = 0) THEN + sql_stmt := 'UPDATE x_policy set name=concat(concat(name,:1),id) where id in (select id from (select id from x_policy where concat(service,name) in (select concat(service,name) from x_policy group by service,name having count(*) >1)))'; + EXECUTE IMMEDIATE sql_stmt USING duplicate; + EXECUTE IMMEDIATE 'ALTER TABLE X_POLICY ADD CONSTRAINT x_policy_UK_name_service UNIQUE (NAME,SERVICE)'; + end if; + commit; + end if; + end if; +end;/ diff --git a/security-admin/db/oracle/patches/035-update-schema-for-x-policy.sql b/security-admin/db/oracle/patches/035-update-schema-for-x-policy.sql new file mode 100644 index 0000000000..745f7f080e --- /dev/null +++ b/security-admin/db/oracle/patches/035-update-schema-for-x-policy.sql @@ -0,0 +1,164 @@ +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +/ +CREATE SEQUENCE X_POLICY_REF_RESOURCE_SEQ START WITH 1 INCREMENT BY 1 NOCACHE NOCYCLE; +CREATE SEQUENCE X_POLICY_REF_ACCESS_TYPE_SEQ START WITH 1 INCREMENT BY 1 NOCACHE NOCYCLE; +CREATE SEQUENCE X_POLICY_REF_CONDITION_SEQ START WITH 1 INCREMENT BY 1 NOCACHE NOCYCLE; +CREATE SEQUENCE X_POLICY_REF_DATAMASK_TYPE_SEQ START WITH 1 INCREMENT BY 1 NOCACHE NOCYCLE; +CREATE SEQUENCE X_POLICY_REF_USER_SEQ START WITH 1 INCREMENT BY 1 NOCACHE NOCYCLE; +CREATE SEQUENCE X_POLICY_REF_GROUP_SEQ START WITH 1 INCREMENT BY 1 NOCACHE NOCYCLE; +commit; +CREATE TABLE x_policy_ref_resource ( +id NUMBER(20) NOT NULL, +guid VARCHAR(1024) DEFAULT NULL NULL, +create_time DATE DEFAULT NULL NULL, +update_time DATE DEFAULT NULL NULL, +added_by_id NUMBER(20) DEFAULT NULL NULL, +upd_by_id NUMBER(20) DEFAULT NULL NULL, +policy_id NUMBER(20) NOT NULL, +resource_def_id NUMBER(20) NOT NULL, +resource_name VARCHAR(4000) DEFAULT NULL NULL, +primary key (id), +CONSTRAINT x_p_ref_res_UK_polId_resDefId UNIQUE (policy_id, resource_def_id), +CONSTRAINT x_p_ref_res_FK_policy_id FOREIGN KEY (policy_id) REFERENCES x_policy (id), +CONSTRAINT x_p_ref_res_FK_resource_def_id FOREIGN KEY (resource_def_id) REFERENCES x_resource_def (id), +CONSTRAINT x_p_ref_res_FK_added_by_id FOREIGN KEY (added_by_id) REFERENCES x_portal_user (id), +CONSTRAINT x_p_ref_res_FK_upd_by_id FOREIGN KEY (upd_by_id) REFERENCES x_portal_user (id) +); +commit; +CREATE TABLE x_policy_ref_access_type ( +id NUMBER(20) NOT NULL, +guid VARCHAR(1024) DEFAULT NULL NULL, +create_time DATE DEFAULT NULL NULL, +update_time DATE DEFAULT NULL NULL, +added_by_id NUMBER(20) DEFAULT NULL NULL, +upd_by_id NUMBER(20) DEFAULT NULL NULL, +policy_id NUMBER(20) NOT NULL, +access_def_id NUMBER(20) NOT NULL, +access_type_name VARCHAR(4000) DEFAULT NULL NULL, +primary key (id), +CONSTRAINT x_p_ref_acc_UK_polId_accDefId UNIQUE(policy_id, access_def_id), +CONSTRAINT x_p_ref_acc_FK_policy_id FOREIGN KEY (policy_id) REFERENCES x_policy (id), +CONSTRAINT x_p_ref_acc_FK_acc_def_id FOREIGN KEY (access_def_id) REFERENCES x_access_type_def (id), +CONSTRAINT x_p_ref_acc_FK_added_by_id FOREIGN KEY (added_by_id) REFERENCES x_portal_user (id), +CONSTRAINT x_p_ref_acc_FK_upd_by_id FOREIGN KEY (upd_by_id) REFERENCES x_portal_user (id) +); +commit; +CREATE TABLE x_policy_ref_condition ( +id NUMBER(20) NOT NULL, +guid VARCHAR(1024) DEFAULT NULL NULL, +create_time DATE DEFAULT NULL NULL, +update_time DATE DEFAULT NULL NULL, +added_by_id NUMBER(20) DEFAULT NULL NULL, +upd_by_id NUMBER(20) DEFAULT NULL NULL, +policy_id NUMBER(20) NOT NULL, +condition_def_id NUMBER(20) NOT NULL, +condition_name VARCHAR(4000) DEFAULT NULL NULL, +primary key (id), +CONSTRAINT x_p_ref_cond_UK_polId_cDefId UNIQUE(policy_id, condition_def_id), +CONSTRAINT x_p_ref_cond_FK_policy_id FOREIGN KEY (policy_id) REFERENCES x_policy (id), +CONSTRAINT x_p_ref_cond_FK_cond_def_id FOREIGN KEY (condition_def_id) REFERENCES x_policy_condition_def (id), +CONSTRAINT x_p_ref_cond_FK_added_by_id FOREIGN KEY (added_by_id) REFERENCES x_portal_user (id), +CONSTRAINT x_p_ref_cond_FK_upd_by_id FOREIGN KEY (upd_by_id) REFERENCES x_portal_user (id) +); +commit; +CREATE TABLE x_policy_ref_datamask_type ( +id NUMBER(20) NOT NULL, +guid VARCHAR(1024) DEFAULT NULL NULL, +create_time DATE DEFAULT NULL NULL, +update_time DATE DEFAULT NULL NULL, +added_by_id NUMBER(20) DEFAULT NULL NULL, +upd_by_id NUMBER(20) DEFAULT NULL NULL, +policy_id NUMBER(20) NOT NULL, +datamask_def_id NUMBER(20) NOT NULL, +datamask_type_name VARCHAR(4000) DEFAULT NULL NULL, +primary key (id), +CONSTRAINT x_p_ref_dmsk_UK_polId_dDefId UNIQUE(policy_id, datamask_def_id), +CONSTRAINT x_p_ref_dmsk_FK_policy_id FOREIGN KEY (policy_id) REFERENCES x_policy (id), +CONSTRAINT x_p_ref_dmsk_FK_dmk_def_id FOREIGN KEY (datamask_def_id) REFERENCES x_datamask_type_def (id), +CONSTRAINT x_p_ref_dmsk_FK_added_by_id FOREIGN KEY (added_by_id) REFERENCES x_portal_user (id), +CONSTRAINT x_p_ref_dmsk_FK_upd_by_id FOREIGN KEY (upd_by_id) REFERENCES x_portal_user (id) +); +commit; +CREATE TABLE x_policy_ref_user ( +id NUMBER(20) NOT NULL, +guid VARCHAR(1024) DEFAULT NULL NULL, +create_time DATE DEFAULT NULL NULL, +update_time DATE DEFAULT NULL NULL, +added_by_id NUMBER(20) DEFAULT NULL NULL, +upd_by_id NUMBER(20) DEFAULT NULL NULL, +policy_id NUMBER(20) NOT NULL, +user_id NUMBER(20) NOT NULL, +user_name VARCHAR(4000) DEFAULT NULL NULL, +primary key (id), +CONSTRAINT x_p_ref_usr_UK_polId_userId UNIQUE(policy_id, user_id), +CONSTRAINT x_p_ref_usr_FK_policy_id FOREIGN KEY (policy_id) REFERENCES x_policy (id), +CONSTRAINT x_p_ref_usr_FK_user_id FOREIGN KEY (user_id) REFERENCES x_user (id), +CONSTRAINT x_p_ref_usr_FK_added_by_id FOREIGN KEY (added_by_id) REFERENCES x_portal_user (id), +CONSTRAINT x_p_ref_usr_FK_upd_by_id FOREIGN KEY (upd_by_id) REFERENCES x_portal_user (id) +); +commit; +CREATE TABLE x_policy_ref_group ( +id NUMBER(20) NOT NULL, +guid VARCHAR(1024) DEFAULT NULL NULL, +create_time DATE DEFAULT NULL NULL, +update_time DATE DEFAULT NULL NULL, +added_by_id NUMBER(20) DEFAULT NULL NULL, +upd_by_id NUMBER(20) DEFAULT NULL NULL, +policy_id NUMBER(20) NOT NULL, +group_id NUMBER(20) NOT NULL, +group_name VARCHAR(4000) DEFAULT NULL NULL, +primary key (id), +CONSTRAINT x_p_ref_grp_UK_polId_grpId UNIQUE(policy_id, group_id), +CONSTRAINT x_p_ref_grp_FK_policy_id FOREIGN KEY (policy_id) REFERENCES x_policy (id), +CONSTRAINT x_p_ref_grp_FK_group_id FOREIGN KEY (group_id) REFERENCES x_group (id), +CONSTRAINT x_p_ref_grp_FK_added_by_id FOREIGN KEY (added_by_id) REFERENCES x_portal_user (id), +CONSTRAINT x_p_ref_grp_FK_upd_by_id FOREIGN KEY (upd_by_id) REFERENCES x_portal_user (id) +); +commit; +DECLARE + v_column_exists number := 0; +BEGIN +Select count(*) into v_column_exists from user_tab_cols where column_name = upper('policy_text') and table_name = upper('x_policy'); + if (v_column_exists = 0) then + execute immediate 'ALTER TABLE x_policy ADD policy_text CLOB DEFAULT NULL NULL'; + commit; + end if; +end;/ + +CREATE OR REPLACE PROCEDURE removeConstraints(ObjName IN varchar2) IS +BEGIN +FOR rec IN( +select owner, constraint_name +from all_constraints +where owner = sys_context('userenv','current_schema') +and table_name = ObjName +and constraint_type = 'R') +LOOP +execute immediate 'ALTER TABLE ' || rec.owner || '.' || ObjName || ' DROP CONSTRAINT ' || rec.constraint_name; +END LOOP; +END;/ +/ + +CALL removeConstraints('X_POLICY_ITEM'); +CALL removeConstraints('X_POLICY_ITEM_ACCESS'); +CALL removeConstraints('X_POLICY_ITEM_CONDITION'); +CALL removeConstraints('X_POLICY_ITEM_DATAMASK'); +CALL removeConstraints('X_POLICY_ITEM_GROUP_PERM'); +CALL removeConstraints('X_POLICY_RESOURCE'); +CALL removeConstraints('X_POLICY_RESOURCE_MAP'); +CALL removeConstraints('X_POLICY_ITEM_USER_PERM'); +CALL removeConstraints('X_POLICY_ITEM_ROWFILTER'); + diff --git a/security-admin/db/oracle/patches/036-denormalize-tag-tables.sql b/security-admin/db/oracle/patches/036-denormalize-tag-tables.sql new file mode 100644 index 0000000000..cae29272c4 --- /dev/null +++ b/security-admin/db/oracle/patches/036-denormalize-tag-tables.sql @@ -0,0 +1,54 @@ +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +DECLARE + v_count number:=0; +BEGIN + select count(*) into v_count from user_tab_cols where table_name='X_TAG_DEF' and column_name='TAG_ATTRS_DEF_TEXT'; + if (v_count = 0) then + execute immediate 'ALTER TABLE X_TAG_DEF ADD TAG_ATTRS_DEF_TEXT CLOB DEFAULT NULL NULL'; + end if; + select count(*) into v_count from user_tab_cols where table_name='X_TAG' and column_name='TAG_ATTRS_TEXT'; + if (v_count = 0) then + execute immediate 'ALTER TABLE X_TAG ADD TAG_ATTRS_TEXT CLOB DEFAULT NULL NULL'; + end if; + select count(*) into v_count from user_tab_cols where table_name='X_SERVICE_RESOURCE' and column_name='SERVICE_RESOURCE_ELEMENTS_TEXT'; + if (v_count = 0) then + execute immediate 'ALTER TABLE X_SERVICE_RESOURCE ADD SERVICE_RESOURCE_ELEMENTS_TEXT CLOB DEFAULT NULL NULL'; + end if; + select count(*) into v_count from user_tab_cols where table_name='X_SERVICE_RESOURCE' and column_name='TAGS_TEXT'; + if (v_count = 0) then + execute immediate 'ALTER TABLE X_SERVICE_RESOURCE ADD TAGS_TEXT CLOB DEFAULT NULL NULL'; + end if; + commit; +END;/ + +CREATE OR REPLACE PROCEDURE removeConstraints(ObjName IN varchar2) IS +BEGIN +FOR rec IN( +select owner, constraint_name +from all_constraints +where owner = sys_context('userenv','current_schema') +and table_name = ObjName +and constraint_type = 'R') +LOOP +execute immediate 'ALTER TABLE ' || rec.owner || '.' || ObjName || ' DROP CONSTRAINT ' || rec.constraint_name; +END LOOP; +END;/ +/ + +CALL removeConstraints('X_TAG_ATTR_DEF'); +CALL removeConstraints('X_TAG_ATTR'); +CALL removeConstraints('X_SERVICE_RESOURCE_ELEMENT'); +CALL removeConstraints('X_SERVICE_RESOURCE_ELEMENT_VAL'); diff --git a/security-admin/db/postgres/patches/028-delete-xgroup-duplicate-references.sql b/security-admin/db/postgres/patches/028-delete-xgroup-duplicate-references.sql new file mode 100644 index 0000000000..6fbb49ea3c --- /dev/null +++ b/security-admin/db/postgres/patches/028-delete-xgroup-duplicate-references.sql @@ -0,0 +1,68 @@ +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +select 'delimiter start'; +CREATE OR REPLACE FUNCTION delete_xgroup_duplicate_references() +RETURNS void AS $$ +DECLARE + donecursor1 BIGINT:=0; + group_name1 VARCHAR(1024); + mingroupid1 BIGINT:=0; + id2 BIGINT:=0; + group_name3 VARCHAR(1024); + user_id3 BIGINT:=0; + minrowid3 BIGINT:=0; + group_name4 VARCHAR(1024); + group_id4 BIGINT:=0; + minrowid4 BIGINT:=0; + + cursor1 cursor for SELECT group_name, min(id) FROM x_group GROUP BY group_name HAVING count(group_name)>1; + cursor2 cursor for SELECT id FROM x_group WHERE group_name = group_name1 AND id > mingroupid1; + cursor3 cursor for SELECT group_name,user_id,min(id) FROM x_group_users GROUP BY group_name,user_id HAVING count(1)>1; + cursor4 cursor for SELECT group_name,min(id) FROM x_group GROUP BY group_name HAVING count(1)>1; + +BEGIN + OPEN cursor1; + LOOP + FETCH cursor1 into group_name1, mingroupid1; + EXIT WHEN NOT FOUND; + OPEN cursor2; + LOOP + FETCH cursor2 INTO id2; + EXIT WHEN NOT FOUND; + UPDATE x_group_users SET p_group_id=mingroupid1 where p_group_id=id2; + END LOOP; + CLOSE cursor2; + END LOOP; + CLOSE cursor1; + + OPEN cursor3; + LOOP + FETCH cursor3 into group_name3, user_id3, minrowid3; + EXIT WHEN NOT FOUND; + DELETE FROM x_group_users WHERE group_name=group_name3 AND user_id=user_id3 AND id > minrowid3; + END LOOP; + CLOSE cursor3; + + OPEN cursor4; + LOOP + FETCH cursor4 into group_name4, minrowid4; + EXIT WHEN NOT FOUND; + DELETE FROM x_group WHERE group_name=group_name4 AND id > minrowid4; + END LOOP; + CLOSE cursor4; +END; +$$ LANGUAGE plpgsql; +select delete_xgroup_duplicate_references(); +select 'delimiter end'; \ No newline at end of file diff --git a/security-admin/db/postgres/patches/029-add-unique-constraint-on-table-x_group.sql b/security-admin/db/postgres/patches/029-add-unique-constraint-on-table-x_group.sql new file mode 100644 index 0000000000..501ec2ea98 --- /dev/null +++ b/security-admin/db/postgres/patches/029-add-unique-constraint-on-table-x_group.sql @@ -0,0 +1,43 @@ +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +select 'delimiter start'; +CREATE OR REPLACE FUNCTION create_unique_constraint_on_username() +RETURNS void AS $$ +DECLARE + v_attnum integer := 0; +gu_attnum integer := 0; +BEGIN + select attnum into v_attnum from pg_attribute where attrelid in(select oid from pg_class where relname='x_group') and attname='group_name'; + IF v_attnum > 0 THEN + IF not exists (select * from pg_constraint where conrelid in(select oid from pg_class where relname='x_group') and conname='x_group_uk_group_name' and contype='u') THEN + IF not exists (select * from pg_index where indrelid in(select oid from pg_class where relname='x_group') and indkey[0]=v_attnum) THEN + ALTER TABLE x_group ALTER COLUMN group_name TYPE VARCHAR(767),ADD CONSTRAINT x_group_UK_group_name UNIQUE(group_name); + END IF; + END IF; + END IF; + +select attnum into gu_attnum from pg_attribute where attrelid in(select oid from pg_class where relname='x_group_users') and attname='group_name'; + IF gu_attnum > 0 THEN + IF not exists (select * from pg_constraint where conrelid in(select oid from pg_class where relname='x_group_users') and conname='x_group_users_UK_uid_gname' and contype='u') THEN + IF not exists (select * from pg_index where indrelid in(select oid from pg_class where relname='x_group_users') and indkey[0]=gu_attnum) THEN + ALTER TABLE x_group_users ALTER COLUMN group_name TYPE VARCHAR(767),ADD CONSTRAINT x_group_users_UK_uid_gname UNIQUE(user_id,group_name); + END IF; + END IF; + END IF; + +END; +$$ LANGUAGE plpgsql; +select create_unique_constraint_on_username(); +select 'delimiter end'; diff --git a/security-admin/db/postgres/patches/033-add-unique-constraint-on-table-x_policy.sql b/security-admin/db/postgres/patches/033-add-unique-constraint-on-table-x_policy.sql new file mode 100644 index 0000000000..e3ac945672 --- /dev/null +++ b/security-admin/db/postgres/patches/033-add-unique-constraint-on-table-x_policy.sql @@ -0,0 +1,34 @@ +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +select 'delimiter start'; +CREATE OR REPLACE FUNCTION create_unique_constraint_on_policyname() +RETURNS void AS $$ +DECLARE + v_attnum integer := 0; +BEGIN + select attnum into v_attnum from pg_attribute where attrelid in(select oid from pg_class where relname='x_policy') and attname in('name'); + IF v_attnum > 0 THEN + IF not exists (select * from pg_constraint where conrelid in(select oid from pg_class where relname='x_policy') and conname='x_policy_uk_name_service' and contype='u') THEN + IF not exists (select * from pg_index where indrelid in(select oid from pg_class where relname='x_policy') and indkey[0]=v_attnum) THEN + UPDATE x_policy set name=(name || '-duplicate-' || id) where id in (select id from (select id from x_policy where service || name in (select service || name from x_policy group by service,name having count(*) >1)) as tmp); + ALTER TABLE x_policy ADD CONSTRAINT x_policy_uk_name_service UNIQUE(name,service); + END IF; + END IF; + END IF; + +END; +$$ LANGUAGE plpgsql; +select create_unique_constraint_on_policyname(); +select 'delimiter end'; diff --git a/security-admin/db/postgres/patches/035-update-schema-for-x-policy.sql b/security-admin/db/postgres/patches/035-update-schema-for-x-policy.sql new file mode 100644 index 0000000000..1414fe300f --- /dev/null +++ b/security-admin/db/postgres/patches/035-update-schema-for-x-policy.sql @@ -0,0 +1,198 @@ +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +DROP TABLE IF EXISTS x_policy_ref_resource CASCADE; +DROP SEQUENCE IF EXISTS x_policy_ref_resource_seq; +CREATE SEQUENCE x_policy_ref_resource_seq; +CREATE TABLE x_policy_ref_resource( +id BIGINT DEFAULT nextval('x_policy_ref_resource_seq'::regclass), +guid VARCHAR(1024) DEFAULT NULL NULL, +create_time TIMESTAMP DEFAULT NULL NULL, +update_time TIMESTAMP DEFAULT NULL NULL, +added_by_id BIGINT DEFAULT NULL NULL, +upd_by_id BIGINT DEFAULT NULL NULL, +policy_id BIGINT NOT NULL, +resource_def_id BIGINT NOT NULL, +resource_name varchar(4000) DEFAULT NULL, +primary key(id), +CONSTRAINT x_p_ref_res_UK_polId_resDefId UNIQUE (policy_id, resource_def_id), +CONSTRAINT x_p_ref_res_FK_policy_id FOREIGN KEY(policy_id) REFERENCES x_policy(id), +CONSTRAINT x_p_ref_res_FK_resource_def_id FOREIGN KEY(resource_def_id) REFERENCES x_resource_def(id), +CONSTRAINT x_p_ref_res_FK_added_by_id FOREIGN KEY(added_by_id) REFERENCES x_portal_user(id), +CONSTRAINT x_p_ref_res_FK_upd_by_id FOREIGN KEY(upd_by_id) REFERENCES x_portal_user(id) +); +commit; +DROP TABLE IF EXISTS x_policy_ref_access_type CASCADE; +DROP SEQUENCE IF EXISTS x_policy_ref_access_type_seq; +CREATE SEQUENCE x_policy_ref_access_type_seq; +CREATE TABLE x_policy_ref_access_type( +id BIGINT DEFAULT nextval('x_policy_ref_access_type_seq'::regclass), +guid VARCHAR(1024) DEFAULT NULL NULL, +create_time TIMESTAMP DEFAULT NULL NULL, +update_time TIMESTAMP DEFAULT NULL NULL, +added_by_id BIGINT DEFAULT NULL NULL, +upd_by_id BIGINT DEFAULT NULL NULL, +policy_id BIGINT NOT NULL, +access_def_id BIGINT NOT NULL, +access_type_name varchar(4000) DEFAULT NULL, +primary key(id), +CONSTRAINT x_p_ref_acc_UK_polId_accDefId UNIQUE(policy_id, access_def_id), +CONSTRAINT x_p_ref_acc_FK_policy_id FOREIGN KEY(policy_id) REFERENCES x_policy(id), +CONSTRAINT x_p_ref_acc_FK_acc_def_id FOREIGN KEY(access_def_id) REFERENCES x_access_type_def(id), +CONSTRAINT x_p_ref_acc_FK_added_by_id FOREIGN KEY(added_by_id) REFERENCES x_portal_user(id), +CONSTRAINT x_p_ref_acc_FK_upd_by_id FOREIGN KEY(upd_by_id) REFERENCES x_portal_user(id) +); +commit; +DROP TABLE IF EXISTS x_policy_ref_condition CASCADE; +DROP SEQUENCE IF EXISTS x_policy_ref_condition_seq; +CREATE SEQUENCE x_policy_ref_condition_seq; +CREATE TABLE x_policy_ref_condition( +id BIGINT DEFAULT nextval('x_policy_ref_condition_seq'::regclass), +guid VARCHAR(1024) DEFAULT NULL NULL, +create_time TIMESTAMP DEFAULT NULL NULL, +update_time TIMESTAMP DEFAULT NULL NULL, +added_by_id BIGINT DEFAULT NULL NULL, +upd_by_id BIGINT DEFAULT NULL NULL, +policy_id BIGINT NOT NULL, +condition_def_id BIGINT NOT NULL, +condition_name varchar(4000) DEFAULT NULL, +primary key(id), +CONSTRAINT x_p_ref_cond_UK_polId_cDefId UNIQUE(policy_id, condition_def_id), +CONSTRAINT x_p_ref_cond_FK_policy_id FOREIGN KEY(policy_id) REFERENCES x_policy(id), +CONSTRAINT x_p_ref_cond_FK_cond_def_id FOREIGN KEY(condition_def_id) REFERENCES x_policy_condition_def(id), +CONSTRAINT x_p_ref_cond_FK_added_by_id FOREIGN KEY(added_by_id) REFERENCES x_portal_user(id), +CONSTRAINT x_p_ref_cond_FK_upd_by_id FOREIGN KEY(upd_by_id) REFERENCES x_portal_user(id) +); +commit; +DROP TABLE IF EXISTS x_policy_ref_datamask_type CASCADE; +DROP SEQUENCE IF EXISTS x_policy_ref_datamask_type_seq; +CREATE SEQUENCE x_policy_ref_datamask_type_seq; +CREATE TABLE x_policy_ref_datamask_type( +id BIGINT DEFAULT nextval('x_policy_ref_datamask_type_seq'::regclass), +guid VARCHAR(1024) DEFAULT NULL NULL, +create_time TIMESTAMP DEFAULT NULL NULL, +update_time TIMESTAMP DEFAULT NULL NULL, +added_by_id BIGINT DEFAULT NULL NULL, +upd_by_id BIGINT DEFAULT NULL NULL, +policy_id BIGINT NOT NULL, +datamask_def_id BIGINT NOT NULL, +datamask_type_name varchar(4000) DEFAULT NULL, +primary key(id), +CONSTRAINT x_p_ref_dmk_UK_polId_dDefId UNIQUE(policy_id, datamask_def_id), +CONSTRAINT x_p_ref_dmk_FK_policy_id FOREIGN KEY(policy_id) REFERENCES x_policy(id), +CONSTRAINT x_p_ref_dmk_FK_dmk_def_id FOREIGN KEY(datamask_def_id) REFERENCES x_datamask_type_def(id), +CONSTRAINT x_p_ref_dmk_FK_added_by_id FOREIGN KEY(added_by_id) REFERENCES x_portal_user(id), +CONSTRAINT x_p_ref_dmk_FK_upd_by_id FOREIGN KEY(upd_by_id) REFERENCES x_portal_user(id) +); +commit; +DROP TABLE IF EXISTS x_policy_ref_user CASCADE; +DROP SEQUENCE IF EXISTS x_policy_ref_user_seq; +CREATE SEQUENCE x_policy_ref_user_seq; +CREATE TABLE x_policy_ref_user( +id BIGINT DEFAULT nextval('x_policy_ref_user_seq'::regclass), +guid VARCHAR(1024) DEFAULT NULL NULL, +create_time TIMESTAMP DEFAULT NULL NULL, +update_time TIMESTAMP DEFAULT NULL NULL, +added_by_id BIGINT DEFAULT NULL NULL, +upd_by_id BIGINT DEFAULT NULL NULL, +policy_id BIGINT NOT NULL, +user_id BIGINT NOT NULL, +user_name varchar(4000) DEFAULT NULL, +primary key(id), +CONSTRAINT x_p_ref_usr_UK_polId_userId UNIQUE(policy_id, user_id), +CONSTRAINT x_p_ref_usr_FK_policy_id FOREIGN KEY(policy_id) REFERENCES x_policy(id), +CONSTRAINT x_p_ref_usr_FK_user_id FOREIGN KEY(user_id) REFERENCES x_user(id), +CONSTRAINT x_p_ref_usr_FK_added_by_id FOREIGN KEY(added_by_id) REFERENCES x_portal_user(id), +CONSTRAINT x_p_ref_usr_FK_upd_by_id FOREIGN KEY(upd_by_id) REFERENCES x_portal_user(id) +); +commit; +DROP TABLE IF EXISTS x_policy_ref_group CASCADE; +DROP SEQUENCE IF EXISTS x_policy_ref_group_seq; +CREATE SEQUENCE x_policy_ref_group_seq; +CREATE TABLE x_policy_ref_group( +id BIGINT DEFAULT nextval('x_policy_ref_group_seq'::regclass), +guid VARCHAR(1024) DEFAULT NULL NULL, +create_time TIMESTAMP DEFAULT NULL NULL, +update_time TIMESTAMP DEFAULT NULL NULL, +added_by_id BIGINT DEFAULT NULL NULL, +upd_by_id BIGINT DEFAULT NULL NULL, +policy_id BIGINT NOT NULL, +group_id BIGINT NOT NULL, +group_name varchar(4000) DEFAULT NULL, +primary key(id), +CONSTRAINT x_p_ref_grp_UK_polId_grpId UNIQUE(policy_id, group_id), +CONSTRAINT x_p_ref_grp_FK_policy_id FOREIGN KEY(policy_id) REFERENCES x_policy(id), +CONSTRAINT x_p_ref_grp_FK_group_id FOREIGN KEY(group_id) REFERENCES x_group(id), +CONSTRAINT x_p_ref_grp_FK_added_by_id FOREIGN KEY(added_by_id) REFERENCES x_portal_user(id), +CONSTRAINT x_p_ref_grp_FK_upd_by_id FOREIGN KEY(upd_by_id) REFERENCES x_portal_user(id) +); +commit; +select 'delimiter start'; +CREATE OR REPLACE FUNCTION add_x_policy_json() +RETURNS void AS $$ +DECLARE + v_column_exists integer := 0; +BEGIN + select count(*) into v_column_exists from pg_attribute where attrelid in(select oid from pg_class where relname='x_policy') and attname='policy_text'; + IF v_column_exists = 0 THEN + ALTER TABLE x_policy ADD COLUMN policy_text TEXT DEFAULT NULL NULL; + END IF; +END; +$$ LANGUAGE plpgsql; +select 'delimiter end'; + +select add_x_policy_json(); +select 'delimiter end'; + +select 'delimiter start'; +CREATE OR REPLACE FUNCTION remove_foreign_key(objName varchar(4000)) +RETURNS void AS $$ +declare + tableName VARCHAR(256); + constraintName VARCHAR(512); + query varchar(4000); + curs CURSOR FOR SELECT table_name,constraint_name from information_schema.key_column_usage where constraint_catalog=current_catalog and table_name=objName and position_in_unique_constraint notnull; +begin + OPEN curs; + loop + FETCH curs INTO tableName,constraintName; + EXIT WHEN NOT FOUND; + query :='ALTER TABLE ' || objName || ' drop constraint ' || constraintName; + execute query; + end loop; + close curs; +END; +$$ LANGUAGE plpgsql; +select 'delimiter end'; + +CREATE OR REPLACE FUNCTION removekeys() +RETURNS void AS +$$ +BEGIN + perform remove_foreign_key('x_policy_item'); + perform remove_foreign_key('x_policy_item_access'); + perform remove_foreign_key('x_policy_item_condition'); + perform remove_foreign_key('x_policy_item_datamask'); + perform remove_foreign_key('x_policy_item_group_perm'); + perform remove_foreign_key('x_policy_resource'); + perform remove_foreign_key('x_policy_resource_map'); + perform remove_foreign_key('x_policy_item_user_perm'); + perform remove_foreign_key('x_policy_item_rowfilter'); + +END; +$$ LANGUAGE plpgsql; +select removekeys(); + +select 'delimiter end'; + diff --git a/security-admin/db/postgres/patches/036-denormalize-tag-tables.sql b/security-admin/db/postgres/patches/036-denormalize-tag-tables.sql new file mode 100644 index 0000000000..e5ed27221c --- /dev/null +++ b/security-admin/db/postgres/patches/036-denormalize-tag-tables.sql @@ -0,0 +1,79 @@ +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +-- function denormalize_tag_tables() +select 'delimiter start'; +CREATE OR REPLACE FUNCTION denormalize_tag_tables() +RETURNS void AS $$ +DECLARE + v_column_exists integer := 0; +BEGIN + select count(*) into v_column_exists from pg_attribute where attrelid in(select oid from pg_class where relname='x_tag_def') and attname='tag_attrs_def_text'; + IF v_column_exists = 0 THEN + ALTER TABLE x_tag_def ADD COLUMN tag_attrs_def_text TEXT DEFAULT NULL NULL; + END IF; + select count(*) into v_column_exists from pg_attribute where attrelid in(select oid from pg_class where relname='x_tag') and attname='tag_attrs_text'; + IF v_column_exists = 0 THEN + ALTER TABLE x_tag ADD COLUMN tag_attrs_text TEXT DEFAULT NULL NULL; + END IF; + select count(*) into v_column_exists from pg_attribute where attrelid in(select oid from pg_class where relname='x_service_resource') and attname='service_resource_elements_text'; + IF v_column_exists = 0 THEN + ALTER TABLE x_service_resource ADD COLUMN service_resource_elements_text TEXT DEFAULT NULL NULL; + END IF; + select count(*) into v_column_exists from pg_attribute where attrelid in(select oid from pg_class where relname='x_service_resource') and attname='tags_text'; + IF v_column_exists = 0 THEN + ALTER TABLE x_service_resource ADD COLUMN tags_text TEXT DEFAULT NULL NULL; + END IF; +END; +$$ LANGUAGE plpgsql; +select 'delimiter end'; + +select denormalize_tag_tables(); +select 'delimiter end'; + +select 'delimiter start'; +CREATE OR REPLACE FUNCTION remove_foreign_key(objName varchar(4000)) +RETURNS void AS $$ +declare + tableName VARCHAR(256); + constraintName VARCHAR(512); + query varchar(4000); + curs CURSOR FOR SELECT table_name,constraint_name from information_schema.key_column_usage where constraint_catalog=current_catalog and table_name=objName and position_in_unique_constraint notnull; +begin + OPEN curs; + loop + FETCH curs INTO tableName,constraintName; + EXIT WHEN NOT FOUND; + query :='ALTER TABLE ' || objName || ' drop constraint ' || constraintName; + execute query; + end loop; + close curs; +END; +$$ LANGUAGE plpgsql; +select 'delimiter end'; + +CREATE OR REPLACE FUNCTION removekeys() +RETURNS void AS +$$ +BEGIN + perform remove_foreign_key('x_tag_attr_def'); + perform remove_foreign_key('x_tag_attr'); + perform remove_foreign_key('x_service_resource_element'); + perform remove_foreign_key('x_service_resource_element_val'); +END; +$$ LANGUAGE plpgsql; +select removekeys(); + +select 'delimiter end'; diff --git a/security-admin/db/sqlanywhere/patches/028-delete-xgroup-duplicate-references.sql b/security-admin/db/sqlanywhere/patches/028-delete-xgroup-duplicate-references.sql new file mode 100644 index 0000000000..46ef9100f0 --- /dev/null +++ b/security-admin/db/sqlanywhere/patches/028-delete-xgroup-duplicate-references.sql @@ -0,0 +1,75 @@ +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +CREATE OR REPLACE PROCEDURE delete_xgroup_duplicate_references() +BEGIN + DECLARE donecursor1 bigint; + DECLARE group_name1 varchar(1024); + DECLARE mingroupid1 bigint; + DECLARE id2 bigint; + DECLARE group_name3 varchar(1024); + DECLARE user_id3 bigint; + DECLARE minrowid3 bigint; + DECLARE group_name4 varchar(1024); + DECLARE group_id4 bigint; + DECLARE minrowid4 bigint; + + DECLARE cursor1 CURSOR FOR SELECT group_name, min(id) FROM x_group GROUP BY group_name HAVING count(group_name)>1; + DECLARE cursor2 CURSOR FOR SELECT id FROM x_group WHERE group_name = group_name1 AND id > mingroupid1; + DECLARE cursor3 CURSOR FOR SELECT group_name,user_id,min(id) FROM x_group_users GROUP BY group_name,user_id HAVING count(1)>1; + DECLARE cursor4 CURSOR FOR SELECT group_name,min(id) FROM x_group GROUP BY group_name HAVING count(1)>1; + + SET donecursor1=0; + SET mingroupid1=0; + SET id2=0; + SET user_id3=0; + SET minrowid3=0; + SET group_id4=0; + SET minrowid4=0; + + OPEN cursor1; + loopc1: LOOP + FETCH cursor1 INTO group_name1, mingroupid1; + IF SQLCODE <> 0 THEN LEAVE loopc1 END IF; + OPEN cursor2; + loopc2: LOOP + FETCH cursor2 INTO id2; + IF SQLCODE <> 0 THEN LEAVE loopc2 END IF; + UPDATE x_group_users SET p_group_id=mingroupid1 where p_group_id=id2; + END LOOP; + CLOSE cursor2; + END LOOP; + CLOSE cursor1; + + OPEN cursor3; + loopc3: LOOP + FETCH cursor3 INTO group_name3, user_id3, minrowid3; + IF SQLCODE <> 0 THEN LEAVE loopc3 END IF; + DELETE FROM x_group_users WHERE group_name=group_name3 AND user_id=user_id3 AND id > minrowid3; + END LOOP; + CLOSE cursor3; + + OPEN cursor4; + loopc4: LOOP + FETCH cursor4 INTO group_name4, minrowid4; + IF SQLCODE <> 0 THEN LEAVE loopc4 END IF; + DELETE FROM x_group WHERE group_name=group_name4 AND id > minrowid4; + END LOOP; + CLOSE cursor4; +END; +GO +EXEC delete_xgroup_duplicate_references; +GO +exit \ No newline at end of file diff --git a/security-admin/db/sqlanywhere/patches/029-add-unique-constraint-on-table-x_group.sql b/security-admin/db/sqlanywhere/patches/029-add-unique-constraint-on-table-x_group.sql new file mode 100644 index 0000000000..c39e68d7c6 --- /dev/null +++ b/security-admin/db/sqlanywhere/patches/029-add-unique-constraint-on-table-x_group.sql @@ -0,0 +1,44 @@ +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +BEGIN +DECLARE tableID INT = 0; +DECLARE columnID INT = 0; +DECLARE guTableID INT = 0; +DECLARE guColumnID INT = 0; + IF EXISTS(select * from SYS.SYSCOLUMNS where tname = 'x_group' and cname='group_name') THEN + IF NOT EXISTS(select * from SYS.SYSCONSTRAINT where constraint_name = 'x_group_UK_group_name') THEN + select table_id into tableID from SYS.SYSTAB where table_name = 'x_group'; + select column_id into columnID from SYS.SYSTABCOL where table_id=tableID and column_name = 'group_name'; + IF NOT EXISTS(select * from SYS.SYSIDXCOL where table_id=tableID and column_id=columnID) THEN + ALTER TABLE dbo.x_group ALTER group_name varchar(767) NOT NULL; + ALTER TABLE dbo.x_group ADD CONSTRAINT x_group_UK_group_name UNIQUE NONCLUSTERED (group_name); + END IF; + END IF; + END IF; + IF EXISTS(select * from SYS.SYSCOLUMNS where tname = 'x_group_users' and cname='group_name') THEN + IF NOT EXISTS(select * from SYS.SYSCONSTRAINT where constraint_name = 'x_group_users_UK_uid_gname') THEN + select table_id into guTableID from SYS.SYSTAB where table_name = 'x_group_users'; + select column_id into guColumnID from SYS.SYSTABCOL where table_id=guTableID and column_name = 'group_name'; + IF NOT EXISTS(select * from SYS.SYSIDXCOL where table_id=guTableID and column_id=guColumnID) THEN + ALTER TABLE dbo.x_group_users ALTER group_name varchar(767) NOT NULL; + alter table dbo.x_group_users drop constraint x_group_users_FK_user_id; + ALTER TABLE dbo.x_group_users ALTER user_id bigint NOT NULL; + ALTER TABLE dbo.x_group_users ADD CONSTRAINT x_group_users_FK_user_id FOREIGN KEY(user_id) REFERENCES dbo.x_user (id); + ALTER TABLE dbo.x_group_users ADD CONSTRAINT x_group_users_UK_uid_gname UNIQUE NONCLUSTERED (user_id,group_name); + END IF; + END IF; + END IF; +END +GO diff --git a/security-admin/db/sqlanywhere/patches/033-add-unique-constraint-on-table-x_policy.sql b/security-admin/db/sqlanywhere/patches/033-add-unique-constraint-on-table-x_policy.sql new file mode 100644 index 0000000000..ace31d68a8 --- /dev/null +++ b/security-admin/db/sqlanywhere/patches/033-add-unique-constraint-on-table-x_policy.sql @@ -0,0 +1,36 @@ +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +BEGIN +DECLARE tableID INT = 0; +DECLARE columnID INT = 0; +DECLARE guTableID INT = 0; +DECLARE guColumnID INT = 0; + IF EXISTS(select * from SYS.SYSCOLUMNS where tname = 'x_policy' and cname='name') THEN + IF NOT EXISTS(select * from SYS.SYSCONSTRAINT where constraint_name = 'x_policy_UK_name_service') THEN + select table_id into tableID from SYS.SYSTAB where table_name = 'x_policy'; + select column_id into columnID from SYS.SYSTABCOL where table_id=tableID and column_name = 'name'; + IF NOT EXISTS(select * from SYS.SYSIDXCOL where table_id=tableID and column_id=columnID) THEN + UPDATE x_policy set name=(name || '-duplicate-' || id) where id in (select id from (select id from x_policy where service || name in (select service || name from x_policy group by service,name having count(*) >1)) as tmp); + DROP INDEX x_policy_service; + ALTER TABLE dbo.x_policy DROP CONSTRAINT x_policy_FK_service; + ALTER TABLE dbo.x_policy ALTER name varchar(512) NOT NULL, ALTER service bigint NOT NULL ; + ALTER TABLE dbo.x_policy ADD CONSTRAINT x_policy_UK_name_service UNIQUE NONCLUSTERED (name,service); + ALTER TABLE dbo.x_policy ADD CONSTRAINT x_policy_FK_service FOREIGN KEY(service) REFERENCES dbo.x_service (id); + CREATE NONCLUSTERED INDEX x_policy_service ON dbo.x_policy(service ASC); + END IF; + END IF; + END IF; +END +GO diff --git a/security-admin/db/sqlanywhere/patches/035-update-schema-for-x-policy.sql b/security-admin/db/sqlanywhere/patches/035-update-schema-for-x-policy.sql new file mode 100644 index 0000000000..5da4538d23 --- /dev/null +++ b/security-admin/db/sqlanywhere/patches/035-update-schema-for-x-policy.sql @@ -0,0 +1,179 @@ + +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +GO +create table dbo.x_policy_ref_resource ( + id bigint IDENTITY NOT NULL, + guid varchar(1024) DEFAULT NULL NULL, + create_time datetime DEFAULT NULL NULL, + update_time datetime DEFAULT NULL NULL, + added_by_id bigint DEFAULT NULL NULL, + upd_by_id bigint DEFAULT NULL NULL, + policy_id bigint NOT NULL, + resource_def_id bigint NOT NULL, + resource_name varchar(4000) DEFAULT NULL NULL, + CONSTRAINT x_policy_ref_res_PK_id PRIMARY KEY CLUSTERED(id), + CONSTRAINT x_p_ref_res_UK_polId_resDefId UNIQUE NONCLUSTERED (policy_id, resource_def_id) +) +GO + +create table dbo.x_policy_ref_access_type ( + id bigint IDENTITY NOT NULL, + guid varchar(1024) DEFAULT NULL NULL, + create_time datetime DEFAULT NULL NULL, + update_time datetime DEFAULT NULL NULL, + added_by_id bigint DEFAULT NULL NULL, + upd_by_id bigint DEFAULT NULL NULL, + policy_id bigint NOT NULL, + access_def_id bigint NOT NULL, + access_type_name varchar(4000) DEFAULT NULL NULL, + CONSTRAINT x_policy_ref_acc_PK_id PRIMARY KEY CLUSTERED(id), + CONSTRAINT x_p_ref_acc_UK_polId_accDefId UNIQUE NONCLUSTERED (policy_id, access_def_id) +) +GO + +create table dbo.x_policy_ref_condition ( + id bigint IDENTITY NOT NULL, + guid varchar(1024) DEFAULT NULL NULL, + create_time datetime DEFAULT NULL NULL, + update_time datetime DEFAULT NULL NULL, + added_by_id bigint DEFAULT NULL NULL, + upd_by_id bigint DEFAULT NULL NULL, + policy_id bigint NOT NULL, + condition_def_id bigint NOT NULL, + condition_name varchar(4000) DEFAULT NULL NULL, + CONSTRAINT x_policy_ref_cond_PK_id PRIMARY KEY CLUSTERED(id), + CONSTRAINT x_p_ref_cond_UK_polId_cDefId UNIQUE NONCLUSTERED (policy_id, condition_def_id) +) +GO + +create table dbo.x_policy_ref_datamask_type ( + id bigint IDENTITY NOT NULL, + guid varchar(1024) DEFAULT NULL NULL, + create_time datetime DEFAULT NULL NULL, + update_time datetime DEFAULT NULL NULL, + added_by_id bigint DEFAULT NULL NULL, + upd_by_id bigint DEFAULT NULL NULL, + policy_id bigint NOT NULL, + datamask_def_id bigint NOT NULL, + datamask_type_name varchar(4000) DEFAULT NULL NULL, + CONSTRAINT x_policy_ref_dmk_PK_id PRIMARY KEY CLUSTERED(id), + CONSTRAINT x_p_ref_dmk_UK_polId_dDefId UNIQUE NONCLUSTERED (policy_id, datamask_def_id) +) +GO + +create table dbo.x_policy_ref_user ( + id bigint IDENTITY NOT NULL, + guid varchar(1024) DEFAULT NULL NULL, + create_time datetime DEFAULT NULL NULL, + update_time datetime DEFAULT NULL NULL, + added_by_id bigint DEFAULT NULL NULL, + upd_by_id bigint DEFAULT NULL NULL, + policy_id bigint NOT NULL, + user_id bigint NOT NULL, + user_name varchar(4000) DEFAULT NULL NULL, + CONSTRAINT x_policy_ref_user_PK_id PRIMARY KEY CLUSTERED(id), + CONSTRAINT x_p_ref_usr_UK_polId_userId UNIQUE NONCLUSTERED (policy_id, user_id) +) +GO + +create table dbo.x_policy_ref_group ( + id bigint IDENTITY NOT NULL, + guid varchar(1024) DEFAULT NULL NULL, + create_time datetime DEFAULT NULL NULL, + update_time datetime DEFAULT NULL NULL, + added_by_id bigint DEFAULT NULL NULL, + upd_by_id bigint DEFAULT NULL NULL, + policy_id bigint NOT NULL, + group_id bigint NOT NULL, + group_name varchar(4000) DEFAULT NULL NULL, + CONSTRAINT x_policy_ref_group_PK_id PRIMARY KEY CLUSTERED(id), + CONSTRAINT x_p_ref_grp_UK_polId_grpId UNIQUE NONCLUSTERED (policy_id, group_id) +) +GO + +IF NOT EXISTS(select * from SYS.SYSCOLUMNS where tname = 'x_policy' and cname='policy_text') THEN + ALTER TABLE dbo.x_policy ADD (policy_text text DEFAULT NULL NULL); +END IF; +GO + +IF EXISTS ( + SELECT 1 + FROM sysobjects + WHERE NAME = 'removeForeignKeyConstraint' + AND TYPE = 'P' +) +BEGIN + drop procedure dbo.removeForeignKeyConstraint +END +GO + +CREATE PROCEDURE dbo.removeForeignKeyConstraint (IN table_name varchar(100)) +AS +BEGIN + DECLARE @stmt VARCHAR(300) + DECLARE cur CURSOR FOR + select 'alter table dbo.' + table_name + ' drop constraint ' + role + from SYS.SYSFOREIGNKEYS + where foreign_creator ='dbo' and foreign_tname = table_name + + OPEN cur WITH HOLD + fetch cur into @stmt + if (@@sqlstatus = 2) + BEGIN + close cur + DEALLOCATE CURSOR cur + END + + WHILE (@@sqlstatus = 0) + BEGIN + + execute(@stmt) + fetch cur into @stmt + END + close cur + DEALLOCATE CURSOR cur + +END +GO + +call dbo.removeForeignKeyConstraint('x_policy_item') +GO + +call dbo.removeForeignKeyConstraint('x_policy_item_access') +GO + +call dbo.removeForeignKeyConstraint('x_policy_item_condition') +GO + +call dbo.removeForeignKeyConstraint('x_policy_item_datamask') +GO + +call dbo.removeForeignKeyConstraint('x_policy_item_group_perm') +GO + +call dbo.removeForeignKeyConstraint('x_policy_item_user_perm') +GO + +call dbo.removeForeignKeyConstraint('x_policy_item_rowfilter') +GO + +call dbo.removeForeignKeyConstraint('x_policy_resource') +GO + +call dbo.removeForeignKeyConstraint('x_policy_resource_map') +GO + +exit diff --git a/security-admin/db/sqlanywhere/patches/036-denormalize-tag-tables.sql b/security-admin/db/sqlanywhere/patches/036-denormalize-tag-tables.sql new file mode 100644 index 0000000000..1fdbfaa539 --- /dev/null +++ b/security-admin/db/sqlanywhere/patches/036-denormalize-tag-tables.sql @@ -0,0 +1,71 @@ +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +IF NOT EXISTS(select * from SYS.SYSCOLUMNS where tname = 'x_tag_def' and cname = 'tag_attrs_def_text') THEN + ALTER TABLE dbo.x_tag_def ADD tag_attrs_def_text text DEFAULT NULL NULL; +END IF; +IF NOT EXISTS(select * from SYS.SYSCOLUMNS where tname = 'x_tag' and cname = 'tag_attrs_text') THEN + ALTER TABLE dbo.x_tag ADD tag_attrs_text text DEFAULT NULL NULL; +END IF; +IF NOT EXISTS(select * from SYS.SYSCOLUMNS where tname = 'x_service_resource' and cname = 'service_resource_elements_text') THEN + ALTER TABLE dbo.x_service_resource ADD service_resource_elements_text text DEFAULT NULL NULL; +END IF; +IF NOT EXISTS(select * from SYS.SYSCOLUMNS where tname = 'x_service_resource' and cname = 'tags_text') THEN + ALTER TABLE dbo.x_service_resource ADD tags_text text DEFAULT NULL NULL; +END IF; +GO + +CREATE PROCEDURE dbo.removeTagForeignKeyConstraint (IN table_name varchar(100)) +AS +BEGIN + DECLARE @stmt VARCHAR(300) + DECLARE cur CURSOR FOR + select 'alter table dbo.' + table_name + ' drop constraint ' + role + from SYS.SYSFOREIGNKEYS + where foreign_creator ='dbo' and foreign_tname = table_name + + OPEN cur WITH HOLD + fetch cur into @stmt + if (@@sqlstatus = 2) + BEGIN + close cur + DEALLOCATE CURSOR cur + END + + WHILE (@@sqlstatus = 0) + BEGIN + + execute(@stmt) + fetch cur into @stmt + END + close cur + DEALLOCATE CURSOR cur + +END +GO + +call dbo.removeTagForeignKeyConstraint('x_tag_attr_def') +GO + +call dbo.removeTagForeignKeyConstraint('x_tag_attr') +GO + +call dbo.removeTagForeignKeyConstraint('x_service_resource_element') +GO + +call dbo.removeTagForeignKeyConstraint('x_service_resource_element_val') +GO + +exit diff --git a/security-admin/db/sqlserver/optimized/current/ranger_core_db_sqlserver.sql b/security-admin/db/sqlserver/optimized/current/ranger_core_db_sqlserver.sql index c4710f2ae0..da726d8817 100644 --- a/security-admin/db/sqlserver/optimized/current/ranger_core_db_sqlserver.sql +++ b/security-admin/db/sqlserver/optimized/current/ranger_core_db_sqlserver.sql @@ -253,6 +253,78 @@ IF (OBJECT_ID('x_enum_def_FK_defid') IS NOT NULL) BEGIN ALTER TABLE [dbo].[x_enum_def] DROP CONSTRAINT x_enum_def_FK_defid END +IF (OBJECT_ID('x_policy_ref_resource_FK_policy_id') IS NOT NULL) +BEGIN + ALTER TABLE [dbo].[x_policy_ref_resource] DROP CONSTRAINT x_policy_ref_resource_FK_policy_id +END +IF (OBJECT_ID('x_policy_ref_resource_FK_res_def_id') IS NOT NULL) +BEGIN + ALTER TABLE [dbo].[x_policy_ref_resource] DROP CONSTRAINT x_policy_ref_resource_FK_res_def_id +END +IF (OBJECT_ID('x_policy_ref_resource') IS NOT NULL) +BEGIN + DROP TABLE [dbo].[x_policy_ref_resource] +END +IF (OBJECT_ID('x_policy_ref_access_type_FK_policy_id') IS NOT NULL) +BEGIN + ALTER TABLE [dbo].[x_policy_ref_access_type] DROP CONSTRAINT x_policy_ref_access_type_FK_policy_id +END +IF (OBJECT_ID('x_policy_ref_access_type_FK_access_def_id') IS NOT NULL) +BEGIN + ALTER TABLE [dbo].[x_policy_ref_access_type] DROP CONSTRAINT x_policy_ref_access_type_FK_access_def_id +END +IF (OBJECT_ID('x_policy_ref_access_type') IS NOT NULL) +BEGIN + DROP TABLE [dbo].[x_policy_ref_access_type] +END +IF (OBJECT_ID('x_policy_ref_condition_FK_policy_id') IS NOT NULL) +BEGIN + ALTER TABLE [dbo].[x_policy_ref_condition] DROP CONSTRAINT x_policy_ref_condition_FK_policy_id +END +IF (OBJECT_ID('x_policy_ref_condition_FK_condition_def_id') IS NOT NULL) +BEGIN + ALTER TABLE [dbo].[x_policy_ref_condition] DROP CONSTRAINT x_policy_ref_condition_FK_condition_def_id +END +IF (OBJECT_ID('x_policy_ref_condition') IS NOT NULL) +BEGIN + DROP TABLE [dbo].[x_policy_ref_condition] +END +IF (OBJECT_ID('x_policy_ref_datamask_type_FK_policy_id') IS NOT NULL) +BEGIN + ALTER TABLE [dbo].[x_policy_ref_datamask_type] DROP CONSTRAINT x_policy_ref_datamask_type_FK_policy_id +END +IF (OBJECT_ID('x_policy_ref_datamask_type_FK_datamask_def_id') IS NOT NULL) +BEGIN + ALTER TABLE [dbo].[x_policy_ref_datamask_type] DROP CONSTRAINT x_policy_ref_datamask_type_FK_datamask_def_id +END +IF (OBJECT_ID('x_policy_ref_datamask_type') IS NOT NULL) +BEGIN + DROP TABLE [dbo].[x_policy_ref_datamask_type] +END +IF (OBJECT_ID('x_policy_ref_user_FK_policy_id') IS NOT NULL) +BEGIN + ALTER TABLE [dbo].[x_policy_ref_user] DROP CONSTRAINT x_policy_ref_user_FK_policy_id +END +IF (OBJECT_ID('x_policy_ref_user_FK_user_id') IS NOT NULL) +BEGIN + ALTER TABLE [dbo].[x_policy_ref_user] DROP CONSTRAINT x_policy_ref_user_FK_user_id +END +IF (OBJECT_ID('x_policy_ref_user') IS NOT NULL) +BEGIN + DROP TABLE [dbo].[x_policy_ref_user] +END +IF (OBJECT_ID('x_policy_ref_group_FK_policy_id') IS NOT NULL) +BEGIN + ALTER TABLE [dbo].[x_policy_ref_group] DROP CONSTRAINT x_policy_ref_group_FK_policy_id +END +IF (OBJECT_ID('x_policy_ref_group_FK_group_id') IS NOT NULL) +BEGIN + ALTER TABLE [dbo].[x_policy_ref_group] DROP CONSTRAINT x_policy_ref_group_FK_group_id +END +IF (OBJECT_ID('x_policy_ref_group') IS NOT NULL) +BEGIN + DROP TABLE [dbo].[x_policy_ref_group] +END IF (OBJECT_ID('x_enum_element_def_FK_defid') IS NOT NULL) BEGIN ALTER TABLE [dbo].[x_enum_element_def] DROP CONSTRAINT x_enum_element_def_FK_defid @@ -453,10 +525,18 @@ IF (OBJECT_ID('x_service_version_info_service_id') IS NOT NULL) BEGIN ALTER TABLE [dbo].[x_service_version_info] DROP CONSTRAINT x_service_version_info_service_id END +IF (OBJECT_ID('x_group_UK_group_name') IS NOT NULL) +BEGIN + ALTER TABLE [dbo].[x_group] DROP CONSTRAINT x_group_UK_group_name +END IF (OBJECT_ID('x_plugin_info_UK') IS NOT NULL) BEGIN ALTER TABLE [dbo].[x_plugin_info] DROP CONSTRAINT x_plugin_info_UK END +IF (OBJECT_ID('x_policy$x_policy_UK_name_service') IS NOT NULL) +BEGIN + ALTER TABLE [dbo].[x_policy] DROP CONSTRAINT x_policy$x_policy_UK_name_service +END IF (OBJECT_ID('vx_trx_log') IS NOT NULL) BEGIN DROP VIEW [dbo].[vx_trx_log] @@ -851,7 +931,7 @@ CREATE TABLE [dbo].[x_group]( [update_time] [datetime2] DEFAULT NULL NULL, [added_by_id] [bigint] DEFAULT NULL NULL, [upd_by_id] [bigint] DEFAULT NULL NULL, - [group_name] [varchar](1024) NOT NULL, + [group_name] [varchar](767) NOT NULL, [descr] [varchar](4000) NOT NULL, [status] [int] DEFAULT 0 NOT NULL, [group_type] [int] DEFAULT 0 NOT NULL, @@ -861,6 +941,10 @@ CREATE TABLE [dbo].[x_group]( PRIMARY KEY CLUSTERED ( [id] ASC +)WITH (PAD_INDEX = OFF,STATISTICS_NORECOMPUTE = OFF,IGNORE_DUP_KEY = OFF,ALLOW_ROW_LOCKS = ON,ALLOW_PAGE_LOCKS = ON) ON [PRIMARY], +CONSTRAINT [x_group$x_group_UK_group_name] UNIQUE NONCLUSTERED +( + [group_name] ASC )WITH (PAD_INDEX = OFF,STATISTICS_NORECOMPUTE = OFF,IGNORE_DUP_KEY = OFF,ALLOW_ROW_LOCKS = ON,ALLOW_PAGE_LOCKS = ON) ON [PRIMARY] ) ON [PRIMARY] @@ -915,12 +999,17 @@ CREATE TABLE [dbo].[x_group_users]( [update_time] [datetime2] DEFAULT NULL NULL, [added_by_id] [bigint] DEFAULT NULL NULL, [upd_by_id] [bigint] DEFAULT NULL NULL, - [group_name] [varchar](1024) NOT NULL, + [group_name] [varchar](767) NOT NULL, [p_group_id] [bigint] DEFAULT NULL NULL, [user_id] [bigint] DEFAULT NULL NULL, PRIMARY KEY CLUSTERED ( [id] ASC +)WITH (PAD_INDEX = OFF,STATISTICS_NORECOMPUTE = OFF,IGNORE_DUP_KEY = OFF,ALLOW_ROW_LOCKS = ON,ALLOW_PAGE_LOCKS = ON) ON [PRIMARY], +CONSTRAINT [[x_group_users$x_group_users_UK_uid_gname] UNIQUE NONCLUSTERED +( + [user_id] ASC, + [group_name] ASC )WITH (PAD_INDEX = OFF,STATISTICS_NORECOMPUTE = OFF,IGNORE_DUP_KEY = OFF,ALLOW_ROW_LOCKS = ON,ALLOW_PAGE_LOCKS = ON) ON [PRIMARY] ) ON [PRIMARY] @@ -1123,17 +1212,22 @@ CREATE TABLE [dbo].[x_policy] ( [added_by_id] [bigint] DEFAULT NULL NULL, [upd_by_id] [bigint] DEFAULT NULL NULL, [version] [bigint] DEFAULT NULL NULL, - [service] [bigint] DEFAULT NULL NULL, - [name] [varchar](512) DEFAULT NULL NULL, + [service] [bigint] NOT NULL, + [name] [varchar](512) NOT NULL, [policy_type] [int] DEFAULT 0 NULL, [description] [varchar](1024) DEFAULT NULL NULL, [resource_signature] [varchar](128) DEFAULT NULL NULL, [is_enabled] [tinyint] DEFAULT 0 NOT NULL, [is_audit_enabled] [tinyint] DEFAULT 0 NOT NULL, + [policy_text] [nvarchar](max) DEFAULT NULL NULL, PRIMARY KEY CLUSTERED ( [id] ASC -)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY] +)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY], +CONSTRAINT [x_policy$x_policy_UK_name_service] UNIQUE NONCLUSTERED +( + [name] ASC, [service] ASC +)WITH (PAD_INDEX = OFF,STATISTICS_NORECOMPUTE = OFF,IGNORE_DUP_KEY = OFF,ALLOW_ROW_LOCKS = ON,ALLOW_PAGE_LOCKS = ON) ON [PRIMARY] ) ON [PRIMARY] SET ANSI_NULLS ON SET QUOTED_IDENTIFIER ON @@ -1566,6 +1660,7 @@ CREATE TABLE [dbo].[x_tag_def]( [name] [varchar](255) NOT NULL, [source] [varchar](128) DEFAULT NULL NULL, [is_enabled] [tinyint] DEFAULT 0 NOT NULL, + [tag_attrs_def_text] [nvarchar](max) DEFAULT NULL NULL, PRIMARY KEY CLUSTERED ( [id] ASC @@ -1592,6 +1687,7 @@ CREATE TABLE [dbo].[x_tag]( [version] [bigint] DEFAULT NULL NULL, [type] [bigint] NOT NULL, [owned_by] [smallint] DEFAULT 0 NOT NULL, + [tag_attrs_text] [nvarchar](max) DEFAULT NULL NULL, PRIMARY KEY CLUSTERED ( [id] ASC @@ -1615,6 +1711,8 @@ CREATE TABLE [dbo].[x_service_resource]( [service_id] [bigint] NOT NULL, [resource_signature] [varchar](128) DEFAULT NULL NULL, [is_enabled] [tinyint] DEFAULT 1 NOT NULL, + [service_resource_elements_text] [nvarchar](max) DEFAULT NULL NULL, + [tags_text] [nvarchar](max) DEFAULT NULL NULL, PRIMARY KEY CLUSTERED ( [id] ASC @@ -1808,6 +1906,138 @@ CONSTRAINT [x_plugin_info$x_plugin_info_UK] UNIQUE NONCLUSTERED SET ANSI_NULLS ON SET QUOTED_IDENTIFIER ON SET ANSI_PADDING ON +CREATE TABLE [dbo].[x_policy_ref_resource] ( + [id] [bigint] IDENTITY (1, 1) NOT NULL, + [guid] [varchar](1024) DEFAULT NULL NULL, + [create_time] [datetime2] DEFAULT NULL NULL, + [update_time] [datetime2] DEFAULT NULL NULL, + [added_by_id] [bigint] DEFAULT NULL NULL, + [upd_by_id] [bigint] DEFAULT NULL NULL, + [policy_id] [bigint] NOT NULL, + [resource_def_id] [bigint] NOT NULL, + [resource_name] [varchar](4000) DEFAULT NULL NULL, + PRIMARY KEY CLUSTERED + ( + [id] ASC + ) WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY], + CONSTRAINT [x_policy_ref_resource$x_policy_ref_resource_UK] UNIQUE NONCLUSTERED + ( + [policy_id] ASC, [resource_def_id] ASC + )WITH (PAD_INDEX = OFF,STATISTICS_NORECOMPUTE = OFF,IGNORE_DUP_KEY = OFF,ALLOW_ROW_LOCKS = ON,ALLOW_PAGE_LOCKS = ON) ON [PRIMARY] +) ON [PRIMARY] +SET ANSI_NULLS ON +SET QUOTED_IDENTIFIER ON +SET ANSI_PADDING ON +CREATE TABLE [dbo].[x_policy_ref_access_type] ( + [id] [bigint] IDENTITY (1, 1) NOT NULL, + [guid] [varchar](1024) DEFAULT NULL NULL, + [create_time] [datetime2] DEFAULT NULL NULL, + [update_time] [datetime2] DEFAULT NULL NULL, + [added_by_id] [bigint] DEFAULT NULL NULL, + [upd_by_id] [bigint] DEFAULT NULL NULL, + [policy_id] [bigint] NOT NULL, + [access_def_id] [bigint] NOT NULL, + [access_type_name] [varchar](4000) DEFAULT NULL NULL, + PRIMARY KEY CLUSTERED + ( + [id] ASC + ) WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY], + CONSTRAINT [x_policy_ref_access_type$x_policy_ref_access_type_UK] UNIQUE NONCLUSTERED +( + [policy_id] ASC, [access_def_id] ASC +)WITH (PAD_INDEX = OFF,STATISTICS_NORECOMPUTE = OFF,IGNORE_DUP_KEY = OFF,ALLOW_ROW_LOCKS = ON,ALLOW_PAGE_LOCKS = ON) ON [PRIMARY] +) ON [PRIMARY] +SET ANSI_NULLS ON +SET QUOTED_IDENTIFIER ON +SET ANSI_PADDING ON +CREATE TABLE [dbo].[x_policy_ref_condition] ( + [id] [bigint] IDENTITY (1, 1) NOT NULL, + [guid] [varchar](1024) DEFAULT NULL NULL, + [create_time] [datetime2] DEFAULT NULL NULL, + [update_time] [datetime2] DEFAULT NULL NULL, + [added_by_id] [bigint] DEFAULT NULL NULL, + [upd_by_id] [bigint] DEFAULT NULL NULL, + [policy_id] [bigint] NOT NULL, + [condition_def_id] [bigint] NOT NULL, + [condition_name] [varchar](4000) DEFAULT NULL NULL, + PRIMARY KEY CLUSTERED + ( + [id] ASC + ) WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY], + CONSTRAINT [x_policy_ref_condition$x_policy_ref_condition_UK] UNIQUE NONCLUSTERED +( + [policy_id] ASC, [condition_def_id] ASC +)WITH (PAD_INDEX = OFF,STATISTICS_NORECOMPUTE = OFF,IGNORE_DUP_KEY = OFF,ALLOW_ROW_LOCKS = ON,ALLOW_PAGE_LOCKS = ON) ON [PRIMARY] +) ON [PRIMARY] +SET ANSI_NULLS ON +SET QUOTED_IDENTIFIER ON +SET ANSI_PADDING ON +CREATE TABLE [dbo].[x_policy_ref_datamask_type] ( + [id] [bigint] IDENTITY (1, 1) NOT NULL, + [guid] [varchar](1024) DEFAULT NULL NULL, + [create_time] [datetime2] DEFAULT NULL NULL, + [update_time] [datetime2] DEFAULT NULL NULL, + [added_by_id] [bigint] DEFAULT NULL NULL, + [upd_by_id] [bigint] DEFAULT NULL NULL, + [policy_id] [bigint] NOT NULL, + [datamask_def_id] [bigint] NOT NULL, + [datamask_type_name] [varchar](4000) DEFAULT NULL NULL, + PRIMARY KEY CLUSTERED + ( + [id] ASC + ) WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY], + CONSTRAINT [x_policy_ref_datamask_type$x_policy_ref_datamask_type_UK] UNIQUE NONCLUSTERED +( + [policy_id] ASC, [datamask_def_id] ASC +)WITH (PAD_INDEX = OFF,STATISTICS_NORECOMPUTE = OFF,IGNORE_DUP_KEY = OFF,ALLOW_ROW_LOCKS = ON,ALLOW_PAGE_LOCKS = ON) ON [PRIMARY] +) ON [PRIMARY] +SET ANSI_NULLS ON +SET QUOTED_IDENTIFIER ON +SET ANSI_PADDING ON +CREATE TABLE [dbo].[x_policy_ref_user] ( + [id] [bigint] IDENTITY (1, 1) NOT NULL, + [guid] [varchar](1024) DEFAULT NULL NULL, + [create_time] [datetime2] DEFAULT NULL NULL, + [update_time] [datetime2] DEFAULT NULL NULL, + [added_by_id] [bigint] DEFAULT NULL NULL, + [upd_by_id] [bigint] DEFAULT NULL NULL, + [policy_id] [bigint] NOT NULL, + [user_id] [bigint] NOT NULL, + [user_name] [varchar](4000) DEFAULT NULL NULL, + PRIMARY KEY CLUSTERED + ( + [id] ASC + ) WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY], + CONSTRAINT [x_policy_ref_user$x_policy_ref_user_UK] UNIQUE NONCLUSTERED +( + [policy_id] ASC, [user_id] ASC +)WITH (PAD_INDEX = OFF,STATISTICS_NORECOMPUTE = OFF,IGNORE_DUP_KEY = OFF,ALLOW_ROW_LOCKS = ON,ALLOW_PAGE_LOCKS = ON) ON [PRIMARY] +) ON [PRIMARY] +SET ANSI_NULLS ON +SET QUOTED_IDENTIFIER ON +SET ANSI_PADDING ON +CREATE TABLE [dbo].[x_policy_ref_group] ( + [id] [bigint] IDENTITY (1, 1) NOT NULL, + [guid] [varchar](1024) DEFAULT NULL NULL, + [create_time] [datetime2] DEFAULT NULL NULL, + [update_time] [datetime2] DEFAULT NULL NULL, + [added_by_id] [bigint] DEFAULT NULL NULL, + [upd_by_id] [bigint] DEFAULT NULL NULL, + [policy_id] [bigint] NOT NULL, + [group_id] [bigint] NOT NULL, + [group_name] [varchar](4000) DEFAULT NULL NULL, + PRIMARY KEY CLUSTERED + ( + [id] ASC + ) WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY], + CONSTRAINT [x_policy_ref_group$x_policy_ref_group_UK] UNIQUE NONCLUSTERED +( + [policy_id] ASC, [group_id] ASC +)WITH (PAD_INDEX = OFF,STATISTICS_NORECOMPUTE = OFF,IGNORE_DUP_KEY = OFF,ALLOW_ROW_LOCKS = ON,ALLOW_PAGE_LOCKS = ON) ON [PRIMARY] +) ON [PRIMARY] +SET ANSI_NULLS ON +SET QUOTED_IDENTIFIER ON +SET ANSI_PADDING ON ALTER TABLE [dbo].[x_asset] WITH CHECK ADD CONSTRAINT [x_asset_FK_added_by_id] FOREIGN KEY([added_by_id]) REFERENCES [dbo].[x_portal_user] ([id]) @@ -2170,6 +2400,73 @@ ALTER TABLE [dbo].[x_policy_item_rowfilter] WITH CHECK ADD CONSTRAINT [x_policy_ ALTER TABLE [dbo].[x_policy_item_rowfilter] WITH CHECK ADD CONSTRAINT [x_policy_item_rowfilter_FK_added_by_id] FOREIGN KEY([added_by_id]) REFERENCES [dbo].[x_portal_user] ([id]) ALTER TABLE [dbo].[x_policy_item_rowfilter] WITH CHECK ADD CONSTRAINT [x_policy_item_rowfilter_FK_upd_by_id] FOREIGN KEY([upd_by_id]) REFERENCES [dbo].[x_portal_user] ([id]) ALTER TABLE [dbo].[x_service_version_info] WITH CHECK ADD CONSTRAINT [x_service_version_info_service_id] FOREIGN KEY([service_id]) REFERENCES [dbo].[x_service] ([id]) +ALTER TABLE [dbo].[x_policy_ref_resource] WITH CHECK ADD CONSTRAINT [x_policy_ref_resource_FK_policy_id] FOREIGN KEY ([policy_id]) +REFERENCES [dbo].[x_policy] ([id]) +ALTER TABLE [dbo].[x_policy_ref_resource] CHECK CONSTRAINT [x_policy_ref_resource_FK_policy_id] +ALTER TABLE [dbo].[x_policy_ref_resource] WITH CHECK ADD CONSTRAINT [x_policy_ref_resource_FK_resource_def_id] FOREIGN KEY ([resource_def_id]) +REFERENCES [dbo].[x_resource_def] ([id]) +ALTER TABLE [dbo].[x_policy_ref_resource] CHECK CONSTRAINT [x_policy_ref_resource_FK_resource_def_id] +ALTER TABLE [dbo].[x_policy_ref_resource] WITH CHECK ADD CONSTRAINT [x_policy_ref_resource_FK_added_by] FOREIGN KEY ([added_by_id]) +REFERENCES [dbo].[x_portal_user] ([id]) +ALTER TABLE [dbo].[x_policy_ref_resource] CHECK CONSTRAINT [x_policy_ref_resource_FK_added_by] +ALTER TABLE [dbo].[x_policy_ref_resource] WITH CHECK ADD CONSTRAINT [x_policy_ref_resource_FK_upd_by] FOREIGN KEY ([upd_by_id]) +REFERENCES [dbo].[x_portal_user] ([id]) +ALTER TABLE [dbo].[x_policy_ref_access_type] WITH CHECK ADD CONSTRAINT [x_policy_ref_access_type_FK_policy_id] FOREIGN KEY ([policy_id]) +REFERENCES [dbo].[x_policy] ([id]) +ALTER TABLE [dbo].[x_policy_ref_access_type] CHECK CONSTRAINT [x_policy_ref_access_type_FK_policy_id] +ALTER TABLE [dbo].[x_policy_ref_access_type] WITH CHECK ADD CONSTRAINT [x_policy_ref_access_type_FK_access_def_id] FOREIGN KEY ([access_def_id]) +REFERENCES [dbo].[x_access_type_def] ([id]) +ALTER TABLE [dbo].[x_policy_ref_access_type] CHECK CONSTRAINT [x_policy_ref_access_type_FK_access_def_id] +ALTER TABLE [dbo].[x_policy_ref_access_type] WITH CHECK ADD CONSTRAINT [x_policy_ref_access_type_FK_added_by] FOREIGN KEY ([added_by_id]) +REFERENCES [dbo].[x_portal_user] ([id]) +ALTER TABLE [dbo].[x_policy_ref_access_type] CHECK CONSTRAINT [x_policy_ref_access_type_FK_added_by] +ALTER TABLE [dbo].[x_policy_ref_access_type] WITH CHECK ADD CONSTRAINT [x_policy_ref_access_type_FK_upd_by] FOREIGN KEY ([upd_by_id]) +REFERENCES [dbo].[x_portal_user] ([id]) +ALTER TABLE [dbo].[x_policy_ref_condition] WITH CHECK ADD CONSTRAINT [x_policy_ref_condition_FK_policy_id] FOREIGN KEY ([policy_id]) +REFERENCES [dbo].[x_policy] ([id]) +ALTER TABLE [dbo].[x_policy_ref_condition] CHECK CONSTRAINT [x_policy_ref_condition_FK_policy_id] +ALTER TABLE [dbo].[x_policy_ref_condition] WITH CHECK ADD CONSTRAINT [x_policy_ref_condition_FK_condition_def_id] FOREIGN KEY ([condition_def_id]) +REFERENCES [dbo].[x_policy_condition_def] ([id]) +ALTER TABLE [dbo].[x_policy_ref_condition] CHECK CONSTRAINT [x_policy_ref_condition_FK_condition_def_id] +ALTER TABLE [dbo].[x_policy_ref_condition] WITH CHECK ADD CONSTRAINT [x_policy_ref_condition_FK_added_by] FOREIGN KEY ([added_by_id]) +REFERENCES [dbo].[x_portal_user] ([id]) +ALTER TABLE [dbo].[x_policy_ref_condition] CHECK CONSTRAINT [x_policy_ref_condition_FK_added_by] +ALTER TABLE [dbo].[x_policy_ref_condition] WITH CHECK ADD CONSTRAINT [x_policy_ref_condition_FK_upd_by] FOREIGN KEY ([upd_by_id]) +REFERENCES [dbo].[x_portal_user] ([id]) +ALTER TABLE [dbo].[x_policy_ref_datamask_type] WITH CHECK ADD CONSTRAINT [x_policy_ref_datamask_type_FK_policy_id] FOREIGN KEY ([policy_id]) +REFERENCES [dbo].[x_policy] ([id]) +ALTER TABLE [dbo].[x_policy_ref_datamask_type] CHECK CONSTRAINT [x_policy_ref_datamask_type_FK_policy_id] +ALTER TABLE [dbo].[x_policy_ref_datamask_type] WITH CHECK ADD CONSTRAINT [x_policy_ref_datamask_type_FK_datamask_def_id] FOREIGN KEY ([datamask_def_id]) +REFERENCES [dbo].[x_datamask_type_def] ([id]) +ALTER TABLE [dbo].[x_policy_ref_datamask_type] CHECK CONSTRAINT [x_policy_ref_datamask_type_FK_datamask_def_id] +ALTER TABLE [dbo].[x_policy_ref_datamask_type] WITH CHECK ADD CONSTRAINT [x_policy_ref_datamask_type_FK_added_by] FOREIGN KEY ([added_by_id]) +REFERENCES [dbo].[x_portal_user] ([id]) +ALTER TABLE [dbo].[x_policy_ref_datamask_type] CHECK CONSTRAINT [x_policy_ref_datamask_type_FK_added_by] +ALTER TABLE [dbo].[x_policy_ref_datamask_type] WITH CHECK ADD CONSTRAINT [x_policy_ref_datamask_type_FK_upd_by] FOREIGN KEY ([upd_by_id]) +REFERENCES [dbo].[x_portal_user] ([id]) +ALTER TABLE [dbo].[x_policy_ref_user] WITH CHECK ADD CONSTRAINT [x_policy_ref_user_FK_policy_id] FOREIGN KEY ([policy_id]) +REFERENCES [dbo].[x_policy] ([id]) +ALTER TABLE [dbo].[x_policy_ref_user] CHECK CONSTRAINT [x_policy_ref_user_FK_policy_id] +ALTER TABLE [dbo].[x_policy_ref_user] WITH CHECK ADD CONSTRAINT [x_policy_ref_user_FK_user_id] FOREIGN KEY ([user_id]) +REFERENCES [dbo].[x_user] ([id]) +ALTER TABLE [dbo].[x_policy_ref_user] CHECK CONSTRAINT [x_policy_ref_user_FK_user_id] +ALTER TABLE [dbo].[x_policy_ref_user] WITH CHECK ADD CONSTRAINT [x_policy_ref_user_FK_added_by] FOREIGN KEY ([added_by_id]) +REFERENCES [dbo].[x_portal_user] ([id]) +ALTER TABLE [dbo].[x_policy_ref_user] CHECK CONSTRAINT [x_policy_ref_user_FK_added_by] +ALTER TABLE [dbo].[x_policy_ref_user] WITH CHECK ADD CONSTRAINT [x_policy_ref_user_FK_upd_by] FOREIGN KEY ([upd_by_id]) +REFERENCES [dbo].[x_portal_user] ([id]) +ALTER TABLE [dbo].[x_policy_ref_user] CHECK CONSTRAINT [x_policy_ref_user_FK_upd_by] +ALTER TABLE [dbo].[x_policy_ref_group] WITH CHECK ADD CONSTRAINT [x_policy_ref_group_FK_policy_id] FOREIGN KEY ([policy_id]) +REFERENCES [dbo].[x_policy] ([id]) +ALTER TABLE [dbo].[x_policy_ref_group] CHECK CONSTRAINT [x_policy_ref_group_FK_policy_id] +ALTER TABLE [dbo].[x_policy_ref_group] WITH CHECK ADD CONSTRAINT [x_policy_ref_group_FK_group_id] FOREIGN KEY ([group_id]) +REFERENCES [dbo].[x_group] ([id]) +ALTER TABLE [dbo].[x_policy_ref_group] CHECK CONSTRAINT [x_policy_ref_group_FK_group_id] +ALTER TABLE [dbo].[x_policy_ref_group] WITH CHECK ADD CONSTRAINT [x_policy_ref_group_FK_added_by] FOREIGN KEY ([added_by_id]) +REFERENCES [dbo].[x_portal_user] ([id]) +ALTER TABLE [dbo].[x_policy_ref_group] CHECK CONSTRAINT [x_policy_ref_group_FK_added_by] +ALTER TABLE [dbo].[x_policy_ref_group] WITH CHECK ADD CONSTRAINT [x_policy_ref_group_FK_upd_by] FOREIGN KEY ([upd_by_id]) +REFERENCES [dbo].[x_portal_user] ([id]) CREATE NONCLUSTERED INDEX [x_asset_cr_time] ON [x_asset] ( [create_time] ASC @@ -2975,6 +3272,11 @@ INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active) VALUES ('025',CURRENT_TIMESTAMP,'Ranger 0.7.0',CURRENT_TIMESTAMP,'localhost','Y'); INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active) VALUES ('026',CURRENT_TIMESTAMP,'Ranger 0.7.0',CURRENT_TIMESTAMP,'localhost','Y'); INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active) VALUES ('027',CURRENT_TIMESTAMP,'Ranger 0.7.0',CURRENT_TIMESTAMP,'localhost','Y'); +INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active) VALUES ('028',CURRENT_TIMESTAMP,'Ranger 0.7.0',CURRENT_TIMESTAMP,'localhost','Y'); +INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active) VALUES ('029',CURRENT_TIMESTAMP,'Ranger 0.7.0',CURRENT_TIMESTAMP,'localhost','Y'); +INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active) VALUES ('033',CURRENT_TIMESTAMP,'Ranger 0.7.0',CURRENT_TIMESTAMP,'localhost','Y'); +INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active) VALUES ('035',CURRENT_TIMESTAMP,'Ranger 0.7.0',CURRENT_TIMESTAMP,'localhost','Y'); +INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active) VALUES ('036',CURRENT_TIMESTAMP,'Ranger 0.7.0',CURRENT_TIMESTAMP,'localhost','Y'); INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active) VALUES ('DB_PATCHES',CURRENT_TIMESTAMP,'Ranger 0.7.0',CURRENT_TIMESTAMP,'localhost','Y'); INSERT INTO x_user_module_perm (user_id,module_id,create_time,update_time,added_by_id,upd_by_id,is_allowed) VALUES (1,3,CURRENT_TIMESTAMP,CURRENT_TIMESTAMP,1,1,1); INSERT INTO x_user_module_perm (user_id,module_id,create_time,update_time,added_by_id,upd_by_id,is_allowed) VALUES (1,1,CURRENT_TIMESTAMP,CURRENT_TIMESTAMP,1,1,1); @@ -3000,9 +3302,16 @@ INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active) VALUES ('J10004',CURRENT_TIMESTAMP,'Ranger 0.7.0',CURRENT_TIMESTAMP,'localhost','Y'); INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active) VALUES ('J10005',CURRENT_TIMESTAMP,'Ranger 0.7.0',CURRENT_TIMESTAMP,'localhost','Y'); INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active) VALUES ('J10006',CURRENT_TIMESTAMP,'Ranger 0.7.0',CURRENT_TIMESTAMP,'localhost','Y'); +INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active) VALUES ('J10007',CURRENT_TIMESTAMP,'Ranger 0.7.0',CURRENT_TIMESTAMP,'localhost','Y'); +INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active) VALUES ('J10008',CURRENT_TIMESTAMP,'Ranger 0.7.0',CURRENT_TIMESTAMP,'localhost','Y'); +INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active) VALUES ('J10009',CURRENT_TIMESTAMP,'Ranger 0.7.0',CURRENT_TIMESTAMP,'localhost','Y'); +INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active) VALUES ('J10010',CURRENT_TIMESTAMP,'Ranger 0.7.0',CURRENT_TIMESTAMP,'localhost','Y'); +INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active) VALUES ('J10011',CURRENT_TIMESTAMP,'Ranger 0.7.0',CURRENT_TIMESTAMP,'localhost','Y'); +INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active) VALUES ('J10017',CURRENT_TIMESTAMP,'Ranger 0.7.0',CURRENT_TIMESTAMP,'localhost','Y'); +INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active) VALUES ('J10020',CURRENT_TIMESTAMP,'Ranger 0.7.0',CURRENT_TIMESTAMP,'localhost','Y'); INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active) VALUES ('JAVA_PATCHES',CURRENT_TIMESTAMP,'Ranger 0.7.0',CURRENT_TIMESTAMP,'localhost','Y'); GO CREATE VIEW [dbo].[vx_trx_log] AS select x_trx_log.id AS id,x_trx_log.create_time AS create_time,x_trx_log.update_time AS update_time,x_trx_log.added_by_id AS added_by_id,x_trx_log.upd_by_id AS upd_by_id,x_trx_log.class_type AS class_type,x_trx_log.object_id AS object_id,x_trx_log.parent_object_id AS parent_object_id,x_trx_log.parent_object_class_type AS parent_object_class_type,x_trx_log.attr_name AS attr_name,x_trx_log.parent_object_name AS parent_object_name,x_trx_log.object_name AS object_name,x_trx_log.prev_val AS prev_val,x_trx_log.new_val AS new_val,x_trx_log.trx_id AS trx_id,x_trx_log.action AS action,x_trx_log.sess_id AS sess_id,x_trx_log.req_id AS req_id,x_trx_log.sess_type AS sess_type from x_trx_log where id in(select min(x_trx_log.id) from x_trx_log group by x_trx_log.trx_id) -GO \ No newline at end of file +GO diff --git a/security-admin/db/sqlserver/patches/028-delete-xgroup-duplicate-references.sql b/security-admin/db/sqlserver/patches/028-delete-xgroup-duplicate-references.sql new file mode 100644 index 0000000000..e8c56b0f90 --- /dev/null +++ b/security-admin/db/sqlserver/patches/028-delete-xgroup-duplicate-references.sql @@ -0,0 +1,81 @@ +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +IF (OBJECT_ID('delete_xgroup_duplicate_references') IS NOT NULL) +BEGIN + DROP PROCEDURE [dbo].[delete_xgroup_duplicate_references] +END +GO +CREATE PROCEDURE delete_xgroup_duplicate_references +AS BEGIN + DECLARE @donecursor1 bigint + DECLARE @group_name1 varchar(1024) + DECLARE @mingroupid1 bigint + DECLARE @id2 bigint + DECLARE @group_name3 varchar(1024) + DECLARE @user_id3 bigint + DECLARE @minrowid3 bigint + DECLARE @group_name4 varchar(1024) + DECLARE @group_id4 bigint + DECLARE @minrowid4 bigint + DECLARE cursor1 CURSOR FOR SELECT group_name, min(id) FROM x_group GROUP BY group_name HAVING count(group_name)>1 + OPEN cursor1 + FETCH NEXT FROM cursor1 INTO @group_name1, @mingroupid1 + WHILE (@@FETCH_STATUS = 0) + BEGIN + DECLARE cursor2 CURSOR FOR SELECT id FROM x_group WHERE group_name = @group_name1 AND id > @mingroupid1 + OPEN cursor2 + FETCH NEXT FROM cursor2 INTO @id2 + WHILE (@@FETCH_STATUS = 0) + BEGIN + UPDATE x_group_users SET p_group_id=@mingroupid1 where p_group_id=@id2 + FETCH NEXT FROM cursor2 INTO @id2 + END + CLOSE cursor2 + DEALLOCATE cursor2 + FETCH NEXT FROM cursor1 INTO @group_name1, @mingroupid1 + END + CLOSE cursor1 + DEALLOCATE cursor1 + + DECLARE cursor3 CURSOR FOR SELECT group_name,user_id,min(id) FROM x_group_users GROUP BY group_name,user_id HAVING count(1)>1 + OPEN cursor3 + FETCH NEXT FROM cursor3 INTO @group_name3, @user_id3, @minrowid3 + WHILE (@@FETCH_STATUS = 0) + BEGIN + DELETE FROM x_group_users WHERE group_name=@group_name3 AND user_id=@user_id3 AND id > @minrowid3 + FETCH NEXT FROM cursor3 INTO @group_name3, @user_id3, @minrowid3 + END + CLOSE cursor3 + DEALLOCATE cursor3 + + DECLARE cursor4 CURSOR FOR SELECT group_name,min(id) FROM x_group GROUP BY group_name HAVING count(1)>1 + OPEN cursor4 + FETCH NEXT FROM cursor4 INTO @group_name4, @minrowid4 + WHILE (@@FETCH_STATUS = 0) + BEGIN + DELETE FROM x_group WHERE group_name=@group_name4 AND id > @minrowid4 + FETCH NEXT FROM cursor4 INTO @group_name4, @minrowid4 + END + CLOSE cursor4 + DEALLOCATE cursor4 +END +GO +IF (OBJECT_ID('delete_xgroup_duplicate_references') IS NOT NULL) +BEGIN + EXEC delete_xgroup_duplicate_references +END +GO +exit \ No newline at end of file diff --git a/security-admin/db/sqlserver/patches/029-add-unique-constraint-on-table-x_group.sql b/security-admin/db/sqlserver/patches/029-add-unique-constraint-on-table-x_group.sql new file mode 100644 index 0000000000..7559976116 --- /dev/null +++ b/security-admin/db/sqlserver/patches/029-add-unique-constraint-on-table-x_group.sql @@ -0,0 +1,40 @@ + +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +IF EXISTS(select * from INFORMATION_SCHEMA.columns where table_name = 'x_group' and column_name = 'group_name') +BEGIN + IF NOT EXISTS(select * from INFORMATION_SCHEMA.CONSTRAINT_COLUMN_USAGE where table_name='x_group' and column_name='group_name' and constraint_name = 'x_group$x_group_UK_group_name') + BEGIN + IF NOT EXISTS(select * from INFORMATION_SCHEMA.TABLE_CONSTRAINTS where table_name='x_group' and constraint_name = 'x_group$x_group_UK_group_name' and CONSTRAINT_TYPE='UNIQUE') + BEGIN + ALTER TABLE [dbo].[x_group] ALTER COLUMN [group_name] [varchar](767) NOT NULL; + ALTER TABLE [dbo].[x_group] ADD CONSTRAINT [x_group$x_group_UK_group_name] UNIQUE ([group_name]); + END + END +END +GO +IF EXISTS(select * from INFORMATION_SCHEMA.columns where table_name = 'x_group_users' and column_name = 'group_name') +BEGIN + IF NOT EXISTS(select * from INFORMATION_SCHEMA.CONSTRAINT_COLUMN_USAGE where table_name='x_group_users' and column_name='group_name' and constraint_name = 'x_group_users$x_group_users_UK_uid_gname') + BEGIN + IF NOT EXISTS(select * from INFORMATION_SCHEMA.TABLE_CONSTRAINTS where table_name='x_group_users' and constraint_name = 'x_group_users$x_group_users_UK_uid_gname' and CONSTRAINT_TYPE='UNIQUE') + BEGIN + ALTER TABLE [dbo].[x_group_users] ALTER COLUMN [group_name] [varchar](767) NOT NULL; + ALTER TABLE [dbo].[x_group_users] ADD CONSTRAINT [x_group_users$x_group_users_UK_uid_gname] UNIQUE (user_id,group_name); + END + END +END +GO +exit diff --git a/security-admin/db/sqlserver/patches/033-add-unique-constraint-on-table-x_policy.sql b/security-admin/db/sqlserver/patches/033-add-unique-constraint-on-table-x_policy.sql new file mode 100644 index 0000000000..075fe8676c --- /dev/null +++ b/security-admin/db/sqlserver/patches/033-add-unique-constraint-on-table-x_policy.sql @@ -0,0 +1,29 @@ + + +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +IF EXISTS(select * from INFORMATION_SCHEMA.columns where table_name = 'x_policy' and column_name = 'name') +BEGIN + IF NOT EXISTS(select * from INFORMATION_SCHEMA.CONSTRAINT_COLUMN_USAGE where table_name='x_policy' and column_name='name' and constraint_name = 'x_policy$x_policy_UK_name_service') + BEGIN + IF NOT EXISTS(select * from INFORMATION_SCHEMA.TABLE_CONSTRAINTS where table_name='x_policy' and constraint_name = 'x_policy$x_policy_UK_name_service' and CONSTRAINT_TYPE='UNIQUE') + BEGIN + UPDATE [dbo].[x_policy] set name=concat(name, '-duplicate-',id) where id in (select id from (select id from [dbo].[x_policy] where concat(service,name) in (select concat(service,name) from [dbo].[x_policy] group by service,name having count(*) >1)) as tmp); + ALTER TABLE [dbo].[x_policy] ADD CONSTRAINT [x_policy$x_policy_UK_name_service] UNIQUE ([name],[service]); + END + END +END +GO +exit diff --git a/security-admin/db/sqlserver/patches/035-update-schema-for-x-policy.sql b/security-admin/db/sqlserver/patches/035-update-schema-for-x-policy.sql new file mode 100644 index 0000000000..ebf44ac836 --- /dev/null +++ b/security-admin/db/sqlserver/patches/035-update-schema-for-x-policy.sql @@ -0,0 +1,453 @@ +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +GO +IF (OBJECT_ID('x_policy_ref_resource_FK_policy_id') IS NOT NULL) +BEGIN + ALTER TABLE [dbo].[x_policy_ref_resource] DROP CONSTRAINT x_policy_ref_resource_FK_policy_id +END +GO +IF (OBJECT_ID('x_policy_ref_resource_FK_resource_def_id') IS NOT NULL) +BEGIN + ALTER TABLE [dbo].[x_policy_ref_resource] DROP CONSTRAINT x_policy_ref_resource_FK_resource_def_id +END +GO +IF (OBJECT_ID('x_policy_ref_resource_UK') IS NOT NULL) +BEGIN + ALTER TABLE [dbo].[x_policy_ref_resource] DROP CONSTRAINT x_policy_ref_resource_UK +END +GO +IF (OBJECT_ID('x_policy_ref_resource') IS NOT NULL) +BEGIN + DROP TABLE [dbo].[x_policy_ref_resource] +END +GO +IF (OBJECT_ID('x_policy_ref_access_type_FK_policy_id') IS NOT NULL) +BEGIN + ALTER TABLE [dbo].[x_policy_ref_access_type] DROP CONSTRAINT x_policy_ref_access_type_FK_policy_id +END +GO +IF (OBJECT_ID('x_policy_ref_access_type_FK_access_def_id') IS NOT NULL) +BEGIN + ALTER TABLE [dbo].[x_policy_ref_access_type] DROP CONSTRAINT x_policy_ref_access_type_FK_access_def_id +END +GO +IF (OBJECT_ID('x_policy_ref_access_type_UK') IS NOT NULL) +BEGIN + ALTER TABLE [dbo].[x_policy_ref_access_type] DROP CONSTRAINT x_policy_ref_access_type_UK +END +GO +IF (OBJECT_ID('x_policy_ref_access_type') IS NOT NULL) +BEGIN + DROP TABLE [dbo].[x_policy_ref_access_type] +END +GO +IF (OBJECT_ID('x_policy_ref_condition_FK_policy_id') IS NOT NULL) +BEGIN + ALTER TABLE [dbo].[x_policy_ref_condition] DROP CONSTRAINT x_policy_ref_condition_FK_policy_id +END +GO +IF (OBJECT_ID('x_policy_ref_condition_FK_condition_def_id') IS NOT NULL) +BEGIN + ALTER TABLE [dbo].[x_policy_ref_condition] DROP CONSTRAINT x_policy_ref_condition_FK_condition_def_id +END +GO +IF (OBJECT_ID('x_policy_ref_condition_UK') IS NOT NULL) +BEGIN + ALTER TABLE [dbo].[x_policy_ref_condition] DROP CONSTRAINT x_policy_ref_condition_UK +END +GO +IF (OBJECT_ID('x_policy_ref_condition') IS NOT NULL) +BEGIN + DROP TABLE [dbo].[x_policy_ref_condition] +END +GO +IF (OBJECT_ID('x_policy_ref_datamask_type_FK_policy_id') IS NOT NULL) +BEGIN + ALTER TABLE [dbo].[x_policy_ref_datamask_type] DROP CONSTRAINT x_policy_ref_datamask_type_FK_policy_id +END +GO +IF (OBJECT_ID('x_policy_ref_datamask_type_FK_datamask_def_id') IS NOT NULL) +BEGIN + ALTER TABLE [dbo].[x_policy_ref_datamask_type] DROP CONSTRAINT x_policy_ref_datamask_type_FK_datamask_def_id +END +GO +IF (OBJECT_ID('x_policy_ref_datamask_type_UK') IS NOT NULL) +BEGIN + ALTER TABLE [dbo].[x_policy_ref_datamask_type] DROP CONSTRAINT x_policy_ref_datamask_type_UK +END +GO +IF (OBJECT_ID('x_policy_ref_datamask_type') IS NOT NULL) +BEGIN + DROP TABLE [dbo].[x_policy_ref_datamask_type] +END +GO +IF (OBJECT_ID('x_policy_ref_user_FK_policy_id') IS NOT NULL) +BEGIN + ALTER TABLE [dbo].[x_policy_ref_user] DROP CONSTRAINT x_policy_ref_user_FK_policy_id +END +GO +IF (OBJECT_ID('x_policy_ref_user_FK_user_id') IS NOT NULL) +BEGIN + ALTER TABLE [dbo].[x_policy_ref_user] DROP CONSTRAINT x_policy_ref_user_FK_user_id +END +GO +IF (OBJECT_ID('x_policy_ref_user_UK') IS NOT NULL) +BEGIN + ALTER TABLE [dbo].[x_policy_ref_user] DROP CONSTRAINT x_policy_ref_user_UK +END +GO +IF (OBJECT_ID('x_policy_ref_user') IS NOT NULL) +BEGIN + DROP TABLE [dbo].[x_policy_ref_user] +END +GO +IF (OBJECT_ID('x_policy_ref_group_FK_policy_id') IS NOT NULL) +BEGIN + ALTER TABLE [dbo].[x_policy_ref_group] DROP CONSTRAINT x_policy_ref_group_FK_policy_id +END +GO +IF (OBJECT_ID('x_policy_ref_group_FK_group_id') IS NOT NULL) +BEGIN + ALTER TABLE [dbo].[x_policy_ref_group] DROP CONSTRAINT x_policy_ref_group_FK_group_id +END +GO +IF (OBJECT_ID('x_policy_ref_group_UK') IS NOT NULL) +BEGIN + ALTER TABLE [dbo].[x_policy_ref_group] DROP CONSTRAINT x_policy_ref_group_UK +END +GO +IF (OBJECT_ID('x_policy_ref_group') IS NOT NULL) +BEGIN + DROP TABLE [dbo].[x_policy_ref_group] +END +GO +SET ANSI_NULLS ON +GO +SET QUOTED_IDENTIFIER ON +GO +SET ANSI_PADDING ON +GO +CREATE TABLE [dbo].[x_policy_ref_resource] ( + [id] [bigint] IDENTITY (1, 1) NOT NULL, + [guid] [varchar](1024) DEFAULT NULL NULL, + [create_time] [datetime2] DEFAULT NULL NULL, + [update_time] [datetime2] DEFAULT NULL NULL, + [added_by_id] [bigint] DEFAULT NULL NULL, + [upd_by_id] [bigint] DEFAULT NULL NULL, + [policy_id] [bigint] NOT NULL, + [resource_def_id] [bigint] NOT NULL, + [resource_name] [varchar](4000) DEFAULT NULL NULL, + PRIMARY KEY CLUSTERED + ( + [id] ASC + ) WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY], + CONSTRAINT [x_policy_ref_resource$x_policy_ref_resource_UK] UNIQUE NONCLUSTERED + ( + [policy_id] ASC, [resource_def_id] ASC + )WITH (PAD_INDEX = OFF,STATISTICS_NORECOMPUTE = OFF,IGNORE_DUP_KEY = OFF,ALLOW_ROW_LOCKS = ON,ALLOW_PAGE_LOCKS = ON) ON [PRIMARY] +) ON [PRIMARY] +GO +SET ANSI_NULLS ON +GO +SET QUOTED_IDENTIFIER ON +GO +SET ANSI_PADDING ON +GO +CREATE TABLE [dbo].[x_policy_ref_access_type] ( + [id] [bigint] IDENTITY (1, 1) NOT NULL, + [guid] [varchar](1024) DEFAULT NULL NULL, + [create_time] [datetime2] DEFAULT NULL NULL, + [update_time] [datetime2] DEFAULT NULL NULL, + [added_by_id] [bigint] DEFAULT NULL NULL, + [upd_by_id] [bigint] DEFAULT NULL NULL, + [policy_id] [bigint] NOT NULL, + [access_def_id] [bigint] NOT NULL, + [access_type_name] [varchar](4000) DEFAULT NULL NULL, + PRIMARY KEY CLUSTERED + ( + [id] ASC + ) WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY], + CONSTRAINT [x_policy_ref_access_type$x_policy_ref_access_type_UK] UNIQUE NONCLUSTERED +( + [policy_id] ASC, [access_def_id] ASC +)WITH (PAD_INDEX = OFF,STATISTICS_NORECOMPUTE = OFF,IGNORE_DUP_KEY = OFF,ALLOW_ROW_LOCKS = ON,ALLOW_PAGE_LOCKS = ON) ON [PRIMARY] +) ON [PRIMARY] +GO +SET ANSI_NULLS ON +GO +SET QUOTED_IDENTIFIER ON +GO +SET ANSI_PADDING ON +GO +CREATE TABLE [dbo].[x_policy_ref_condition] ( + [id] [bigint] IDENTITY (1, 1) NOT NULL, + [guid] [varchar](1024) DEFAULT NULL NULL, + [create_time] [datetime2] DEFAULT NULL NULL, + [update_time] [datetime2] DEFAULT NULL NULL, + [added_by_id] [bigint] DEFAULT NULL NULL, + [upd_by_id] [bigint] DEFAULT NULL NULL, + [policy_id] [bigint] NOT NULL, + [condition_def_id] [bigint] NOT NULL, + [condition_name] [varchar](4000) DEFAULT NULL NULL, + PRIMARY KEY CLUSTERED + ( + [id] ASC + ) WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY], + CONSTRAINT [x_policy_ref_condition$x_policy_ref_condition_UK] UNIQUE NONCLUSTERED +( + [policy_id] ASC, [condition_def_id] ASC +)WITH (PAD_INDEX = OFF,STATISTICS_NORECOMPUTE = OFF,IGNORE_DUP_KEY = OFF,ALLOW_ROW_LOCKS = ON,ALLOW_PAGE_LOCKS = ON) ON [PRIMARY] +) ON [PRIMARY] +GO +SET ANSI_NULLS ON +GO +SET QUOTED_IDENTIFIER ON +GO +SET ANSI_PADDING ON +GO +CREATE TABLE [dbo].[x_policy_ref_datamask_type] ( + [id] [bigint] IDENTITY (1, 1) NOT NULL, + [guid] [varchar](1024) DEFAULT NULL NULL, + [create_time] [datetime2] DEFAULT NULL NULL, + [update_time] [datetime2] DEFAULT NULL NULL, + [added_by_id] [bigint] DEFAULT NULL NULL, + [upd_by_id] [bigint] DEFAULT NULL NULL, + [policy_id] [bigint] NOT NULL, + [datamask_def_id] [bigint] NOT NULL, + [datamask_type_name] [varchar](4000) DEFAULT NULL NULL, + PRIMARY KEY CLUSTERED + ( + [id] ASC + ) WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY], + CONSTRAINT [x_policy_ref_datamask_type$x_policy_ref_datamask_type_UK] UNIQUE NONCLUSTERED +( + [policy_id] ASC, [datamask_def_id] ASC +)WITH (PAD_INDEX = OFF,STATISTICS_NORECOMPUTE = OFF,IGNORE_DUP_KEY = OFF,ALLOW_ROW_LOCKS = ON,ALLOW_PAGE_LOCKS = ON) ON [PRIMARY] +) ON [PRIMARY] +GO +SET ANSI_NULLS ON +GO +SET QUOTED_IDENTIFIER ON +GO +SET ANSI_PADDING ON +GO +CREATE TABLE [dbo].[x_policy_ref_user] ( + [id] [bigint] IDENTITY (1, 1) NOT NULL, + [guid] [varchar](1024) DEFAULT NULL NULL, + [create_time] [datetime2] DEFAULT NULL NULL, + [update_time] [datetime2] DEFAULT NULL NULL, + [added_by_id] [bigint] DEFAULT NULL NULL, + [upd_by_id] [bigint] DEFAULT NULL NULL, + [policy_id] [bigint] NOT NULL, + [user_id] [bigint] NOT NULL, + [user_name] [varchar](4000) DEFAULT NULL NULL, + PRIMARY KEY CLUSTERED + ( + [id] ASC + ) WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY], + CONSTRAINT [x_policy_ref_user$x_policy_ref_user_UK] UNIQUE NONCLUSTERED +( + [policy_id] ASC, [user_id] ASC +)WITH (PAD_INDEX = OFF,STATISTICS_NORECOMPUTE = OFF,IGNORE_DUP_KEY = OFF,ALLOW_ROW_LOCKS = ON,ALLOW_PAGE_LOCKS = ON) ON [PRIMARY] +) ON [PRIMARY] +GO +SET ANSI_NULLS ON +GO +SET QUOTED_IDENTIFIER ON +GO +SET ANSI_PADDING ON +GO +CREATE TABLE [dbo].[x_policy_ref_group] ( + [id] [bigint] IDENTITY (1, 1) NOT NULL, + [guid] [varchar](1024) DEFAULT NULL NULL, + [create_time] [datetime2] DEFAULT NULL NULL, + [update_time] [datetime2] DEFAULT NULL NULL, + [added_by_id] [bigint] DEFAULT NULL NULL, + [upd_by_id] [bigint] DEFAULT NULL NULL, + [policy_id] [bigint] NOT NULL, + [group_id] [bigint] NOT NULL, + [group_name] [varchar](4000) DEFAULT NULL NULL, + PRIMARY KEY CLUSTERED + ( + [id] ASC + ) WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY], + CONSTRAINT [x_policy_ref_group$x_policy_ref_group_UK] UNIQUE NONCLUSTERED +( + [policy_id] ASC, [group_id] ASC +)WITH (PAD_INDEX = OFF,STATISTICS_NORECOMPUTE = OFF,IGNORE_DUP_KEY = OFF,ALLOW_ROW_LOCKS = ON,ALLOW_PAGE_LOCKS = ON) ON [PRIMARY] +) ON [PRIMARY] +GO +ALTER TABLE [dbo].[x_policy_ref_resource] WITH CHECK ADD CONSTRAINT [x_policy_ref_resource_FK_policy_id] FOREIGN KEY ([policy_id]) +REFERENCES [dbo].[x_policy] ([id]) +ALTER TABLE [dbo].[x_policy_ref_resource] CHECK CONSTRAINT [x_policy_ref_resource_FK_policy_id] +ALTER TABLE [dbo].[x_policy_ref_resource] WITH CHECK ADD CONSTRAINT [x_policy_ref_resource_FK_resource_def_id] FOREIGN KEY ([resource_def_id]) +REFERENCES [dbo].[x_resource_def] ([id]) +ALTER TABLE [dbo].[x_policy_ref_resource] CHECK CONSTRAINT [x_policy_ref_resource_FK_resource_def_id] +ALTER TABLE [dbo].[x_policy_ref_resource] WITH CHECK ADD CONSTRAINT [x_policy_ref_resource_FK_added_by] FOREIGN KEY ([added_by_id]) +REFERENCES [dbo].[x_portal_user] ([id]) +ALTER TABLE [dbo].[x_policy_ref_resource] CHECK CONSTRAINT [x_policy_ref_resource_FK_added_by] +ALTER TABLE [dbo].[x_policy_ref_resource] WITH CHECK ADD CONSTRAINT [x_policy_ref_resource_FK_upd_by] FOREIGN KEY ([upd_by_id]) +REFERENCES [dbo].[x_portal_user] ([id]) +ALTER TABLE [dbo].[x_policy_ref_access_type] WITH CHECK ADD CONSTRAINT [x_policy_ref_access_type_FK_policy_id] FOREIGN KEY ([policy_id]) +REFERENCES [dbo].[x_policy] ([id]) +ALTER TABLE [dbo].[x_policy_ref_access_type] CHECK CONSTRAINT [x_policy_ref_access_type_FK_policy_id] +ALTER TABLE [dbo].[x_policy_ref_access_type] WITH CHECK ADD CONSTRAINT [x_policy_ref_access_type_FK_access_def_id] FOREIGN KEY ([access_def_id]) +REFERENCES [dbo].[x_access_type_def] ([id]) +ALTER TABLE [dbo].[x_policy_ref_access_type] CHECK CONSTRAINT [x_policy_ref_access_type_FK_access_def_id] +ALTER TABLE [dbo].[x_policy_ref_access_type] WITH CHECK ADD CONSTRAINT [x_policy_ref_access_type_FK_added_by] FOREIGN KEY ([added_by_id]) +REFERENCES [dbo].[x_portal_user] ([id]) +ALTER TABLE [dbo].[x_policy_ref_access_type] CHECK CONSTRAINT [x_policy_ref_access_type_FK_added_by] +ALTER TABLE [dbo].[x_policy_ref_access_type] WITH CHECK ADD CONSTRAINT [x_policy_ref_access_type_FK_upd_by] FOREIGN KEY ([upd_by_id]) +REFERENCES [dbo].[x_portal_user] ([id]) +ALTER TABLE [dbo].[x_policy_ref_condition] WITH CHECK ADD CONSTRAINT [x_policy_ref_condition_FK_policy_id] FOREIGN KEY ([policy_id]) +REFERENCES [dbo].[x_policy] ([id]) +ALTER TABLE [dbo].[x_policy_ref_condition] CHECK CONSTRAINT [x_policy_ref_condition_FK_policy_id] +ALTER TABLE [dbo].[x_policy_ref_condition] WITH CHECK ADD CONSTRAINT [x_policy_ref_condition_FK_condition_def_id] FOREIGN KEY ([condition_def_id]) +REFERENCES [dbo].[x_policy_condition_def] ([id]) +ALTER TABLE [dbo].[x_policy_ref_condition] CHECK CONSTRAINT [x_policy_ref_condition_FK_condition_def_id] +ALTER TABLE [dbo].[x_policy_ref_condition] WITH CHECK ADD CONSTRAINT [x_policy_ref_condition_FK_added_by] FOREIGN KEY ([added_by_id]) +REFERENCES [dbo].[x_portal_user] ([id]) +ALTER TABLE [dbo].[x_policy_ref_condition] CHECK CONSTRAINT [x_policy_ref_condition_FK_added_by] +ALTER TABLE [dbo].[x_policy_ref_condition] WITH CHECK ADD CONSTRAINT [x_policy_ref_condition_FK_upd_by] FOREIGN KEY ([upd_by_id]) +REFERENCES [dbo].[x_portal_user] ([id]) +ALTER TABLE [dbo].[x_policy_ref_datamask_type] WITH CHECK ADD CONSTRAINT [x_policy_ref_datamask_type_FK_policy_id] FOREIGN KEY ([policy_id]) +REFERENCES [dbo].[x_policy] ([id]) +ALTER TABLE [dbo].[x_policy_ref_datamask_type] CHECK CONSTRAINT [x_policy_ref_datamask_type_FK_policy_id] +ALTER TABLE [dbo].[x_policy_ref_datamask_type] WITH CHECK ADD CONSTRAINT [x_policy_ref_datamask_type_FK_datamask_def_id] FOREIGN KEY ([datamask_def_id]) +REFERENCES [dbo].[x_datamask_type_def] ([id]) +ALTER TABLE [dbo].[x_policy_ref_datamask_type] CHECK CONSTRAINT [x_policy_ref_datamask_type_FK_datamask_def_id] +ALTER TABLE [dbo].[x_policy_ref_datamask_type] WITH CHECK ADD CONSTRAINT [x_policy_ref_datamask_type_FK_added_by] FOREIGN KEY ([added_by_id]) +REFERENCES [dbo].[x_portal_user] ([id]) +ALTER TABLE [dbo].[x_policy_ref_datamask_type] CHECK CONSTRAINT [x_policy_ref_datamask_type_FK_added_by] +ALTER TABLE [dbo].[x_policy_ref_datamask_type] WITH CHECK ADD CONSTRAINT [x_policy_ref_datamask_type_FK_upd_by] FOREIGN KEY ([upd_by_id]) +REFERENCES [dbo].[x_portal_user] ([id]) + + +ALTER TABLE [dbo].[x_policy_ref_user] WITH CHECK ADD CONSTRAINT [x_policy_ref_user_FK_policy_id] FOREIGN KEY ([policy_id]) +REFERENCES [dbo].[x_policy] ([id]) +ALTER TABLE [dbo].[x_policy_ref_user] CHECK CONSTRAINT [x_policy_ref_user_FK_policy_id] + +ALTER TABLE [dbo].[x_policy_ref_user] WITH CHECK ADD CONSTRAINT [x_policy_ref_user_FK_user_id] FOREIGN KEY ([user_id]) +REFERENCES [dbo].[x_user] ([id]) +ALTER TABLE [dbo].[x_policy_ref_user] CHECK CONSTRAINT [x_policy_ref_user_FK_user_id] + +ALTER TABLE [dbo].[x_policy_ref_user] WITH CHECK ADD CONSTRAINT [x_policy_ref_user_FK_added_by] FOREIGN KEY ([added_by_id]) +REFERENCES [dbo].[x_portal_user] ([id]) +ALTER TABLE [dbo].[x_policy_ref_user] CHECK CONSTRAINT [x_policy_ref_user_FK_added_by] + +ALTER TABLE [dbo].[x_policy_ref_user] WITH CHECK ADD CONSTRAINT [x_policy_ref_user_FK_upd_by] FOREIGN KEY ([upd_by_id]) +REFERENCES [dbo].[x_portal_user] ([id]) +ALTER TABLE [dbo].[x_policy_ref_user] CHECK CONSTRAINT [x_policy_ref_user_FK_upd_by] + +ALTER TABLE [dbo].[x_policy_ref_group] WITH CHECK ADD CONSTRAINT [x_policy_ref_group_FK_policy_id] FOREIGN KEY ([policy_id]) +REFERENCES [dbo].[x_policy] ([id]) +ALTER TABLE [dbo].[x_policy_ref_group] CHECK CONSTRAINT [x_policy_ref_group_FK_policy_id] +ALTER TABLE [dbo].[x_policy_ref_group] WITH CHECK ADD CONSTRAINT [x_policy_ref_group_FK_group_id] FOREIGN KEY ([group_id]) +REFERENCES [dbo].[x_group] ([id]) +ALTER TABLE [dbo].[x_policy_ref_group] CHECK CONSTRAINT [x_policy_ref_group_FK_group_id] +ALTER TABLE [dbo].[x_policy_ref_group] WITH CHECK ADD CONSTRAINT [x_policy_ref_group_FK_added_by] FOREIGN KEY ([added_by_id]) +REFERENCES [dbo].[x_portal_user] ([id]) +ALTER TABLE [dbo].[x_policy_ref_group] CHECK CONSTRAINT [x_policy_ref_group_FK_added_by] +ALTER TABLE [dbo].[x_policy_ref_group] WITH CHECK ADD CONSTRAINT [x_policy_ref_group_FK_upd_by] FOREIGN KEY ([upd_by_id]) +REFERENCES [dbo].[x_portal_user] ([id]) +GO +IF NOT EXISTS (SELECT + * + FROM INFORMATION_SCHEMA.columns + WHERE table_name = 'x_policy' + AND column_name = 'policy_text') +BEGIN + ALTER TABLE [dbo].[x_policy] ADD [policy_text] [nvarchar](max) DEFAULT NULL NULL; +END +GO + +SET ANSI_NULLS ON +GO +SET QUOTED_IDENTIFIER ON +GO +SET ANSI_PADDING ON +GO +IF EXISTS ( + SELECT type_desc, type + FROM sys.procedures WITH(NOLOCK) + WHERE NAME = 'removeConstraints' + AND type = 'P' + ) +BEGIN + PRINT 'Proc exist with name dbo.removeConstraints' + DROP PROCEDURE dbo.removeConstraints + PRINT 'Proc dropped dbo.removeConstraints' +END +GO +CREATE PROCEDURE dbo.removeConstraints + -- Add the parameters for the stored procedure here + @tablename nvarchar(100) +AS +BEGIN + + DECLARE @stmt VARCHAR(300); + + -- Cursor to generate ALTER TABLE DROP CONSTRAINT statements + DECLARE cur CURSOR FOR + SELECT 'ALTER TABLE ' + OBJECT_SCHEMA_NAME(parent_object_id) + '.' + OBJECT_NAME(parent_object_id) + + ' DROP CONSTRAINT ' + name + FROM sys.foreign_keys + WHERE OBJECT_SCHEMA_NAME(referenced_object_id) = 'dbo' AND + OBJECT_NAME(referenced_object_id) = @tablename; + + OPEN cur; + FETCH cur INTO @stmt; + + -- Drop each found foreign key constraint + WHILE @@FETCH_STATUS = 0 + BEGIN + EXEC (@stmt); + FETCH cur INTO @stmt; + END + + CLOSE cur; + DEALLOCATE cur; + +END +GO + +EXEC dbo.removeConstraints 'x_policy_item' +GO + +EXEC dbo.removeConstraints 'x_policy_item_access' +GO + +EXEC dbo.removeConstraints 'x_policy_item_condition' +GO + +EXEC dbo.removeConstraints 'x_policy_item_datamask' +GO + +EXEC dbo.removeConstraints 'x_policy_item_group_perm' +GO + +EXEC dbo.removeConstraints 'x_policy_item_user_perm' +GO + +EXEC dbo.removeConstraints 'x_policy_item_rowfilter' +GO + +EXEC dbo.removeConstraints 'x_policy_resource' +GO + +EXEC dbo.removeConstraints 'x_policy_resource_map' +GO + +EXIT diff --git a/security-admin/db/sqlserver/patches/036-denormalize-tag-tables.sql b/security-admin/db/sqlserver/patches/036-denormalize-tag-tables.sql new file mode 100644 index 0000000000..9bfae30f21 --- /dev/null +++ b/security-admin/db/sqlserver/patches/036-denormalize-tag-tables.sql @@ -0,0 +1,97 @@ +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +GO +IF NOT EXISTS(select * from INFORMATION_SCHEMA.columns where table_name = 'x_tag_def' and column_name = 'tag_attrs_def_text') +BEGIN + ALTER TABLE [dbo].[x_tag_def] ADD [tag_attrs_def_text] [nvarchar](max) DEFAULT NULL NULL; +END +IF NOT EXISTS(select * from INFORMATION_SCHEMA.columns where table_name = 'x_tag' and column_name = 'tag_attrs_text') +BEGIN + ALTER TABLE [dbo].[x_tag] ADD [tag_attrs_text] [nvarchar](max) DEFAULT NULL NULL; +END +IF NOT EXISTS(select * from INFORMATION_SCHEMA.columns where table_name = 'x_service_resource' and column_name = 'service_resource_elements_text') +BEGIN + ALTER TABLE [dbo].[x_service_resource] ADD [service_resource_elements_text] [nvarchar](max) DEFAULT NULL NULL; +END +IF NOT EXISTS(select * from INFORMATION_SCHEMA.columns where table_name = 'x_service_resource' and column_name = 'tags_text') +BEGIN + ALTER TABLE [dbo].[x_service_resource] ADD [tags_text] [nvarchar](max) DEFAULT NULL NULL; +END +GO + +SET ANSI_NULLS ON +GO +SET QUOTED_IDENTIFIER ON +GO +SET ANSI_PADDING ON +GO +IF EXISTS ( + SELECT type_desc, type + FROM sys.procedures WITH(NOLOCK) + WHERE NAME = 'removeConstraints' + AND type = 'P' + ) +BEGIN + PRINT 'Proc exist with name dbo.removeConstraints' + DROP PROCEDURE dbo.removeConstraints + PRINT 'Proc dropped dbo.removeConstraints' +END +GO +CREATE PROCEDURE dbo.removeConstraints + -- Add the parameters for the stored procedure here + @tablename nvarchar(100) +AS +BEGIN + + DECLARE @stmt VARCHAR(300); + + -- Cursor to generate ALTER TABLE DROP CONSTRAINT statements + DECLARE cur CURSOR FOR + SELECT 'ALTER TABLE ' + OBJECT_SCHEMA_NAME(parent_object_id) + '.' + OBJECT_NAME(parent_object_id) + + ' DROP CONSTRAINT ' + name + FROM sys.foreign_keys + WHERE OBJECT_SCHEMA_NAME(referenced_object_id) = 'dbo' AND + OBJECT_NAME(referenced_object_id) = @tablename; + + OPEN cur; + FETCH cur INTO @stmt; + + -- Drop each found foreign key constraint + WHILE @@FETCH_STATUS = 0 + BEGIN + EXEC (@stmt); + FETCH cur INTO @stmt; + END + + CLOSE cur; + DEALLOCATE cur; + +END +GO + +EXEC dbo.removeConstraints 'x_tag_attr_def' +GO + +EXEC dbo.removeConstraints 'x_tag_attr' +GO + +EXEC dbo.removeConstraints 'x_service_resource_element' +GO + +EXEC dbo.removeConstraints 'x_service_resource_element_val' +GO + +EXIT diff --git a/security-admin/pom.xml b/security-admin/pom.xml index 90b8cab802..51d958e466 100644 --- a/security-admin/pom.xml +++ b/security-admin/pom.xml @@ -43,7 +43,7 @@ ${asm.version} - c3p0 + com.mchange c3p0 ${c3p0.version} @@ -54,13 +54,8 @@ commons-beanutils - commons-beanutils-bean-collections - ${commons.beanutils.collections.version} - - - commons-beanutils - commons-beanutils-core - ${commons.beanutils.core.version} + commons-beanutils + ${commons.beanutils.version} commons-cli @@ -223,9 +218,14 @@ ${sun.jersey.core.version} - org.apache.hadoop - hadoop-mapreduce-client-core - ${hadoop.version} + org.apache.hadoop + hadoop-mapreduce-client-core + ${hadoop.version} + + + io.netty + netty + ${netty.version} com.sun.jersey.contribs @@ -281,6 +281,12 @@ com.googlecode.owasp-java-html-sanitizer owasp-java-html-sanitizer ${owasp-java-html-sanitizer.version} + + + com.google.guava + guava + + org.springframework.ldap @@ -395,12 +401,42 @@ tomcat jasper-runtime + + com.sun.jersey + jersey-core + + + com.sun.jersey + jersey-server + + + com.sun.jersey + jersey-json + + + com.google.guava + guava + org.apache.hadoop hadoop-hdfs ${hadoop.version} + + + com.sun.jersey + jersey-core + + + com.sun.jersey + jersey-server + + + com.google.guava + guava + + org.apache.ranger @@ -420,20 +456,27 @@ com.nimbusds nimbus-jose-jwt - 3.9 + 4.41.2 compile - - - org.bouncycastle - bcprov-jdk15on - - + + + + net.minidev + json-smart + 2.3 com.google.inject guice 3.0 + + com.webcohesion.enunciate + enunciate-core-annotations + @@ -484,7 +527,8 @@ WEB-INF/lib/spring-*.SEC03.jar, WEB-INF/lib/spring-*.RC3.jar, - WEB-INF/lib/spring-2.*.jar + WEB-INF/lib/spring-2.*.jar, + WEB-INF/lib/asm-5*.jar ${project.build.directory}/${project.build.finalName} @@ -516,6 +560,17 @@ + + com.webcohesion.enunciate + enunciate-maven-plugin + + enunciate.xml + + docs/target/ + 1.8 + 1.8 + + diff --git a/security-admin/scripts/db_setup.py b/security-admin/scripts/db_setup.py index 6e791512b2..7b17f9ae11 100644 --- a/security-admin/scripts/db_setup.py +++ b/security-admin/scripts/db_setup.py @@ -35,7 +35,7 @@ ranger_version='' jisql_debug=True retryPatchAfterSeconds=120 - +java_patch_regex="^Patch.*?J\d{5}.class$" is_unix = os_name == "LINUX" or os_name == "DARWIN" if is_unix: @@ -93,6 +93,8 @@ def populate_global_dict(): value = '' value = value.strip() globalDict[key] = value + if 'ranger_admin_max_heap_size' not in globalDict: + globalDict['ranger_admin_max_heap_size']='1g' def jisql_log(query, db_password): if jisql_debug == True: @@ -183,13 +185,14 @@ def import_core_db_schema(self, db_name, db_user, db_password, file_name,first_t class MysqlConf(BaseDB): # Constructor - def __init__(self, host,SQL_CONNECTOR_JAR,JAVA_BIN,db_ssl_enabled,db_ssl_required,db_ssl_verifyServerCertificate,javax_net_ssl_keyStore,javax_net_ssl_keyStorePassword,javax_net_ssl_trustStore,javax_net_ssl_trustStorePassword): + def __init__(self, host,SQL_CONNECTOR_JAR,JAVA_BIN,db_ssl_enabled,db_ssl_required,db_ssl_verifyServerCertificate,javax_net_ssl_keyStore,javax_net_ssl_keyStorePassword,javax_net_ssl_trustStore,javax_net_ssl_trustStorePassword,db_ssl_auth_type): self.host = host self.SQL_CONNECTOR_JAR = SQL_CONNECTOR_JAR self.JAVA_BIN = JAVA_BIN self.db_ssl_enabled=db_ssl_enabled.lower() self.db_ssl_required=db_ssl_required.lower() self.db_ssl_verifyServerCertificate=db_ssl_verifyServerCertificate.lower() + self.db_ssl_auth_type=db_ssl_auth_type.lower() self.javax_net_ssl_keyStore=javax_net_ssl_keyStore self.javax_net_ssl_keyStorePassword=javax_net_ssl_keyStorePassword self.javax_net_ssl_trustStore=javax_net_ssl_trustStore @@ -202,7 +205,10 @@ def get_jisql_cmd(self, user, password ,db_name): if self.db_ssl_enabled == 'true': db_ssl_param="?useSSL=%s&requireSSL=%s&verifyServerCertificate=%s" %(self.db_ssl_enabled,self.db_ssl_required,self.db_ssl_verifyServerCertificate) if self.db_ssl_verifyServerCertificate == 'true': - db_ssl_cert_param=" -Djavax.net.ssl.keyStore=%s -Djavax.net.ssl.keyStorePassword=%s -Djavax.net.ssl.trustStore=%s -Djavax.net.ssl.trustStorePassword=%s " %(self.javax_net_ssl_keyStore,self.javax_net_ssl_keyStorePassword,self.javax_net_ssl_trustStore,self.javax_net_ssl_trustStorePassword) + if self.db_ssl_auth_type == '1-way': + db_ssl_cert_param=" -Djavax.net.ssl.trustStore=%s -Djavax.net.ssl.trustStorePassword=%s " %(self.javax_net_ssl_trustStore,self.javax_net_ssl_trustStorePassword) + else: + db_ssl_cert_param=" -Djavax.net.ssl.keyStore=%s -Djavax.net.ssl.keyStorePassword=%s -Djavax.net.ssl.trustStore=%s -Djavax.net.ssl.trustStorePassword=%s " %(self.javax_net_ssl_keyStore,self.javax_net_ssl_keyStorePassword,self.javax_net_ssl_trustStore,self.javax_net_ssl_trustStorePassword) self.JAVA_BIN = self.JAVA_BIN.strip("'") if is_unix: jisql_cmd = "%s %s -cp %s:%s/jisql/lib/* org.apache.util.sql.Jisql -driver mysqlconj -cstring jdbc:mysql://%s/%s%s -u '%s' -p '%s' -noheader -trim -c \;" %(self.JAVA_BIN,db_ssl_cert_param,self.SQL_CONNECTOR_JAR,path,self.host,db_name,db_ssl_param,user,password) @@ -481,6 +487,7 @@ def auditdb_operation(self, xa_db_host, audit_db_host, db_name, audit_db_name, d self.grant_audit_db_user(db_user, audit_db_name, audit_db_user, audit_db_password, db_password,TABLE_NAME) def execute_java_patches(self, xa_db_host, db_user, db_password, db_name): + global globalDict my_dict = {} version = "" className = "" @@ -494,7 +501,7 @@ def execute_java_patches(self, xa_db_host, db_user, db_password, db_name): files = os.listdir(javaFiles) if files: for filename in files: - f = re.match("^Patch.*?.class$",filename) + f = re.match(java_patch_regex,filename) if f: className = re.match("(Patch.*?)_.*.class",filename) className = className.group(1) @@ -551,7 +558,7 @@ def execute_java_patches(self, xa_db_host, db_user, db_password, db_name): path = os.path.join("%s","WEB-INF","classes","conf:%s","WEB-INF","classes","lib","*:%s","WEB-INF",":%s","META-INF",":%s","WEB-INF","lib","*:%s","WEB-INF","classes",":%s","WEB-INF","classes","META-INF:%s" )%(app_home ,app_home ,app_home, app_home, app_home, app_home ,app_home ,self.SQL_CONNECTOR_JAR) elif os_name == "WINDOWS": path = os.path.join("%s","WEB-INF","classes","conf;%s","WEB-INF","classes","lib","*;%s","WEB-INF",";%s","META-INF",";%s","WEB-INF","lib","*;%s","WEB-INF","classes",";%s","WEB-INF","classes","META-INF;%s" )%(app_home ,app_home ,app_home, app_home, app_home, app_home ,app_home ,self.SQL_CONNECTOR_JAR) - get_java_cmd = "%s -Dlogdir=%s -Dlog4j.configuration=db_patch.log4j.xml -cp %s org.apache.ranger.patch.%s"%(self.JAVA_BIN,ranger_log,path,className) + get_java_cmd = "%s -XX:MetaspaceSize=100m -XX:MaxMetaspaceSize=200m -Xmx%s -Xms1g -Dlogdir=%s -Dlog4j.configuration=db_patch.log4j.xml -cp %s org.apache.ranger.patch.%s"%(self.JAVA_BIN,globalDict['ranger_admin_max_heap_size'],ranger_log,path,className) if is_unix: ret = subprocess.call(shlex.split(get_java_cmd)) elif os_name == "WINDOWS": @@ -1171,6 +1178,7 @@ def auditdb_operation(self, xa_db_host , audit_db_host , db_name ,audit_db_name, self.grant_audit_db_user( audit_db_name ,db_user, audit_db_user, db_password,audit_db_password) def execute_java_patches(self, xa_db_host, db_user, db_password, db_name): + global globalDict my_dict = {} version = "" className = "" @@ -1184,7 +1192,7 @@ def execute_java_patches(self, xa_db_host, db_user, db_password, db_name): files = os.listdir(javaFiles) if files: for filename in files: - f = re.match("^Patch.*?.class$",filename) + f = re.match(java_patch_regex,filename) if f: className = re.match("(Patch.*?)_.*.class",filename) className = className.group(1) @@ -1266,7 +1274,7 @@ def execute_java_patches(self, xa_db_host, db_user, db_password, db_name): path = os.path.join("%s","WEB-INF","classes","conf:%s","WEB-INF","classes","lib","*:%s","WEB-INF",":%s","META-INF",":%s","WEB-INF","lib","*:%s","WEB-INF","classes",":%s","WEB-INF","classes","META-INF:%s" )%(app_home ,app_home ,app_home, app_home, app_home, app_home ,app_home ,self.SQL_CONNECTOR_JAR) elif os_name == "WINDOWS": path = os.path.join("%s","WEB-INF","classes","conf;%s","WEB-INF","classes","lib","*;%s","WEB-INF",";%s","META-INF",";%s","WEB-INF","lib","*;%s","WEB-INF","classes",";%s","WEB-INF","classes","META-INF;%s" )%(app_home ,app_home ,app_home, app_home, app_home, app_home ,app_home ,self.SQL_CONNECTOR_JAR) - get_java_cmd = "%s -Djava.security.egd=file:///dev/urandom -Dlogdir=%s -Dlog4j.configuration=db_patch.log4j.xml -cp %s org.apache.ranger.patch.%s"%(self.JAVA_BIN,ranger_log,path,className) + get_java_cmd = "%s -XX:MetaspaceSize=100m -XX:MaxMetaspaceSize=200m -Xmx%s -Xms1g -Djava.security.egd=file:///dev/urandom -Dlogdir=%s -Dlog4j.configuration=db_patch.log4j.xml -cp %s org.apache.ranger.patch.%s"%(self.JAVA_BIN,globalDict['ranger_admin_max_heap_size'],ranger_log,path,className) if is_unix: ret = subprocess.call(shlex.split(get_java_cmd)) elif os_name == "WINDOWS": @@ -1306,6 +1314,7 @@ def execute_java_patches(self, xa_db_host, db_user, db_password, db_name): sys.exit(1) def change_admin_default_password(self, xa_db_host, db_user, db_password, db_name,userName,oldPassword,newPassword): + global globalDict my_dict = {} version = "" className = "ChangePasswordUtil" @@ -1355,7 +1364,7 @@ def change_admin_default_password(self, xa_db_host, db_user, db_password, db_nam path = os.path.join("%s","WEB-INF","classes","conf:%s","WEB-INF","classes","lib","*:%s","WEB-INF",":%s","META-INF",":%s","WEB-INF","lib","*:%s","WEB-INF","classes",":%s","WEB-INF","classes","META-INF:%s" )%(app_home ,app_home ,app_home, app_home, app_home, app_home ,app_home ,self.SQL_CONNECTOR_JAR) elif os_name == "WINDOWS": path = os.path.join("%s","WEB-INF","classes","conf;%s","WEB-INF","classes","lib","*;%s","WEB-INF",";%s","META-INF",";%s","WEB-INF","lib","*;%s","WEB-INF","classes",";%s","WEB-INF","classes","META-INF;%s" )%(app_home ,app_home ,app_home, app_home, app_home, app_home ,app_home ,self.SQL_CONNECTOR_JAR) - get_java_cmd = "%s -Dlogdir=%s -Dlog4j.configuration=db_patch.log4j.xml -cp %s org.apache.ranger.patch.cliutil.%s %s %s %s -default"%(self.JAVA_BIN,ranger_log,path,className,userName,oldPassword,newPassword) + get_java_cmd = "%s -XX:MetaspaceSize=100m -XX:MaxMetaspaceSize=200m -Xmx%s -Xms1g -Dlogdir=%s -Dlog4j.configuration=db_patch.log4j.xml -cp %s org.apache.ranger.patch.cliutil.%s %s %s %s -default"%(self.JAVA_BIN,globalDict['ranger_admin_max_heap_size'],ranger_log,path,className,userName,oldPassword,newPassword) if is_unix: status = subprocess.call(shlex.split(get_java_cmd)) elif os_name == "WINDOWS": @@ -1866,6 +1875,7 @@ def auditdb_operation(self, xa_db_host, audit_db_host, db_name, audit_db_name, d self.grant_audit_db_user(audit_db_name ,db_user, audit_db_user, db_password,audit_db_password) def execute_java_patches(self, xa_db_host, db_user, db_password, db_name): + global globalDict my_dict = {} version = "" className = "" @@ -1879,7 +1889,7 @@ def execute_java_patches(self, xa_db_host, db_user, db_password, db_name): files = os.listdir(javaFiles) if files: for filename in files: - f = re.match("^Patch.*?.class$",filename) + f = re.match(java_patch_regex,filename) if f: className = re.match("(Patch.*?)_.*.class",filename) className = className.group(1) @@ -1936,7 +1946,7 @@ def execute_java_patches(self, xa_db_host, db_user, db_password, db_name): path = os.path.join("%s","WEB-INF","classes","conf:%s","WEB-INF","classes","lib","*:%s","WEB-INF",":%s","META-INF",":%s","WEB-INF","lib","*:%s","WEB-INF","classes",":%s","WEB-INF","classes","META-INF:%s" )%(app_home ,app_home ,app_home, app_home, app_home, app_home ,app_home ,self.SQL_CONNECTOR_JAR) elif os_name == "WINDOWS": path = os.path.join("%s","WEB-INF","classes","conf;%s","WEB-INF","classes","lib","*;%s","WEB-INF",";%s","META-INF",";%s","WEB-INF","lib","*;%s","WEB-INF","classes",";%s","WEB-INF","classes","META-INF;%s" )%(app_home ,app_home ,app_home, app_home, app_home, app_home ,app_home ,self.SQL_CONNECTOR_JAR) - get_java_cmd = "%s -Dlogdir=%s -Dlog4j.configuration=db_patch.log4j.xml -cp %s org.apache.ranger.patch.%s"%(self.JAVA_BIN,ranger_log,path,className) + get_java_cmd = "%s -XX:MetaspaceSize=100m -XX:MaxMetaspaceSize=200m -Xmx%s -Xms1g -Dlogdir=%s -Dlog4j.configuration=db_patch.log4j.xml -cp %s org.apache.ranger.patch.%s"%(self.JAVA_BIN,globalDict['ranger_admin_max_heap_size'],ranger_log,path,className) if is_unix: ret = subprocess.call(shlex.split(get_java_cmd)) elif os_name == "WINDOWS": @@ -1976,6 +1986,7 @@ def execute_java_patches(self, xa_db_host, db_user, db_password, db_name): sys.exit(1) def change_admin_default_password(self, xa_db_host, db_user, db_password, db_name,userName,oldPassword,newPassword): + global globalDict my_dict = {} version = "" className = "ChangePasswordUtil" @@ -2025,7 +2036,7 @@ def change_admin_default_password(self, xa_db_host, db_user, db_password, db_nam path = os.path.join("%s","WEB-INF","classes","conf:%s","WEB-INF","classes","lib","*:%s","WEB-INF",":%s","META-INF",":%s","WEB-INF","lib","*:%s","WEB-INF","classes",":%s","WEB-INF","classes","META-INF:%s" )%(app_home ,app_home ,app_home, app_home, app_home, app_home ,app_home ,self.SQL_CONNECTOR_JAR) elif os_name == "WINDOWS": path = os.path.join("%s","WEB-INF","classes","conf;%s","WEB-INF","classes","lib","*;%s","WEB-INF",";%s","META-INF",";%s","WEB-INF","lib","*;%s","WEB-INF","classes",";%s","WEB-INF","classes","META-INF;%s" )%(app_home ,app_home ,app_home, app_home, app_home, app_home ,app_home ,self.SQL_CONNECTOR_JAR) - get_java_cmd = "%s -Dlogdir=%s -Dlog4j.configuration=db_patch.log4j.xml -cp %s org.apache.ranger.patch.cliutil.%s %s %s %s -default"%(self.JAVA_BIN,ranger_log,path,className,userName,oldPassword,newPassword) + get_java_cmd = "%s -XX:MetaspaceSize=100m -XX:MaxMetaspaceSize=200m -Xmx%s -Xms1g -Dlogdir=%s -Dlog4j.configuration=db_patch.log4j.xml -cp %s org.apache.ranger.patch.cliutil.%s %s %s %s -default"%(self.JAVA_BIN,globalDict['ranger_admin_max_heap_size'],ranger_log,path,className,userName,oldPassword,newPassword) if is_unix: status = subprocess.call(shlex.split(get_java_cmd)) elif os_name == "WINDOWS": @@ -2498,6 +2509,7 @@ def auditdb_operation(self, xa_db_host, audit_db_host, db_name, audit_db_name,db self.grant_audit_db_user( audit_db_name ,db_user, audit_db_user, db_password,audit_db_password,TABLE_NAME) def execute_java_patches(self, xa_db_host, db_user, db_password, db_name): + global globalDict my_dict = {} version = "" className = "" @@ -2511,7 +2523,7 @@ def execute_java_patches(self, xa_db_host, db_user, db_password, db_name): files = os.listdir(javaFiles) if files: for filename in files: - f = re.match("^Patch.*?.class$",filename) + f = re.match(java_patch_regex,filename) if f: className = re.match("(Patch.*?)_.*.class",filename) className = className.group(1) @@ -2568,7 +2580,7 @@ def execute_java_patches(self, xa_db_host, db_user, db_password, db_name): path = os.path.join("%s","WEB-INF","classes","conf:%s","WEB-INF","classes","lib","*:%s","WEB-INF",":%s","META-INF",":%s","WEB-INF","lib","*:%s","WEB-INF","classes",":%s","WEB-INF","classes","META-INF:%s" )%(app_home ,app_home ,app_home, app_home, app_home, app_home ,app_home ,self.SQL_CONNECTOR_JAR) elif os_name == "WINDOWS": path = os.path.join("%s","WEB-INF","classes","conf;%s","WEB-INF","classes","lib","*;%s","WEB-INF",";%s","META-INF",";%s","WEB-INF","lib","*;%s","WEB-INF","classes",";%s","WEB-INF","classes","META-INF;%s" )%(app_home ,app_home ,app_home, app_home, app_home, app_home ,app_home ,self.SQL_CONNECTOR_JAR) - get_java_cmd = "%s -Dlogdir=%s -Dlog4j.configuration=db_patch.log4j.xml -cp %s org.apache.ranger.patch.%s"%(self.JAVA_BIN,ranger_log,path,className) + get_java_cmd = "%s -XX:MetaspaceSize=100m -XX:MaxMetaspaceSize=200m -Xmx%s -Xms1g -Dlogdir=%s -Dlog4j.configuration=db_patch.log4j.xml -cp %s org.apache.ranger.patch.%s"%(self.JAVA_BIN,globalDict['ranger_admin_max_heap_size'],ranger_log,path,className) if is_unix: ret = subprocess.call(shlex.split(get_java_cmd)) elif os_name == "WINDOWS": @@ -2608,6 +2620,7 @@ def execute_java_patches(self, xa_db_host, db_user, db_password, db_name): sys.exit(1) def change_admin_default_password(self, xa_db_host, db_user, db_password, db_name,userName,oldPassword,newPassword): + global globalDict my_dict = {} version = "" className = "ChangePasswordUtil" @@ -2657,7 +2670,7 @@ def change_admin_default_password(self, xa_db_host, db_user, db_password, db_nam path = os.path.join("%s","WEB-INF","classes","conf:%s","WEB-INF","classes","lib","*:%s","WEB-INF",":%s","META-INF",":%s","WEB-INF","lib","*:%s","WEB-INF","classes",":%s","WEB-INF","classes","META-INF:%s" )%(app_home ,app_home ,app_home, app_home, app_home, app_home ,app_home ,self.SQL_CONNECTOR_JAR) elif os_name == "WINDOWS": path = os.path.join("%s","WEB-INF","classes","conf;%s","WEB-INF","classes","lib","*;%s","WEB-INF",";%s","META-INF",";%s","WEB-INF","lib","*;%s","WEB-INF","classes",";%s","WEB-INF","classes","META-INF;%s" )%(app_home ,app_home ,app_home, app_home, app_home, app_home ,app_home ,self.SQL_CONNECTOR_JAR) - get_java_cmd = "%s -Dlogdir=%s -Dlog4j.configuration=db_patch.log4j.xml -cp %s org.apache.ranger.patch.cliutil.%s %s %s %s -default"%(self.JAVA_BIN,ranger_log,path,className,userName,oldPassword,newPassword) + get_java_cmd = "%s -XX:MetaspaceSize=100m -XX:MaxMetaspaceSize=200m -Xmx%s -Xms1g -Dlogdir=%s -Dlog4j.configuration=db_patch.log4j.xml -cp %s org.apache.ranger.patch.cliutil.%s %s %s %s -default"%(self.JAVA_BIN,globalDict['ranger_admin_max_heap_size'],ranger_log,path,className,userName,oldPassword,newPassword) if is_unix: status = subprocess.call(shlex.split(get_java_cmd)) elif os_name == "WINDOWS": @@ -3122,6 +3135,7 @@ def auditdb_operation(self, xa_db_host, audit_db_host, db_name, audit_db_name,db self.grant_audit_db_user( audit_db_name ,db_user, audit_db_user, db_password,audit_db_password,TABLE_NAME) def execute_java_patches(self, xa_db_host, db_user, db_password, db_name): + global globalDict my_dict = {} version = "" className = "" @@ -3135,7 +3149,7 @@ def execute_java_patches(self, xa_db_host, db_user, db_password, db_name): files = os.listdir(javaFiles) if files: for filename in files: - f = re.match("^Patch.*?.class$",filename) + f = re.match(java_patch_regex,filename) if f: className = re.match("(Patch.*?)_.*.class",filename) className = className.group(1) @@ -3192,7 +3206,7 @@ def execute_java_patches(self, xa_db_host, db_user, db_password, db_name): path = os.path.join("%s","WEB-INF","classes","conf:%s","WEB-INF","classes","lib","*:%s","WEB-INF",":%s","META-INF",":%s","WEB-INF","lib","*:%s","WEB-INF","classes",":%s","WEB-INF","classes","META-INF:%s" )%(app_home ,app_home ,app_home, app_home, app_home, app_home ,app_home ,self.SQL_CONNECTOR_JAR) elif os_name == "WINDOWS": path = os.path.join("%s","WEB-INF","classes","conf;%s","WEB-INF","classes","lib","*;%s","WEB-INF",";%s","META-INF",";%s","WEB-INF","lib","*;%s","WEB-INF","classes",";%s","WEB-INF","classes","META-INF;%s" )%(app_home ,app_home ,app_home, app_home, app_home, app_home ,app_home ,self.SQL_CONNECTOR_JAR) - get_java_cmd = "%s -Dlogdir=%s -Dlog4j.configuration=db_patch.log4j.xml -cp %s org.apache.ranger.patch.%s"%(self.JAVA_BIN,ranger_log,path,className) + get_java_cmd = "%s -XX:MetaspaceSize=100m -XX:MaxMetaspaceSize=200m -Xmx%s -Xms1g -Dlogdir=%s -Dlog4j.configuration=db_patch.log4j.xml -cp %s org.apache.ranger.patch.%s"%(self.JAVA_BIN,globalDict['ranger_admin_max_heap_size'],ranger_log,path,className) if is_unix: ret = subprocess.call(shlex.split(get_java_cmd)) elif os_name == "WINDOWS": @@ -3253,6 +3267,7 @@ def set_options(self, db_name, db_user, db_password, TABLE_NAME): ret = subprocessCallWithRetry(shlex.split(query)) def change_admin_default_password(self, xa_db_host, db_user, db_password, db_name,userName,oldPassword,newPassword): + global globalDict my_dict = {} version = "" className = "ChangePasswordUtil" @@ -3302,7 +3317,7 @@ def change_admin_default_password(self, xa_db_host, db_user, db_password, db_nam path = os.path.join("%s","WEB-INF","classes","conf:%s","WEB-INF","classes","lib","*:%s","WEB-INF",":%s","META-INF",":%s","WEB-INF","lib","*:%s","WEB-INF","classes",":%s","WEB-INF","classes","META-INF:%s" )%(app_home ,app_home ,app_home, app_home, app_home, app_home ,app_home ,self.SQL_CONNECTOR_JAR) elif os_name == "WINDOWS": path = os.path.join("%s","WEB-INF","classes","conf;%s","WEB-INF","classes","lib","*;%s","WEB-INF",";%s","META-INF",";%s","WEB-INF","lib","*;%s","WEB-INF","classes",";%s","WEB-INF","classes","META-INF;%s" )%(app_home ,app_home ,app_home, app_home, app_home, app_home ,app_home ,self.SQL_CONNECTOR_JAR) - get_java_cmd = "%s -Dlogdir=%s -Dlog4j.configuration=db_patch.log4j.xml -cp %s org.apache.ranger.patch.cliutil.%s %s %s %s -default"%(self.JAVA_BIN,ranger_log,path,className,userName,oldPassword,newPassword) + get_java_cmd = "%s -XX:MetaspaceSize=100m -XX:MaxMetaspaceSize=200m -Xmx%s -Xms1g -Dlogdir=%s -Dlog4j.configuration=db_patch.log4j.xml -cp %s org.apache.ranger.patch.cliutil.%s %s %s %s -default"%(self.JAVA_BIN,globalDict['ranger_admin_max_heap_size'],ranger_log,path,className,userName,oldPassword,newPassword) if is_unix: status = subprocess.call(shlex.split(get_java_cmd)) elif os_name == "WINDOWS": @@ -3604,6 +3619,7 @@ def main(argv): db_ssl_enabled='false' db_ssl_required='false' db_ssl_verifyServerCertificate='false' + db_ssl_auth_type='2-way' javax_net_ssl_keyStore='' javax_net_ssl_keyStorePassword='' javax_net_ssl_trustStore='' @@ -3617,30 +3633,33 @@ def main(argv): db_ssl_required=globalDict['db_ssl_required'].lower() if 'db_ssl_verifyServerCertificate' in globalDict: db_ssl_verifyServerCertificate=globalDict['db_ssl_verifyServerCertificate'].lower() + if 'db_ssl_auth_type' in globalDict: + db_ssl_auth_type=globalDict['db_ssl_auth_type'].lower() if db_ssl_verifyServerCertificate == 'true': - if 'javax_net_ssl_keyStore' in globalDict: - javax_net_ssl_keyStore=globalDict['javax_net_ssl_keyStore'] - if 'javax_net_ssl_keyStorePassword' in globalDict: - javax_net_ssl_keyStorePassword=globalDict['javax_net_ssl_keyStorePassword'] if 'javax_net_ssl_trustStore' in globalDict: javax_net_ssl_trustStore=globalDict['javax_net_ssl_trustStore'] if 'javax_net_ssl_trustStorePassword' in globalDict: javax_net_ssl_trustStorePassword=globalDict['javax_net_ssl_trustStorePassword'] - if not os.path.exists(javax_net_ssl_keyStore): - log("[E] Invalid file Name! Unable to find keystore file:"+javax_net_ssl_keyStore,"error") - sys.exit(1) if not os.path.exists(javax_net_ssl_trustStore): log("[E] Invalid file Name! Unable to find truststore file:"+javax_net_ssl_trustStore,"error") sys.exit(1) - if javax_net_ssl_keyStorePassword is None or javax_net_ssl_keyStorePassword =="": - log("[E] Invalid ssl keystore password!","error") - sys.exit(1) if javax_net_ssl_trustStorePassword is None or javax_net_ssl_trustStorePassword =="": log("[E] Invalid ssl truststore password!","error") sys.exit(1) + if db_ssl_auth_type == '2-way': + if 'javax_net_ssl_keyStore' in globalDict: + javax_net_ssl_keyStore=globalDict['javax_net_ssl_keyStore'] + if 'javax_net_ssl_keyStorePassword' in globalDict: + javax_net_ssl_keyStorePassword=globalDict['javax_net_ssl_keyStorePassword'] + if not os.path.exists(javax_net_ssl_keyStore): + log("[E] Invalid file Name! Unable to find keystore file:"+javax_net_ssl_keyStore,"error") + sys.exit(1) + if javax_net_ssl_keyStorePassword is None or javax_net_ssl_keyStorePassword =="": + log("[E] Invalid ssl keystore password!","error") + sys.exit(1) MYSQL_CONNECTOR_JAR=globalDict['SQL_CONNECTOR_JAR'] - xa_sqlObj = MysqlConf(xa_db_host, MYSQL_CONNECTOR_JAR, JAVA_BIN,db_ssl_enabled,db_ssl_required,db_ssl_verifyServerCertificate,javax_net_ssl_keyStore,javax_net_ssl_keyStorePassword,javax_net_ssl_trustStore,javax_net_ssl_trustStorePassword) + xa_sqlObj = MysqlConf(xa_db_host, MYSQL_CONNECTOR_JAR, JAVA_BIN,db_ssl_enabled,db_ssl_required,db_ssl_verifyServerCertificate,javax_net_ssl_keyStore,javax_net_ssl_keyStorePassword,javax_net_ssl_trustStore,javax_net_ssl_trustStorePassword,db_ssl_auth_type) xa_db_version_file = os.path.join(RANGER_ADMIN_HOME , mysql_dbversion_catalog) xa_db_core_file = os.path.join(RANGER_ADMIN_HOME , mysql_core_file) xa_patch_file = os.path.join(RANGER_ADMIN_HOME ,mysql_patches) @@ -3700,7 +3719,7 @@ def main(argv): if AUDIT_DB_FLAVOR == "MYSQL": MYSQL_CONNECTOR_JAR=globalDict['SQL_CONNECTOR_JAR'] - audit_sqlObj = MysqlConf(audit_db_host,MYSQL_CONNECTOR_JAR,JAVA_BIN,db_ssl_enabled,db_ssl_required,db_ssl_verifyServerCertificate,javax_net_ssl_keyStore,javax_net_ssl_keyStorePassword,javax_net_ssl_trustStore,javax_net_ssl_trustStorePassword) + audit_sqlObj = MysqlConf(audit_db_host,MYSQL_CONNECTOR_JAR,JAVA_BIN,db_ssl_enabled,db_ssl_required,db_ssl_verifyServerCertificate,javax_net_ssl_keyStore,javax_net_ssl_keyStorePassword,javax_net_ssl_trustStore,javax_net_ssl_trustStorePassword,db_ssl_auth_type) audit_db_file = os.path.join(RANGER_ADMIN_HOME ,mysql_audit_file) elif AUDIT_DB_FLAVOR == "ORACLE": diff --git a/security-admin/scripts/dba_script.py b/security-admin/scripts/dba_script.py index 83d6fe728b..6843aa80b6 100644 --- a/security-admin/scripts/dba_script.py +++ b/security-admin/scripts/dba_script.py @@ -157,13 +157,14 @@ def create_auditdb_user(self, xa_db_host, audit_db_host, db_name, audit_db_name, class MysqlConf(BaseDB): # Constructor - def __init__(self, host,SQL_CONNECTOR_JAR,JAVA_BIN,db_ssl_enabled,db_ssl_required,db_ssl_verifyServerCertificate,javax_net_ssl_keyStore,javax_net_ssl_keyStorePassword,javax_net_ssl_trustStore,javax_net_ssl_trustStorePassword): + def __init__(self, host,SQL_CONNECTOR_JAR,JAVA_BIN,db_ssl_enabled,db_ssl_required,db_ssl_verifyServerCertificate,javax_net_ssl_keyStore,javax_net_ssl_keyStorePassword,javax_net_ssl_trustStore,javax_net_ssl_trustStorePassword,db_ssl_auth_type): self.host = host self.SQL_CONNECTOR_JAR = SQL_CONNECTOR_JAR self.JAVA_BIN = JAVA_BIN self.db_ssl_enabled=db_ssl_enabled.lower() self.db_ssl_required=db_ssl_required.lower() self.db_ssl_verifyServerCertificate=db_ssl_verifyServerCertificate.lower() + self.db_ssl_auth_type=db_ssl_auth_type.lower() self.javax_net_ssl_keyStore=javax_net_ssl_keyStore self.javax_net_ssl_keyStorePassword=javax_net_ssl_keyStorePassword self.javax_net_ssl_trustStore=javax_net_ssl_trustStore @@ -177,7 +178,10 @@ def get_jisql_cmd(self, user, password ,db_name): if self.db_ssl_enabled == 'true': db_ssl_param="?useSSL=%s&requireSSL=%s&verifyServerCertificate=%s" %(self.db_ssl_enabled,self.db_ssl_required,self.db_ssl_verifyServerCertificate) if self.db_ssl_verifyServerCertificate == 'true': - db_ssl_cert_param=" -Djavax.net.ssl.keyStore=%s -Djavax.net.ssl.keyStorePassword=%s -Djavax.net.ssl.trustStore=%s -Djavax.net.ssl.trustStorePassword=%s " %(self.javax_net_ssl_keyStore,self.javax_net_ssl_keyStorePassword,self.javax_net_ssl_trustStore,self.javax_net_ssl_trustStorePassword) + if self.db_ssl_auth_type == '1-way': + db_ssl_cert_param=" -Djavax.net.ssl.trustStore=%s -Djavax.net.ssl.trustStorePassword=%s " %(self.javax_net_ssl_trustStore,self.javax_net_ssl_trustStorePassword) + else: + db_ssl_cert_param=" -Djavax.net.ssl.keyStore=%s -Djavax.net.ssl.keyStorePassword=%s -Djavax.net.ssl.trustStore=%s -Djavax.net.ssl.trustStorePassword=%s " %(self.javax_net_ssl_keyStore,self.javax_net_ssl_keyStorePassword,self.javax_net_ssl_trustStore,self.javax_net_ssl_trustStorePassword) if is_unix: jisql_cmd = "%s %s -cp %s:%s/jisql/lib/* org.apache.util.sql.Jisql -driver mysqlconj -cstring jdbc:mysql://%s/%s%s -u %s -p '%s' -noheader -trim -c \;" %(self.JAVA_BIN,db_ssl_cert_param,self.SQL_CONNECTOR_JAR,path,self.host,db_name,db_ssl_param,user,password) elif os_name == "WINDOWS": @@ -1644,6 +1648,7 @@ def main(argv): db_ssl_enabled='false' db_ssl_required='false' db_ssl_verifyServerCertificate='false' + db_ssl_auth_type='2-way' javax_net_ssl_keyStore='' javax_net_ssl_keyStorePassword='' javax_net_ssl_trustStore='' @@ -1656,30 +1661,33 @@ def main(argv): db_ssl_required=globalDict['db_ssl_required'].lower() if 'db_ssl_verifyServerCertificate' in globalDict: db_ssl_verifyServerCertificate=globalDict['db_ssl_verifyServerCertificate'].lower() + if 'db_ssl_auth_type' in globalDict: + db_ssl_auth_type=globalDict['db_ssl_auth_type'].lower() if db_ssl_verifyServerCertificate == 'true': - if 'javax_net_ssl_keyStore' in globalDict: - javax_net_ssl_keyStore=globalDict['javax_net_ssl_keyStore'] - if 'javax_net_ssl_keyStorePassword' in globalDict: - javax_net_ssl_keyStorePassword=globalDict['javax_net_ssl_keyStorePassword'] if 'javax_net_ssl_trustStore' in globalDict: javax_net_ssl_trustStore=globalDict['javax_net_ssl_trustStore'] if 'javax_net_ssl_trustStorePassword' in globalDict: javax_net_ssl_trustStorePassword=globalDict['javax_net_ssl_trustStorePassword'] - if not os.path.exists(javax_net_ssl_keyStore): - log("[E] Invalid file Name! Unable to find keystore file:"+javax_net_ssl_keyStore,"error") - sys.exit(1) if not os.path.exists(javax_net_ssl_trustStore): log("[E] Invalid file Name! Unable to find truststore file:"+javax_net_ssl_trustStore,"error") sys.exit(1) - if javax_net_ssl_keyStorePassword is None or javax_net_ssl_keyStorePassword =="": - log("[E] Invalid ssl keystore password!","error") - sys.exit(1) if javax_net_ssl_trustStorePassword is None or javax_net_ssl_trustStorePassword =="": log("[E] Invalid ssl truststore password!","error") sys.exit(1) + if db_ssl_auth_type == '2-way': + if 'javax_net_ssl_keyStore' in globalDict: + javax_net_ssl_keyStore=globalDict['javax_net_ssl_keyStore'] + if 'javax_net_ssl_keyStorePassword' in globalDict: + javax_net_ssl_keyStorePassword=globalDict['javax_net_ssl_keyStorePassword'] + if not os.path.exists(javax_net_ssl_keyStore): + log("[E] Invalid file Name! Unable to find keystore file:"+javax_net_ssl_keyStore,"error") + sys.exit(1) + if javax_net_ssl_keyStorePassword is None or javax_net_ssl_keyStorePassword =="": + log("[E] Invalid ssl keystore password!","error") + sys.exit(1) MYSQL_CONNECTOR_JAR=CONNECTOR_JAR - xa_sqlObj = MysqlConf(xa_db_host, MYSQL_CONNECTOR_JAR, JAVA_BIN,db_ssl_enabled,db_ssl_required,db_ssl_verifyServerCertificate,javax_net_ssl_keyStore,javax_net_ssl_keyStorePassword,javax_net_ssl_trustStore,javax_net_ssl_trustStorePassword) + xa_sqlObj = MysqlConf(xa_db_host, MYSQL_CONNECTOR_JAR, JAVA_BIN,db_ssl_enabled,db_ssl_required,db_ssl_verifyServerCertificate,javax_net_ssl_keyStore,javax_net_ssl_keyStorePassword,javax_net_ssl_trustStore,javax_net_ssl_trustStorePassword,db_ssl_auth_type) xa_db_version_file = os.path.join(RANGER_ADMIN_HOME,mysql_dbversion_catalog) xa_db_core_file = os.path.join(RANGER_ADMIN_HOME,mysql_core_file) xa_patch_file = os.path.join(RANGER_ADMIN_HOME,mysql_patches) @@ -1726,7 +1734,7 @@ def main(argv): if AUDIT_DB_FLAVOR == "MYSQL": MYSQL_CONNECTOR_JAR=CONNECTOR_JAR - audit_sqlObj = MysqlConf(audit_db_host,MYSQL_CONNECTOR_JAR,JAVA_BIN,db_ssl_enabled,db_ssl_required,db_ssl_verifyServerCertificate,javax_net_ssl_keyStore,javax_net_ssl_keyStorePassword,javax_net_ssl_trustStore,javax_net_ssl_trustStorePassword) + audit_sqlObj = MysqlConf(audit_db_host,MYSQL_CONNECTOR_JAR,JAVA_BIN,db_ssl_enabled,db_ssl_required,db_ssl_verifyServerCertificate,javax_net_ssl_keyStore,javax_net_ssl_keyStorePassword,javax_net_ssl_trustStore,javax_net_ssl_trustStorePassword,db_ssl_auth_type) audit_db_file = os.path.join(RANGER_ADMIN_HOME,mysql_audit_file) elif AUDIT_DB_FLAVOR == "ORACLE": diff --git a/security-admin/scripts/install.properties b/security-admin/scripts/install.properties index f323c955da..d7f098b30f 100644 --- a/security-admin/scripts/install.properties +++ b/security-admin/scripts/install.properties @@ -56,6 +56,8 @@ db_host=localhost db_ssl_enabled=false db_ssl_required=false db_ssl_verifyServerCertificate=false +#db_ssl_auth_type=1-way|2-way, where 1-way represents standard one way ssl authentication and 2-way represents mutual ssl authentication +db_ssl_auth_type=2-way javax_net_ssl_keyStore= javax_net_ssl_keyStorePassword= javax_net_ssl_trustStore= @@ -219,6 +221,8 @@ JAVA_BIN='java' JAVA_VERSION_REQUIRED='1.7' JAVA_ORACLE='Java(TM) SE Runtime Environment' +ranger_admin_max_heap_size=1g + #mysql_create_user_file=${PWD}/db/mysql/create_dev_user.sql mysql_core_file=db/mysql/xa_core_db.sql mysql_audit_file=db/mysql/xa_audit_db.sql diff --git a/security-admin/scripts/rolebasedusersearchutil.py b/security-admin/scripts/rolebasedusersearchutil.py new file mode 100644 index 0000000000..d651461f66 --- /dev/null +++ b/security-admin/scripts/rolebasedusersearchutil.py @@ -0,0 +1,159 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. See accompanying LICENSE file. +# + +import os +import re +import sys +import errno +import shlex +import logging +import subprocess +import platform +import fileinput +import getpass +import shutil +from xml.etree import ElementTree as ET +from os.path import basename +from subprocess import Popen,PIPE +from datetime import date +from datetime import datetime +from operator import contains + + +os_name = platform.system() +os_name = os_name.upper() + +msgPrompt = "Enter the below options" +msgCommand = "Usage : python rolebasedusersearchutil.py -u -p -r " +msgRoleList = " can be ROLE_USER/ROLE_SYS_ADMIN/ROLE_KEY_ADMIN" + + + +if os_name == "LINUX": + RANGER_ADMIN_HOME = os.getenv("RANGER_ADMIN_HOME") + if RANGER_ADMIN_HOME is None: + RANGER_ADMIN_HOME = os.getcwd() +elif os_name == "WINDOWS": + RANGER_ADMIN_HOME = os.getenv("RANGER_ADMIN_HOME") + +def log(msg,type): + if type == 'info': + logging.info(" %s",msg) + if type == 'debug': + logging.debug(" %s",msg) + if type == 'warning': + logging.warning(" %s",msg) + if type == 'exception': + logging.exception(" %s",msg) + if type == 'error': + logging.error(" %s",msg) + +def main(argv): + FORMAT = '%(asctime)-15s %(message)s' + logging.basicConfig(format=FORMAT, level=logging.DEBUG) + ews_lib = os.path.join(RANGER_ADMIN_HOME,"ews","lib") + app_home = os.path.join(RANGER_ADMIN_HOME,"ews","webapp") + ranger_log = os.path.join(RANGER_ADMIN_HOME,"ews","logs") + + if os.environ['JAVA_HOME'] == "": + log("[E] ---------- JAVA_HOME environment property not defined, aborting installation. ----------", "error") + sys.exit(1) + JAVA_BIN=os.path.join(os.environ['JAVA_HOME'],'bin','java') + if os_name == "WINDOWS" : + JAVA_BIN = JAVA_BIN+'.exe' + if os.path.isfile(JAVA_BIN): + pass + else: + while os.path.isfile(JAVA_BIN) == False: + log("Enter java executable path: :","info") + JAVA_BIN=raw_input() + log("[I] Using Java:" + str(JAVA_BIN),"info") + userName = "" + password = "" + userRole = "" + userNameMsgFlag = False + passwordMsgFlag = False + userRoleMsgFlag = False + userroleFlag = False + + if len(argv) == 1: + print msgPrompt + " or \n" + msgCommand + "\n " +msgRoleList + userName = raw_input('Enter a user name: ') + password = getpass.getpass('Enter a user password:') + userRole = raw_input('Enter a role: ') + elif len(argv) > 1 and len(argv) < 8 : + for i in range(1, len(sys.argv)) : + if sys.argv[i] == "-u" : + if len(argv)-1 > i+1 or len(argv)-1 == i+1: + userName = sys.argv[i+1] + continue + if sys.argv[i] == "-p" : + if len(argv)-1 > i+1 or len(argv)-1 == i+1: + password = sys.argv[i+1] + continue + if sys.argv[i] == "-r" : + if len(argv)-1 > i+1 or len(argv)-1 == i+1: + userRole = sys.argv[i+1] + userroleFlag = True + continue + else: + log("[E] Invalid argument list.", "error") + log("[I] " + msgCommand + "\n " + msgRoleList, "info") + sys.exit(1) + + if userName == "" : + userNameMsgFlag = True + elif userName != "" : + if userName.lower() == "-p" or userName.lower() == "-r" or userName.lower() == "-u" : + userNameMsgFlag = True + if password == "" : + passwordMsgFlag = True + elif password.lower() == "-p" or password.lower() == "-r" or password.lower() == "-u" : + passwordMsgFlag = True + if userroleFlag == True : + if userRole == "": + userRoleMsgFlag = True + elif userRole != "": + if userRole.lower() == "-p" or userRole.lower() == "-r" or userRole.lower() == "-u": + userRoleMsgFlag = True + if userNameMsgFlag == True or passwordMsgFlag == True or userRoleMsgFlag == True : + print msgPrompt + " or \n" + msgCommand + "\n " +msgRoleList + if userNameMsgFlag == True : + userName = raw_input('Enter a user name: ') + if passwordMsgFlag == True : + password = getpass.getpass("Enter user password:") + if userRoleMsgFlag == True : + userRole = raw_input('Enter a role: ') + if userName != "" and password != "" : + if os_name == "LINUX": + path = os.path.join("%s","WEB-INF","classes","conf:%s","WEB-INF","classes","lib","*:%s","WEB-INF",":%s","META-INF",":%s","WEB-INF","lib","*:%s","WEB-INF","classes",":%s","WEB-INF","classes","META-INF:%s/*")%(app_home ,app_home ,app_home, app_home, app_home, app_home ,app_home,ews_lib) + elif os_name == "WINDOWS": + path = os.path.join("%s","WEB-INF","classes","conf;%s","WEB-INF","classes","lib","*;%s","WEB-INF",";%s","META-INF",";%s","WEB-INF","lib","*;%s","WEB-INF","classes",";%s","WEB-INF","classes","META-INF" )%(app_home ,app_home ,app_home, app_home, app_home, app_home ,app_home) + if userRole != "" : + get_java_cmd = "%s -Dlogdir=%s -Dlog4j.configuration=db_patch.log4j.xml -cp %s org.apache.ranger.patch.cliutil.%s %s %s %s"%(JAVA_BIN,ranger_log,path,'RoleBasedUserSearchUtil',userName,password,userRole) + if userRole == "" : + get_java_cmd = "%s -Dlogdir=%s -Dlog4j.configuration=db_patch.log4j.xml -cp %s org.apache.ranger.patch.cliutil.%s %s %s "%(JAVA_BIN,ranger_log,path,'RoleBasedUserSearchUtil',userName,password) + if os_name == "LINUX": + ret = subprocess.call(shlex.split(get_java_cmd)) + elif os_name == "WINDOWS": + ret = subprocess.call(get_java_cmd) + if ret == 0: + log("[I] List fetched successfully","info") + else: + log("[E] Unable to fetch user list of given role ","error") + sys.exit(1) + else: + log("[E] Input Error","error") + +main(sys.argv) diff --git a/security-admin/scripts/setup.sh b/security-admin/scripts/setup.sh index 87be127589..633d363acc 100755 --- a/security-admin/scripts/setup.sh +++ b/security-admin/scripts/setup.sh @@ -68,6 +68,7 @@ db_password=$(get_prop 'db_password' $PROPFILE) db_ssl_enabled=$(get_prop 'db_ssl_enabled' $PROPFILE) db_ssl_required=$(get_prop 'db_ssl_required' $PROPFILE) db_ssl_verifyServerCertificate=$(get_prop 'db_ssl_verifyServerCertificate' $PROPFILE) +db_ssl_auth_type=$(get_prop 'db_ssl_auth_type' $PROPFILE) javax_net_ssl_keyStore=$(get_prop 'javax_net_ssl_keyStore' $PROPFILE) javax_net_ssl_keyStorePassword=$(get_prop 'javax_net_ssl_keyStorePassword' $PROPFILE) javax_net_ssl_trustStore=$(get_prop 'javax_net_ssl_trustStore' $PROPFILE) @@ -254,11 +255,13 @@ init_variables(){ db_ssl_enabled="false" db_ssl_required="false" db_ssl_verifyServerCertificate="false" + db_ssl_auth_type="2-way" fi if [ "${db_ssl_enabled}" == "true" ] then db_ssl_required=`echo $db_ssl_required | tr '[:upper:]' '[:lower:]'` db_ssl_verifyServerCertificate=`echo $db_ssl_verifyServerCertificate | tr '[:upper:]' '[:lower:]'` + db_ssl_auth_type=`echo $db_ssl_auth_type | tr '[:upper:]' '[:lower:]'` if [ "${db_ssl_required}" != "true" ] then db_ssl_required="false" @@ -267,6 +270,10 @@ init_variables(){ then db_ssl_verifyServerCertificate="false" fi + if [ "${db_ssl_auth_type}" != "1-way" ] + then + db_ssl_auth_type="2-way" + fi fi } @@ -485,6 +492,10 @@ update_properties() { propertyName=ranger.db.ssl.verifyServerCertificate newPropertyValue="${db_ssl_verifyServerCertificate}" updatePropertyToFilePy $propertyName $newPropertyValue $to_file_default + + propertyName=ranger.db.ssl.auth.type + newPropertyValue="${db_ssl_auth_type}" + updatePropertyToFilePy $propertyName $newPropertyValue $to_file_default fi if [ "${DB_FLAVOR}" == "MYSQL" ] @@ -1377,7 +1388,12 @@ setup_install_files(){ if [ "${db_ssl_verifyServerCertificate}" == "true" ] then - DB_SSL_PARAM="' -Djavax.net.ssl.keyStore=${javax_net_ssl_keyStore} -Djavax.net.ssl.keyStorePassword=${javax_net_ssl_keyStorePassword} -Djavax.net.ssl.trustStore=${javax_net_ssl_trustStore} -Djavax.net.ssl.trustStorePassword=${javax_net_ssl_trustStorePassword} '" + if [ "${db_ssl_auth_type}" == "1-way" ] + then + DB_SSL_PARAM="' -Djavax.net.ssl.trustStore=${javax_net_ssl_trustStore} -Djavax.net.ssl.trustStorePassword=${javax_net_ssl_trustStorePassword} '" + else + DB_SSL_PARAM="' -Djavax.net.ssl.keyStore=${javax_net_ssl_keyStore} -Djavax.net.ssl.keyStorePassword=${javax_net_ssl_keyStorePassword} -Djavax.net.ssl.trustStore=${javax_net_ssl_trustStore} -Djavax.net.ssl.trustStorePassword=${javax_net_ssl_trustStorePassword} '" + fi echo "export DB_SSL_PARAM=${DB_SSL_PARAM}" > ${WEBAPP_ROOT}/WEB-INF/classes/conf/ranger-admin-env-dbsslparam.sh chmod a+rx ${WEBAPP_ROOT}/WEB-INF/classes/conf/ranger-admin-env-dbsslparam.sh else diff --git a/security-admin/src/main/java/org/apache/ranger/biz/AssetMgr.java b/security-admin/src/main/java/org/apache/ranger/biz/AssetMgr.java index 60df91afa7..5ee2616d1f 100644 --- a/security-admin/src/main/java/org/apache/ranger/biz/AssetMgr.java +++ b/security-admin/src/main/java/org/apache/ranger/biz/AssetMgr.java @@ -897,55 +897,57 @@ private String getRemoteAddress(final HttpServletRequest request) { } public VXTrxLogList getReportLogs(SearchCriteria searchCriteria) { - if (!xaBizUtil.isAdmin()) { - throw restErrorUtil.create403RESTException("Permission Denied !"); - } - - if (searchCriteria == null) { - searchCriteria = new SearchCriteria(); - } - - if (searchCriteria.getParamList() != null - && searchCriteria.getParamList().size() > 0) { - int clientTimeOffsetInMinute = RestUtil.getClientTimeOffset(); - Date temp = null; - DateUtil dateUtil = new DateUtil(); - if (searchCriteria.getParamList().containsKey("startDate")) { - temp = (Date) searchCriteria.getParamList().get( - "startDate"); - temp = dateUtil.getDateFromGivenDate(temp, 0, 0, 0, 0); - temp = dateUtil.addTimeOffset(temp, clientTimeOffsetInMinute); - searchCriteria.getParamList().put("startDate", temp); - } - if (searchCriteria.getParamList().containsKey("endDate")) { - temp = (Date) searchCriteria.getParamList().get( - "endDate"); - temp = dateUtil.getDateFromGivenDate(temp, 0, 23, 59, 59); - temp = dateUtil.addTimeOffset(temp, clientTimeOffsetInMinute); - searchCriteria.getParamList().put("endDate", temp); + if (xaBizUtil.isAdmin() || xaBizUtil.isKeyAdmin()) { + if (searchCriteria == null) { + searchCriteria = new SearchCriteria(); } - if (searchCriteria.getParamList().containsKey("owner")) { - XXPortalUser xXPortalUser = rangerDaoManager.getXXPortalUser().findByLoginId( - (searchCriteria.getParamList().get("owner").toString())); - if(xXPortalUser != null) { - searchCriteria.getParamList().put("owner", xXPortalUser.getId()); - } else { - searchCriteria.getParamList().put("owner", 0); + + if (searchCriteria.getParamList() != null + && searchCriteria.getParamList().size() > 0) { + int clientTimeOffsetInMinute = RestUtil.getClientTimeOffset(); + Date temp = null; + DateUtil dateUtil = new DateUtil(); + if (searchCriteria.getParamList().containsKey("startDate")) { + temp = (Date) searchCriteria.getParamList().get( + "startDate"); + temp = dateUtil.getDateFromGivenDate(temp, 0, 0, 0, 0); + temp = dateUtil.addTimeOffset(temp, clientTimeOffsetInMinute); + searchCriteria.getParamList().put("startDate", temp); } - + if (searchCriteria.getParamList().containsKey("endDate")) { + temp = (Date) searchCriteria.getParamList().get( + "endDate"); + temp = dateUtil.getDateFromGivenDate(temp, 0, 23, 59, 59); + temp = dateUtil.addTimeOffset(temp, clientTimeOffsetInMinute); + searchCriteria.getParamList().put("endDate", temp); + } + if (searchCriteria.getParamList().containsKey("owner")) { + XXPortalUser xXPortalUser = rangerDaoManager.getXXPortalUser().findByLoginId( + (searchCriteria.getParamList().get("owner").toString())); + if(xXPortalUser != null) { + searchCriteria.getParamList().put("owner", xXPortalUser.getId()); + } else { + searchCriteria.getParamList().put("owner", 0); + } + + } + } + VXTrxLogList vXTrxLogList = xTrxLogService + .searchXTrxLogs(searchCriteria); + Long count = xTrxLogService + .searchXTrxLogsCount(searchCriteria); + vXTrxLogList.setTotalCount(count); + + List newList = validateXXTrxLogList(vXTrxLogList.getVXTrxLogs()); + vXTrxLogList.setVXTrxLogs(newList); + return vXTrxLogList; + } else { + throw restErrorUtil.create403RESTException("Permission Denied !"); } - VXTrxLogList vXTrxLogList = xTrxLogService - .searchXTrxLogs(searchCriteria); - Long count = xTrxLogService - .searchXTrxLogsCount(searchCriteria); - vXTrxLogList.setTotalCount(count); - List newList = validateXXTrxLogList(vXTrxLogList.getVXTrxLogs()); - vXTrxLogList.setVXTrxLogs(newList); - return vXTrxLogList; } public VXAccessAuditList getAccessLogs(SearchCriteria searchCriteria) { @@ -1057,9 +1059,7 @@ public List validateXXTrxLogList(List xTrxLogList) { } } } - if(vXTrxLog.getPreviousValue() != null && !vXTrxLog.getPreviousValue().isEmpty() || vXTrxLog.getNewValue() != null && !vXTrxLog.getNewValue().isEmpty()) { - vXTrxLogs.add(vXTrxLog); - } + vXTrxLogs.add(vXTrxLog); } return vXTrxLogs; } diff --git a/security-admin/src/main/java/org/apache/ranger/biz/PolicyRefUpdater.java b/security-admin/src/main/java/org/apache/ranger/biz/PolicyRefUpdater.java new file mode 100644 index 0000000000..25b48bb508 --- /dev/null +++ b/security-admin/src/main/java/org/apache/ranger/biz/PolicyRefUpdater.java @@ -0,0 +1,286 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ranger.biz; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.ranger.db.RangerDaoManager; +import org.apache.ranger.db.XXPolicyRefAccessTypeDao; +import org.apache.ranger.db.XXPolicyRefConditionDao; +import org.apache.ranger.db.XXPolicyRefDataMaskTypeDao; +import org.apache.ranger.db.XXPolicyRefGroupDao; +import org.apache.ranger.db.XXPolicyRefResourceDao; +import org.apache.ranger.db.XXPolicyRefUserDao; +import org.apache.ranger.entity.XXAccessTypeDef; +import org.apache.ranger.entity.XXDataMaskTypeDef; +import org.apache.ranger.entity.XXGroup; +import org.apache.ranger.entity.XXPolicy; +import org.apache.ranger.entity.XXPolicyConditionDef; +import org.apache.ranger.entity.XXPolicyRefAccessType; +import org.apache.ranger.entity.XXPolicyRefCondition; +import org.apache.ranger.entity.XXPolicyRefDataMaskType; +import org.apache.ranger.entity.XXPolicyRefGroup; +import org.apache.ranger.entity.XXPolicyRefResource; +import org.apache.ranger.entity.XXPolicyRefUser; +import org.apache.ranger.entity.XXResourceDef; +import org.apache.ranger.entity.XXServiceDef; +import org.apache.ranger.entity.XXUser; +import org.apache.ranger.plugin.model.RangerPolicy; +import org.apache.ranger.plugin.model.RangerPolicy.RangerDataMaskPolicyItem; +import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyItem; +import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyItemAccess; +import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyItemCondition; +import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyItemDataMaskInfo; +import org.apache.ranger.service.RangerAuditFields; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +@Component +public class PolicyRefUpdater { + + @Autowired + RangerDaoManager daoMgr; + + @Autowired + RangerAuditFields rangerAuditFields; + + public void createNewPolMappingForRefTable(RangerPolicy policy, XXPolicy xPolicy, XXServiceDef xServiceDef) throws Exception { + if(policy == null) { + return; + } + + cleanupRefTables(policy); + + final Set resourceNames = policy.getResources().keySet(); + final Set groupNames = new HashSet<>(); + final Set userNames = new HashSet<>(); + final Set accessTypes = new HashSet<>(); + final Set conditionTypes = new HashSet<>(); + final Set dataMaskTypes = new HashSet<>(); + + for (List policyItems : getAllPolicyItems(policy)) { + if (CollectionUtils.isEmpty(policyItems)) { + continue; + } + + for (RangerPolicyItem policyItem : policyItems) { + groupNames.addAll(policyItem.getGroups()); + userNames.addAll(policyItem.getUsers()); + + if (CollectionUtils.isNotEmpty(policyItem.getAccesses())) { + for (RangerPolicyItemAccess access : policyItem.getAccesses()) { + accessTypes.add(access.getType()); + } + } + + if (CollectionUtils.isNotEmpty(policyItem.getConditions())) { + for (RangerPolicyItemCondition condition : policyItem.getConditions()) { + conditionTypes.add(condition.getType()); + } + } + + if (policyItem instanceof RangerDataMaskPolicyItem) { + RangerPolicyItemDataMaskInfo dataMaskInfo = ((RangerDataMaskPolicyItem) policyItem).getDataMaskInfo(); + + dataMaskTypes.add(dataMaskInfo.getDataMaskType()); + } + } + } + + for (String resource : resourceNames) { + XXResourceDef xResDef = daoMgr.getXXResourceDef().findByNameAndPolicyId(resource, policy.getId()); + + if (xResDef == null) { + throw new Exception(resource + ": is not a valid resource-type. policy='"+ policy.getName() + "' service='"+ policy.getService() + "'"); + } + + XXPolicyRefResource xPolRes = rangerAuditFields.populateAuditFields(new XXPolicyRefResource(), xPolicy); + + xPolRes.setPolicyId(policy.getId()); + xPolRes.setResourceDefId(xResDef.getId()); + xPolRes.setResourceName(resource); + + daoMgr.getXXPolicyRefResource().create(xPolRes); + } + + for (String group : groupNames) { + if (StringUtils.isBlank(group)) { + continue; + } + + XXGroup xGroup = daoMgr.getXXGroup().findByGroupName(group); + + if (xGroup == null) { + throw new Exception(group + ": group does not exist. policy='"+ policy.getName() + "' service='"+ policy.getService() + "' group='" + group + "'"); + } + + XXPolicyRefGroup xPolGroup = rangerAuditFields.populateAuditFields(new XXPolicyRefGroup(), xPolicy); + + xPolGroup.setPolicyId(policy.getId()); + xPolGroup.setGroupId(xGroup.getId()); + xPolGroup.setGroupName(group); + + daoMgr.getXXPolicyRefGroup().create(xPolGroup); + } + + for (String user : userNames) { + if (StringUtils.isBlank(user)) { + continue; + } + + XXUser xUser = daoMgr.getXXUser().findByUserName(user); + + if (xUser == null) { + throw new Exception(user + ": user does not exist. policy='"+ policy.getName() + "' service='"+ policy.getService() + "' user='" + user +"'"); + } + + XXPolicyRefUser xPolUser = rangerAuditFields.populateAuditFields(new XXPolicyRefUser(), xPolicy); + + xPolUser.setPolicyId(policy.getId()); + xPolUser.setUserId(xUser.getId()); + xPolUser.setUserName(user); + + daoMgr.getXXPolicyRefUser().create(xPolUser); + } + + for (String accessType : accessTypes) { + XXAccessTypeDef xAccTypeDef = daoMgr.getXXAccessTypeDef().findByNameAndServiceId(accessType, xPolicy.getService()); + + if (xAccTypeDef == null) { + throw new Exception(accessType + ": is not a valid access-type. policy='" + policy.getName() + "' service='" + policy.getService() + "'"); + } + + XXPolicyRefAccessType xPolAccess = rangerAuditFields.populateAuditFields(new XXPolicyRefAccessType(), xPolicy); + + xPolAccess.setPolicyId(policy.getId()); + xPolAccess.setAccessDefId(xAccTypeDef.getId()); + xPolAccess.setAccessTypeName(accessType); + + daoMgr.getXXPolicyRefAccessType().create(xPolAccess); + } + + for (String condition : conditionTypes) { + XXPolicyConditionDef xPolCondDef = daoMgr.getXXPolicyConditionDef().findByServiceDefIdAndName(xServiceDef.getId(), condition); + + if (xPolCondDef == null) { + throw new Exception(condition + ": is not a valid condition-type. policy='"+ xPolicy.getName() + "' service='"+ xPolicy.getService() + "'"); + } + + XXPolicyRefCondition xPolCond = rangerAuditFields.populateAuditFields(new XXPolicyRefCondition(), xPolicy); + + xPolCond.setPolicyId(policy.getId()); + xPolCond.setConditionDefId(xPolCondDef.getId()); + xPolCond.setConditionName(condition); + + daoMgr.getXXPolicyRefCondition().create(xPolCond); + } + + for (String dataMaskType : dataMaskTypes ) { + XXDataMaskTypeDef dataMaskDef = daoMgr.getXXDataMaskTypeDef().findByNameAndServiceId(dataMaskType, xPolicy.getService()); + + if (dataMaskDef == null) { + throw new Exception(dataMaskType + ": is not a valid datamask-type. policy='" + policy.getName() + "' service='" + policy.getService() + "'"); + } + + XXPolicyRefDataMaskType xxDataMaskInfo = new XXPolicyRefDataMaskType(); + + xxDataMaskInfo.setPolicyId(policy.getId()); + xxDataMaskInfo.setDataMaskDefId(dataMaskDef.getId()); + xxDataMaskInfo.setDataMaskTypeName(dataMaskType); + + daoMgr.getXXPolicyRefDataMaskType().create(xxDataMaskInfo); + } + } + + public Boolean cleanupRefTables(RangerPolicy policy) { + final Long policyId = policy == null ? null : policy.getId(); + + if (policyId == null) { + return false; + } + + XXPolicyRefResourceDao xPolResDao = daoMgr.getXXPolicyRefResource(); + XXPolicyRefGroupDao xPolGroupDao = daoMgr.getXXPolicyRefGroup(); + XXPolicyRefUserDao xPolUserDao = daoMgr.getXXPolicyRefUser(); + XXPolicyRefAccessTypeDao xPolAccessDao = daoMgr.getXXPolicyRefAccessType(); + XXPolicyRefConditionDao xPolCondDao = daoMgr.getXXPolicyRefCondition(); + XXPolicyRefDataMaskTypeDao xPolDataMaskDao = daoMgr.getXXPolicyRefDataMaskType(); + + for (XXPolicyRefResource resource : xPolResDao.findByPolicyId(policyId)) { + xPolResDao.remove(resource); + } + + for(XXPolicyRefGroup group : xPolGroupDao.findByPolicyId(policyId)) { + xPolGroupDao.remove(group); + } + + for(XXPolicyRefUser user : xPolUserDao.findByPolicyId(policyId)) { + xPolUserDao.remove(user); + } + + for(XXPolicyRefAccessType access : xPolAccessDao.findByPolicyId(policyId)) { + xPolAccessDao.remove(access); + } + + for(XXPolicyRefCondition condVal : xPolCondDao.findByPolicyId(policyId)) { + xPolCondDao.remove(condVal); + } + + for(XXPolicyRefDataMaskType dataMask : xPolDataMaskDao.findByPolicyId(policyId)) { + xPolDataMaskDao.remove(dataMask); + } + + return true; + } + + static List> getAllPolicyItems(RangerPolicy policy) { + List> ret = new ArrayList<>(); + + if (CollectionUtils.isNotEmpty(policy.getPolicyItems())) { + ret.add(policy.getPolicyItems()); + } + + if (CollectionUtils.isNotEmpty(policy.getDenyPolicyItems())) { + ret.add(policy.getDenyPolicyItems()); + } + + if (CollectionUtils.isNotEmpty(policy.getAllowExceptions())) { + ret.add(policy.getAllowExceptions()); + } + + if (CollectionUtils.isNotEmpty(policy.getDenyExceptions())) { + ret.add(policy.getDenyExceptions()); + } + + if (CollectionUtils.isNotEmpty(policy.getDataMaskPolicyItems())) { + ret.add(policy.getDataMaskPolicyItems()); + } + + if (CollectionUtils.isNotEmpty(policy.getRowFilterPolicyItems())) { + ret.add(policy.getRowFilterPolicyItems()); + } + + return ret; + } +} diff --git a/security-admin/src/main/java/org/apache/ranger/biz/RangerBizUtil.java b/security-admin/src/main/java/org/apache/ranger/biz/RangerBizUtil.java index 4d6227d2e5..8b8ce36da4 100644 --- a/security-admin/src/main/java/org/apache/ranger/biz/RangerBizUtil.java +++ b/security-admin/src/main/java/org/apache/ranger/biz/RangerBizUtil.java @@ -1452,14 +1452,12 @@ public Boolean hasAccess(XXDBBase xxDbBase, RangerBaseModelObject baseModel) { if (xxDbBase != null && xxDbBase instanceof XXServiceDef) { XXServiceDef xServiceDef = (XXServiceDef) xxDbBase; String implClass = xServiceDef.getImplclassname(); - if (implClass == null) { - return false; - } - - if (isKeyAdmin && implClass.equals(EmbeddedServiceDefsUtil.KMS_IMPL_CLASS_NAME)) { - return true; - } else if ((isSysAdmin || isUser) && !implClass.equals(EmbeddedServiceDefsUtil.KMS_IMPL_CLASS_NAME)) { - return true; + if (EmbeddedServiceDefsUtil.KMS_IMPL_CLASS_NAME.equals(implClass)) { + // KMS case + return isKeyAdmin; + } else { + // Other cases - implClass can be null! + return isSysAdmin || isUser; } } @@ -1474,18 +1472,13 @@ public Boolean hasAccess(XXDBBase xxDbBase, RangerBaseModelObject baseModel) { XXService xService = (XXService) xxDbBase; XXServiceDef xServiceDef = daoManager.getXXServiceDef().getById(xService.getType()); String implClass = xServiceDef.getImplclassname(); - if (implClass == null) { - return false; - } - - if (isKeyAdmin && implClass.equals(EmbeddedServiceDefsUtil.KMS_IMPL_CLASS_NAME)) { - return true; - } else if (isUser && !implClass.equals(EmbeddedServiceDefsUtil.KMS_IMPL_CLASS_NAME)) { - return true; + if (EmbeddedServiceDefsUtil.KMS_IMPL_CLASS_NAME.equals(implClass)) { + // KMS case + return isKeyAdmin; + } else { + // Other cases - implClass can be null! + return isUser; } - // else if ((isSysAdmin || isUser) && !implClass.equals(EmbeddedServiceDefsUtil.KMS_IMPL_CLASS_NAME)) { - // return true; - // } } return false; } @@ -1517,7 +1510,7 @@ public void hasKMSPermissions(String objType, String implClassName) { // TODO: As of now we are allowing SYS_ADMIN to create/update/read/delete all the // services including KMS - if (objType.equalsIgnoreCase("Service-Def") && session.isUserAdmin() && implClassName.equals(EmbeddedServiceDefsUtil.KMS_IMPL_CLASS_NAME)) { + if (objType.equalsIgnoreCase("Service-Def") && session.isUserAdmin() && EmbeddedServiceDefsUtil.KMS_IMPL_CLASS_NAME.equals(implClassName)) { throw restErrorUtil.createRESTException("System Admin cannot create/update/delete KMS " + objType, MessageEnums.OPER_NO_PERMISSION); } diff --git a/security-admin/src/main/java/org/apache/ranger/biz/RangerPolicyRetriever.java b/security-admin/src/main/java/org/apache/ranger/biz/RangerPolicyRetriever.java index 1b6f4407f3..b0734e4f30 100644 --- a/security-admin/src/main/java/org/apache/ranger/biz/RangerPolicyRetriever.java +++ b/security-admin/src/main/java/org/apache/ranger/biz/RangerPolicyRetriever.java @@ -26,35 +26,53 @@ import java.util.Map; import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.collections.MapUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.ranger.authorization.utils.JsonUtils; import org.apache.ranger.authorization.utils.StringUtil; import org.apache.ranger.db.RangerDaoManager; -import org.apache.ranger.entity.*; +import org.apache.ranger.entity.XXPolicy; +import org.apache.ranger.entity.XXPortalUser; +import org.apache.ranger.entity.XXService; import org.apache.ranger.plugin.model.RangerPolicy; import org.apache.ranger.plugin.model.RangerPolicy.RangerDataMaskPolicyItem; import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyItem; import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyItemAccess; import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyItemCondition; -import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyItemDataMaskInfo; -import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyItemRowFilterInfo; import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyResource; -import org.apache.ranger.plugin.model.RangerPolicy.RangerRowFilterPolicyItem; -import org.apache.ranger.plugin.policyevaluator.RangerPolicyItemEvaluator; import org.apache.ranger.plugin.util.RangerPerfTracer; +import org.springframework.transaction.PlatformTransactionManager; +import org.springframework.transaction.TransactionStatus; +import org.springframework.transaction.support.TransactionCallback; +import org.springframework.transaction.support.TransactionTemplate; public class RangerPolicyRetriever { static final Log LOG = LogFactory.getLog(RangerPolicyRetriever.class); static final Log PERF_LOG = RangerPerfTracer.getPerfLogger("db.RangerPolicyRetriever"); - final RangerDaoManager daoMgr; - final LookupCache lookupCache; + private final RangerDaoManager daoMgr; + private final LookupCache lookupCache = new LookupCache(); + + private final PlatformTransactionManager txManager; + private final TransactionTemplate txTemplate; + + public RangerPolicyRetriever(RangerDaoManager daoMgr, PlatformTransactionManager txManager) { + this.daoMgr = daoMgr; + this.txManager = txManager; + if (this.txManager != null) { + this.txTemplate = new TransactionTemplate(this.txManager); + this.txTemplate.setReadOnly(true); + } else { + this.txTemplate = null; + } + } public RangerPolicyRetriever(RangerDaoManager daoMgr) { this.daoMgr = daoMgr; - this.lookupCache = new LookupCache(); + this.txManager = null; + this.txTemplate = null; } public List getServicePolicies(Long serviceId) { @@ -93,7 +111,42 @@ public List getServicePolicies(String serviceName) { return ret; } - public List getServicePolicies(XXService xService) { + private class PolicyLoaderThread extends Thread { + final TransactionTemplate txTemplate; + final XXService xService; + List policies; + + PolicyLoaderThread(TransactionTemplate txTemplate, final XXService xService) { + this.txTemplate = txTemplate; + this.xService = xService; + } + + public List getPolicies() { return policies; } + + @Override + public void run() { + try { + txTemplate.setReadOnly(true); + policies = txTemplate.execute(new TransactionCallback>() { + @Override + public List doInTransaction(TransactionStatus status) { + try { + RetrieverContext ctx = new RetrieverContext(xService); + return ctx.getAllPolicies(); + } catch (Exception ex) { + LOG.error("RangerPolicyRetriever.getServicePolicies(): Failed to get policies for service:[" + xService.getName() + "] in a new transaction", ex); + status.setRollbackOnly(); + return null; + } + } + }); + } catch (Throwable ex) { + LOG.error("RangerPolicyRetriever.getServicePolicies(): Failed to get policies for service:[" + xService.getName() + "] in a new transaction", ex); + } + } + } + + public List getServicePolicies(final XXService xService) { String serviceName = xService == null ? null : xService.getName(); Long serviceId = xService == null ? null : xService.getId(); @@ -109,9 +162,27 @@ public List getServicePolicies(XXService xService) { } if(xService != null) { - RetrieverContext ctx = new RetrieverContext(xService); + if (txTemplate == null) { + if (LOG.isDebugEnabled()) { + LOG.debug("Transaction Manager is null; Retrieving policies in the existing transaction"); + } + RetrieverContext ctx = new RetrieverContext(xService); + ret = ctx.getAllPolicies(); + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("Retrieving policies in a new, read-only transaction"); + } - ret = ctx.getAllPolicies(); + PolicyLoaderThread t = new PolicyLoaderThread(txTemplate, xService); + t.setDaemon(true); + t.start(); + try { + t.join(); + ret = t.getPolicies(); + } catch (InterruptedException ie) { + LOG.error("Failed to retrieve policies in a new, read-only thread.", ie); + } + } } else { if(LOG.isDebugEnabled()) { LOG.debug("RangerPolicyRetriever.getServicePolicies(xService=" + xService + "): invalid parameter"); @@ -228,33 +299,13 @@ private XXPolicy getXXPolicy(Long policyId) { } class LookupCache { - final Map userNames = new HashMap(); - final Map userScreenNames = new HashMap(); - final Map groupNames = new HashMap(); - final Map accessTypes = new HashMap(); - final Map conditions = new HashMap(); - final Map resourceDefs = new HashMap(); - final Map dataMasks = new HashMap(); - - String getUserName(Long userId) { - String ret = null; - - if(userId != null) { - ret = userNames.get(userId); - - if(ret == null) { - XXUser user = daoMgr.getXXUser().getById(userId); - - if(user != null) { - ret = user.getName(); // Name is `loginId` - - userNames.put(userId, ret); - } - } - } - - return ret; - } + final Map userScreenNames = new HashMap(); + final Map> groupMappingsPerPolicy = new HashMap<>(); + final Map> userMappingsPerPolicy = new HashMap<>(); + final Map> accessMappingsPerPolicy = new HashMap<>(); + final Map> resourceMappingsPerPolicy = new HashMap<>(); + final Map> dataMaskMappingsPerPolicy = new HashMap<>(); + final Map> conditionMappingsPerPolicy = new HashMap<>(); String getUserScreenName(Long userId) { String ret = null; @@ -290,257 +341,135 @@ String getUserScreenName(Long userId) { return ret; } - String getGroupName(Long groupId) { - String ret = null; - - if(groupId != null) { - ret = groupNames.get(groupId); + void setNameMapping(Map> nameMappingContainer, List nameMappings) { + nameMappingContainer.clear(); - if(ret == null) { - XXGroup group = daoMgr.getXXGroup().getById(groupId); + for (PolicyTextNameMap nameMapping : nameMappings) { + Map policyNameMap = nameMappingContainer.get(nameMapping.policyId); - if(group != null) { - ret = group.getName(); + if (policyNameMap == null) { + policyNameMap = new HashMap<>(); - groupNames.put(groupId, ret); - } + nameMappingContainer.put(nameMapping.policyId, policyNameMap); } - } - return ret; - } - - String getAccessType(Long accessTypeId) { - String ret = null; - - if(accessTypeId != null) { - ret = accessTypes.get(accessTypeId); - - if(ret == null) { - XXAccessTypeDef xAccessType = daoMgr.getXXAccessTypeDef().getById(accessTypeId); - - if(xAccessType != null) { - ret = xAccessType.getName(); - - accessTypes.put(accessTypeId, ret); - } - } + policyNameMap.put(nameMapping.oldName, nameMapping.currentName); } - - return ret; } - String getConditionType(Long conditionDefId) { - String ret = null; + String getMappedName(Map> nameMappingContainer, Long policyId, String nameToMap) { + Map policyNameMap = nameMappingContainer.get(policyId); - if(conditionDefId != null) { - ret = conditions.get(conditionDefId); - - if(ret == null) { - XXPolicyConditionDef xPolicyConditionDef = daoMgr.getXXPolicyConditionDef().getById(conditionDefId); - - if(xPolicyConditionDef != null) { - ret = xPolicyConditionDef.getName(); - - conditions.put(conditionDefId, ret); - } - } - } - - return ret; + return policyNameMap != null ? policyNameMap.get(nameToMap) : null; } - String getResourceName(Long resourceDefId) { - String ret = null; - - if(resourceDefId != null) { - ret = resourceDefs.get(resourceDefId); - - if(ret == null) { - XXResourceDef xResourceDef = daoMgr.getXXResourceDef().getById(resourceDefId); - - if(xResourceDef != null) { - ret = xResourceDef.getName(); + void setGroupNameMapping(List groupNameMapping) { + setNameMapping(groupMappingsPerPolicy, groupNameMapping); + } - resourceDefs.put(resourceDefId, ret); - } - } - } + void setUserNameMapping(List userNameMapping) { + setNameMapping(userMappingsPerPolicy, userNameMapping); + } - return ret; + void setAccessNameMapping(List accessNameMapping) { + setNameMapping(accessMappingsPerPolicy, accessNameMapping); } - String getDataMaskName(Long dataMaskDefId) { - String ret = null; + public void setResourceNameMapping(List resourceNameMapping) { + setNameMapping(resourceMappingsPerPolicy, resourceNameMapping); + } - if(dataMaskDefId != null) { - ret = dataMasks.get(dataMaskDefId); + public void setDataMaskNameMapping(List dataMaskMapping) { + setNameMapping(dataMaskMappingsPerPolicy, dataMaskMapping); + } - if(ret == null) { - XXDataMaskTypeDef xDataMaskDef = daoMgr.getXXDataMaskTypeDef().getById(dataMaskDefId); + public void setConditionNameMapping(List conditionNameMapping) { + setNameMapping(conditionMappingsPerPolicy, conditionNameMapping); + } - if(xDataMaskDef != null) { - ret = xDataMaskDef.getName(); + } - dataMasks.put(dataMaskDefId, ret); - } - } - } + public static class PolicyTextNameMap { + final Long policyId; + final String oldName; + final String currentName; - return ret; + public PolicyTextNameMap(Long policyId, String oldName, String currentName) { + this.policyId = policyId; + this.oldName = oldName; + this.currentName = currentName; } } - static List asList(XXPolicy policy) { - List ret = new ArrayList(); + static List asList(XXPolicy policy) { + List ret = new ArrayList<>(); - if(policy != null) { - ret.add(policy); - } + if (policy != null) { + ret.add(policy); + } - return ret; - } + return ret; + } class RetrieverContext { - final XXService service; - final ListIterator iterPolicy; - final ListIterator iterResources; - final ListIterator iterResourceMaps; - final ListIterator iterPolicyItems; - final ListIterator iterUserPerms; - final ListIterator iterGroupPerms; - final ListIterator iterAccesses; - final ListIterator iterConditions; - final ListIterator iterDataMaskInfos; - final ListIterator iterRowFilterInfos; + final XXService service; + final ListIterator iterPolicy; RetrieverContext(XXService xService) { - Long serviceId = xService == null ? null : xService.getId(); - - List xPolicies = daoMgr.getXXPolicy().findByServiceId(serviceId); - List xResources = daoMgr.getXXPolicyResource().findByServiceId(serviceId); - List xResourceMaps = daoMgr.getXXPolicyResourceMap().findByServiceId(serviceId); - List xPolicyItems = daoMgr.getXXPolicyItem().findByServiceId(serviceId); - List xUserPerms = daoMgr.getXXPolicyItemUserPerm().findByServiceId(serviceId); - List xGroupPerms = daoMgr.getXXPolicyItemGroupPerm().findByServiceId(serviceId); - List xAccesses = daoMgr.getXXPolicyItemAccess().findByServiceId(serviceId); - List xConditions = daoMgr.getXXPolicyItemCondition().findByServiceId(serviceId); - List xDataMaskInfos = daoMgr.getXXPolicyItemDataMaskInfo().findByServiceId(serviceId); - List xRowFilterInfos = daoMgr.getXXPolicyItemRowFilterInfo().findByServiceId(serviceId); - - this.service = xService; - this.iterPolicy = xPolicies.listIterator(); - this.iterResources = xResources.listIterator(); - this.iterResourceMaps = xResourceMaps.listIterator(); - this.iterPolicyItems = xPolicyItems.listIterator(); - this.iterUserPerms = xUserPerms.listIterator(); - this.iterGroupPerms = xGroupPerms.listIterator(); - this.iterAccesses = xAccesses.listIterator(); - this.iterConditions = xConditions.listIterator(); - this.iterDataMaskInfos = xDataMaskInfos.listIterator(); - this.iterRowFilterInfos = xRowFilterInfos.listIterator(); - } - - RetrieverContext(XXPolicy xPolicy) { - this(xPolicy, getXXService(xPolicy.getService())); + if (xService != null) { + Long serviceId = xService.getId(); + + lookupCache.setGroupNameMapping(daoMgr.getXXPolicyRefGroup().findUpdatedGroupNamesByService(serviceId)); + lookupCache.setUserNameMapping(daoMgr.getXXPolicyRefUser().findUpdatedUserNamesByService(serviceId)); + lookupCache.setAccessNameMapping(daoMgr.getXXPolicyRefAccessType().findUpdatedAccessNamesByService(serviceId)); + lookupCache.setResourceNameMapping(daoMgr.getXXPolicyRefResource().findUpdatedResourceNamesByService(serviceId)); + lookupCache.setDataMaskNameMapping(daoMgr.getXXPolicyRefDataMaskType().findUpdatedDataMaskNamesByService(serviceId)); + lookupCache.setConditionNameMapping(daoMgr.getXXPolicyRefCondition().findUpdatedConditionNamesByService(serviceId)); + + this.service = xService; + this.iterPolicy = daoMgr.getXXPolicy().findByServiceId(serviceId).listIterator(); + } else { + this.service = null; + this.iterPolicy = null; + } } RetrieverContext(XXPolicy xPolicy, XXService xService) { - Long policyId = xPolicy == null ? null : xPolicy.getId(); - - List xPolicies = asList(xPolicy); - List xResources = daoMgr.getXXPolicyResource().findByPolicyId(policyId); - List xResourceMaps = daoMgr.getXXPolicyResourceMap().findByPolicyId(policyId); - List xPolicyItems = daoMgr.getXXPolicyItem().findByPolicyId(policyId); - List xUserPerms = daoMgr.getXXPolicyItemUserPerm().findByPolicyId(policyId); - List xGroupPerms = daoMgr.getXXPolicyItemGroupPerm().findByPolicyId(policyId); - List xAccesses = daoMgr.getXXPolicyItemAccess().findByPolicyId(policyId); - List xConditions = daoMgr.getXXPolicyItemCondition().findByPolicyId(policyId); - List xDataMaskInfos = daoMgr.getXXPolicyItemDataMaskInfo().findByPolicyId(policyId); - List xRowFilterInfos = daoMgr.getXXPolicyItemRowFilterInfo().findByPolicyId(policyId); - - this.service = xService; - this.iterPolicy = xPolicies.listIterator(); - this.iterResources = xResources.listIterator(); - this.iterResourceMaps = xResourceMaps.listIterator(); - this.iterPolicyItems = xPolicyItems.listIterator(); - this.iterUserPerms = xUserPerms.listIterator(); - this.iterGroupPerms = xGroupPerms.listIterator(); - this.iterAccesses = xAccesses.listIterator(); - this.iterConditions = xConditions.listIterator(); - this.iterDataMaskInfos = xDataMaskInfos.listIterator(); - this.iterRowFilterInfos = xRowFilterInfos.listIterator(); + Long policyId = xPolicy.getId(); + + lookupCache.setGroupNameMapping(daoMgr.getXXPolicyRefGroup().findUpdatedGroupNamesByPolicy(policyId)); + lookupCache.setUserNameMapping(daoMgr.getXXPolicyRefUser().findUpdatedUserNamesByPolicy(policyId)); + lookupCache.setAccessNameMapping(daoMgr.getXXPolicyRefAccessType().findUpdatedAccessNamesByPolicy(policyId)); + lookupCache.setResourceNameMapping(daoMgr.getXXPolicyRefResource().findUpdatedResourceNamesByPolicy(policyId)); + lookupCache.setDataMaskNameMapping(daoMgr.getXXPolicyRefDataMaskType().findUpdatedDataMaskNamesByPolicy(policyId)); + lookupCache.setConditionNameMapping(daoMgr.getXXPolicyRefCondition().findUpdatedConditionNamesByPolicy(policyId)); + + this.service = xService; + this.iterPolicy = asList(xPolicy).listIterator(); } RangerPolicy getNextPolicy() { RangerPolicy ret = null; - if(iterPolicy.hasNext()) { + if (service != null && iterPolicy != null && iterPolicy.hasNext()) { XXPolicy xPolicy = iterPolicy.next(); - if(xPolicy != null) { - ret = new RangerPolicy(); - - ret.setId(xPolicy.getId()); - ret.setGuid(xPolicy.getGuid()); - ret.setIsEnabled(xPolicy.getIsEnabled()); - ret.setCreatedBy(lookupCache.getUserScreenName(xPolicy.getAddedByUserId())); - ret.setUpdatedBy(lookupCache.getUserScreenName(xPolicy.getUpdatedByUserId())); - ret.setCreateTime(xPolicy.getCreateTime()); - ret.setUpdateTime(xPolicy.getUpdateTime()); - ret.setVersion(xPolicy.getVersion()); - ret.setService(service == null ? null : service.getName()); - ret.setName(StringUtils.trim(xPolicy.getName())); - ret.setPolicyType(xPolicy.getPolicyType() == null ? RangerPolicy.POLICY_TYPE_ACCESS : xPolicy.getPolicyType()); - ret.setDescription(xPolicy.getDescription()); - ret.setResourceSignature(xPolicy.getResourceSignature()); - ret.setIsAuditEnabled(xPolicy.getIsAuditEnabled()); - - getResource(ret); - getPolicyItems(ret); - } - } - - return ret; - } - - List getAllPolicies() { - List ret = new ArrayList(); - - while(iterPolicy.hasNext()) { - RangerPolicy policy = getNextPolicy(); - - if(policy != null) { - ret.add(policy); - } - } - - if(! hasProcessedAll()) { - LOG.warn("getAllPolicies(): perhaps one or more policies got updated during retrieval. Falling back to secondary method"); - - ret = getAllPoliciesBySecondary(); - } - - return ret; - } - - List getAllPoliciesBySecondary() { - List ret = null; - - if(service != null) { - List xPolicies = daoMgr.getXXPolicy().findByServiceId(service.getId()); - - if(CollectionUtils.isNotEmpty(xPolicies)) { - ret = new ArrayList(xPolicies.size()); - - for(XXPolicy xPolicy : xPolicies) { - RetrieverContext ctx = new RetrieverContext(xPolicy, service); - - RangerPolicy policy = ctx.getNextPolicy(); - - if(policy != null) { - ret.add(policy); - } + if (xPolicy != null) { + String policyText = xPolicy.getPolicyText(); + + ret = JsonUtils.jsonToObject(policyText, RangerPolicy.class); + + if (ret != null) { + ret.setId(xPolicy.getId()); + ret.setGuid(xPolicy.getGuid()); + ret.setCreatedBy(lookupCache.getUserScreenName(xPolicy.getAddedByUserId())); + ret.setUpdatedBy(lookupCache.getUserScreenName(xPolicy.getUpdatedByUserId())); + ret.setCreateTime(xPolicy.getCreateTime()); + ret.setUpdateTime(xPolicy.getUpdateTime()); + ret.setVersion(xPolicy.getVersion()); + ret.setPolicyType(xPolicy.getPolicyType() == null ? RangerPolicy.POLICY_TYPE_ACCESS : xPolicy.getPolicyType()); + ret.setService(service.getName()); + updatePolicyReferenceFields(ret); } } } @@ -548,206 +477,121 @@ List getAllPoliciesBySecondary() { return ret; } - private boolean hasProcessedAll() { - boolean moreToProcess = iterPolicy.hasNext() - || iterResources.hasNext() - || iterResourceMaps.hasNext() - || iterPolicyItems.hasNext() - || iterUserPerms.hasNext() - || iterGroupPerms.hasNext() - || iterAccesses.hasNext() - || iterConditions.hasNext() - || iterDataMaskInfos.hasNext() - || iterRowFilterInfos.hasNext(); - - return !moreToProcess; - } + void updatePolicyReferenceFields(final RangerPolicy policy) { + final Long policyId = policy.getId(); - private void getResource(RangerPolicy policy) { - while(iterResources.hasNext()) { - XXPolicyResource xResource = iterResources.next(); + Map policyResourceNameMap = lookupCache.resourceMappingsPerPolicy.get(policyId); - if(xResource.getPolicyid().equals(policy.getId())) { - RangerPolicyResource resource = new RangerPolicyResource(); + if (MapUtils.isNotEmpty(policyResourceNameMap) && CollectionUtils.containsAny(policyResourceNameMap.keySet(), policy.getResources().keySet())) { + Map updatedResources = new HashMap<>(); - resource.setIsExcludes(xResource.getIsexcludes()); - resource.setIsRecursive(xResource.getIsrecursive()); + for (Map.Entry entry : policy.getResources().entrySet()) { + String resourceName = entry.getKey(); + RangerPolicyResource policyResource = entry.getValue(); + String updatedName = policyResourceNameMap.get(resourceName); - while(iterResourceMaps.hasNext()) { - XXPolicyResourceMap xResourceMap = iterResourceMaps.next(); - - if(xResourceMap.getResourceid().equals(xResource.getId())) { - resource.getValues().add(xResourceMap.getValue()); - } else { - if(iterResourceMaps.hasPrevious()) { - iterResourceMaps.previous(); - } - break; - } + if (updatedName == null) { + updatedName = resourceName; } - policy.getResources().put(lookupCache.getResourceName(xResource.getResdefid()), resource); - } else if(xResource.getPolicyid().compareTo(policy.getId()) > 0) { - if(iterResources.hasPrevious()) { - iterResources.previous(); - } - break; + updatedResources.put(updatedName, policyResource); } + + policy.setResources(updatedResources); } - } - private void getPolicyItems(RangerPolicy policy) { - while(iterPolicyItems.hasNext()) { - XXPolicyItem xPolicyItem = iterPolicyItems.next(); + for (List policyItems : PolicyRefUpdater.getAllPolicyItems(policy)) { + if (CollectionUtils.isEmpty(policyItems)) { + continue; + } - if(xPolicyItem.getPolicyid().equals(policy.getId())) { - final RangerPolicyItem policyItem; - final RangerDataMaskPolicyItem dataMaskPolicyItem; - final RangerRowFilterPolicyItem rowFilterPolicyItem; + for (RangerPolicyItem policyItem : policyItems) { + if (lookupCache.groupMappingsPerPolicy.containsKey(policyId)) { + List updatedGroups = getUpdatedNames(lookupCache.groupMappingsPerPolicy, policyId, policyItem.getGroups()); - if(xPolicyItem.getItemType() == RangerPolicyItemEvaluator.POLICY_ITEM_TYPE_DATAMASK) { - dataMaskPolicyItem = new RangerDataMaskPolicyItem(); - rowFilterPolicyItem = null; - policyItem = dataMaskPolicyItem; - } else if(xPolicyItem.getItemType() == RangerPolicyItemEvaluator.POLICY_ITEM_TYPE_ROWFILTER) { - dataMaskPolicyItem = null; - rowFilterPolicyItem = new RangerRowFilterPolicyItem(); - policyItem = rowFilterPolicyItem; - } else { - dataMaskPolicyItem = null; - rowFilterPolicyItem = null; - policyItem = new RangerPolicyItem(); + if (updatedGroups != null) { + policyItem.setGroups(updatedGroups); + } } + if (lookupCache.userMappingsPerPolicy.containsKey(policyId)) { + List updatedUsers = getUpdatedNames(lookupCache.userMappingsPerPolicy, policyId, policyItem.getUsers()); - while(iterAccesses.hasNext()) { - XXPolicyItemAccess xAccess = iterAccesses.next(); - - if(xAccess.getPolicyitemid().equals(xPolicyItem.getId())) { - policyItem.getAccesses().add(new RangerPolicyItemAccess(lookupCache.getAccessType(xAccess.getType()), xAccess.getIsallowed())); - } else { - if(iterAccesses.hasPrevious()) { - iterAccesses.previous(); - } - break; + if (updatedUsers != null) { + policyItem.setUsers(updatedUsers); } } - while(iterUserPerms.hasNext()) { - XXPolicyItemUserPerm xUserPerm = iterUserPerms.next(); + if (lookupCache.accessMappingsPerPolicy.containsKey(policyId)) { + for (RangerPolicyItemAccess itemAccess : policyItem.getAccesses()) { + String updatedName = lookupCache.getMappedName(lookupCache.accessMappingsPerPolicy, policyId, itemAccess.getType()); - if(xUserPerm.getPolicyitemid().equals(xPolicyItem.getId())) { - String userName = lookupCache.getUserName(xUserPerm.getUserid()); - if (userName != null) { - policyItem.getUsers().add(userName); - } - } else { - if(iterUserPerms.hasPrevious()) { - iterUserPerms.previous(); + if (updatedName != null) { + itemAccess.setType(updatedName); } - break; } } - while(iterGroupPerms.hasNext()) { - XXPolicyItemGroupPerm xGroupPerm = iterGroupPerms.next(); + if (lookupCache.conditionMappingsPerPolicy.containsKey(policyId)) { + for (RangerPolicyItemCondition condition : policyItem.getConditions()) { + String updatedName = lookupCache.getMappedName(lookupCache.conditionMappingsPerPolicy, policyId, condition.getType()); - if(xGroupPerm.getPolicyitemid().equals(xPolicyItem.getId())) { - String groupName = lookupCache.getGroupName(xGroupPerm.getGroupid()); - if (groupName != null) { - policyItem.getGroups().add(groupName); + if (updatedName != null) { + condition.setType(updatedName); } - } else { - if(iterGroupPerms.hasPrevious()) { - iterGroupPerms.previous(); - } - break; } } - RangerPolicyItemCondition condition = null; - Long prevConditionType = null; - while(iterConditions.hasNext()) { - XXPolicyItemCondition xCondition = iterConditions.next(); - - if(xCondition.getPolicyitemid().equals(xPolicyItem.getId())) { - if(! xCondition.getType().equals(prevConditionType)) { - condition = new RangerPolicyItemCondition(); - condition.setType(lookupCache.getConditionType(xCondition.getType())); - condition.getValues().add(xCondition.getValue()); + if (policyItem instanceof RangerDataMaskPolicyItem && lookupCache.dataMaskMappingsPerPolicy.containsKey(policyId)) { + RangerDataMaskPolicyItem dataMaskItem = (RangerDataMaskPolicyItem) policyItem; + String updatedName = lookupCache.getMappedName(lookupCache.dataMaskMappingsPerPolicy, policyId, dataMaskItem.getDataMaskInfo().getDataMaskType()); - policyItem.getConditions().add(condition); - - prevConditionType = xCondition.getType(); - } else { - condition.getValues().add(xCondition.getValue()); - } - } else { - if(iterConditions.hasPrevious()) { - iterConditions.previous(); - } - break; + if (updatedName != null) { + dataMaskItem.getDataMaskInfo().setDataMaskType(updatedName); } } + } + } + } - policyItem.setDelegateAdmin(xPolicyItem.getDelegateAdmin()); + List getUpdatedNames(final Map> nameMappingContainer, final Long policyId, final List namesToMap) { + List ret = null; + Map policyNameMap = nameMappingContainer.get(policyId); - if(dataMaskPolicyItem != null) { - while (iterDataMaskInfos.hasNext()) { - XXPolicyItemDataMaskInfo xDataMaskInfo = iterDataMaskInfos.next(); + if (MapUtils.isNotEmpty(policyNameMap) && CollectionUtils.containsAny(policyNameMap.keySet(), namesToMap)) { + ret = new ArrayList<>(); - if (xDataMaskInfo.getPolicyItemId().equals(xPolicyItem.getId())) { - dataMaskPolicyItem.setDataMaskInfo(new RangerPolicyItemDataMaskInfo(lookupCache.getDataMaskName(xDataMaskInfo.getType()), xDataMaskInfo.getConditionExpr(), xDataMaskInfo.getValueExpr())); - } else { - if (iterDataMaskInfos.hasPrevious()) { - iterDataMaskInfos.previous(); - } - break; - } - } + for (String nameToMap : namesToMap) { + String mappedName = policyNameMap.get(nameToMap); + + if (mappedName != null) { + ret.add(mappedName); + } else { + ret.add(nameToMap); } + } - if(rowFilterPolicyItem != null) { - while (iterRowFilterInfos.hasNext()) { - XXPolicyItemRowFilterInfo xRowFilterInfo = iterRowFilterInfos.next(); + } - if (xRowFilterInfo.getPolicyItemId().equals(xPolicyItem.getId())) { - rowFilterPolicyItem.setRowFilterInfo(new RangerPolicyItemRowFilterInfo(xRowFilterInfo.getFilterExpr())); - } else { - if (iterRowFilterInfos.hasPrevious()) { - iterRowFilterInfos.previous(); - } - break; - } - } - } + return ret; + } + List getAllPolicies() { + List ret = new ArrayList<>(); - int itemType = xPolicyItem.getItemType() == null ? RangerPolicyItemEvaluator.POLICY_ITEM_TYPE_ALLOW : xPolicyItem.getItemType(); - - if(itemType == RangerPolicyItemEvaluator.POLICY_ITEM_TYPE_ALLOW) { - policy.getPolicyItems().add(policyItem); - } else if(itemType == RangerPolicyItemEvaluator.POLICY_ITEM_TYPE_DENY) { - policy.getDenyPolicyItems().add(policyItem); - } else if(itemType == RangerPolicyItemEvaluator.POLICY_ITEM_TYPE_ALLOW_EXCEPTIONS) { - policy.getAllowExceptions().add(policyItem); - } else if(itemType == RangerPolicyItemEvaluator.POLICY_ITEM_TYPE_DENY_EXCEPTIONS) { - policy.getDenyExceptions().add(policyItem); - } else if(itemType == RangerPolicyItemEvaluator.POLICY_ITEM_TYPE_DATAMASK) { - policy.getDataMaskPolicyItems().add(dataMaskPolicyItem); - } else if(itemType == RangerPolicyItemEvaluator.POLICY_ITEM_TYPE_ROWFILTER) { - policy.getRowFilterPolicyItems().add(rowFilterPolicyItem); - } else { // unknown itemType - LOG.warn("RangerPolicyRetriever.getPolicy(policyId=" + policy.getId() + "): ignoring unknown policyItemType " + itemType); - } - } else if(xPolicyItem.getPolicyid().compareTo(policy.getId()) > 0) { - if(iterPolicyItems.hasPrevious()) { - iterPolicyItems.previous(); + if (iterPolicy != null) { + while (iterPolicy.hasNext()) { + RangerPolicy policy = getNextPolicy(); + + if (policy != null) { + ret.add(policy); } - break; } } + + return ret; } } + } diff --git a/security-admin/src/main/java/org/apache/ranger/biz/RangerTagDBRetriever.java b/security-admin/src/main/java/org/apache/ranger/biz/RangerTagDBRetriever.java index 52c12882c4..09494fc4be 100644 --- a/security-admin/src/main/java/org/apache/ranger/biz/RangerTagDBRetriever.java +++ b/security-admin/src/main/java/org/apache/ranger/biz/RangerTagDBRetriever.java @@ -19,47 +19,63 @@ package org.apache.ranger.biz; +import java.lang.reflect.Type; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.ListIterator; import java.util.Map; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.reflect.TypeToken; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.ranger.authorization.hadoop.config.RangerConfiguration; import org.apache.ranger.authorization.utils.StringUtil; import org.apache.ranger.db.RangerDaoManager; import org.apache.ranger.entity.*; import org.apache.ranger.plugin.model.*; -import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyResource; import org.apache.ranger.plugin.util.RangerPerfTracer; - +import org.apache.ranger.service.RangerServiceResourceService; +import org.springframework.transaction.PlatformTransactionManager; +import org.springframework.transaction.TransactionStatus; +import org.springframework.transaction.support.TransactionCallback; +import org.springframework.transaction.support.TransactionTemplate; public class RangerTagDBRetriever { static final Log LOG = LogFactory.getLog(RangerTagDBRetriever.class); static final Log PERF_LOG = RangerPerfTracer.getPerfLogger("db.RangerTagDBRetriever"); public static final String OPTION_RANGER_FILTER_TAGS_FOR_SERVICE_PLUGIN = "ranger.filter.tags.for.service.plugin"; + public static final Type subsumedDataType = new TypeToken>() {}.getType(); + + public static final Gson gsonBuilder = new GsonBuilder().setDateFormat("yyyyMMdd-HH:mm:ss.SSS-Z") + .create(); + private final RangerDaoManager daoMgr; - private final XXService xService; private final LookupCache lookupCache; private List serviceResources; private Map tagDefs; - private Map tags; - private List tagResourceMaps; - private boolean filterForServicePlugin; + RangerTagDBRetriever(final RangerDaoManager daoMgr, final PlatformTransactionManager txManager, final XXService xService) { - public RangerTagDBRetriever(final RangerDaoManager daoMgr, final XXService xService) { this.daoMgr = daoMgr; - this.xService = xService; + + final TransactionTemplate txTemplate; + + if (txManager != null) { + txTemplate = new TransactionTemplate(txManager); + txTemplate.setReadOnly(true); + } else { + txTemplate = null; + } this.lookupCache = new LookupCache(); - if (this.daoMgr != null && this.xService != null) { + if (this.daoMgr != null && xService != null) { RangerPerfTracer perf = null; @@ -67,81 +83,99 @@ public RangerTagDBRetriever(final RangerDaoManager daoMgr, final XXService xServ perf = RangerPerfTracer.getPerfTracer(PERF_LOG, "RangerTagDBReceiver.getTags(serviceName=" + xService.getName()); } - filterForServicePlugin = RangerConfiguration.getInstance().getBoolean(OPTION_RANGER_FILTER_TAGS_FOR_SERVICE_PLUGIN, false); - TagRetrieverServiceResourceContext serviceResourceContext = new TagRetrieverServiceResourceContext(xService); - TagRetrieverTagDefContext tagDefContext = new TagRetrieverTagDefContext(xService); - TagRetrieverTagContext tagContext = new TagRetrieverTagContext(xService); + if (txTemplate == null) { + if (LOG.isDebugEnabled()) { + LOG.debug("Load Tags in the same thread and using an existing transaction"); + } + if (!initializeTagCache(xService)) { + LOG.error("Failed to get tags for service:[" + xService.getName() + "] in the same thread and using an existing transaction"); + } + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("Load Tags in a separate thread and using a new transaction"); + } - serviceResources = serviceResourceContext.getAllServiceResources(); - tagDefs = tagDefContext.getAllTagDefs(); - tags = tagContext.getAllTags(); - tagResourceMaps = getAllTagResourceMaps(); + TagLoaderThread t = new TagLoaderThread(txTemplate, xService); + t.setDaemon(true); + t.start(); + try { + t.join(); + } catch (InterruptedException ie) { + LOG.error("Failed to get Tags in a separate thread and using a new transaction", ie); + } + } RangerPerfTracer.log(perf); } } - public List getTagResourceMaps() { - return tagResourceMaps; - } - - public List getServiceResources() { + List getServiceResources() { return serviceResources; } - public Map getTagDefs() { + Map getTagDefs() { return tagDefs; } - public Map getTags() { - return tags; - } - - private List getAllTagResourceMaps() { - - List xTagResourceMaps = filterForServicePlugin ? daoMgr.getXXTagResourceMap().findForServicePlugin(xService.getId()) : daoMgr.getXXTagResourceMap().findByServiceId(xService.getId()); + Map getTags() { - ListIterator iterTagResourceMap = xTagResourceMaps.listIterator(); + Map ret = new HashMap<>(); - List ret = new ArrayList(); - - while (iterTagResourceMap.hasNext()) { - - XXTagResourceMap xTagResourceMap = iterTagResourceMap.next(); - - if (xTagResourceMap != null) { - - RangerTagResourceMap tagResourceMap = new RangerTagResourceMap(); + if (CollectionUtils.isNotEmpty(serviceResources)) { + for (RangerServiceResource serviceResource : serviceResources) { + List tags = lookupCache.serviceResourceToTags.get(serviceResource.getId()); + if (CollectionUtils.isNotEmpty(tags)) { + for (RangerTag tag : tags) { + ret.put(tag.getId(), tag); + } + } + } + } - tagResourceMap.setId(xTagResourceMap.getId()); - tagResourceMap.setGuid(xTagResourceMap.getGuid()); - tagResourceMap.setCreatedBy(lookupCache.getUserScreenName(xTagResourceMap.getAddedByUserId())); - tagResourceMap.setUpdatedBy(lookupCache.getUserScreenName(xTagResourceMap.getUpdatedByUserId())); - tagResourceMap.setCreateTime(xTagResourceMap.getCreateTime()); - tagResourceMap.setUpdateTime(xTagResourceMap.getUpdateTime()); - tagResourceMap.setResourceId(xTagResourceMap.getResourceId()); - tagResourceMap.setTagId(xTagResourceMap.getTagId()); + return ret; + } - ret.add(tagResourceMap); + Map> getResourceToTagIds() { + Map> ret = new HashMap<>(); + + if (CollectionUtils.isNotEmpty(serviceResources)) { + for (RangerServiceResource serviceResource : serviceResources) { + List tags = lookupCache.serviceResourceToTags.get(serviceResource.getId()); + if (CollectionUtils.isNotEmpty(tags)) { + List tagIds = new ArrayList<>(); + ret.put(serviceResource.getId(), tagIds); + for (RangerTag tag : tags) { + tagIds.add(tag.getId()); + } + } } } return ret; } - static List asList(T obj) { - List ret = new ArrayList(); - - if (obj != null) { - ret.add(obj); + private boolean initializeTagCache(XXService xService) { + boolean ret; + try { + TagRetrieverServiceResourceContext serviceResourceContext = new TagRetrieverServiceResourceContext(xService); + TagRetrieverTagDefContext tagDefContext = new TagRetrieverTagDefContext(xService); + + serviceResources = serviceResourceContext.getAllServiceResources(); + tagDefs = tagDefContext.getAllTagDefs(); + + ret = true; + } catch (Exception ex) { + LOG.error("Failed to get tags for service:[" + xService.getName() + "]", ex); + serviceResources = null; + tagDefs = null; + ret = false; } - return ret; } private class LookupCache { - final Map userScreenNames = new HashMap(); - final Map resourceDefs = new HashMap(); + final Map userScreenNames = new HashMap<>(); + final Map> serviceResourceToTags = new HashMap<>(); String getUserScreenName(Long userId) { String ret = null; @@ -177,24 +211,38 @@ String getUserScreenName(Long userId) { return ret; } - String getResourceName(Long resourceDefId) { - String ret = null; - - if (resourceDefId != null) { - ret = resourceDefs.get(resourceDefId); + } - if (ret == null) { - XXResourceDef xResourceDef = daoMgr.getXXResourceDef().getById(resourceDefId); + private class TagLoaderThread extends Thread { + final TransactionTemplate txTemplate; + final XXService xService; - if (xResourceDef != null) { - ret = xResourceDef.getName(); + TagLoaderThread(TransactionTemplate txTemplate, final XXService xService) { + this.txTemplate = txTemplate; + this.xService = xService; + } - resourceDefs.put(resourceDefId, ret); + @Override + public void run() { + try { + txTemplate.setReadOnly(true); + Boolean result = txTemplate.execute(new TransactionCallback() { + @Override + public Boolean doInTransaction(TransactionStatus status) { + boolean ret = initializeTagCache(xService); + if (!ret) { + status.setRollbackOnly(); + LOG.error("Failed to get tags for service:[" + xService.getName() + "] in a new transaction"); + } + return ret; } - } + }); + if (LOG.isDebugEnabled()) { + LOG.debug("transaction result:[" + result +"]"); + } + } catch (Throwable ex) { + LOG.error("Failed to get tags for service:[" + xService.getName() + "] in a new transaction", ex); } - - return ret; } } @@ -202,39 +250,19 @@ private class TagRetrieverServiceResourceContext { final XXService service; final ListIterator iterServiceResource; - final ListIterator iterServiceResourceElement; - final ListIterator iterServiceResourceElementValue; TagRetrieverServiceResourceContext(XXService xService) { Long serviceId = xService == null ? null : xService.getId(); - - List xServiceResources = filterForServicePlugin ? daoMgr.getXXServiceResource().findForServicePlugin(serviceId) : daoMgr.getXXServiceResource().findTaggedResourcesInServiceId(serviceId); - List xServiceResourceElements = filterForServicePlugin ? daoMgr.getXXServiceResourceElement().findForServicePlugin(serviceId) : daoMgr.getXXServiceResourceElement().findTaggedResourcesInServiceId(serviceId); - List xServiceResourceElementValues = filterForServicePlugin ? daoMgr.getXXServiceResourceElementValue().findForServicePlugin(serviceId) : daoMgr.getXXServiceResourceElementValue().findTaggedResourcesInServiceId(serviceId); - this.service = xService; - this.iterServiceResource = xServiceResources.listIterator(); - this.iterServiceResourceElement = xServiceResourceElements.listIterator(); - this.iterServiceResourceElementValue = xServiceResourceElementValues.listIterator(); - } - - TagRetrieverServiceResourceContext(XXServiceResource xServiceResource, XXService xService) { - Long resourceId = xServiceResource == null ? null : xServiceResource.getId(); - - List xServiceResources = asList(xServiceResource); - List xServiceResourceElements = daoMgr.getXXServiceResourceElement().findByResourceId(resourceId); - List xServiceResourceElementValues = daoMgr.getXXServiceResourceElementValue().findByResourceId(resourceId); + List xServiceResources = daoMgr.getXXServiceResource().findTaggedResourcesInServiceId(serviceId); - this.service = xService; this.iterServiceResource = xServiceResources.listIterator(); - this.iterServiceResourceElement = xServiceResourceElements.listIterator(); - this.iterServiceResourceElementValue = xServiceResourceElementValues.listIterator(); } List getAllServiceResources() { - List ret = new ArrayList(); + List ret = new ArrayList<>(); while (iterServiceResource.hasNext()) { RangerServiceResource serviceResource = getNextServiceResource(); @@ -244,12 +272,6 @@ List getAllServiceResources() { } } - if (!hasProcessedAll()) { - LOG.warn("getAllServiceResources(): perhaps one or more serviceResources got updated during retrieval. Using fallback ... "); - - ret = getServiceResourcesBySecondary(); - } - return ret; } @@ -259,7 +281,7 @@ RangerServiceResource getNextServiceResource() { if (iterServiceResource.hasNext()) { XXServiceResource xServiceResource = iterServiceResource.next(); - if (xServiceResource != null) { + if (xServiceResource != null && StringUtils.isNotEmpty(xServiceResource.getTags())) { ret = new RangerServiceResource(); ret.setId(xServiceResource.getId()); @@ -272,108 +294,35 @@ RangerServiceResource getNextServiceResource() { ret.setVersion(xServiceResource.getVersion()); ret.setResourceSignature(xServiceResource.getResourceSignature()); - getServiceResourceElements(ret); - } - } + Map serviceResourceElements = gsonBuilder.fromJson(xServiceResource.getServiceResourceElements(), RangerServiceResourceService.subsumedDataType); + ret.setResourceElements(serviceResourceElements); - return ret; - } - - void getServiceResourceElements(RangerServiceResource serviceResource) { - while (iterServiceResourceElement.hasNext()) { - XXServiceResourceElement xServiceResourceElement = iterServiceResourceElement.next(); - - if (xServiceResourceElement.getResourceId().equals(serviceResource.getId())) { - RangerPolicyResource resource = new RangerPolicyResource(); - - resource.setIsExcludes(xServiceResourceElement.getIsExcludes()); - resource.setIsRecursive(xServiceResourceElement.getIsRecursive()); - - while (iterServiceResourceElementValue.hasNext()) { - XXServiceResourceElementValue xServiceResourceElementValue = iterServiceResourceElementValue.next(); - - if (xServiceResourceElementValue.getResElementId().equals(xServiceResourceElement.getId())) { - resource.getValues().add(xServiceResourceElementValue.getValue()); - } else { - if (iterServiceResourceElementValue.hasPrevious()) { - iterServiceResourceElementValue.previous(); - } - break; - } - } - - serviceResource.getResourceElements().put(lookupCache.getResourceName(xServiceResourceElement.getResDefId()), resource); - } else if (xServiceResourceElement.getResourceId().compareTo(serviceResource.getId()) > 0) { - if (iterServiceResourceElement.hasPrevious()) { - iterServiceResourceElement.previous(); - } - break; + List tags = gsonBuilder.fromJson(xServiceResource.getTags(), RangerServiceResourceService.duplicatedDataType); + lookupCache.serviceResourceToTags.put(xServiceResource.getId(), tags); } } - } - - boolean hasProcessedAll() { - boolean moreToProcess = iterServiceResource.hasNext() - || iterServiceResourceElement.hasNext() - || iterServiceResourceElementValue.hasNext(); - return !moreToProcess; - } - - List getServiceResourcesBySecondary() { - List ret = null; - - if (service != null) { - List xServiceResources = filterForServicePlugin ? daoMgr.getXXServiceResource().findForServicePlugin(service.getId()) : daoMgr.getXXServiceResource().findTaggedResourcesInServiceId(service.getId()); - - if (CollectionUtils.isNotEmpty(xServiceResources)) { - ret = new ArrayList(xServiceResources.size()); - for (XXServiceResource xServiceResource : xServiceResources) { - TagRetrieverServiceResourceContext ctx = new TagRetrieverServiceResourceContext(xServiceResource, service); - - RangerServiceResource serviceResource = ctx.getNextServiceResource(); - - if (serviceResource != null) { - ret.add(serviceResource); - } - } - } - } return ret; } + } private class TagRetrieverTagDefContext { final XXService service; final ListIterator iterTagDef; - final ListIterator iterTagAttributeDef; - TagRetrieverTagDefContext(XXService xService) { Long serviceId = xService == null ? null : xService.getId(); - List xTagDefs = filterForServicePlugin ? daoMgr.getXXTagDef().findForServicePlugin(serviceId) : daoMgr.getXXTagDef().findByServiceId(serviceId); - List xTagAttributeDefs = filterForServicePlugin ? daoMgr.getXXTagAttributeDef().findForServicePlugin(serviceId) : daoMgr.getXXTagAttributeDef().findByServiceId(serviceId); - - this.service = xService; - this.iterTagDef = xTagDefs.listIterator(); - this.iterTagAttributeDef = xTagAttributeDefs.listIterator(); - } - - TagRetrieverTagDefContext(XXTagDef xTagDef, XXService xService) { - Long tagDefId = xTagDef == null ? null : xTagDef.getId(); - - List xTagDefs = asList(xTagDef); - List xTagAttributeDefs = daoMgr.getXXTagAttributeDef().findByTagDefId(tagDefId); + List xTagDefs = daoMgr.getXXTagDef().findByServiceId(serviceId); this.service = xService; this.iterTagDef = xTagDefs.listIterator(); - this.iterTagAttributeDef = xTagAttributeDefs.listIterator(); } Map getAllTagDefs() { - Map ret = new HashMap(); + Map ret = new HashMap<>(); while (iterTagDef.hasNext()) { RangerTagDef tagDef = getNextTagDef(); @@ -383,13 +332,6 @@ Map getAllTagDefs() { } } - if (!hasProcessedAllTagDefs()) { - LOG.warn("getAllTagDefs(): perhaps one or more tag-definitions got updated during retrieval. Using fallback ... "); - - ret = getTagDefsBySecondary(); - - } - return ret; } @@ -412,192 +354,15 @@ RangerTagDef getNextTagDef() { ret.setVersion(xTagDef.getVersion()); ret.setName(xTagDef.getName()); ret.setSource(xTagDef.getSource()); - - getTagAttributeDefs(ret); + List attributeDefs = gsonBuilder.fromJson(xTagDef.getTagAttrDefs(), RangerTagDBRetriever.subsumedDataType); + ret.setAttributeDefs(attributeDefs); } } return ret; } - void getTagAttributeDefs(RangerTagDef tagDef) { - while (iterTagAttributeDef.hasNext()) { - XXTagAttributeDef xTagAttributeDef = iterTagAttributeDef.next(); - - if (xTagAttributeDef.getTagDefId().equals(tagDef.getId())) { - RangerTagDef.RangerTagAttributeDef tagAttributeDef = new RangerTagDef.RangerTagAttributeDef(); - - tagAttributeDef.setName(xTagAttributeDef.getName()); - tagAttributeDef.setType(xTagAttributeDef.getType()); - - tagDef.getAttributeDefs().add(tagAttributeDef); - } else if (xTagAttributeDef.getTagDefId().compareTo(tagDef.getId()) > 0) { - if (iterTagAttributeDef.hasPrevious()) { - iterTagAttributeDef.previous(); - } - break; - } - } - } - - boolean hasProcessedAllTagDefs() { - boolean moreToProcess = iterTagAttributeDef.hasNext(); - return !moreToProcess; - } - - Map getTagDefsBySecondary() { - Map ret = null; - - if (service != null) { - List xTagDefs = daoMgr.getXXTagDef().findByServiceId(service.getId()); - - if (CollectionUtils.isNotEmpty(xTagDefs)) { - ret = new HashMap(xTagDefs.size()); - - for (XXTagDef xTagDef : xTagDefs) { - TagRetrieverTagDefContext ctx = new TagRetrieverTagDefContext(xTagDef, service); - - RangerTagDef tagDef = ctx.getNextTagDef(); - - if (tagDef != null) { - ret.put(tagDef.getId(), tagDef); - } - } - } - } - return ret; - } } - private class TagRetrieverTagContext { - - final XXService service; - final ListIterator iterTag; - final ListIterator iterTagAttribute; - - TagRetrieverTagContext(XXService xService) { - Long serviceId = xService == null ? null : xService.getId(); - - List xTags = filterForServicePlugin ? daoMgr.getXXTag().findForServicePlugin(serviceId) : daoMgr.getXXTag().findByServiceId(serviceId); - List xTagAttributes = filterForServicePlugin ? daoMgr.getXXTagAttribute().findForServicePlugin(serviceId) : daoMgr.getXXTagAttribute().findByServiceId(serviceId); - - this.service = xService; - this.iterTag = xTags.listIterator(); - this.iterTagAttribute = xTagAttributes.listIterator(); - - } - - TagRetrieverTagContext(XXTag xTag, XXService xService) { - Long tagId = xTag == null ? null : xTag.getId(); - - List xTags = asList(xTag); - List xTagAttributes = daoMgr.getXXTagAttribute().findByTagId(tagId); - - this.service = xService; - this.iterTag = xTags.listIterator(); - this.iterTagAttribute = xTagAttributes.listIterator(); - } - - - Map getAllTags() { - Map ret = new HashMap(); - - while (iterTag.hasNext()) { - RangerTag tag = getNextTag(); - - if (tag != null) { - ret.put(tag.getId(), tag); - } - } - - if (!hasProcessedAllTags()) { - LOG.warn("getAllTags(): perhaps one or more tags got updated during retrieval. Using fallback ... "); - - ret = getTagsBySecondary(); - } - - return ret; - } - - RangerTag getNextTag() { - RangerTag ret = null; - - if (iterTag.hasNext()) { - XXTag xTag = iterTag.next(); - - if (xTag != null) { - ret = new RangerTag(); - - ret.setId(xTag.getId()); - ret.setGuid(xTag.getGuid()); - ret.setOwner(xTag.getOwner()); - ret.setCreatedBy(lookupCache.getUserScreenName(xTag.getAddedByUserId())); - ret.setUpdatedBy(lookupCache.getUserScreenName(xTag.getUpdatedByUserId())); - ret.setCreateTime(xTag.getCreateTime()); - ret.setUpdateTime(xTag.getUpdateTime()); - ret.setVersion(xTag.getVersion()); - - Map tagDefs = getTagDefs(); - if (tagDefs != null) { - RangerTagDef tagDef = tagDefs.get(xTag.getType()); - if (tagDef != null) { - ret.setType(tagDef.getName()); - } - } - - getTagAttributes(ret); - } - } - - return ret; - } - - void getTagAttributes(RangerTag tag) { - while (iterTagAttribute.hasNext()) { - XXTagAttribute xTagAttribute = iterTagAttribute.next(); - - if (xTagAttribute.getTagId().equals(tag.getId())) { - String attributeName = xTagAttribute.getName(); - String attributeValue = xTagAttribute.getValue(); - - - tag.getAttributes().put(attributeName, attributeValue); - } else if (xTagAttribute.getTagId().compareTo(tag.getId()) > 0) { - if (iterTagAttribute.hasPrevious()) { - iterTagAttribute.previous(); - } - break; - } - } - } - - boolean hasProcessedAllTags() { - boolean moreToProcess = iterTagAttribute.hasNext(); - return !moreToProcess; - } - - Map getTagsBySecondary() { - Map ret = null; - - if (service != null) { - List xTags = daoMgr.getXXTag().findByServiceId(service.getId()); - - if (CollectionUtils.isNotEmpty(xTags)) { - ret = new HashMap(xTags.size()); - - for (XXTag xTag : xTags) { - TagRetrieverTagContext ctx = new TagRetrieverTagContext(xTag, service); - - RangerTag tag = ctx.getNextTag(); - - if (tag != null) { - ret.put(tag.getId(), tag); - } - } - } - } - return ret; - } - } } diff --git a/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java b/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java index 8132357d75..52d0130430 100644 --- a/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java +++ b/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java @@ -19,6 +19,12 @@ package org.apache.ranger.biz; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.UnknownHostException; +import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -29,15 +35,9 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import java.util.Set; import java.util.Map.Entry; +import java.util.Set; import java.util.StringTokenizer; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.net.UnknownHostException; -import java.text.SimpleDateFormat; import javax.annotation.PostConstruct; import javax.servlet.ServletOutputStream; @@ -49,21 +49,34 @@ import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.security.SecureClientLogin; +import org.apache.hadoop.security.authentication.util.KerberosName; +import org.apache.poi.hssf.usermodel.HSSFWorkbook; +import org.apache.poi.ss.usermodel.Cell; +import org.apache.poi.ss.usermodel.CellStyle; +import org.apache.poi.ss.usermodel.Font; +import org.apache.poi.ss.usermodel.Row; +import org.apache.poi.ss.usermodel.Sheet; +import org.apache.poi.ss.usermodel.Workbook; import org.apache.ranger.audit.provider.MiscUtil; import org.apache.ranger.authorization.hadoop.config.RangerConfiguration; import org.apache.ranger.common.AppConstants; import org.apache.ranger.common.ContextUtil; import org.apache.ranger.common.MessageEnums; -import org.apache.ranger.common.RangerCommonEnums; +import org.apache.ranger.common.db.RangerTransactionSynchronizationAdapter; +import org.apache.ranger.entity.*; +import org.apache.ranger.plugin.model.validation.RangerServiceDefValidator; +import org.apache.ranger.plugin.model.validation.RangerValidator; +import org.apache.ranger.plugin.model.validation.ValidationFailureDetails; import org.apache.ranger.plugin.policyengine.RangerPolicyEngine; import org.apache.ranger.plugin.policyresourcematcher.RangerDefaultPolicyResourceMatcher; import org.apache.ranger.plugin.policyresourcematcher.RangerPolicyResourceMatcher; -import org.apache.ranger.plugin.resourcematcher.RangerAbstractResourceMatcher; -import org.apache.ranger.plugin.service.RangerBaseService; +import org.apache.ranger.plugin.store.ServiceStore; import org.apache.ranger.plugin.util.PasswordUtils; import org.apache.ranger.common.JSONUtil; import org.apache.ranger.common.PropertiesUtil; import org.apache.ranger.common.RESTErrorUtil; +import org.apache.ranger.common.RangerConstants; import org.apache.ranger.common.RangerFactory; import org.apache.ranger.common.RangerServicePoliciesCache; import org.apache.ranger.common.RangerVersionInfo; @@ -77,15 +90,6 @@ import org.apache.ranger.db.XXEnumDefDao; import org.apache.ranger.db.XXEnumElementDefDao; import org.apache.ranger.db.XXPolicyConditionDefDao; -import org.apache.ranger.db.XXPolicyItemAccessDao; -import org.apache.ranger.db.XXPolicyItemConditionDao; -import org.apache.ranger.db.XXPolicyItemDao; -import org.apache.ranger.db.XXPolicyItemDataMaskInfoDao; -import org.apache.ranger.db.XXPolicyItemGroupPermDao; -import org.apache.ranger.db.XXPolicyItemRowFilterInfoDao; -import org.apache.ranger.db.XXPolicyItemUserPermDao; -import org.apache.ranger.db.XXPolicyResourceDao; -import org.apache.ranger.db.XXPolicyResourceMapDao; import org.apache.ranger.db.XXResourceDefDao; import org.apache.ranger.db.XXServiceConfigDefDao; import org.apache.ranger.db.XXServiceConfigMapDao; @@ -99,18 +103,11 @@ import org.apache.ranger.entity.XXDataMaskTypeDef; import org.apache.ranger.entity.XXEnumDef; import org.apache.ranger.entity.XXEnumElementDef; -import org.apache.ranger.entity.XXGroup; import org.apache.ranger.entity.XXPolicy; import org.apache.ranger.entity.XXPolicyConditionDef; -import org.apache.ranger.entity.XXPolicyItem; -import org.apache.ranger.entity.XXPolicyItemAccess; -import org.apache.ranger.entity.XXPolicyItemCondition; -import org.apache.ranger.entity.XXPolicyItemDataMaskInfo; -import org.apache.ranger.entity.XXPolicyItemGroupPerm; -import org.apache.ranger.entity.XXPolicyItemRowFilterInfo; -import org.apache.ranger.entity.XXPolicyItemUserPerm; -import org.apache.ranger.entity.XXPolicyResource; -import org.apache.ranger.entity.XXPolicyResourceMap; +import org.apache.ranger.entity.XXPolicyRefAccessType; +import org.apache.ranger.entity.XXPolicyRefCondition; +import org.apache.ranger.entity.XXPolicyRefResource; import org.apache.ranger.entity.XXResourceDef; import org.apache.ranger.entity.XXService; import org.apache.ranger.entity.XXServiceConfigDef; @@ -121,13 +118,11 @@ import org.apache.ranger.entity.XXUser; import org.apache.ranger.plugin.model.RangerPolicy; import org.apache.ranger.plugin.model.RangerPolicy.RangerDataMaskPolicyItem; -import org.apache.ranger.plugin.model.RangerPolicy.RangerRowFilterPolicyItem; import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyItem; import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyItemAccess; import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyItemCondition; -import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyItemDataMaskInfo; -import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyItemRowFilterInfo; import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyResource; +import org.apache.ranger.plugin.model.RangerPolicy.RangerRowFilterPolicyItem; import org.apache.ranger.plugin.model.RangerPolicyResourceSignature; import org.apache.ranger.plugin.model.RangerService; import org.apache.ranger.plugin.model.RangerServiceDef; @@ -142,7 +137,6 @@ import org.apache.ranger.plugin.model.RangerServiceDef.RangerRowFilterDef; import org.apache.ranger.plugin.model.RangerServiceDef.RangerServiceConfigDef; import org.apache.ranger.plugin.model.validation.RangerServiceDefHelper; -import org.apache.ranger.plugin.policyevaluator.RangerPolicyItemEvaluator; import org.apache.ranger.plugin.store.AbstractServiceStore; import org.apache.ranger.plugin.store.EmbeddedServiceDefsUtil; import org.apache.ranger.plugin.store.PList; @@ -159,15 +153,14 @@ import org.apache.ranger.service.RangerServiceDefWithAssignedIdService; import org.apache.ranger.service.RangerServiceService; import org.apache.ranger.service.RangerServiceWithAssignedIdService; -import org.apache.ranger.service.XGroupService; import org.apache.ranger.service.XUserService; import org.apache.ranger.view.RangerExportPolicyList; import org.apache.ranger.view.RangerPolicyList; import org.apache.ranger.view.RangerServiceDefList; import org.apache.ranger.view.RangerServiceList; -import org.apache.ranger.view.VXGroup; import org.apache.ranger.view.VXString; import org.apache.ranger.view.VXUser; +import org.codehaus.jettison.json.JSONException; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Component; @@ -175,14 +168,6 @@ import org.springframework.transaction.TransactionStatus; import org.springframework.transaction.support.TransactionCallback; import org.springframework.transaction.support.TransactionTemplate; -import org.apache.poi.hssf.usermodel.HSSFWorkbook; -import org.apache.poi.ss.usermodel.Cell; -import org.apache.poi.ss.usermodel.CellStyle; -import org.apache.poi.ss.usermodel.Font; -import org.apache.poi.ss.usermodel.Row; -import org.apache.poi.ss.usermodel.Sheet; -import org.apache.poi.ss.usermodel.Workbook; -import org.codehaus.jettison.json.JSONException; import com.google.gson.Gson; @@ -190,25 +175,38 @@ public class ServiceDBStore extends AbstractServiceStore { private static final Log LOG = LogFactory.getLog(ServiceDBStore.class); + public static final String RANGER_TAG_EXPIRY_CONDITION_NAME = "accessed-after-expiry"; + + private static final String ADMIN_USER_PRINCIPAL = "ranger.admin.kerberos.principal"; + private static final String ADMIN_USER_KEYTAB = "ranger.admin.kerberos.keytab"; + private static final String LOOKUP_PRINCIPAL = "ranger.lookup.kerberos.principal"; + private static final String LOOKUP_KEYTAB = "ranger.lookup.kerberos.keytab"; + private static final String RANGER_AUTH_TYPE = "hadoop.security.authentication"; + private static final String AMBARI_SERVICE_CHECK_USER = "ambari.service.check.user"; + + private static final String KERBEROS_TYPE = "kerberos"; + private static final String POLICY_ALLOW_EXCLUDE = "Policy Allow:Exclude"; - //private static final String POLICY_ALLOW_INCLUDE = "Policy Allow:Include"; - private static final String POLICY_DENY_EXCLUDE = "Policy Deny:Exclude"; - private static final String POLICY_DENY_INCLUDE = "Policy Deny:Include"; - - private static String LOCAL_HOSTNAME = "unknown"; - private static final String HOSTNAME = "Host name"; - private static final String USER_NAME = "Exported by"; + private static final String POLICY_ALLOW_INCLUDE = "Policy Allow:Include"; + private static final String POLICY_DENY_EXCLUDE = "Policy Deny:Exclude"; + private static final String POLICY_DENY_INCLUDE = "Policy Deny:Include"; + + private static String LOCAL_HOSTNAME = "unknown"; + private static final String HOSTNAME = "Host name"; + private static final String USER_NAME = "Exported by"; private static final String RANGER_VERSION = "Ranger apache version"; - private static final String TIMESTAMP = "Export time"; + private static final String TIMESTAMP = "Export time"; - private static final String AMBARI_SERVICE_CHECK_USER = "ambari.service.check.user"; - - public static final String CRYPT_ALGO = PropertiesUtil.getProperty("ranger.password.encryption.algorithm", PasswordUtils.DEFAULT_CRYPT_ALGO); - public static final String ENCRYPT_KEY = PropertiesUtil.getProperty("ranger.password.encryption.key", PasswordUtils.DEFAULT_ENCRYPT_KEY); - public static final String SALT = PropertiesUtil.getProperty("ranger.password.salt", PasswordUtils.DEFAULT_SALT); - public static final Integer ITERATION_COUNT = PropertiesUtil.getIntProperty("ranger.password.iteration.count", PasswordUtils.DEFAULT_ITERATION_COUNT); + private static final String AUDITTOHDFS_KMS_PATH = "/ranger/audit/kms"; + private static final String AUDITTOHDFS_POLICY_NAME = "kms-audit-path"; + private static final String SERVICE_ADMIN_USERS = "service.admin.users"; - static { + public static final String CRYPT_ALGO = PropertiesUtil.getProperty("ranger.password.encryption.algorithm", PasswordUtils.DEFAULT_CRYPT_ALGO); + public static final String ENCRYPT_KEY = PropertiesUtil.getProperty("ranger.password.encryption.key", PasswordUtils.DEFAULT_ENCRYPT_KEY); + public static final String SALT = PropertiesUtil.getProperty("ranger.password.salt", PasswordUtils.DEFAULT_SALT); + public static final Integer ITERATION_COUNT = PropertiesUtil.getIntProperty("ranger.password.iteration.count", PasswordUtils.DEFAULT_ITERATION_COUNT); + + static { try { LOCAL_HOSTNAME = java.net.InetAddress.getLocalHost().getCanonicalHostName(); } catch (UnknownHostException e) { @@ -244,7 +242,7 @@ public class ServiceDBStore extends AbstractServiceStore { XUserMgr xUserMgr; @Autowired - XGroupService xGroupService; + PolicyRefUpdater policyRefUpdater; @Autowired RangerDataHistService dataHistService; @@ -274,7 +272,14 @@ public class ServiceDBStore extends AbstractServiceStore { @Autowired ServiceMgr serviceMgr; + @Autowired + AssetMgr assetMgr; + + @Autowired + RangerTransactionSynchronizationAdapter transactionSynchronizationAdapter; + private static volatile boolean legacyServiceDefsInitDone = false; + private Boolean populateExistingBaseFields = false; public static final String HIDDEN_PASSWORD_STR = "*****"; @@ -285,7 +290,6 @@ public class ServiceDBStore extends AbstractServiceStore { private ServicePredicateUtil predicateUtil = null; - @Override public void init() throws Exception { if (LOG.isDebugEnabled()) { @@ -353,9 +357,21 @@ public RangerServiceDef createServiceDef(RangerServiceDef serviceDef) throws Exc + serviceDef.getName() + " already exists", MessageEnums.ERROR_DUPLICATE_OBJECT); } - + List configs = serviceDef.getConfigs(); List resources = serviceDef.getResources(); + + if (CollectionUtils.isNotEmpty(resources)) { + RangerServiceDefValidator validator = new RangerServiceDefValidator(this); + List failures = new ArrayList<>(); + boolean isValidResources = validator.isValidResources(serviceDef, failures, RangerValidator.Action.CREATE); + if (!isValidResources) { + throw restErrorUtil.createRESTException("service-def with name: " + + serviceDef.getName() + " has invalid resources:[" + failures.toString() + "]", + MessageEnums.INVALID_INPUT_DATA); + } + } + List accessTypes = serviceDef.getAccessTypes(); List policyConditions = serviceDef.getPolicyConditions(); List contextEnrichers = serviceDef.getContextEnrichers(); @@ -732,8 +748,8 @@ private void updateChildObjectsOfServiceDef(XXServiceDef createdSvcDef, List policyResList = daoMgr.getXXPolicyResource().findByResDefId(xRes.getId()); - if (!stringUtil.isEmpty(policyResList)) { + List xxPolicyRefResource = daoMgr.getXXPolicyRefResource().findByResourceDefID(xRes.getId()); + if (!stringUtil.isEmpty(xxPolicyRefResource)) { throw restErrorUtil.createRESTException("Policy/Policies are referring to this resource: " + xRes.getName() + ". Please remove such references from policy before updating service-def.", MessageEnums.DATA_NOT_UPDATABLE); @@ -819,8 +835,8 @@ private void updateChildObjectsOfServiceDef(XXServiceDef createdSvcDef, List polItemAccessList = daoMgr.getXXPolicyItemAccess().findByType(xAccess.getId()); - if(!stringUtil.isEmpty(polItemAccessList)) { + List policyRefAccessTypeList = daoMgr.getXXPolicyRefAccessType().findByAccessTypeDefId(xAccess.getId()); + if(!stringUtil.isEmpty(policyRefAccessTypeList)) { throw restErrorUtil.createRESTException("Policy/Policies are referring to this access-type: " + xAccess.getName() + ". Please remove such references from policy before updating service-def.", MessageEnums.DATA_NOT_UPDATABLE); @@ -862,15 +878,14 @@ private void updateChildObjectsOfServiceDef(XXServiceDef createdSvcDef, List policyItemCondList = daoMgr.getXXPolicyItemCondition() - .findByPolicyConditionDefId(xCondition.getId()); - if(!stringUtil.isEmpty(policyItemCondList)) { + List xxPolicyRefConditions = daoMgr.getXXPolicyRefCondition().findByConditionDefId(xCondition.getId()); + if(!stringUtil.isEmpty(xxPolicyRefConditions)) { throw restErrorUtil.createRESTException("Policy/Policies are referring to this policy-condition: " + xCondition.getName() + ". Please remove such references from policy before updating service-def.", MessageEnums.DATA_NOT_UPDATABLE); } - for(XXPolicyItemCondition policyItemCond : policyItemCondList) { - daoMgr.getXXPolicyItemCondition().remove(policyItemCond); + for(XXPolicyRefCondition xxPolicyRefCondition : xxPolicyRefConditions) { + daoMgr.getXXPolicyRefCondition().remove(xxPolicyRefCondition); } xxPolCondDao.remove(xCondition); } @@ -1203,9 +1218,9 @@ public void deleteServiceDef(Long serviceDefId, Boolean forceDelete) throws Exce List policyCondList = policyCondDao.findByServiceDefId(serviceDefId); for (XXPolicyConditionDef policyCond : policyCondList) { - List policyItemCondList = daoMgr.getXXPolicyItemCondition().findByPolicyConditionDefId(policyCond.getId()); - for (XXPolicyItemCondition policyItemCond : policyItemCondList) { - daoMgr.getXXPolicyItemCondition().remove(policyItemCond); + List xxPolicyRefConditions = daoMgr.getXXPolicyRefCondition().findByConditionDefId(policyCond.getId()); + for (XXPolicyRefCondition XXPolicyRefCondition : xxPolicyRefConditions) { + daoMgr.getXXPolicyRefCondition().remove(XXPolicyRefCondition); } policyCondDao.remove(policyCond); } @@ -1255,37 +1270,25 @@ public void deleteXXAccessTypeDef(XXAccessTypeDef xAccess) { daoMgr.getXXAccessTypeDefGrants().remove(atdGrant); } - List policyItemAccessList = daoMgr.getXXPolicyItemAccess().findByType(xAccess.getId()); - for (XXPolicyItemAccess policyItemAccess : policyItemAccessList) { - daoMgr.getXXPolicyItemAccess().remove(policyItemAccess); + List policyRefAccessTypeList = daoMgr.getXXPolicyRefAccessType().findByAccessTypeDefId(xAccess.getId()); + for (XXPolicyRefAccessType xxPolicyRefAccessType : policyRefAccessTypeList) { + daoMgr.getXXPolicyRefAccessType().remove(xxPolicyRefAccessType); } daoMgr.getXXAccessTypeDef().remove(xAccess); } public void deleteXXResourceDef(XXResourceDef xRes) { - List xChildObjs = daoMgr.getXXResourceDef().findByParentResId(xRes.getId()); for(XXResourceDef childRes : xChildObjs) { deleteXXResourceDef(childRes); } - - List xxResources = daoMgr.getXXPolicyResource().findByResDefId(xRes.getId()); - for (XXPolicyResource xPolRes : xxResources) { - deleteXXPolicyResource(xPolRes); + List xxPolicyRefResources = daoMgr.getXXPolicyRefResource().findByResourceDefID(xRes.getId()); + for (XXPolicyRefResource xPolRefRes : xxPolicyRefResources) { + daoMgr.getXXPolicyRefResource().remove(xPolRefRes); } - daoMgr.getXXResourceDef().remove(xRes); } - public void deleteXXPolicyResource(XXPolicyResource xPolRes) { - List polResMapList = daoMgr.getXXPolicyResourceMap().findByPolicyResId(xPolRes.getId()); - XXPolicyResourceMapDao polResMapDao = daoMgr.getXXPolicyResourceMap(); - for (XXPolicyResourceMap xxPolResMap : polResMapList) { - polResMapDao.remove(xxPolResMap); - } - daoMgr.getXXPolicyResource().remove(xPolRes); - } - @Override public RangerServiceDef getServiceDef(Long id) throws Exception { if (LOG.isDebugEnabled()) { @@ -1436,10 +1439,7 @@ public RangerService createService(RangerService service) throws Exception { xConfMap.setServiceId(xCreatedService.getId()); xConfMap.setConfigkey(configKey); xConfMap.setConfigvalue(configValue); - xConfMapDao.create(xConfMap); - } - if (LOG.isDebugEnabled()) { - LOG.debug("vXUser:[" + vXUser + "]"); + xConfMap = xConfMapDao.create(xConfMap); } RangerService createdService = svcService.getPopulatedViewObject(xCreatedService); @@ -1454,7 +1454,7 @@ public RangerService createService(RangerService service) throws Exception { bizUtil.createTrxLog(trxLogList); if (createDefaultPolicy) { - createDefaultPolicies(createdService); + createDefaultPolicies(xCreatedService, vXUser); } return createdService; @@ -1462,7 +1462,7 @@ public RangerService createService(RangerService service) throws Exception { } @Override - public RangerService updateService(RangerService service) throws Exception { + public RangerService updateService(RangerService service, Map options) throws Exception { if(LOG.isDebugEnabled()) { LOG.debug("==> ServiceDBStore.updateService()"); } @@ -1480,13 +1480,26 @@ public RangerService updateService(RangerService service) throws Exception { boolean renamed = !StringUtils.equalsIgnoreCase(service.getName(), existingName); if(renamed) { - XXService newNameService = daoMgr.getXXService().findByName(service.getName()); + XXService newNameService = daoMgr.getXXService().findByName(service.getName()); - if(newNameService != null) { - throw restErrorUtil.createRESTException("another service already exists with name '" - + service.getName() + "'. ID=" + newNameService.getId(), MessageEnums.DATA_NOT_UPDATABLE); - } - } + if (newNameService != null) { + throw restErrorUtil.createRESTException("another service already exists with name '" + + service.getName() + "'. ID=" + newNameService.getId(), MessageEnums.DATA_NOT_UPDATABLE); + } + + long countOfTaggedResources = daoMgr.getXXServiceResource().countTaggedResourcesInServiceId(existing.getId()); + + Boolean isForceRename = options != null && options.get(ServiceStore.OPTION_FORCE_RENAME) != null ? (Boolean) options.get(ServiceStore.OPTION_FORCE_RENAME) : Boolean.FALSE; + + if (countOfTaggedResources != 0L) { + if (isForceRename) { + LOG.warn("Forcing the renaming of service from " + existingName + " to " + service.getName() + " although it is associated with " + countOfTaggedResources + + " service-resources!"); + } else { + throw restErrorUtil.createRESTException("Service " + existingName + " cannot be renamed, as it has associated service-resources", MessageEnums.DATA_NOT_UPDATABLE); + } + } + } Map configs = service.getConfigs(); Map validConfigs = validateRequiredConfigParams(service, configs); @@ -1583,25 +1596,21 @@ public RangerService updateService(RangerService service) throws Exception { + userName + "] please use existing user", MessageEnums.OPER_NO_PERMISSION); } vXUser = xUserMgr.createServiceConfigUser(userName); + if (LOG.isDebugEnabled()) { + LOG.debug("Service config user created:[" + vXUser + "]"); + } } } - + if (StringUtils.equalsIgnoreCase(configKey, CONFIG_KEY_PASSWORD)) { if (StringUtils.equalsIgnoreCase(configValue, HIDDEN_PASSWORD_STR)) { - String[] crypt_algo_array = null; - if (configValue.contains(",")) { - crypt_algo_array = configValue.split(","); - } - if (oldPassword != null && oldPassword.contains(",")) { - String encryptKey = null; - String salt = null; - int iterationCount = 0; - crypt_algo_array = oldPassword.split(","); - String OLD_CRYPT_ALGO = crypt_algo_array[0]; - encryptKey = crypt_algo_array[1]; - salt = crypt_algo_array[2]; - iterationCount = Integer.parseInt(crypt_algo_array[3]); - + if (oldPassword != null && oldPassword.contains(",")) { + String[] crypt_algo_array = oldPassword.split(","); + String OLD_CRYPT_ALGO = crypt_algo_array[0]; + String encryptKey = crypt_algo_array[1]; + String salt = crypt_algo_array[2]; + int iterationCount = Integer.parseInt(crypt_algo_array[3]); + if (!OLD_CRYPT_ALGO.equalsIgnoreCase(CRYPT_ALGO)) { String decryptedPwd = PasswordUtils.decryptPassword(oldPassword); String paddingString = CRYPT_ALGO + "," + encryptKey + "," + salt + "," + iterationCount; @@ -1626,15 +1635,13 @@ public RangerService updateService(RangerService service) throws Exception { } } } + XXServiceConfigMap xConfMap = new XXServiceConfigMap(); xConfMap = (XXServiceConfigMap) rangerAuditFields.populateAuditFields(xConfMap, xUpdService); xConfMap.setServiceId(service.getId()); xConfMap.setConfigkey(configKey); xConfMap.setConfigvalue(configValue); - xConfMapDao.create(xConfMap); - } - if (LOG.isDebugEnabled()) { - LOG.debug("vXUser:[" + vXUser + "]"); + xConfMap = xConfMapDao.create(xConfMap); } RangerService updService = svcService.getPopulatedViewObject(xUpdService); dataHistService.createObjectDataHistory(updService, RangerDataHistService.ACTION_UPDATE); @@ -1656,8 +1663,10 @@ public void deleteService(Long id) throws Exception { } List policies = daoMgr.getXXPolicy().findByServiceId(service.getId()); + //RangerPolicy rangerPolicy =null; for(XXPolicy policy : policies) { LOG.info("Deleting Policy, policyName: " + policy.getName()); + //rangerPolicy = getPolicy(policy.getId()); deletePolicy(policy.getId()); } @@ -1816,14 +1825,6 @@ public RangerPolicy createPolicy(RangerPolicy policy) throws Exception { throw new Exception("policy already exists: ServiceName=" + policy.getService() + "; PolicyName=" + policy.getName() + ". ID=" + existing.getId()); } - Map resources = policy.getResources(); - List policyItems = policy.getPolicyItems(); - List denyPolicyItems = policy.getDenyPolicyItems(); - List allowExceptions = policy.getAllowExceptions(); - List denyExceptions = policy.getDenyExceptions(); - List dataMaskItems = policy.getDataMaskPolicyItems(); - List rowFilterItems = policy.getRowFilterPolicyItems(); - policy.setVersion(Long.valueOf(1)); updatePolicySignature(policy); @@ -1841,14 +1842,8 @@ public RangerPolicy createPolicy(RangerPolicy policy) throws Exception { } XXPolicy xCreatedPolicy = daoMgr.getXXPolicy().getById(policy.getId()); + policyRefUpdater.createNewPolMappingForRefTable(policy, xCreatedPolicy, xServiceDef); - createNewResourcesForPolicy(policy, xCreatedPolicy, resources); - createNewPolicyItemsForPolicy(policy, xCreatedPolicy, policyItems, xServiceDef, RangerPolicyItemEvaluator.POLICY_ITEM_TYPE_ALLOW); - createNewPolicyItemsForPolicy(policy, xCreatedPolicy, denyPolicyItems, xServiceDef, RangerPolicyItemEvaluator.POLICY_ITEM_TYPE_DENY); - createNewPolicyItemsForPolicy(policy, xCreatedPolicy, allowExceptions, xServiceDef, RangerPolicyItemEvaluator.POLICY_ITEM_TYPE_ALLOW_EXCEPTIONS); - createNewPolicyItemsForPolicy(policy, xCreatedPolicy, denyExceptions, xServiceDef, RangerPolicyItemEvaluator.POLICY_ITEM_TYPE_DENY_EXCEPTIONS); - createNewDataMaskPolicyItemsForPolicy(policy, xCreatedPolicy, dataMaskItems, xServiceDef, RangerPolicyItemEvaluator.POLICY_ITEM_TYPE_DATAMASK); - createNewRowFilterPolicyItemsForPolicy(policy, xCreatedPolicy, rowFilterItems, xServiceDef, RangerPolicyItemEvaluator.POLICY_ITEM_TYPE_ROWFILTER); handlePolicyUpdate(service, true); RangerPolicy createdPolicy = policyService.getPopulatedViewObject(xCreatedPolicy); dataHistService.createObjectDataHistory(createdPolicy, RangerDataHistService.ACTION_CREATE); @@ -1896,19 +1891,12 @@ public RangerPolicy updatePolicy(RangerPolicy policy) throws Exception { throw new Exception("another policy already exists with name '" + policy.getName() + "'. ID=" + newNamePolicy.getId()); } } - Map newResources = policy.getResources(); - List policyItems = policy.getPolicyItems(); - List denyPolicyItems = policy.getDenyPolicyItems(); - List allowExceptions = policy.getAllowExceptions(); - List denyExceptions = policy.getDenyExceptions(); - List dataMaskPolicyItems = policy.getDataMaskPolicyItems(); - List rowFilterItems = policy.getRowFilterPolicyItems(); - + policy.setCreateTime(xxExisting.getCreateTime()); policy.setGuid(xxExisting.getGuid()); policy.setVersion(xxExisting.getVersion()); - List trxLogList = policyService.getTransactionLog(policy, xxExisting, RangerPolicyService.OPERATION_UPDATE_CONTEXT); + List trxLogList = policyService.getTransactionLog(policy, xxExisting, existing, RangerPolicyService.OPERATION_UPDATE_CONTEXT); updatePolicySignature(policy); @@ -1917,26 +1905,19 @@ public RangerPolicy updatePolicy(RangerPolicy policy) throws Exception { isTagVersionUpdateNeeded = existing.getIsEnabled() ? !policy.getIsEnabled() : policy.getIsEnabled(); isTagVersionUpdateNeeded = isTagVersionUpdateNeeded || !StringUtils.equals(existing.getResourceSignature(), policy.getResourceSignature()); } + policy = policyService.update(policy); XXPolicy newUpdPolicy = daoMgr.getXXPolicy().getById(policy.getId()); - deleteExistingPolicyResources(policy); - deleteExistingPolicyItems(policy); - - createNewResourcesForPolicy(policy, newUpdPolicy, newResources); - createNewPolicyItemsForPolicy(policy, newUpdPolicy, policyItems, xServiceDef, RangerPolicyItemEvaluator.POLICY_ITEM_TYPE_ALLOW); - createNewPolicyItemsForPolicy(policy, newUpdPolicy, denyPolicyItems, xServiceDef, RangerPolicyItemEvaluator.POLICY_ITEM_TYPE_DENY); - createNewPolicyItemsForPolicy(policy, newUpdPolicy, allowExceptions, xServiceDef, RangerPolicyItemEvaluator.POLICY_ITEM_TYPE_ALLOW_EXCEPTIONS); - createNewPolicyItemsForPolicy(policy, newUpdPolicy, denyExceptions, xServiceDef, RangerPolicyItemEvaluator.POLICY_ITEM_TYPE_DENY_EXCEPTIONS); - createNewDataMaskPolicyItemsForPolicy(policy, newUpdPolicy, dataMaskPolicyItems, xServiceDef, RangerPolicyItemEvaluator.POLICY_ITEM_TYPE_DATAMASK); - createNewRowFilterPolicyItemsForPolicy(policy, newUpdPolicy, rowFilterItems, xServiceDef, RangerPolicyItemEvaluator.POLICY_ITEM_TYPE_ROWFILTER); - - handlePolicyUpdate(service, isTagVersionUpdateNeeded); + policyRefUpdater.cleanupRefTables(policy); + policyRefUpdater.createNewPolMappingForRefTable(policy, newUpdPolicy, xServiceDef); + + handlePolicyUpdate(service, isTagVersionUpdateNeeded); RangerPolicy updPolicy = policyService.getPopulatedViewObject(newUpdPolicy); dataHistService.createObjectDataHistory(updPolicy, RangerDataHistService.ACTION_UPDATE); bizUtil.createTrxLog(trxLogList); - + return updPolicy; } @@ -1970,10 +1951,7 @@ public void deletePolicy(Long policyId) throws Exception { policy.setVersion(version); List trxLogList = policyService.getTransactionLog(policy, RangerPolicyService.OPERATION_DELETE_CONTEXT); - - deleteExistingPolicyItems(policy); - deleteExistingPolicyResources(policy); - + policyRefUpdater.cleanupRefTables(policy); policyService.delete(policy); handlePolicyUpdate(service, true); @@ -1983,6 +1961,32 @@ public void deletePolicy(Long policyId) throws Exception { LOG.info("Policy Deleted Successfully. PolicyName : " + policyName); } +/* + public void deletePolicy(RangerPolicy policy) throws Exception { + if(policy == null) { + return; + } + if(LOG.isDebugEnabled()) { + LOG.debug("==> ServiceDBStore.deletePolicy(" + policy.getId() + ")"); + } + RangerService service = getServiceByName(policy.getService()); + if(service == null) { + throw new Exception("service does not exist - name='" + policy.getService()); + } + Long version = policy.getVersion(); + if(version == null) { + version = Long.valueOf(1); + LOG.info("Found Version Value: `null`, so setting value of version to 1, While updating object, version should not be null."); + } else { + version = Long.valueOf(version.longValue() + 1); + } + policy.setVersion(version); + List trxLogList = policyService.getTransactionLog(policy, RangerPolicyService.OPERATION_DELETE_CONTEXT); + daoMgr.getXXPolicy().deletePolicyIDReference("id",policy.getId()); + handlePolicyUpdate(service, true); + dataHistService.createObjectDataHistory(policy, RangerDataHistService.ACTION_DELETE); + bizUtil.createTrxLog(trxLogList); + }*/ @Override public RangerPolicy getPolicy(Long id) throws Exception { @@ -2002,6 +2006,22 @@ public List getPolicies(SearchFilter filter) throws Exception { return ret; } + @Override + public Long getPolicyId(final Long serviceId, final String policyName) { + if(LOG.isDebugEnabled()) { + LOG.debug("==> ServiceDBStore.getPolicyId()"); + } + Long ret = null; + XXPolicy xxPolicy = daoMgr.getXXPolicy().findByNameAndServiceId(policyName, serviceId); + if (xxPolicy != null) { + ret = xxPolicy.getId(); + } + if(LOG.isDebugEnabled()) { + LOG.debug("<== ServiceDBStore.getPolicyId()"); + } + return ret; + } + public void getPoliciesInExcel(List policies, HttpServletResponse response) throws Exception { if (LOG.isDebugEnabled()) { LOG.debug("==> ServiceDBStore.getPoliciesInExcel()"); @@ -2266,48 +2286,47 @@ List getMatchers(RangerServiceDef serviceDef, Map policyTypes = new ArrayList<>(); if (StringUtils.isNotBlank(policyTypeStr)) { - policyType = Integer.parseInt(policyTypeStr); - } - - Set> validResourceHierarchies = serviceDefHelper.getResourceHierarchies(policyType, filterResources.keySet()); - - if (LOG.isDebugEnabled()) { - LOG.debug("Found " + validResourceHierarchies.size() + " valid resource hierarchies for key-set " + filterResources.keySet()); + policyTypes.add(Integer.parseInt(policyTypeStr)); + } else { + policyTypes.add(RangerPolicy.POLICY_TYPE_ACCESS); + policyTypes.add(RangerPolicy.POLICY_TYPE_DATAMASK); + policyTypes.add(RangerPolicy.POLICY_TYPE_ROWFILTER); } - List> resourceHierarchies = new ArrayList>(validResourceHierarchies); - - for (List validResourceHierarchy : resourceHierarchies) { + for (Integer policyType : policyTypes) { + Set> validResourceHierarchies = serviceDefHelper.getResourceHierarchies(policyType, filterResources.keySet()); if (LOG.isDebugEnabled()) { - LOG.debug("validResourceHierarchy:[" + validResourceHierarchy + "]"); + LOG.debug("Found " + validResourceHierarchies.size() + " valid resource hierarchies for key-set " + filterResources.keySet()); } - Map policyResources = new HashMap(); - - for (RangerResourceDef resourceDef : validResourceHierarchy) { + List> resourceHierarchies = new ArrayList>(validResourceHierarchies); - String resourceValue = filterResources.get(resourceDef.getName()); + for (List validResourceHierarchy : resourceHierarchies) { - if (StringUtils.isBlank(resourceValue)) { - resourceValue = RangerAbstractResourceMatcher.WILDCARD_ASTERISK; + if (LOG.isDebugEnabled()) { + LOG.debug("validResourceHierarchy:[" + validResourceHierarchy + "]"); } - policyResources.put(resourceDef.getName(), new RangerPolicyResource(resourceValue, false, resourceDef.getRecursiveSupported())); - } + Map policyResources = new HashMap(); - RangerDefaultPolicyResourceMatcher matcher = new RangerDefaultPolicyResourceMatcher(); - matcher.setServiceDef(serviceDef); - matcher.setPolicyResources(policyResources); - matcher.init(); + for (RangerResourceDef resourceDef : validResourceHierarchy) { + policyResources.put(resourceDef.getName(), new RangerPolicyResource(filterResources.get(resourceDef.getName()), false, resourceDef.getRecursiveSupported())); + } - ret.add(matcher); + RangerDefaultPolicyResourceMatcher matcher = new RangerDefaultPolicyResourceMatcher(); + matcher.setServiceDef(serviceDef); + matcher.setPolicyResources(policyResources, policyType); + matcher.init(); - if (LOG.isDebugEnabled()) { - LOG.debug("Added matcher:[" + matcher + "]"); + ret.add(matcher); + + if (LOG.isDebugEnabled()) { + LOG.debug("Added matcher:[" + matcher + "]"); + } } } @@ -2323,7 +2342,7 @@ private List getServicePoliciesFromDb(XXService service) throws Ex LOG.debug("==> ServiceDBStore.getServicePoliciesFromDb(" + service.getName() + ")"); } - RangerPolicyRetriever policyRetriever = new RangerPolicyRetriever(daoMgr); + RangerPolicyRetriever policyRetriever = new RangerPolicyRetriever(daoMgr, txManager); List ret = policyRetriever.getServicePolicies(service); @@ -2485,127 +2504,428 @@ public ServicePolicies getServicePolicies(String serviceName) throws Exception { return ret; } - void createDefaultPolicies(RangerService createdService) throws Exception { + void createDefaultPolicies(XXService createdService, VXUser vXUser) throws Exception { + RangerServiceDef serviceDef = getServiceDef(createdService.getType()); - RangerBaseService svc = serviceMgr.getRangerServiceByService(createdService, this); + if (serviceDef.getName().equals(EmbeddedServiceDefsUtil.EMBEDDED_SERVICEDEF_TAG_NAME)) { + createDefaultTagPolicy(createdService); + } else { + // we need to create one policy for each resource hierarchy + RangerServiceDefHelper serviceDefHelper = new RangerServiceDefHelper(serviceDef); + for (List aHierarchy : serviceDefHelper.getResourceHierarchies(RangerPolicy.POLICY_TYPE_ACCESS)) { + RangerPolicy policy = new RangerPolicy(); + createDefaultPolicy(policy, createdService, vXUser, aHierarchy); + policy = createPolicy(policy); + RangerPolicy policyAudit = new RangerPolicy(); + createPolicyForKeyAdmin(policyAudit, serviceDef, aHierarchy, createdService); + } + } + } - if (svc != null) { + private void createPolicyForKeyAdmin(RangerPolicy policyAudit, RangerServiceDef serviceDef, List aHierarchy, XXService createdService) { + if (serviceDef.getName().equals(EmbeddedServiceDefsUtil.EMBEDDED_SERVICEDEF_HDFS_NAME)) { + try { + // we need to create one policy for keyadmin user for audit to HDFS + RangerPolicy policy = getPolicyForKMSAudit(aHierarchy , createdService.getName(), serviceDef); + if (policy != null) { + createPolicy(policy); + } + } catch (Exception e) { + LOG.error("Error creating policy for keyadmin for audit to HDFS : " + serviceDef.getName(), e); + } + } + } - List serviceCheckUsers = getServiceCheckUsers(createdService); + private RangerPolicy getPolicyForKMSAudit(List resourceHierarchy, String serviceName, RangerServiceDef serviceDef) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> ServiceDBStore.getPolicyForKMSAudit()"); + } - List allAccesses = svc.getAndAllowAllAccesses(); + RangerPolicy policy = new RangerPolicy(); - List defaultPolicies = svc.getDefaultRangerPolicies(); + policy.setIsEnabled(true); + policy.setVersion(1L); + policy.setName(AUDITTOHDFS_POLICY_NAME); + policy.setService(serviceName); + policy.setDescription("Policy for " + AUDITTOHDFS_POLICY_NAME); + policy.setIsAuditEnabled(true); + policy.setResources(createKMSAuditResource(resourceHierarchy)); - if (CollectionUtils.isNotEmpty(defaultPolicies)) { + List policyItems = new ArrayList(); + //Create policy item for keyadmin + RangerPolicy.RangerPolicyItem policyItem = new RangerPolicy.RangerPolicyItem(); + List userKeyAdmin = new ArrayList(); + userKeyAdmin.add("keyadmin"); + policyItem.setUsers(userKeyAdmin); + policyItem.setAccesses(getAndAllowAllAccesses(serviceDef)); + policyItem.setDelegateAdmin(false); - createDefaultPolicyUsersAndGroups(defaultPolicies); + policyItems.add(policyItem); + policy.setPolicyItems(policyItems); - for (RangerPolicy defaultPolicy : defaultPolicies) { - if (CollectionUtils.isNotEmpty(serviceCheckUsers) - && StringUtils.equalsIgnoreCase(defaultPolicy.getService(), createdService.getName())) { + if (LOG.isDebugEnabled()) { + LOG.debug("<== ServiceDBStore.getPolicyForKMSAudit()" + policy); + } - RangerPolicy.RangerPolicyItem policyItem = new RangerPolicy.RangerPolicyItem(); + return policy; + } - policyItem.setUsers(serviceCheckUsers); - policyItem.setAccesses(allAccesses); - policyItem.setDelegateAdmin(true); + public List getAndAllowAllAccesses(RangerServiceDef serviceDef) { + List ret = new ArrayList(); - defaultPolicy.getPolicyItems().add(policyItem); - } - createPolicy(defaultPolicy); - } - } + for (RangerServiceDef.RangerAccessTypeDef accessTypeDef : serviceDef.getAccessTypes()) { + RangerPolicy.RangerPolicyItemAccess access = new RangerPolicy.RangerPolicyItemAccess(); + access.setType(accessTypeDef.getName()); + access.setIsAllowed(true); + ret.add(access); } + return ret; } - void createDefaultPolicyUsersAndGroups(List defaultPolicies) { - Set defaultPolicyUsers = new HashSet(); - Set defaultPolicyGroups = new HashSet(); + private Map createKMSAuditResource( + List resourceHierarchy) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> ServiceDBStore.createKMSAuditResource()"); + } + Map resourceMap = new HashMap<>(); - for (RangerPolicy defaultPolicy : defaultPolicies) { + for (RangerServiceDef.RangerResourceDef resourceDef : resourceHierarchy) { + RangerPolicy.RangerPolicyResource polRes = new RangerPolicy.RangerPolicyResource(); - for (RangerPolicyItem defaultPolicyItem : defaultPolicy.getPolicyItems()) { - defaultPolicyUsers.addAll(defaultPolicyItem.getUsers()); - defaultPolicyGroups.addAll(defaultPolicyItem.getGroups()); - } - for (RangerPolicyItem defaultPolicyItem : defaultPolicy.getAllowExceptions()) { - defaultPolicyUsers.addAll(defaultPolicyItem.getUsers()); - defaultPolicyGroups.addAll(defaultPolicyItem.getGroups()); - } - for (RangerPolicyItem defaultPolicyItem : defaultPolicy.getDenyPolicyItems()) { - defaultPolicyUsers.addAll(defaultPolicyItem.getUsers()); - defaultPolicyGroups.addAll(defaultPolicyItem.getGroups()); + polRes.setIsExcludes(false); + polRes.setIsRecursive(resourceDef.getRecursiveSupported()); + polRes.setValue(AUDITTOHDFS_KMS_PATH); + + resourceMap.put(resourceDef.getName(), polRes); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== ServiceDBStore.createKMSAuditResource():" + + resourceMap); + } + return resourceMap; + } + + private void createDefaultTagPolicy(XXService createdService) throws Exception { + if (LOG.isDebugEnabled()) { + LOG.debug("==> ServiceDBStore.createDefaultTagPolicy() "); + } + + String tagResourceDefName = null; + boolean isConditionDefFound = false; + + RangerServiceDef tagServiceDef = getServiceDef(createdService.getType()); + List tagResourceDef = tagServiceDef.getResources(); + if (tagResourceDef != null && tagResourceDef.size() > 0) { + // Assumption : First (and perhaps the only) resourceDef is the name of the tag resource + RangerResourceDef theTagResourceDef = tagResourceDef.get(0); + tagResourceDefName = theTagResourceDef.getName(); + } else { + LOG.error("ServiceDBStore.createService() - Cannot create default TAG policy: Cannot get tagResourceDef Name."); + } + + List policyConditionDefs = tagServiceDef.getPolicyConditions(); + + if (CollectionUtils.isNotEmpty(policyConditionDefs)) { + for (RangerPolicyConditionDef conditionDef : policyConditionDefs) { + if (conditionDef.getName().equals(RANGER_TAG_EXPIRY_CONDITION_NAME)) { + isConditionDefFound = true; + break; + } } - for (RangerPolicyItem defaultPolicyItem : defaultPolicy.getDenyExceptions()) { - defaultPolicyUsers.addAll(defaultPolicyItem.getUsers()); - defaultPolicyGroups.addAll(defaultPolicyItem.getGroups()); + } + if (!isConditionDefFound) { + LOG.error("ServiceDBStore.createService() - Cannot create default TAG policy: Cannot get tagPolicyConditionDef with name=" + RANGER_TAG_EXPIRY_CONDITION_NAME); + } + + if (tagResourceDefName != null && isConditionDefFound) { + + String tagType = "EXPIRES_ON"; + + String policyName = tagType; + + RangerPolicy policy = new RangerPolicy(); + + policy.setIsEnabled(true); + policy.setVersion(1L); + policy.setName(StringUtils.trim(policyName)); + policy.setService(createdService.getName()); + policy.setDescription("Policy for data with " + tagType + " tag"); + policy.setIsAuditEnabled(true); + + Map resourceMap = new HashMap(); + + RangerPolicyResource polRes = new RangerPolicyResource(); + polRes.setIsExcludes(false); + polRes.setIsRecursive(false); + polRes.setValue(tagType); + resourceMap.put(tagResourceDefName, polRes); + + policy.setResources(resourceMap); + + List policyItems = new ArrayList(); + + RangerPolicyItem policyItem = new RangerPolicyItem(); + + List groups = new ArrayList(); + groups.add(RangerConstants.GROUP_PUBLIC); + policyItem.setGroups(groups); + + List accessTypeDefs = daoMgr.getXXAccessTypeDef().findByServiceDefId(createdService.getType()); + List accesses = new ArrayList(); + for (XXAccessTypeDef accessTypeDef : accessTypeDefs) { + RangerPolicyItemAccess access = new RangerPolicyItemAccess(); + access.setType(accessTypeDef.getName()); + access.setIsAllowed(true); + accesses.add(access); } - for (RangerPolicyItem defaultPolicyItem : defaultPolicy.getDataMaskPolicyItems()) { - defaultPolicyUsers.addAll(defaultPolicyItem.getUsers()); - defaultPolicyGroups.addAll(defaultPolicyItem.getGroups()); + policyItem.setAccesses(accesses); + + List policyItemConditions = new ArrayList(); + List values = new ArrayList(); + values.add("yes"); + RangerPolicyItemCondition policyItemCondition = new RangerPolicyItemCondition(RANGER_TAG_EXPIRY_CONDITION_NAME, values); + policyItemConditions.add(policyItemCondition); + + policyItem.setConditions(policyItemConditions); + policyItem.setDelegateAdmin(Boolean.FALSE); + + policyItems.add(policyItem); + + policy.setDenyPolicyItems(policyItems); + + policy = createPolicy(policy); + } else { + LOG.error("ServiceDBStore.createService() - Cannot create default TAG policy, tagResourceDefName=" + tagResourceDefName + + ", tagPolicyConditionName=" + RANGER_TAG_EXPIRY_CONDITION_NAME); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== ServiceDBStore.createDefaultTagPolicy()"); + } + } + + private String buildPolicyName(List resourceHierarchy) { + String ret = "all"; + if (CollectionUtils.isNotEmpty(resourceHierarchy)) { + int resourceDefCount = 0; + for (RangerResourceDef resourceDef : resourceHierarchy) { + if (resourceDefCount > 0) { + ret += ", "; + } else { + ret += " - "; + } + ret += resourceDef.getName(); + resourceDefCount++; } - for (RangerPolicyItem defaultPolicyItem : defaultPolicy.getRowFilterPolicyItems()) { - defaultPolicyUsers.addAll(defaultPolicyItem.getUsers()); - defaultPolicyGroups.addAll(defaultPolicyItem.getGroups()); + } + return ret; + } + + void createDefaultPolicy(RangerPolicy policy, XXService createdService, VXUser vXUser, List resourceHierarchy) throws Exception { + + String policyName=buildPolicyName(resourceHierarchy); + + policy.setIsEnabled(true); + policy.setVersion(1L); + policy.setName(StringUtils.trim(policyName)); + policy.setService(createdService.getName()); + policy.setDescription("Policy for " + policyName); + policy.setIsAuditEnabled(true); + policy.setPolicyType(policy.getPolicyType() == null ? RangerPolicy.POLICY_TYPE_ACCESS : policy.getPolicyType()); + + policy.setResources(createDefaultPolicyResource(resourceHierarchy)); + + if (vXUser != null) { + List policyItems = new ArrayList(); + List accessTypeDefs = daoMgr.getXXAccessTypeDef().findByServiceDefId(createdService.getType()); + //Create Default policy item for the service user + RangerPolicyItem policyItem = createDefaultPolicyItem(createdService, vXUser, accessTypeDefs); + policyItems.add(policyItem); + // For KMS add default policies for HDFS & HIVE users. + XXServiceDef xServiceDef = daoMgr.getXXServiceDef().getById(createdService.getType()); + if (EmbeddedServiceDefsUtil.KMS_IMPL_CLASS_NAME.equals(xServiceDef.getImplclassname())) { + List hdfsAccessTypeDefs = new ArrayList(); + List hiveAccessTypeDefs = new ArrayList(); + for(XXAccessTypeDef accessTypeDef : accessTypeDefs) { + if (accessTypeDef.getName().equalsIgnoreCase(ACCESS_TYPE_GET_METADATA)) { + hdfsAccessTypeDefs.add(accessTypeDef); + hiveAccessTypeDefs.add(accessTypeDef); + } else if (accessTypeDef.getName().equalsIgnoreCase(ACCESS_TYPE_GENERATE_EEK)) { + hdfsAccessTypeDefs.add(accessTypeDef); + } else if (accessTypeDef.getName().equalsIgnoreCase(ACCESS_TYPE_DECRYPT_EEK)) { + hiveAccessTypeDefs.add(accessTypeDef); + } + } + + String hdfsUser = PropertiesUtil.getProperty("ranger.kms.service.user.hdfs", "hdfs"); + if (hdfsUser != null && !hdfsUser.isEmpty()) { + XXUser xxUser = daoMgr.getXXUser().findByUserName(hdfsUser); + if (xxUser != null) { + vXUser = xUserService.populateViewBean(xxUser); + } else { + vXUser = xUserMgr.createServiceConfigUser(hdfsUser); + } + if (vXUser != null) { + LOG.info("Creating default KMS policy item for " + hdfsUser); + policyItem = createDefaultPolicyItem(createdService, vXUser, hdfsAccessTypeDefs); + policyItems.add(policyItem); + } + } + + String hiveUser = PropertiesUtil.getProperty("ranger.kms.service.user.hive", "hive"); + if (hiveUser != null && !hiveUser.isEmpty()) { + XXUser xxUser = daoMgr.getXXUser().findByUserName(hiveUser); + if (xxUser != null) { + vXUser = xUserService.populateViewBean(xxUser); + } else { + vXUser = xUserMgr.createServiceConfigUser(hiveUser); + } + if (vXUser != null) { + LOG.info("Creating default KMS policy item for " + hiveUser); + policyItem = createDefaultPolicyItem(createdService, vXUser, hiveAccessTypeDefs); + policyItems.add(policyItem); + } + } } + policy.setPolicyItems(policyItems); } - for (String policyUser : defaultPolicyUsers) { - if (LOG.isDebugEnabled()) { - LOG.debug("Checking policyUser:[" + policyUser + "] for existence"); + } + + private RangerPolicyItem createDefaultPolicyItem(XXService createdService, VXUser vXUser, List accessTypeDefs) throws Exception { + String adminPrincipal = PropertiesUtil.getProperty(ADMIN_USER_PRINCIPAL); + String adminKeytab = PropertiesUtil.getProperty(ADMIN_USER_KEYTAB); + String authType = PropertiesUtil.getProperty(RANGER_AUTH_TYPE,"simple"); + String lookupPrincipal = PropertiesUtil.getProperty(LOOKUP_PRINCIPAL); + String lookupKeytab = PropertiesUtil.getProperty(LOOKUP_KEYTAB); + + RangerPolicyItem policyItem = new RangerPolicyItem(); + + List users = new ArrayList(); + users.add(vXUser.getName()); + VXUser vXLookupUser = getLookupUser(authType, lookupPrincipal, lookupKeytab); + + XXService xService = daoMgr.getXXService().findByName(createdService.getName()); + XXServiceDef xServiceDef = daoMgr.getXXServiceDef().getById(xService.getType()); + if (StringUtils.equals(xServiceDef.getImplclassname(), EmbeddedServiceDefsUtil.KMS_IMPL_CLASS_NAME)){ + VXUser vXAdminUser = getLookupUser(authType, adminPrincipal, adminKeytab); + if(vXAdminUser != null){ + users.add(vXAdminUser.getName()); + } + }else if(vXLookupUser != null){ + users.add(vXLookupUser.getName()); + }else{ + // do nothing + } + + if (StringUtils.equals(xServiceDef.getImplclassname(), EmbeddedServiceDefsUtil.ATLAS_IMPL_CLASS_NAME)){ + VXUser vXUserAdmin = chkAdminUserExists("admin"); + if(vXUserAdmin != null){ + users.add(vXUserAdmin.getName()); } - if (StringUtils.isNotBlank(policyUser) && !StringUtils.equals(policyUser, RangerPolicyEngine.USER_CURRENT) - && !StringUtils.equals(policyUser, RangerPolicyEngine.RESOURCE_OWNER)) { - XXUser xxUser = daoMgr.getXXUser().findByUserName(policyUser); - if (xxUser == null) { - UserSessionBase usb = ContextUtil.getCurrentUserSession(); - if (usb != null && !usb.isKeyAdmin() && !usb.isUserAdmin() && !usb.isSpnegoEnabled()) { - throw restErrorUtil.createRESTException("User does not exist with given username: [" - + policyUser + "] please use existing user", MessageEnums.OPER_NO_PERMISSION); + } + + RangerService rangerService = getServiceByName(createdService.getName()); + if (rangerService != null){ + Map map = rangerService.getConfigs(); + if (map != null && map.containsKey(AMBARI_SERVICE_CHECK_USER)){ + String userNames = map.get(AMBARI_SERVICE_CHECK_USER); + String[] userList = userNames.split(","); + if(userList != null){ + for (String userName : userList) { + if(!StringUtils.isEmpty(userName)){ + XXUser xxUser = daoMgr.getXXUser().findByUserName(userName); + if (xxUser != null) { + vXUser = xUserService.populateViewBean(xxUser); + } else { + vXUser = xUserMgr.createServiceConfigUser(userName); + LOG.info("Creating Ambari Service Check User : "+vXUser.getName()); + } + if(vXUser != null){ + users.add(vXUser.getName()); + } + } } - xUserMgr.createServiceConfigUser(policyUser); } } } - for (String policyGroup : defaultPolicyGroups) { - if (LOG.isDebugEnabled()) { - LOG.debug("Checking policyGroup:[" + policyGroup + "] for existence"); + policyItem.setUsers(users); + + List accesses = new ArrayList(); + for(XXAccessTypeDef accessTypeDef : accessTypeDefs) { + RangerPolicyItemAccess access = new RangerPolicyItemAccess(); + access.setType(accessTypeDef.getName()); + access.setIsAllowed(true); + accesses.add(access); + } + policyItem.setAccesses(accesses); + + policyItem.setDelegateAdmin(true); + return policyItem; + } + + private VXUser chkAdminUserExists(String adminUser) { + VXUser vXUser = null; + if(!StringUtils.isEmpty(adminUser)){ + XXUser xxUser = daoMgr.getXXUser().findByUserName(adminUser); + if (xxUser != null) { + vXUser = xUserService.populateViewBean(xxUser); } - if (StringUtils.isNotBlank(policyGroup)) { - XXGroup xxGroup = daoMgr.getXXGroup().findByGroupName(policyGroup); - if (xxGroup == null) { - UserSessionBase usb = ContextUtil.getCurrentUserSession(); - if (usb != null && !usb.isKeyAdmin() && !usb.isUserAdmin() && !usb.isSpnegoEnabled()) { - throw restErrorUtil.createRESTException("Group does not exist with given groupname: [" - + policyGroup + "] please use existing group", MessageEnums.OPER_NO_PERMISSION); + } + return vXUser; + } + + private VXUser getLookupUser(String authType, String lookupPrincipal, String lookupKeytab) { + VXUser vXUser = null; + if(!StringUtils.isEmpty(authType) && authType.equalsIgnoreCase(KERBEROS_TYPE)){ + if(SecureClientLogin.isKerberosCredentialExists(lookupPrincipal, lookupKeytab)){ + KerberosName krbName = new KerberosName(lookupPrincipal); + String lookupUser=null; + try { + lookupUser = krbName.getShortName(); + } catch (IOException e) { + throw restErrorUtil.createRESTException("Please provide proper value of lookup user principal : "+ lookupPrincipal, MessageEnums.INVALID_INPUT_DATA); + } + + if(LOG.isDebugEnabled()){ + LOG.debug("Checking for Lookup User : "+lookupUser); + } + if(!StringUtils.isEmpty(lookupUser)){ + XXUser xxUser = daoMgr.getXXUser().findByUserName(lookupUser); + if (xxUser != null) { + vXUser = xUserService.populateViewBean(xxUser); + } else { + vXUser = xUserMgr.createServiceConfigUser(lookupUser); + LOG.info("Creating Lookup User : "+vXUser.getName()); } - VXGroup vXGroup = new VXGroup(); - vXGroup.setName(policyGroup); - vXGroup.setDescription(policyGroup); - vXGroup.setGroupSource(RangerCommonEnums.GROUP_INTERNAL); - vXGroup.setIsVisible(RangerCommonEnums.IS_VISIBLE); - xGroupService.createResource(vXGroup); } } } + return vXUser; } - List getServiceCheckUsers(RangerService createdService) { - List ret = new ArrayList(); - Map serviceConfig = createdService.getConfigs(); + Map createDefaultPolicyResource(List resourceHierarchy) throws Exception { + Map resourceMap = new HashMap<>(); - if (serviceConfig.containsKey(AMBARI_SERVICE_CHECK_USER)) { - String userNames = serviceConfig.get(AMBARI_SERVICE_CHECK_USER); - String[] userList = userNames.split(","); - for (String userName : userList) { - if (!StringUtils.isEmpty(userName)) { - ret.add(userName); - } + for (RangerResourceDef resourceDef : resourceHierarchy) { + RangerPolicyResource polRes = new RangerPolicyResource(); + polRes.setIsExcludes(false); + polRes.setIsRecursive(false); + + String value = "*"; + if("path".equalsIgnoreCase(resourceDef.getName())) { + value = "/*"; } - } - return ret; + if(resourceDef.getRecursiveSupported()) { + polRes.setIsRecursive(Boolean.TRUE); + } + + polRes.setValue(value); + resourceMap.put(resourceDef.getName(), polRes); + } + return resourceMap; } private Map validateRequiredConfigParams(RangerService service, Map configs) { @@ -2640,345 +2960,77 @@ private void handlePolicyUpdate(RangerService service, boolean isTagVersionUpdat updatePolicyVersion(service, isTagVersionUpdateNeeded); } + public enum VERSION_TYPE { POLICY_VERSION, TAG_VERSION, POLICY_AND_TAG_VERSION } + private void updatePolicyVersion(RangerService service, boolean isTagVersionUpdateNeeded) throws Exception { if(service == null || service.getId() == null) { return; } - boolean filterForServicePlugin = RangerConfiguration.getInstance().getBoolean(RangerTagDBRetriever.OPTION_RANGER_FILTER_TAGS_FOR_SERVICE_PLUGIN, false); - XXServiceDao serviceDao = daoMgr.getXXService(); - XXService serviceDbObj = serviceDao.getById(service.getId()); + final XXService serviceDbObj = serviceDao.getById(service.getId()); if(serviceDbObj == null) { LOG.warn("updatePolicyVersion(serviceId=" + service.getId() + "): service not found"); return; } - XXServiceVersionInfoDao serviceVersionInfoDao = daoMgr.getXXServiceVersionInfo(); - - XXServiceVersionInfo serviceVersionInfoDbObj = serviceVersionInfoDao.findByServiceId(service.getId()); + final RangerDaoManager daoManager = daoMgr; + final Long serviceId = serviceDbObj.getId(); + final VERSION_TYPE versionType = VERSION_TYPE.POLICY_VERSION; - if(serviceVersionInfoDbObj != null) { - serviceVersionInfoDbObj.setPolicyVersion(getNextVersion(serviceVersionInfoDbObj.getPolicyVersion())); - serviceVersionInfoDbObj.setPolicyUpdateTime(new Date()); - - serviceVersionInfoDao.update(serviceVersionInfoDbObj); - - } else { - LOG.warn("updatePolicyVersion(service=" + serviceDbObj.getName() + "): serviceVersionInfo not found, creating it.."); - - serviceVersionInfoDbObj = new XXServiceVersionInfo(); - serviceVersionInfoDbObj.setServiceId(serviceDbObj.getId()); - serviceVersionInfoDbObj.setPolicyVersion(getNextVersion(serviceDbObj.getPolicyVersion())); - serviceVersionInfoDbObj.setPolicyUpdateTime(new Date()); - serviceVersionInfoDbObj.setTagVersion(serviceDbObj.getTagVersion()); - serviceVersionInfoDbObj.setTagUpdateTime(serviceDbObj.getTagUpdateTime()); - - serviceVersionInfoDao.create(serviceVersionInfoDbObj); - } + Runnable serviceVersionUpdater = new ServiceVersionUpdater(daoManager, serviceId, versionType); + transactionSynchronizationAdapter.executeOnTransactionCommit(serviceVersionUpdater); // if this is a tag service, update all services that refer to this tag service // so that next policy-download from plugins will get updated tag policies boolean isTagService = serviceDbObj.getType() == EmbeddedServiceDefsUtil.instance().getTagServiceDefId(); if(isTagService) { - List referringServices = serviceDao.findByTagServiceId(serviceDbObj.getId()); + List referringServices = serviceDao.findByTagServiceId(serviceId); if(CollectionUtils.isNotEmpty(referringServices)) { for(XXService referringService : referringServices) { - serviceVersionInfoDbObj = serviceVersionInfoDao.findByServiceId(referringService.getId()); - if (serviceVersionInfoDbObj != null) { + final Long referringServiceId = referringService.getId(); + final VERSION_TYPE tagServiceversionType = VERSION_TYPE.POLICY_VERSION; - serviceVersionInfoDbObj.setPolicyVersion(getNextVersion(serviceVersionInfoDbObj.getPolicyVersion())); - serviceVersionInfoDbObj.setPolicyUpdateTime(new Date()); - - if (filterForServicePlugin && isTagVersionUpdateNeeded) { - serviceVersionInfoDbObj.setTagVersion(getNextVersion(serviceVersionInfoDbObj.getTagVersion())); - serviceVersionInfoDbObj.setTagUpdateTime(new Date()); - } - serviceVersionInfoDao.update(serviceVersionInfoDbObj); - } else { - LOG.warn("updatePolicyVersion(service=" + referringService.getName() + "): serviceVersionInfo not found, creating it.."); - serviceVersionInfoDbObj = new XXServiceVersionInfo(); - serviceVersionInfoDbObj.setServiceId(referringService.getId()); - serviceVersionInfoDbObj.setPolicyVersion(getNextVersion(referringService.getPolicyVersion())); - serviceVersionInfoDbObj.setPolicyUpdateTime(new Date()); - if (filterForServicePlugin && isTagVersionUpdateNeeded) { - serviceVersionInfoDbObj.setTagVersion(getNextVersion(referringService.getTagVersion())); - serviceVersionInfoDbObj.setTagUpdateTime(new Date()); - } else { - serviceVersionInfoDbObj.setTagVersion(referringService.getTagVersion()); - serviceVersionInfoDbObj.setTagUpdateTime(referringService.getTagUpdateTime()); - } - serviceVersionInfoDao.create(serviceVersionInfoDbObj); - } + Runnable tagServiceVersionUpdater = new ServiceVersionUpdater(daoManager, referringServiceId, tagServiceversionType); + transactionSynchronizationAdapter.executeOnTransactionCommit(tagServiceVersionUpdater); } } } } - private XXPolicyItem createNewPolicyItemForPolicy(RangerPolicy policy, XXPolicy xPolicy, RangerPolicyItem policyItem, XXServiceDef xServiceDef, int itemOrder, int policyItemType) throws Exception { - XXPolicyItem xPolicyItem = new XXPolicyItem(); - - xPolicyItem = (XXPolicyItem) rangerAuditFields.populateAuditFields(xPolicyItem, xPolicy); - - xPolicyItem.setDelegateAdmin(policyItem.getDelegateAdmin()); - xPolicyItem.setItemType(policyItemType); - xPolicyItem.setIsEnabled(Boolean.TRUE); - xPolicyItem.setComments(null); - xPolicyItem.setPolicyId(policy.getId()); - xPolicyItem.setOrder(itemOrder); - xPolicyItem = daoMgr.getXXPolicyItem().create(xPolicyItem); - - List accesses = policyItem.getAccesses(); - for (int i = 0; i < accesses.size(); i++) { - RangerPolicyItemAccess access = accesses.get(i); - - XXAccessTypeDef xAccTypeDef = daoMgr.getXXAccessTypeDef() - .findByNameAndServiceId(access.getType(), - xPolicy.getService()); - if (xAccTypeDef == null) { - throw new Exception(access.getType() + ": is not a valid access-type. policy='" + policy.getName() + "' service='" + policy.getService() + "'"); - } - - XXPolicyItemAccess xPolItemAcc = new XXPolicyItemAccess(); - - xPolItemAcc = (XXPolicyItemAccess) rangerAuditFields.populateAuditFields(xPolItemAcc, xPolicyItem); - xPolItemAcc.setIsAllowed(access.getIsAllowed()); - xPolItemAcc.setType(xAccTypeDef.getId()); - xPolItemAcc.setPolicyitemid(xPolicyItem.getId()); - xPolItemAcc.setOrder(i); - - daoMgr.getXXPolicyItemAccess().create(xPolItemAcc); - } - - List users = policyItem.getUsers(); - for(int i = 0; i < users.size(); i++) { - String user = users.get(i); - if (StringUtils.isBlank(user)) { - continue; - } - XXUser xUser = daoMgr.getXXUser().findByUserName(user); - if(xUser == null) { - throw new Exception(user + ": user does not exist. policy='"+ policy.getName() + "' service='"+ policy.getService() + "' user='" + user +"'"); - } - XXPolicyItemUserPerm xUserPerm = new XXPolicyItemUserPerm(); - xUserPerm = (XXPolicyItemUserPerm) rangerAuditFields.populateAuditFields(xUserPerm, xPolicyItem); - xUserPerm.setUserId(xUser.getId()); - xUserPerm.setPolicyItemId(xPolicyItem.getId()); - xUserPerm.setOrder(i); - xUserPerm = daoMgr.getXXPolicyItemUserPerm().create(xUserPerm); - } - - List groups = policyItem.getGroups(); - for(int i = 0; i < groups.size(); i++) { - String group = groups.get(i); - if (StringUtils.isBlank(group)) { - continue; - } - XXGroup xGrp = daoMgr.getXXGroup().findByGroupName(group); - if(xGrp == null) { - throw new Exception(group + ": group does not exist. policy='"+ policy.getName() + "' service='"+ policy.getService() + "' group='" + group + "'"); - } - XXPolicyItemGroupPerm xGrpPerm = new XXPolicyItemGroupPerm(); - xGrpPerm = (XXPolicyItemGroupPerm) rangerAuditFields.populateAuditFields(xGrpPerm, xPolicyItem); - xGrpPerm.setGroupId(xGrp.getId()); - xGrpPerm.setPolicyItemId(xPolicyItem.getId()); - xGrpPerm.setOrder(i); - xGrpPerm = daoMgr.getXXPolicyItemGroupPerm().create(xGrpPerm); - } - - List conditions = policyItem.getConditions(); - for(RangerPolicyItemCondition condition : conditions) { - XXPolicyConditionDef xPolCond = daoMgr - .getXXPolicyConditionDef().findByServiceDefIdAndName( - xServiceDef.getId(), condition.getType()); - - if(xPolCond == null) { - throw new Exception(condition.getType() + ": is not a valid condition-type. policy='"+ xPolicy.getName() + "' service='"+ xPolicy.getService() + "'"); - } - - for(int i = 0; i < condition.getValues().size(); i++) { - String value = condition.getValues().get(i); - XXPolicyItemCondition xPolItemCond = new XXPolicyItemCondition(); - xPolItemCond = (XXPolicyItemCondition) rangerAuditFields.populateAuditFields(xPolItemCond, xPolicyItem); - xPolItemCond.setPolicyItemId(xPolicyItem.getId()); - xPolItemCond.setType(xPolCond.getId()); - xPolItemCond.setValue(value); - xPolItemCond.setOrder(i); - - daoMgr.getXXPolicyItemCondition().create(xPolItemCond); - } - } - - return xPolicyItem; - } - - private void createNewPolicyItemsForPolicy(RangerPolicy policy, XXPolicy xPolicy, List policyItems, XXServiceDef xServiceDef, int policyItemType) throws Exception { - if(CollectionUtils.isNotEmpty(policyItems)) { - for (int itemOrder = 0; itemOrder < policyItems.size(); itemOrder++) { - RangerPolicyItem policyItem = policyItems.get(itemOrder); - createNewPolicyItemForPolicy(policy, xPolicy, policyItem, xServiceDef, itemOrder, policyItemType); - } - } - } - - private void createNewDataMaskPolicyItemsForPolicy(RangerPolicy policy, XXPolicy xPolicy, List policyItems, XXServiceDef xServiceDef, int policyItemType) throws Exception { - if(CollectionUtils.isNotEmpty(policyItems)) { - for (int itemOrder = 0; itemOrder < policyItems.size(); itemOrder++) { - RangerDataMaskPolicyItem policyItem = policyItems.get(itemOrder); - - XXPolicyItem xPolicyItem = createNewPolicyItemForPolicy(policy, xPolicy, policyItem, xServiceDef, itemOrder, policyItemType); - - RangerPolicyItemDataMaskInfo dataMaskInfo = policyItem.getDataMaskInfo(); - - if(dataMaskInfo != null) { - XXDataMaskTypeDef dataMaskDef = daoMgr.getXXDataMaskTypeDef().findByNameAndServiceId(dataMaskInfo.getDataMaskType(), xPolicy.getService()); - - if(dataMaskDef == null) { - throw new Exception(dataMaskInfo.getDataMaskType() + ": is not a valid datamask-type. policy='" + policy.getName() + "' service='" + policy.getService() + "'"); - } - - XXPolicyItemDataMaskInfo xxDataMaskInfo = new XXPolicyItemDataMaskInfo(); - - xxDataMaskInfo.setPolicyItemId(xPolicyItem.getId()); - xxDataMaskInfo.setType(dataMaskDef.getId()); - xxDataMaskInfo.setConditionExpr(dataMaskInfo.getConditionExpr()); - xxDataMaskInfo.setValueExpr(dataMaskInfo.getValueExpr()); - - daoMgr.getXXPolicyItemDataMaskInfo().create(xxDataMaskInfo); - } - } - } - } - - private void createNewRowFilterPolicyItemsForPolicy(RangerPolicy policy, XXPolicy xPolicy, List policyItems, XXServiceDef xServiceDef, int policyItemType) throws Exception { - if(CollectionUtils.isNotEmpty(policyItems)) { - for (int itemOrder = 0; itemOrder < policyItems.size(); itemOrder++) { - RangerRowFilterPolicyItem policyItem = policyItems.get(itemOrder); - - XXPolicyItem xPolicyItem = createNewPolicyItemForPolicy(policy, xPolicy, policyItem, xServiceDef, itemOrder, policyItemType); - - RangerPolicyItemRowFilterInfo dataMaskInfo = policyItem.getRowFilterInfo(); - - if(dataMaskInfo != null) { - XXPolicyItemRowFilterInfo xxRowFilterInfo = new XXPolicyItemRowFilterInfo(); - - xxRowFilterInfo.setPolicyItemId(xPolicyItem.getId()); - xxRowFilterInfo.setFilterExpr(dataMaskInfo.getFilterExpr()); - - xxRowFilterInfo = daoMgr.getXXPolicyItemRowFilterInfo().create(xxRowFilterInfo); - } - } - } - } + public static void persistVersionChange(RangerDaoManager daoMgr, Long id, VERSION_TYPE versionType) { + XXServiceVersionInfoDao serviceVersionInfoDao = daoMgr.getXXServiceVersionInfo(); - private void createNewResourcesForPolicy(RangerPolicy policy, XXPolicy xPolicy, Map resources) throws Exception { - - for (Entry resource : resources.entrySet()) { - RangerPolicyResource policyRes = resource.getValue(); - - XXResourceDef xResDef = daoMgr.getXXResourceDef() - .findByNameAndPolicyId(resource.getKey(), policy.getId()); - if (xResDef == null) { - throw new Exception(resource.getKey() + ": is not a valid resource-type. policy='"+ policy.getName() + "' service='"+ policy.getService() + "'"); - } - - XXPolicyResource xPolRes = new XXPolicyResource(); - xPolRes = (XXPolicyResource) rangerAuditFields.populateAuditFields(xPolRes, xPolicy); - - xPolRes.setIsExcludes(policyRes.getIsExcludes()); - xPolRes.setIsRecursive(policyRes.getIsRecursive()); - xPolRes.setPolicyId(policy.getId()); - xPolRes.setResDefId(xResDef.getId()); - xPolRes = daoMgr.getXXPolicyResource().create(xPolRes); - - List values = policyRes.getValues(); - if(CollectionUtils.isNotEmpty(values)){ - for(int i = 0; i < values.size(); i++) { - if(values.get(i)!=null){ - XXPolicyResourceMap xPolResMap = new XXPolicyResourceMap(); - xPolResMap = (XXPolicyResourceMap) rangerAuditFields.populateAuditFields(xPolResMap, xPolRes); - xPolResMap.setResourceId(xPolRes.getId()); - xPolResMap.setValue(values.get(i)); - xPolResMap.setOrder(i); - xPolResMap = daoMgr.getXXPolicyResourceMap().create(xPolResMap); - } - } - } - } - } + XXServiceVersionInfo serviceVersionInfoDbObj = serviceVersionInfoDao.findByServiceId(id); - private Boolean deleteExistingPolicyItems(RangerPolicy policy) { - if(policy == null) { - return false; - } - - XXPolicyItemDao policyItemDao = daoMgr.getXXPolicyItem(); - List policyItems = policyItemDao.findByPolicyId(policy.getId()); - for(XXPolicyItem policyItem : policyItems) { - Long polItemId = policyItem.getId(); - - XXPolicyItemConditionDao polCondDao = daoMgr.getXXPolicyItemCondition(); - List conditions = polCondDao.findByPolicyItemId(polItemId); - for(XXPolicyItemCondition condition : conditions) { - polCondDao.remove(condition); - } - - XXPolicyItemGroupPermDao grpPermDao = daoMgr.getXXPolicyItemGroupPerm(); - List groups = grpPermDao.findByPolicyItemId(polItemId); - for(XXPolicyItemGroupPerm group : groups) { - grpPermDao.remove(group); - } - - XXPolicyItemUserPermDao userPermDao = daoMgr.getXXPolicyItemUserPerm(); - List users = userPermDao.findByPolicyItemId(polItemId); - for(XXPolicyItemUserPerm user : users) { - userPermDao.remove(user); - } - - XXPolicyItemAccessDao polItemAccDao = daoMgr.getXXPolicyItemAccess(); - List accesses = polItemAccDao.findByPolicyItemId(polItemId); - for(XXPolicyItemAccess access : accesses) { - polItemAccDao.remove(access); + if(serviceVersionInfoDbObj != null) { + if (versionType == VERSION_TYPE.POLICY_VERSION || versionType == VERSION_TYPE.POLICY_AND_TAG_VERSION) { + serviceVersionInfoDbObj.setPolicyVersion(getNextVersion(serviceVersionInfoDbObj.getPolicyVersion())); + serviceVersionInfoDbObj.setPolicyUpdateTime(new Date()); } - - XXPolicyItemDataMaskInfoDao polItemDataMaskInfoDao = daoMgr.getXXPolicyItemDataMaskInfo(); - List dataMaskInfos = polItemDataMaskInfoDao.findByPolicyItemId(polItemId); - for(XXPolicyItemDataMaskInfo dataMaskInfo : dataMaskInfos) { - polItemDataMaskInfoDao.remove(dataMaskInfo); + if (versionType == VERSION_TYPE.TAG_VERSION || versionType == VERSION_TYPE.POLICY_AND_TAG_VERSION) { + serviceVersionInfoDbObj.setTagVersion(getNextVersion(serviceVersionInfoDbObj.getTagVersion())); + serviceVersionInfoDbObj.setTagUpdateTime(new Date()); } - XXPolicyItemRowFilterInfoDao polItemRowFilterInfoDao = daoMgr.getXXPolicyItemRowFilterInfo(); - List rowFilterInfos = polItemRowFilterInfoDao.findByPolicyItemId(polItemId); - for(XXPolicyItemRowFilterInfo rowFilterInfo : rowFilterInfos) { - polItemRowFilterInfoDao.remove(rowFilterInfo); - } + serviceVersionInfoDao.update(serviceVersionInfoDbObj); - policyItemDao.remove(policyItem); - } - return true; - } + } else { + XXService service = daoMgr.getXXService().getById(id); + if (service != null) { + serviceVersionInfoDbObj = new XXServiceVersionInfo(); + serviceVersionInfoDbObj.setServiceId(service.getId()); + serviceVersionInfoDbObj.setPolicyVersion(1L); + serviceVersionInfoDbObj.setPolicyUpdateTime(new Date()); + serviceVersionInfoDbObj.setTagVersion(1L); + serviceVersionInfoDbObj.setTagUpdateTime(new Date()); - private Boolean deleteExistingPolicyResources(RangerPolicy policy) { - if(policy == null) { - return false; - } - - List resources = daoMgr.getXXPolicyResource().findByPolicyId(policy.getId()); - - XXPolicyResourceDao resDao = daoMgr.getXXPolicyResource(); - for(XXPolicyResource resource : resources) { - List resMapList = daoMgr.getXXPolicyResourceMap().findByPolicyResId(resource.getId()); - - XXPolicyResourceMapDao resMapDao = daoMgr.getXXPolicyResourceMap(); - for(XXPolicyResourceMap resMap : resMapList) { - resMapDao.remove(resMap); + serviceVersionInfoDao.create(serviceVersionInfoDbObj); } - resDao.remove(resource); } - return true; } @Override @@ -3049,55 +3101,34 @@ protected void updateServicesForServiceDefUpdate(RangerServiceDef serviceDef) th return; } + final RangerDaoManager daoManager = daoMgr; + boolean isTagServiceDef = StringUtils.equals(serviceDef.getName(), EmbeddedServiceDefsUtil.EMBEDDED_SERVICEDEF_TAG_NAME); XXServiceDao serviceDao = daoMgr.getXXService(); - XXServiceVersionInfoDao serviceVersionInfoDao = daoMgr.getXXServiceVersionInfo(); List services = serviceDao.findByServiceDefId(serviceDef.getId()); if(CollectionUtils.isNotEmpty(services)) { for(XXService service : services) { - XXServiceVersionInfo serviceVersionInfo = serviceVersionInfoDao.findByServiceId(service.getId()); - if (serviceVersionInfo != null) { - serviceVersionInfo.setPolicyVersion(getNextVersion(serviceVersionInfo.getPolicyVersion())); - serviceVersionInfo.setPolicyUpdateTime(serviceDef.getUpdateTime()); - serviceVersionInfoDao.update(serviceVersionInfo); - } else { - LOG.warn("updateServicesForServiceDefUpdate(service=" + service.getName() + "): serviceVersionInfo not found, creating it.."); - serviceVersionInfo = new XXServiceVersionInfo(); - serviceVersionInfo.setServiceId(service.getId()); - serviceVersionInfo.setPolicyVersion(getNextVersion(service.getPolicyVersion())); - serviceVersionInfo.setTagVersion(service.getTagVersion()); - serviceVersionInfo.setPolicyUpdateTime(new Date()); - serviceVersionInfo.setTagUpdateTime(service.getTagUpdateTime()); + final Long serviceId = service.getId(); + final VERSION_TYPE versionType = VERSION_TYPE.POLICY_VERSION; - serviceVersionInfoDao.create(serviceVersionInfo); - } + Runnable serviceVersionUpdater = new ServiceVersionUpdater(daoManager, serviceId, versionType); + transactionSynchronizationAdapter.executeOnTransactionCommit(serviceVersionUpdater); if(isTagServiceDef) { - List referrringServices = serviceDao.findByTagServiceId(service.getId()); + List referringServices = serviceDao.findByTagServiceId(service.getId()); - if(CollectionUtils.isNotEmpty(referrringServices)) { - for(XXService referringService : referrringServices) { - serviceVersionInfo = serviceVersionInfoDao.findByServiceId(referringService.getId()); - if (serviceVersionInfo != null) { - serviceVersionInfo.setPolicyVersion(getNextVersion(serviceVersionInfo.getPolicyVersion())); - serviceVersionInfo.setPolicyUpdateTime(serviceDef.getUpdateTime()); + if(CollectionUtils.isNotEmpty(referringServices)) { + for(XXService referringService : referringServices) { - serviceVersionInfoDao.update(serviceVersionInfo); - } else { - LOG.warn("updateServicesForServiceDefUpdate(service=" + referringService.getName() + "): serviceVersionInfo not found, creating it.."); - serviceVersionInfo = new XXServiceVersionInfo(); - serviceVersionInfo.setServiceId(referringService.getId()); - serviceVersionInfo.setPolicyVersion(getNextVersion(referringService.getPolicyVersion())); - serviceVersionInfo.setTagVersion(referringService.getTagVersion()); - serviceVersionInfo.setPolicyUpdateTime(new Date()); - serviceVersionInfo.setTagUpdateTime(referringService.getTagUpdateTime()); - - serviceVersionInfoDao.create(serviceVersionInfo); - } + final Long referringServiceId = referringService.getId(); + final VERSION_TYPE tagServiceVersionType = VERSION_TYPE.POLICY_VERSION; + + Runnable tagServiceVersionUpdater = new ServiceVersionUpdater(daoManager, referringServiceId, tagServiceVersionType); + transactionSynchronizationAdapter.executeOnTransactionCommit(tagServiceVersionUpdater); } } } @@ -3583,10 +3614,6 @@ public Map createPolicyMap( private void writeBookForPolicyItems(RangerPolicy policy, RangerPolicyItem policyItem, RangerDataMaskPolicyItem dataMaskPolicyItem, RangerRowFilterPolicyItem rowFilterPolicyItem, Row row, String policyConditonType) { - if (LOG.isDebugEnabled()) { - // To avoid PMD violation - LOG.debug("policyConditonType:[" + policyConditonType + "]"); - } List groups = new ArrayList(); List users = new ArrayList(); String groupNames = ""; @@ -3638,12 +3665,16 @@ private void writeBookForPolicyItems(RangerPolicy policy, RangerPolicyItem polic if (CollectionUtils.isNotEmpty(groups)) { groupNames = groupNames + groups.toString(); StringTokenizer groupToken = new StringTokenizer(groupNames, "[]"); - groupNames = groupToken.nextToken().toString(); + while(groupToken.hasMoreTokens()) { + groupNames = groupToken.nextToken().toString(); + } } if (CollectionUtils.isNotEmpty(users)) { userNames = userNames + users.toString(); StringTokenizer userToken = new StringTokenizer(userNames, "[]"); - userNames = userToken.nextToken().toString(); + while(userToken.hasMoreTokens()) { + userNames = userToken.nextToken().toString(); + } } cell = row.createCell(3); cell.setCellValue(groupNames); @@ -3826,7 +3857,7 @@ private void updateServiceWithCustomProperty() { chkServiceUpdate = true; } if(chkServiceUpdate){ - updateService(rangerService); + updateService(rangerService, null); if(LOG.isDebugEnabled()){ LOG.debug("Updated service "+rangerService.getName()+" with custom properties in secure environment"); } @@ -3835,7 +3866,6 @@ private void updateServiceWithCustomProperty() { } } catch (Throwable e) { LOG.fatal("updateServiceWithCustomProperty failed with exception : "+e.getMessage()); - return; } } @@ -3867,4 +3897,35 @@ private void createGenericUsers() { genericUser.setDescription(RangerPolicyEngine.RESOURCE_OWNER); xUserService.createXUserWithOutLogin(genericUser); } + + public boolean isServiceAdminUser(String serviceName, String userName) { + boolean ret=false; + XXServiceConfigMap cfgSvcAdminUsers = daoMgr.getXXServiceConfigMap().findByServiceNameAndConfigKey(serviceName, SERVICE_ADMIN_USERS); + String svcAdminUsers = cfgSvcAdminUsers != null ? cfgSvcAdminUsers.getConfigvalue() : null; + if (svcAdminUsers != null) { + for (String svcAdminUser : svcAdminUsers.split(",")) { + if (userName.equals(svcAdminUser)) { + ret=true; + break; + } + } + } + return ret; + } + + public static class ServiceVersionUpdater implements Runnable { + final Long serviceId; + final RangerDaoManager daoManager; + final VERSION_TYPE versionType; + + public ServiceVersionUpdater(RangerDaoManager daoManager, Long serviceId, VERSION_TYPE versionType ) { + this.serviceId = serviceId; + this.daoManager = daoManager; + this.versionType = versionType; + } + @Override + public void run() { + ServiceDBStore.persistVersionChange(this.daoManager, this.serviceId, this.versionType); + } + } } diff --git a/security-admin/src/main/java/org/apache/ranger/biz/TagDBStore.java b/security-admin/src/main/java/org/apache/ranger/biz/TagDBStore.java index fa97bc96ad..793ddf4cf0 100644 --- a/security-admin/src/main/java/org/apache/ranger/biz/TagDBStore.java +++ b/security-admin/src/main/java/org/apache/ranger/biz/TagDBStore.java @@ -20,35 +20,27 @@ package org.apache.ranger.biz; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; import java.util.Map; import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.collections.MapUtils; import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.ranger.authorization.utils.JsonUtils; import org.apache.ranger.common.GUIDUtil; import org.apache.ranger.common.MessageEnums; import org.apache.ranger.common.RESTErrorUtil; +import org.apache.ranger.common.RangerAdminTagEnricher; import org.apache.ranger.common.RangerServiceTagsCache; import org.apache.ranger.db.RangerDaoManager; import org.apache.ranger.entity.XXDBBase; -import org.apache.ranger.entity.XXResourceDef; import org.apache.ranger.entity.XXService; -import org.apache.ranger.entity.XXServiceDef; import org.apache.ranger.entity.XXServiceResource; import org.apache.ranger.entity.XXServiceVersionInfo; import org.apache.ranger.entity.XXTag; -import org.apache.ranger.entity.XXTagAttribute; -import org.apache.ranger.entity.XXTagAttributeDef; -import org.apache.ranger.entity.XXServiceResourceElement; -import org.apache.ranger.entity.XXServiceResourceElementValue; import org.apache.ranger.entity.XXTagResourceMap; import org.apache.ranger.plugin.model.*; -import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyResource; -import org.apache.ranger.plugin.model.RangerTagDef.RangerTagAttributeDef; import org.apache.ranger.plugin.store.AbstractTagStore; import org.apache.ranger.plugin.store.PList; import org.apache.ranger.plugin.store.RangerServiceResourceSignature; @@ -61,8 +53,11 @@ import org.apache.ranger.service.RangerTagService; import org.apache.ranger.service.RangerServiceResourceService; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Component; +import org.springframework.transaction.PlatformTransactionManager; +import javax.annotation.PostConstruct; import javax.servlet.http.HttpServletResponse; @Component @@ -84,6 +79,10 @@ public class TagDBStore extends AbstractTagStore { @Autowired RangerDaoManager daoManager; + @Autowired + @Qualifier(value = "transactionManager") + PlatformTransactionManager txManager; + @Autowired RESTErrorUtil errorUtil; @@ -96,6 +95,11 @@ public class TagDBStore extends AbstractTagStore { @Autowired RESTErrorUtil restErrorUtil; + @PostConstruct + public void initStore() { + RangerAdminTagEnricher.setTagStore(this); + } + @Override public RangerTagDef createTagDef(RangerTagDef tagDef) throws Exception { if (LOG.isDebugEnabled()) { @@ -104,8 +108,6 @@ public RangerTagDef createTagDef(RangerTagDef tagDef) throws Exception { RangerTagDef ret = rangerTagDefService.create(tagDef); - createTagAttributeDefs(ret.getId(), tagDef.getAttributeDefs()); - ret = rangerTagDefService.read(ret.getId()); if (LOG.isDebugEnabled()) { @@ -125,6 +127,8 @@ public RangerTagDef updateTagDef(RangerTagDef tagDef) throws Exception { if (existing == null) { throw errorUtil.createRESTException("failed to update tag-def [" + tagDef.getName() + "], Reason: No TagDef found with id: [" + tagDef.getId() + "]", MessageEnums.DATA_NOT_UPDATABLE); + } else if (!existing.getName().equals(tagDef.getName())) { + throw errorUtil.createRESTException("Cannot change tag-def name; existing-name:[" + existing.getName() + "], new-name:[" + tagDef.getName() + "]", MessageEnums.DATA_NOT_UPDATABLE); } tagDef.setCreatedBy(existing.getCreatedBy()); @@ -134,10 +138,6 @@ public RangerTagDef updateTagDef(RangerTagDef tagDef) throws Exception { RangerTagDef ret = rangerTagDefService.update(tagDef); - // TODO: delete attributes might fail; so instead of delete+create, following should be updated to deal with only attributes that changed - deleteTagAttributeDefs(ret.getId()); - createTagAttributeDefs(ret.getId(), tagDef.getAttributeDefs()); - ret = rangerTagDefService.read(ret.getId()); if (LOG.isDebugEnabled()) { @@ -161,7 +161,6 @@ public void deleteTagDefByName(String name) throws Exception { LOG.debug("Deleting tag-def [name=" + name + "; id=" + tagDef.getId() + "]"); } - deleteTagAttributeDefs(tagDef.getId()); rangerTagDefService.delete(tagDef); } } @@ -181,7 +180,6 @@ public void deleteTagDef(Long id) throws Exception { RangerTagDef tagDef = rangerTagDefService.read(id); if(tagDef != null) { - deleteTagAttributeDefs(tagDef.getId()); rangerTagDefService.delete(tagDef); } } @@ -294,8 +292,6 @@ public RangerTag createTag(RangerTag tag) throws Exception { RangerTag ret = rangerTagService.create(tag); - createTagAttributes(ret.getId(), tag.getAttributes()); - ret = rangerTagService.read(ret.getId()); if (LOG.isDebugEnabled()) { @@ -324,9 +320,6 @@ public RangerTag updateTag(RangerTag tag) throws Exception { RangerTag ret = rangerTagService.update(tag); - deleteTagAttributes(existing.getId()); - createTagAttributes(existing.getId(), tag.getAttributes()); - ret = rangerTagService.read(ret.getId()); if (LOG.isDebugEnabled()) { @@ -344,8 +337,6 @@ public void deleteTag(Long id) throws Exception { RangerTag tag = rangerTagService.read(id); - deleteTagAttributes(id); - rangerTagService.delete(tag); if (LOG.isDebugEnabled()) { @@ -485,8 +476,6 @@ public RangerServiceResource createServiceResource(RangerServiceResource resourc RangerServiceResource ret = rangerServiceResourceService.create(resource); - createResourceForServiceResource(ret.getId(), resource); - ret = rangerServiceResourceService.read(ret.getId()); if (LOG.isDebugEnabled()) { @@ -514,9 +503,6 @@ public RangerServiceResource updateServiceResource(RangerServiceResource resourc resource.setResourceSignature(serializer.getSignature()); } - boolean serviceResourceElementUpdateNeeded = - !StringUtils.equals(existing.getResourceSignature(), resource.getResourceSignature()); - resource.setCreatedBy(existing.getCreatedBy()); resource.setCreateTime(existing.getCreateTime()); resource.setGuid(existing.getGuid()); @@ -524,11 +510,6 @@ public RangerServiceResource updateServiceResource(RangerServiceResource resourc rangerServiceResourceService.update(resource); - if (serviceResourceElementUpdateNeeded) { - deleteResourceForServiceResource(existing.getId()); - createResourceForServiceResource(existing.getId(), resource); - } - RangerServiceResource ret = rangerServiceResourceService.read(existing.getId()); if (LOG.isDebugEnabled()) { @@ -538,6 +519,24 @@ public RangerServiceResource updateServiceResource(RangerServiceResource resourc return ret; } + + @Override + public void refreshServiceResource(Long resourceId) throws Exception { + XXServiceResource serviceResourceEntity = daoManager.getXXServiceResource().getById(resourceId); + String tagsText = null; + + List tagResourceMaps = getTagResourceMapsForResourceId(resourceId); + if (tagResourceMaps != null) { + List associatedTags = new ArrayList<>(); + for (RangerTagResourceMap element : tagResourceMaps) { + associatedTags.add(getTag(element.getTagId())); + } + tagsText = JsonUtils.listToJson(associatedTags); + } + serviceResourceEntity.setTags(tagsText); + daoManager.getXXServiceResource().update(serviceResourceEntity); + } + @Override public void deleteServiceResource(Long id) throws Exception { if (LOG.isDebugEnabled()) { @@ -547,7 +546,6 @@ public void deleteServiceResource(Long id) throws Exception { RangerServiceResource resource = getServiceResource(id); if(resource != null) { - deleteResourceForServiceResource(resource.getId()); rangerServiceResourceService.delete(resource); } @@ -565,7 +563,6 @@ public void deleteServiceResourceByGuid(String guid) throws Exception { RangerServiceResource resource = getServiceResourceByGuid(guid); if(resource != null) { - deleteResourceForServiceResource(resource.getId()); rangerServiceResourceService.delete(resource); } @@ -706,6 +703,9 @@ public RangerTagResourceMap createTagResourceMap(RangerTagResourceMap tagResourc RangerTagResourceMap ret = rangerTagResourceMapService.create(tagResourceMap); + // We also need to update tags stored with the resource + refreshServiceResource(tagResourceMap.getResourceId()); + if (LOG.isDebugEnabled()) { LOG.debug("<== TagDBStore.createTagResourceMap(" + tagResourceMap + "): " + ret); } @@ -728,6 +728,8 @@ public void deleteTagResourceMap(Long id) throws Exception { if (tag.getOwner() == RangerTag.OWNER_SERVICERESOURCE) { deleteTag(tagId); } + // We also need to update tags stored with the resource + refreshServiceResource(tagResourceMap.getResourceId()); if (LOG.isDebugEnabled()) { LOG.debug("<== TagDBStore.deleteTagResourceMap(" + id + ")"); @@ -978,36 +980,12 @@ public ServiceTags getServiceTags(String serviceName) throws Exception { throw new Exception("service-def does not exist. id=" + xxService.getType()); } - RangerTagDBRetriever tagDBRetriever = new RangerTagDBRetriever(daoManager, xxService); + RangerTagDBRetriever tagDBRetriever = new RangerTagDBRetriever(daoManager, txManager, xxService); Map tagDefMap = tagDBRetriever.getTagDefs(); Map tagMap = tagDBRetriever.getTags(); List resources = tagDBRetriever.getServiceResources(); - List tagResourceMaps = tagDBRetriever.getTagResourceMaps(); - - Map> resourceToTagIds = new HashMap>(); - - if (CollectionUtils.isNotEmpty(tagResourceMaps)) { - Long resourceId = null; - List tagIds = null; - - for (RangerTagResourceMap tagResourceMap : tagResourceMaps) { - if (!tagResourceMap.getResourceId().equals(resourceId)) { - if (resourceId != null) { - resourceToTagIds.put(resourceId, tagIds); - } - - resourceId = tagResourceMap.getResourceId(); - tagIds = new ArrayList(); - } - - tagIds.add(tagResourceMap.getTagId()); - } - - if (resourceId != null) { - resourceToTagIds.put(resourceId, tagIds); - } - } + Map> resourceToTagIds = tagDBRetriever.getResourceToTagIds(); ret = new ServiceTags(); @@ -1026,159 +1004,6 @@ public ServiceTags getServiceTags(String serviceName) throws Exception { } - private List createTagAttributeDefs(Long tagDefId, List tagAttrDefList) { - if (LOG.isDebugEnabled()) { - LOG.debug("==> TagDBStore.createTagAttributeDefs(" + tagDefId + ", attributeDefCount=" + (tagAttrDefList == null ? 0 : tagAttrDefList.size()) + ")"); - } - - if (tagDefId == null) { - throw errorUtil.createRESTException("TagDBStore.createTagAttributeDefs(): Error creating tag-attr def. tagDefId can not be null.", MessageEnums.ERROR_CREATING_OBJECT); - } - - List ret = new ArrayList(); - - if (CollectionUtils.isNotEmpty(tagAttrDefList)) { - for (RangerTagDef.RangerTagAttributeDef attrDef : tagAttrDefList) { - XXTagAttributeDef xAttrDef = new XXTagAttributeDef(); - - xAttrDef.setTagDefId(tagDefId); - xAttrDef.setName(attrDef.getName()); - xAttrDef.setType(attrDef.getType()); - xAttrDef = (XXTagAttributeDef) rangerAuditFields.populateAuditFieldsForCreate(xAttrDef); - - xAttrDef = daoManager.getXXTagAttributeDef().create(xAttrDef); - - ret.add(xAttrDef); - } - } - - if (LOG.isDebugEnabled()) { - LOG.debug("<== TagDBStore.createTagAttributeDefs(" + tagDefId + ", attributeDefCount=" + (tagAttrDefList == null ? 0 : tagAttrDefList.size()) + "): retCount=" + ret.size()); - } - - return ret; - } - - private void deleteTagAttributeDefs(Long tagDefId) { - if (LOG.isDebugEnabled()) { - LOG.debug("==> TagDBStore.deleteTagAttributeDefs(" + tagDefId + ")"); - } - - if (tagDefId != null) { - List tagAttrDefList = daoManager.getXXTagAttributeDef().findByTagDefId(tagDefId); - - if (CollectionUtils.isNotEmpty(tagAttrDefList)) { - for (XXTagAttributeDef xAttrDef : tagAttrDefList) { - if (LOG.isDebugEnabled()) { - LOG.debug("Deleting tag-attribute def [name=" + xAttrDef.getName() + "; id=" + xAttrDef.getId() + "]"); - } - daoManager.getXXTagAttributeDef().remove(xAttrDef); - } - } - } - - if (LOG.isDebugEnabled()) { - LOG.debug("<== TagDBStore.deleteTagAttributeDefs(" + tagDefId + ")"); - } - } - - private List createTagAttributes(Long tagId, Map attributes) { - List ret = new ArrayList(); - - if(MapUtils.isNotEmpty(attributes)) { - for (Map.Entry attr : attributes.entrySet()) { - XXTagAttribute xTagAttr = new XXTagAttribute(); - - xTagAttr.setTagId(tagId); - xTagAttr.setName(attr.getKey()); - xTagAttr.setValue(attr.getValue()); - xTagAttr = (XXTagAttribute) rangerAuditFields.populateAuditFieldsForCreate(xTagAttr); - - xTagAttr = daoManager.getXXTagAttribute().create(xTagAttr); - - ret.add(xTagAttr); - } - } - - return ret; - } - - private void deleteTagAttributes(Long tagId) { - List tagAttrList = daoManager.getXXTagAttribute().findByTagId(tagId); - for (XXTagAttribute tagAttr : tagAttrList) { - daoManager.getXXTagAttribute().remove(tagAttr); - } - } - - private void deleteResourceForServiceResource(Long resourceId) { - List resElements = daoManager.getXXServiceResourceElement().findByResourceId(resourceId); - - if(CollectionUtils.isNotEmpty(resElements)) { - for(XXServiceResourceElement resElement : resElements) { - List elementValues = daoManager.getXXServiceResourceElementValue().findByResValueId(resElement.getId()); - - if(CollectionUtils.isNotEmpty(elementValues)) { - for(XXServiceResourceElementValue elementValue : elementValues) { - daoManager.getXXServiceResourceElementValue().remove(elementValue.getId()); - } - } - - daoManager.getXXServiceResourceElement().remove(resElement.getId()); - } - } - } - - private void createResourceForServiceResource(Long resourceId, RangerServiceResource serviceResource) { - String serviceName = serviceResource.getServiceName(); - - XXService xService = daoManager.getXXService().findByName(serviceName); - - if (xService == null) { - throw errorUtil.createRESTException("No Service found with name: " + serviceName, MessageEnums.ERROR_CREATING_OBJECT); - } - - XXServiceDef xServiceDef = daoManager.getXXServiceDef().getById(xService.getType()); - - if (xServiceDef == null) { - throw errorUtil.createRESTException("No Service-Def found with ID: " + xService.getType(), MessageEnums.ERROR_CREATING_OBJECT); - } - - Map resElements = serviceResource.getResourceElements(); - - for (Map.Entry resElement : resElements.entrySet()) { - XXResourceDef xResDef = daoManager.getXXResourceDef().findByNameAndServiceDefId(resElement.getKey(), xServiceDef.getId()); - - if (xResDef == null) { - LOG.error("TagDBStore.createResource: ResourceType is not valid [" + resElement.getKey() + "]"); - throw errorUtil.createRESTException("Resource Type is not valid [" + resElement.getKey() + "]", MessageEnums.DATA_NOT_FOUND); - } - - RangerPolicyResource policyRes = resElement.getValue(); - - XXServiceResourceElement resourceElement = new XXServiceResourceElement(); - resourceElement.setIsExcludes(policyRes.getIsExcludes()); - resourceElement.setIsRecursive(policyRes.getIsRecursive()); - resourceElement.setResDefId(xResDef.getId()); - resourceElement.setResourceId(resourceId); - - resourceElement = (XXServiceResourceElement) rangerAuditFields.populateAuditFieldsForCreate(resourceElement); - - resourceElement = daoManager.getXXServiceResourceElement().create(resourceElement); - - int sortOrder = 1; - for (String resVal : policyRes.getValues()) { - XXServiceResourceElementValue resourceElementValue = new XXServiceResourceElementValue(); - resourceElementValue.setResElementId(resourceElement.getId()); - resourceElementValue.setValue(resVal); - resourceElementValue.setSortOrder(sortOrder); - resourceElementValue = (XXServiceResourceElementValue) rangerAuditFields.populateAuditFieldsForCreate(resourceElementValue); - - resourceElementValue = daoManager.getXXServiceResourceElementValue().create(resourceElementValue); - sortOrder++; - } - } - } - @Override public void deleteAllTagObjectsForService(String serviceName) throws Exception { @@ -1191,8 +1016,6 @@ public void deleteAllTagObjectsForService(String serviceName) throws Exception { if (service != null) { Long serviceId = service.getId(); - List xxTagAttributes = daoManager.getXXTagAttribute().findByServiceIdAndOwner(serviceId, RangerTag.OWNER_SERVICERESOURCE); - List xxTags = daoManager.getXXTag().findByServiceIdAndOwner(serviceId, RangerTag.OWNER_SERVICERESOURCE); List xxTagResourceMaps = daoManager.getXXTagResourceMap().findByServiceId(serviceId); @@ -1208,17 +1031,6 @@ public void deleteAllTagObjectsForService(String serviceName) throws Exception { } } - if (CollectionUtils.isNotEmpty(xxTagAttributes)) { - for (XXTagAttribute xxTagAttribute : xxTagAttributes) { - try { - daoManager.getXXTagAttribute().remove(xxTagAttribute); - } catch (Exception e) { - LOG.error("Error deleting RangerTagAttribute with id=" + xxTagAttribute.getId(), e); - throw e; - } - } - } - if (CollectionUtils.isNotEmpty(xxTags)) { for (XXTag xxTag : xxTags) { try { @@ -1230,32 +1042,6 @@ public void deleteAllTagObjectsForService(String serviceName) throws Exception { } } - List xxServiceResourceElementValues = daoManager.getXXServiceResourceElementValue().findByServiceId(serviceId); - - if (CollectionUtils.isNotEmpty(xxServiceResourceElementValues)) { - for (XXServiceResourceElementValue xxServiceResourceElementValue : xxServiceResourceElementValues) { - try { - daoManager.getXXServiceResourceElementValue().remove(xxServiceResourceElementValue); - } catch (Exception e) { - LOG.error("Error deleting ServiceResourceElementValue with id=" + xxServiceResourceElementValue.getId(), e); - throw e; - } - } - } - - List xxServiceResourceElements = daoManager.getXXServiceResourceElement().findByServiceId(serviceId); - - if (CollectionUtils.isNotEmpty(xxServiceResourceElements)) { - for (XXServiceResourceElement xxServiceResourceElement : xxServiceResourceElements) { - try { - daoManager.getXXServiceResourceElement().remove(xxServiceResourceElement); - } catch (Exception e) { - LOG.error("Error deleting ServiceResourceElement with id=" + xxServiceResourceElement.getId(), e); - throw e; - } - } - } - List xxServiceResources = daoManager.getXXServiceResource().findByServiceId(serviceId); if (CollectionUtils.isNotEmpty(xxServiceResources)) { diff --git a/security-admin/src/main/java/org/apache/ranger/biz/UserMgr.java b/security-admin/src/main/java/org/apache/ranger/biz/UserMgr.java index be16f75864..f17c6d1cbe 100644 --- a/security-admin/src/main/java/org/apache/ranger/biz/UserMgr.java +++ b/security-admin/src/main/java/org/apache/ranger/biz/UserMgr.java @@ -142,6 +142,8 @@ public XXPortalUser createUser(VXPortalUser userProfile, int userStatus, Collection userRoleList) { XXPortalUser user = mapVXPortalUserToXXPortalUser(userProfile); checkAdminAccess(); + List userRolesList = new ArrayList(userRoleList); + xUserMgr.checkAccessRoles(userRolesList); user = createUser(user, userStatus, userRoleList); return user; @@ -175,7 +177,11 @@ public XXPortalUser createUser(VXPortalUser userProfile, int userStatus) { Collection reqRoleList = userProfile.getUserRoleList(); if (reqRoleList != null && reqRoleList.size() > 0) { for (String role : reqRoleList) { - roleList.add(role); + if (role != null) { + roleList.add(role); + } else { + roleList.add(RangerConstants.ROLE_USER); + } } } else { roleList.add(RangerConstants.ROLE_USER); @@ -395,7 +401,13 @@ public VXResponse changePassword(VXPasswordChange pwdChange) { logger.warn("SECURITY:changePassword(). User not found. LoginId="+ pwdChange.getLoginId()); throw restErrorUtil.createRESTException("serverMsg.userMgrInvalidUser",MessageEnums.DATA_NOT_FOUND, null, null,pwdChange.getLoginId()); } - + if (gjUser.getUserSource() == RangerCommonEnums.USER_EXTERNAL) { + logger.info("SECURITY:changePassword().Ranger External Users cannot change password. LoginId=" + pwdChange.getLoginId()); + VXResponse vXResponse = new VXResponse(); + vXResponse.setStatusCode(HttpServletResponse.SC_FORBIDDEN); + vXResponse.setMsgDesc("SECURITY:changePassword().Ranger External Users cannot change password. LoginId=" + pwdChange.getLoginId()); + throw restErrorUtil.generateRESTException(vXResponse); + } //check current password and provided old password is same or not String encryptedOldPwd = encrypt(pwdChange.getLoginId(),pwdChange.getOldPassword()); if (!stringUtil.equals(encryptedOldPwd, gjUser.getPassword())) { @@ -444,16 +456,13 @@ public VXPortalUser changeEmailAddress(XXPortalUser gjUser, VXPasswordChange changeEmail) { checkAccessForUpdate(gjUser); if (StringUtils.isEmpty(changeEmail.getEmailAddress())) { - throw restErrorUtil.createRESTException( - "serverMsg.userMgrInvalidEmail", - MessageEnums.INVALID_INPUT_DATA, changeEmail.getId(), - "emailAddress", changeEmail.toString()); + changeEmail.setEmailAddress(null); } String encryptedOldPwd = encrypt(gjUser.getLoginId(), changeEmail.getOldPassword()); - if (!stringUtil.validateEmail(changeEmail.getEmailAddress())) { + if (!StringUtils.isEmpty(changeEmail.getEmailAddress()) && !stringUtil.validateEmail(changeEmail.getEmailAddress())) { logger.info("Invalid email address." + changeEmail); throw restErrorUtil.createRESTException( "serverMsg.userMgrInvalidEmail", @@ -478,9 +487,12 @@ public VXPortalUser changeEmailAddress(XXPortalUser gjUser, String saltEncodedpasswd = encrypt(gjUser.getLoginId(), changeEmail.getOldPassword()); - + if (gjUser.getUserSource() == RangerCommonEnums.USER_APP) { gjUser.setPassword(saltEncodedpasswd); - + } + else if (gjUser.getUserSource() == RangerCommonEnums.USER_EXTERNAL) { + gjUser.setPassword(gjUser.getPassword()); + } daoManager.getXXPortalUser().update(gjUser); return mapXXPortalUserVXPortalUser(gjUser); } @@ -1109,6 +1121,8 @@ public VXPortalUser createDefaultAccountUser(VXPortalUser userProfile) { checkAdminAccess(); logger.info("create:" + userProfile.getLoginId()); XXPortalUser xXPortalUser = null; + Collection existingRoleList = null; + Collection reqRoleList = null; String loginId = userProfile.getLoginId(); String emailAddress = userProfile.getEmailAddress(); @@ -1143,13 +1157,59 @@ public VXPortalUser createDefaultAccountUser(VXPortalUser userProfile) { */ } } + VXPortalUser userProfileRes = null; if (xXPortalUser != null) { - return mapXXPortalUserToVXPortalUserForDefaultAccount(xXPortalUser); - } else { - return null; - } + userProfileRes = mapXXPortalUserToVXPortalUserForDefaultAccount(xXPortalUser); + if (userProfile.getUserRoleList() != null + && userProfile.getUserRoleList().size() > 0 + && ((List) userProfile.getUserRoleList()).get(0) != null) { + reqRoleList = userProfile.getUserRoleList(); + existingRoleList = this.getRolesByLoginId(loginId); + XXPortalUser xxPortalUser = daoManager.getXXPortalUser() + .findByLoginId(userProfile.getLoginId()); + if (xxPortalUser != null && xxPortalUser.getUserSource() == RangerCommonEnums.USER_EXTERNAL) { + userProfileRes = updateRoleForExternalUsers(reqRoleList, existingRoleList, userProfileRes); + } + } + } + return userProfileRes; } + protected VXPortalUser updateRoleForExternalUsers(Collection reqRoleList, Collection existingRoleList, VXPortalUser userProfileRes) { + UserSessionBase session = ContextUtil.getCurrentUserSession(); + if ("rangerusersync".equals(session.getXXPortalUser().getLoginId()) + && reqRoleList != null && !reqRoleList.isEmpty() + && existingRoleList != null && !existingRoleList.isEmpty()) { + if (!reqRoleList.equals(existingRoleList)) { + userProfileRes.setUserRoleList(reqRoleList); + userProfileRes.setUserSource(RangerCommonEnums.USER_EXTERNAL); + List xuserPermissionList = daoManager.getXXUserPermission().findByUserPermissionId(userProfileRes.getId()); + + if (xuserPermissionList!=null && xuserPermissionList.size()>0){ + + for (XXUserPermission xXUserPermission : xuserPermissionList) { + if (xXUserPermission != null) { + try { + xUserPermissionService.deleteResource(xXUserPermission.getId()); + } catch (Exception e) { + logger.error(e.getMessage()); + } + } + + } + } + updateUser(userProfileRes); + } + } else { + if (logger.isDebugEnabled()) { + logger.debug("Permission" + " denied. LoggedInUser=" + + (session != null ? session.getXXPortalUser().getId() : "") + + " isn't permitted to perform the action."); + } + } + return userProfileRes; + } + protected VXPortalUser mapXXPortalUserToVXPortalUserForDefaultAccount( XXPortalUser user) { @@ -1189,7 +1249,7 @@ public boolean isUserInRole(Long userId, String role) { public XXPortalUser updateUserWithPass(VXPortalUser userProfile) { String updatedPassword = userProfile.getPassword(); - XXPortalUser xXPortalUser = this.updateUser(userProfile); + XXPortalUser xXPortalUser = this.updateUser(userProfile); if (xXPortalUser == null) { return null; @@ -1210,8 +1270,13 @@ public XXPortalUser updateUserWithPass(VXPortalUser userProfile) { String encryptedNewPwd = encrypt(xXPortalUser.getLoginId(), updatedPassword); - xXPortalUser.setPassword(encryptedNewPwd); - xXPortalUser = daoManager.getXXPortalUser().update(xXPortalUser); + if (xXPortalUser.getUserSource() != RangerCommonEnums.USER_EXTERNAL) { + xXPortalUser.setPassword(encryptedNewPwd); + } + else if (xXPortalUser.getUserSource() != RangerCommonEnums.USER_EXTERNAL) { + xXPortalUser.setPassword(xXPortalUser.getPassword()); + } + xXPortalUser = daoManager.getXXPortalUser().update(xXPortalUser); } return xXPortalUser; } @@ -1229,7 +1294,13 @@ public XXPortalUser updatePasswordInSHA256(String userName,String userPassword,b } String dbOldPwd =xXPortalUser.getPassword(); String encryptedNewPwd = encrypt(xXPortalUser.getLoginId(),userPassword); - xXPortalUser.setPassword(encryptedNewPwd); + if (xXPortalUser.getUserSource() != RangerCommonEnums.USER_EXTERNAL) { + xXPortalUser.setPassword(encryptedNewPwd); + } + else if (xXPortalUser.getUserSource() != RangerCommonEnums.USER_EXTERNAL) { + xXPortalUser.setPassword(xXPortalUser.getPassword()); + } + xXPortalUser = daoManager.getXXPortalUser().update(xXPortalUser); if(xXPortalUser!=null && logAudits){ String dbNewPwd=xXPortalUser.getPassword(); @@ -1306,7 +1377,12 @@ public XXPortalUser updateOldUserName(String userLoginId,String newUserName, Str xXPortalUser.setLoginId(newUserName); // The old password needs to be encrypted by the new user name String updatedPwd = encrypt(newUserName,currentPassword); - xXPortalUser.setPassword(updatedPwd); + if (xXPortalUser.getUserSource() == RangerCommonEnums.USER_APP) { + xXPortalUser.setPassword(updatedPwd); + } + else if (xXPortalUser.getUserSource() == RangerCommonEnums.USER_EXTERNAL) { + xXPortalUser.setPassword(xXPortalUser.getPassword()); + } xXPortalUser = daoManager.getXXPortalUser().update(xXPortalUser); List trxLogList = new ArrayList(); XXTrxLog xTrxLog = new XXTrxLog(); diff --git a/security-admin/src/main/java/org/apache/ranger/biz/XUserMgr.java b/security-admin/src/main/java/org/apache/ranger/biz/XUserMgr.java index cd1de9ffdd..9d48531c34 100644 --- a/security-admin/src/main/java/org/apache/ranger/biz/XUserMgr.java +++ b/security-admin/src/main/java/org/apache/ranger/biz/XUserMgr.java @@ -23,11 +23,14 @@ import java.util.Collection; import java.util.HashMap; import java.util.HashSet; +import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang.StringUtils; import org.apache.ranger.common.ContextUtil; import org.apache.ranger.common.GUIDUtil; import org.apache.ranger.common.RangerCommonEnums; @@ -53,6 +56,7 @@ import org.apache.ranger.common.MessageEnums; import org.apache.ranger.common.PropertiesUtil; import org.apache.ranger.common.RangerConstants; +import org.apache.ranger.common.RangerServicePoliciesCache; import org.apache.ranger.common.SearchCriteria; import org.apache.ranger.common.UserSessionBase; import org.apache.ranger.db.RangerDaoManager; @@ -156,6 +160,9 @@ public class XUserMgr extends XUserMgrBase { @Autowired GUIDUtil guidUtil; + @Autowired + UserMgr userManager; + static final Logger logger = Logger.getLogger(XUserMgr.class); @@ -180,6 +187,7 @@ public VXGroup getGroupByGroupName(String groupName) { public VXUser createXUser(VXUser vXUser) { checkAdminAccess(); + validatePassword(vXUser); String userName = vXUser.getName(); if (userName == null || "null".equalsIgnoreCase(userName) || userName.trim().isEmpty()) { @@ -348,11 +356,15 @@ public VXUser updateXUser(VXUser vXUser) { checkAccess(vXUser.getName()); VXPortalUser oldUserProfile = userMgr.getUserProfileByLoginId(vXUser .getName()); + if (oldUserProfile == null) { + throw restErrorUtil.createRESTException( + "user " + vXUser.getName() + " does not exist.", + MessageEnums.INVALID_INPUT_DATA); + } VXPortalUser vXPortalUser = new VXPortalUser(); if (oldUserProfile != null && oldUserProfile.getId() != null) { vXPortalUser.setId(oldUserProfile.getId()); } - // TODO : There is a possibility that old user may not exist. vXPortalUser.setFirstName(vXUser.getFirstName()); if("null".equalsIgnoreCase(vXPortalUser.getFirstName())){ @@ -382,8 +394,14 @@ public VXUser updateXUser(VXUser vXUser) { && password.equals(hiddenPasswordString)) { vXPortalUser.setPassword(oldUserProfile.getPassword()); } - vXPortalUser.setPassword(password); - + else if(oldUserProfile != null && oldUserProfile.getUserSource() == RangerCommonEnums.USER_EXTERNAL && password != null){ + vXPortalUser.setPassword(oldUserProfile.getPassword()); + logger.debug("User is trrying to change external user password which we are not allowing it to change"); + } + else if(password != null){ + validatePassword(vXUser); + vXPortalUser.setPassword(password); + } Collection groupIdList = vXUser.getGroupIdList(); XXPortalUser xXPortalUser = new XXPortalUser(); xXPortalUser = userMgr.updateUserWithPass(vXPortalUser); @@ -435,7 +453,13 @@ public VXUser updateXUser(VXUser vXUser) { // There is nothing to log anything in XXUser so far. vXUser = xUserService.updateResource(vXUser); vXUser.setUserRoleList(roleList); + if (oldUserProfile.getUserSource() == RangerCommonEnums.USER_APP) { vXUser.setPassword(password); + } + else if (oldUserProfile.getUserSource() == RangerCommonEnums.USER_EXTERNAL) { + vXUser.setPassword(oldUserProfile.getPassword()); + } + List trxLogList = xUserService.getTransactionLog(vXUser, oldUserProfile, "update"); vXUser.setPassword(hiddenPasswordString); @@ -517,7 +541,13 @@ public VXUserGroupInfo createXUserGroupFromMap( VXUserGroupInfo vxUGInfo = new VXUserGroupInfo(); VXUser vXUser = vXUserGroupInfo.getXuserInfo(); - + VXPortalUser vXPortalUser = userMgr.getUserProfileByLoginId(vXUser.getName()); + XXPortalUser xxPortalUser = daoManager.getXXPortalUser().findByLoginId(vXUser.getName()); + Collection reqRoleList = vXUser.getUserRoleList(); + List existingRole = daoManager.getXXPortalUserRole().findXPortalUserRolebyXPortalUserId(xxPortalUser.getId()); + if (xxPortalUser.getUserSource() == RangerCommonEnums.USER_EXTERNAL) { + vXPortalUser = userManager.updateRoleForExternalUsers(reqRoleList,existingRole, vXPortalUser); + } vXUser = xUserService.createXUserWithOutLogin(vXUser); vxUGInfo.setXuserInfo(vXUser); @@ -533,9 +563,7 @@ public VXUserGroupInfo createXUserGroupFromMap( vXGroupUser = xGroupUserService .createXGroupUserWithOutLogin(vXGroupUser); } - VXPortalUser vXPortalUser = userMgr.getUserProfileByLoginId(vXUser - .getName()); - if(vXPortalUser!=null){ + if (vXPortalUser != null) { assignPermissionToUser(vXPortalUser, true); } vxUGInfo.setXgroupInfo(vxg); @@ -559,17 +587,40 @@ public VXGroupUserInfo createXGroupUserFromMap( List vxu = new ArrayList(); for (VXUser vXUser : vXGroupUserInfo.getXuserInfo()) { - XXUser xUser = daoManager.getXXUser().findByUserName(vXUser.getName()); + XXUser xUser = daoManager.getXXUser().findByUserName( + vXUser.getName()); + XXPortalUser xXPortalUser = daoManager.getXXPortalUser() + .findByLoginId(vXUser.getName()); if (xUser != null) { - // Add or update group user mapping only if the user already exists in x_user table. + // Add or update group user mapping only if the user already + // exists in x_user table. + logger.debug(String.format("createXGroupUserFromMap(): Create or update group %s ", vXGroup.getName())); vXGroup = xGroupService.createXGroupWithOutLogin(vXGroup); vxGUInfo.setXgroupInfo(vXGroup); vxu.add(vXUser); VXGroupUser vXGroupUser = new VXGroupUser(); vXGroupUser.setUserId(xUser.getId()); vXGroupUser.setName(vXGroup.getName()); - vXGroupUser = xGroupUserService - .createXGroupUserWithOutLogin(vXGroupUser); + if (xXPortalUser.getUserSource() == RangerCommonEnums.USER_EXTERNAL) { + vXGroupUser = xGroupUserService + .createXGroupUserWithOutLogin(vXGroupUser); + logger.debug(String.format("createXGroupUserFromMap(): Create or update group user mapping with groupname = " + vXGroup.getName() + + " username = %s userId = %d", xXPortalUser.getLoginId(), xUser.getId())); + } + Collection reqRoleList = vXUser.getUserRoleList(); + + XXPortalUser xxPortalUser = daoManager.getXXPortalUser() + .findByLoginId(vXUser.getName()); + List existingRole = daoManager.getXXPortalUserRole() + .findXPortalUserRolebyXPortalUserId( + xxPortalUser.getId()); + VXPortalUser vxPortalUser = userManager + .mapXXPortalUserToVXPortalUserForDefaultAccount(xxPortalUser); + if (xxPortalUser.getUserSource() == RangerCommonEnums.USER_EXTERNAL) { + vxPortalUser = userManager.updateRoleForExternalUsers( + reqRoleList, existingRole, vxPortalUser); + assignPermissionToUser(vxPortalUser, true); + } } } @@ -614,6 +665,7 @@ public VXGroupUserInfo getXGroupUserFromMap( public VXUser createXUserWithOutLogin(VXUser vXUser) { checkAdminAccess(); + validatePassword(vXUser); return xUserService.createXUserWithOutLogin(vXUser); } @@ -645,6 +697,12 @@ public VXGroupUser createXGroupUser(VXGroupUser vXGroupUser) { public VXUser getXUser(Long id) { VXUser vXUser=null; vXUser=xUserService.readResourceWithOutLogin(id); + if(vXUser != null){ + if(!hasAccessToGetUserInfo(vXUser)){ + logger.info("Logged-In user is not allowed to access requested user data."); + throw restErrorUtil.create403RESTException("Logged-In user is not allowed to access requested user data."); + } + } if(vXUser!=null && !hasAccessToModule(RangerConstants.MODULE_USER_GROUPS)){ vXUser=getMaskedVXUser(vXUser); } @@ -658,6 +716,27 @@ public VXGroupUser getXGroupUser(Long id) { public VXGroup getXGroup(Long id) { VXGroup vXGroup=null; + UserSessionBase userSession = ContextUtil.getCurrentUserSession(); + if (userSession != null && userSession.getLoginId() != null) { + VXUser loggedInVXUser = xUserService.getXUserByUserName(userSession + .getLoginId()); + if (loggedInVXUser != null) { + if (loggedInVXUser.getUserRoleList().size() == 1 + && loggedInVXUser.getUserRoleList().contains( + RangerConstants.ROLE_USER)) { + + List listGroupId = daoManager.getXXGroupUser() + .findGroupIdListByUserId(loggedInVXUser.getId()); + + if (!listGroupId.contains(id)) { + logger.info("Logged-In user is not allowed to access requested user data."); + throw restErrorUtil + .create403RESTException("Logged-In user is not allowed to access requested group data."); + } + + } + } + } vXGroup=xGroupService.readResourceWithOutLogin(id); if(vXGroup!=null && !hasAccessToModule(RangerConstants.MODULE_USER_GROUPS)){ vXGroup=getMaskedVXGroup(vXGroup); @@ -796,12 +875,33 @@ private void setUserDesc(VXUser vXUser) { public VXGroup updateXGroup(VXGroup vXGroup) { checkAdminAccess(); XXGroup xGroup = daoManager.getXXGroup().getById(vXGroup.getId()); + if (vXGroup != null && xGroup != null && !vXGroup.getName().equals(xGroup.getName())) { + throw restErrorUtil.createRESTException( + "group name updates are not allowed.", + MessageEnums.INVALID_INPUT_DATA); + } List trxLogList = xGroupService.getTransactionLog(vXGroup, xGroup, "update"); xaBizUtil.createTrxLog(trxLogList); vXGroup = (VXGroup) xGroupService.updateResource(vXGroup); + if (vXGroup != null) { + updateXgroupUserForGroupUpdate(vXGroup); + RangerServicePoliciesCache.sInstance=null; + } return vXGroup; } + + private void updateXgroupUserForGroupUpdate(VXGroup vXGroup) { + List grpUsers = daoManager.getXXGroupUser().findByGroupId(vXGroup.getId()); + if(CollectionUtils.isNotEmpty(grpUsers)){ + for (XXGroupUser grpUser : grpUsers) { + VXGroupUser vXGroupUser = xGroupUserService.populateViewBean(grpUser); + vXGroupUser.setName(vXGroup.getName()); + updateXGroupUser(vXGroupUser); + } + } + } + public VXGroupUser updateXGroupUser(VXGroupUser vXGroupUser) { checkAdminAccess(); return super.updateXGroupUser(vXGroupUser); @@ -1267,30 +1367,42 @@ private void populatePageList(List auditMapList, int startIndex, int public void checkAccessRoles(List stringRolesList) { UserSessionBase session = ContextUtil.getCurrentUserSession(); - if (session != null && stringRolesList!=null) { + if (session != null && stringRolesList != null) { if (!session.isUserAdmin() && !session.isKeyAdmin()) { throw restErrorUtil.create403RESTException("Permission" + " denied. LoggedInUser=" + (session != null ? session.getXXPortalUser().getId() : "Not Logged In") + " ,isn't permitted to perform the action."); - }else{ - if (session.isUserAdmin() && stringRolesList.contains(RangerConstants.ROLE_KEY_ADMIN)) { - throw restErrorUtil.create403RESTException("Permission" - + " denied. LoggedInUser=" - + (session != null ? session.getXXPortalUser().getId() - : "") - + " isn't permitted to perform the action."); - } - if (session.isKeyAdmin() && stringRolesList.contains(RangerConstants.ROLE_SYS_ADMIN)) { - throw restErrorUtil.create403RESTException("Permission" - + " denied. LoggedInUser=" - + (session != null ? session.getXXPortalUser().getId() - : "") - + " isn't permitted to perform the action."); + } else { + if (!"rangerusersync".equals(session.getXXPortalUser() + .getLoginId())) {// new logic for rangerusersync user + if (session.isUserAdmin() + && stringRolesList + .contains(RangerConstants.ROLE_KEY_ADMIN)) { + throw restErrorUtil.create403RESTException("Permission" + + " denied. LoggedInUser=" + + (session != null ? session.getXXPortalUser() + .getId() : "") + + " isn't permitted to perform the action."); + } + if (session.isKeyAdmin() + && stringRolesList + .contains(RangerConstants.ROLE_SYS_ADMIN)) { + throw restErrorUtil.create403RESTException("Permission" + + " denied. LoggedInUser=" + + (session != null ? session.getXXPortalUser() + .getId() : "") + + " isn't permitted to perform the action."); + } + } else { + logger.info("LoggedInUser=" + + (session != null ? session.getXXPortalUser() + .getId() : "") + + " is permitted to perform the action"); } } - }else{ + } else { VXResponse vXResponse = new VXResponse(); vXResponse.setStatusCode(HttpServletResponse.SC_UNAUTHORIZED); vXResponse.setMsgDesc("Bad Credentials"); @@ -1311,6 +1423,7 @@ public VXStringList setUserRolesByExternalID(Long userId, List vString if(vXUser!=null && roleListNewProfile.size()>0){ VXPortalUser oldUserProfile = userMgr.getUserProfileByLoginId(vXUser.getName()); if(oldUserProfile!=null){ + denySelfRoleChange(oldUserProfile.getLoginId()); updateUserRolesPermissions(oldUserProfile,roleListNewProfile); portalUserRoleList = daoManager.getXXPortalUserRole().findByUserId(oldUserProfile.getId()); return getStringListFromUserRoleList(portalUserRoleList); @@ -1333,6 +1446,7 @@ public VXStringList setUserRolesByName(String userName, List vStringRo if(userName!=null && roleListNewProfile.size()>0){ VXPortalUser oldUserProfile = userMgr.getUserProfileByLoginId(userName); if(oldUserProfile!=null){ + denySelfRoleChange(oldUserProfile.getLoginId()); updateUserRolesPermissions(oldUserProfile,roleListNewProfile); List portalUserRoleList = daoManager.getXXPortalUserRole().findByUserId(oldUserProfile.getId()); return getStringListFromUserRoleList(portalUserRoleList); @@ -1650,6 +1764,31 @@ public VXGroupList searchXGroups(SearchCriteria searchCriteria) { searchCriteria.setSortBy("id"); vXGroupList=xGroupService.searchXGroups(searchCriteria); } + UserSessionBase userSession = ContextUtil.getCurrentUserSession(); + if (userSession != null && userSession.getLoginId() != null) { + VXUser loggedInVXUser = xUserService.getXUserByUserName(userSession + .getLoginId()); + if (loggedInVXUser != null) { + if (loggedInVXUser.getUserRoleList().size() == 1 + && loggedInVXUser.getUserRoleList().contains( + RangerConstants.ROLE_USER)) { + + List updatedList = new ArrayList(); + + List listGroupId = daoManager.getXXGroupUser() + .findGroupIdListByUserId(loggedInVXUser.getId()); + + for (VXGroup group : vXGroupList.getList()) { + if (listGroupId.contains(group.getId())) { + updatedList.add(group); + } + } + logger.info("Logged-In user having user role will be able to fetch his own groups details."); + vXGroupList.setVXGroups(updatedList); + + } + } + } if(vXGroupList!=null && !hasAccessToModule(RangerConstants.MODULE_USER_GROUPS)){ if(vXGroupList!=null && vXGroupList.getListSize()>0){ List listMasked=new ArrayList(); @@ -1663,6 +1802,99 @@ public VXGroupList searchXGroups(SearchCriteria searchCriteria) { return vXGroupList; } + + public VXGroupList lookupXGroups(SearchCriteria searchCriteria) { + VXGroupList ret = null; + + try { + HashMap searchParams = searchCriteria.getParamList(); + String nameToLookFor = searchParams != null ? (String) searchParams.get("name") : null; + VXGroup exactMatch = null; + + if (StringUtils.isEmpty(searchCriteria.getSortBy())) { + searchCriteria.setSortBy(nameToLookFor != null ? "name" : "id"); + } + + if(nameToLookFor != null) { + exactMatch = getGroupByGroupName(nameToLookFor); + + for (Map.Entry entry : searchParams.entrySet()) { + if(exactMatch == null) { + break; + } + + String paramName = entry.getKey(); + Object paramValue = entry.getValue(); + + switch (paramName.toLowerCase()) { + case "isvisible": + if (!Objects.equals(exactMatch.getIsVisible(), paramValue)) { + exactMatch = null; + } + break; + + case "groupsource": + if (!Objects.equals(exactMatch.getGroupSource(), paramValue)) { + exactMatch = null; + } + break; + + default: + // ignore + break; + } + } + } + + VXGroupList searchResult = xGroupService.searchXGroups(searchCriteria); + + if (exactMatch != null && exactMatch.getId() != null) { + List groups = searchResult.getList(); + + if (!groups.isEmpty()) { // remove exactMatch from groups if it is present + boolean removed = false; + + for (Iterator iter = groups.iterator(); iter.hasNext(); ) { + VXGroup group = iter.next(); + + if (group != null && exactMatch.getId().equals(group.getId())) { + iter.remove(); + removed = true; + + break; + } + } + + if (!removed) { // remove the last entry, if exactMatch was not removed above - to accomodate for add() below + groups.remove(groups.size() - 1); + } + } + + groups.add(0, exactMatch); + + ret = new VXGroupList(groups); + + ret.setStartIndex(searchCriteria.getStartIndex()); + ret.setTotalCount(searchResult.getTotalCount()); + ret.setPageSize(searchCriteria.getMaxRows()); + ret.setSortBy(searchCriteria.getSortBy()); + ret.setSortType(searchCriteria.getSortType()); + } else { + ret = searchResult; + } + } catch (Exception e) { + logger.error("Error getting the exact match of group =>"+e); + } + + if (ret != null && ret.getListSize() > 0 && !hasAccessToModule(RangerConstants.MODULE_USER_GROUPS)) { + for(VXGroup vXGroup : ret.getList()) { + getMaskedVXGroup(vXGroup); + } + } + + return ret; + } + public Collection getMaskedCollection(Collection listunMasked){ List listMasked=new ArrayList(); if(listunMasked!=null) { @@ -1855,7 +2087,7 @@ public void deleteXGroup(Long id, boolean force) { } } - public void deleteXUser(Long id, boolean force) { + public synchronized void deleteXUser(Long id, boolean force) { checkAdminAccess(); XXUserDao xXUserDao = daoManager.getXXUser(); XXUser xXUser = xXUserDao.getById(id); @@ -2145,5 +2377,51 @@ public VXUser createServiceConfigUser(String userName){ } } return createdXUser; -} + } + private void validatePassword(VXUser vXUser) { + if (vXUser.getPassword() != null && !vXUser.getPassword().isEmpty()) { + boolean checkPassword = false; + String pattern = "(?=.*[0-9])(?=.*[a-zA-Z]).{8,}"; + checkPassword = vXUser.getPassword().trim().matches(pattern); + if (!checkPassword) { + logger.warn("validatePassword(). Password should be minimum 8 characters with min one alphabet and one numeric."); + throw restErrorUtil.createRESTException("serverMsg.xuserMgrValidatePassword", MessageEnums.INVALID_PASSWORD, null, "Password should be minimum 8 characters with min one alphabet and one numeric", null); + } + } else { + logger.warn("validatePassword(). Password cannot be blank/null."); + throw restErrorUtil.createRESTException("serverMsg.xuserMgrValidatePassword", MessageEnums.INVALID_PASSWORD, null, "Password cannot be blank/null", null); + } + } + + private boolean hasAccessToGetUserInfo(VXUser requestedVXUser) { + UserSessionBase userSession = ContextUtil.getCurrentUserSession(); + if (userSession != null && userSession.getLoginId() != null) { + VXUser loggedInVXUser = xUserService.getXUserByUserName(userSession + .getLoginId()); + if (loggedInVXUser != null) { + if (loggedInVXUser.getUserRoleList().size() == 1 + && loggedInVXUser.getUserRoleList().contains( + RangerConstants.ROLE_USER)) { + + return requestedVXUser.getId().equals(loggedInVXUser.getId()) ? true : false; + + }else{ + return true; + } + } + } + return false; + } + public void denySelfRoleChange(String userName) { + UserSessionBase session = ContextUtil.getCurrentUserSession(); + if (session != null && session.getXXPortalUser()!=null) { + if (userName.equals(session.getXXPortalUser().getLoginId())) { + throw restErrorUtil.create403RESTException("Permission" + + " denied. LoggedInUser=" + + (session != null ? session.getXXPortalUser().getId() + : "Not Logged In") + + " ,isn't permitted to change its own role."); + } + } + } } diff --git a/security-admin/src/main/java/org/apache/ranger/common/PropertiesUtil.java b/security-admin/src/main/java/org/apache/ranger/common/PropertiesUtil.java index 537d556a20..0dc5df8112 100644 --- a/security-admin/src/main/java/org/apache/ranger/common/PropertiesUtil.java +++ b/security-admin/src/main/java/org/apache/ranger/common/PropertiesUtil.java @@ -268,12 +268,18 @@ protected void processProperties( db_ssl_verifyServerCertificate="false"; } db_ssl_verifyServerCertificate=db_ssl_verifyServerCertificate.toLowerCase(); + String db_ssl_auth_type=propertiesMap.get("ranger.db.ssl.auth.type"); + if(StringUtils.isEmpty(db_ssl_auth_type)|| !"1-way".equalsIgnoreCase(db_ssl_auth_type)){ + db_ssl_auth_type="2-way"; + } propertiesMap.put("ranger.db.ssl.enabled", db_ssl_enabled); props.put("ranger.db.ssl.enabled", db_ssl_enabled); propertiesMap.put("ranger.db.ssl.required", db_ssl_required); props.put("ranger.db.ssl.required", db_ssl_required); propertiesMap.put("ranger.db.ssl.verifyServerCertificate", db_ssl_verifyServerCertificate); props.put("ranger.db.ssl.verifyServerCertificate", db_ssl_verifyServerCertificate); + propertiesMap.put("ranger.db.ssl.auth.type", db_ssl_auth_type); + props.put("ranger.db.ssl.auth.type", db_ssl_auth_type); String ranger_jpa_jdbc_url=propertiesMap.get("ranger.jpa.jdbc.url"); if(!StringUtils.isEmpty(ranger_jpa_jdbc_url)){ StringBuffer ranger_jpa_jdbc_url_ssl=new StringBuffer(ranger_jpa_jdbc_url); diff --git a/security-admin/src/main/java/org/apache/ranger/common/RangerAdminTagEnricher.java b/security-admin/src/main/java/org/apache/ranger/common/RangerAdminTagEnricher.java new file mode 100644 index 0000000000..f81184d138 --- /dev/null +++ b/security-admin/src/main/java/org/apache/ranger/common/RangerAdminTagEnricher.java @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.common; + +import org.apache.ranger.plugin.contextenricher.RangerTagEnricher; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.ranger.plugin.model.RangerService; +import org.apache.ranger.plugin.policyengine.RangerAccessRequest; +import org.apache.ranger.plugin.store.ServiceStore; +import org.apache.ranger.plugin.store.TagStore; +import org.apache.ranger.plugin.util.ServiceTags; + +public class RangerAdminTagEnricher extends RangerTagEnricher { + private static final Log LOG = LogFactory.getLog(RangerAdminTagEnricher.class); + + private static TagStore tagStore = null; + + private Long serviceId; + + public static void setTagStore(TagStore tagStore) { + RangerAdminTagEnricher.tagStore = tagStore; + } + + @Override + public void init() { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerAdminTagEnricher.init()"); + } + super.init(); + + ServiceStore svcStore = tagStore != null ? tagStore.getServiceStore() : null; + + if (tagStore == null || svcStore == null) { + LOG.error("ServiceDBStore/TagDBStore is not initialized!! Internal Error!"); + } else { + try { + RangerService service = svcStore.getServiceByName(serviceName); + serviceId = service.getId(); + } catch (Exception e) { + LOG.error("Cannot find service with name:[" + serviceName + "]", e); + LOG.error("This will cause tag-enricher in Ranger-Admin to fail!!"); + } + } + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerAdminTagEnricher.init()"); + } + } + + @Override + public void enrich(RangerAccessRequest request) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerAdminTagEnricher.enrich(" + request + ")"); + } + + refreshTagsIfNeeded(); + super.enrich(request); + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerAdminTagEnricher.enrich(" + request + ")"); + } + } + + private void refreshTagsIfNeeded() { + ServiceTags serviceTags = null; + try { + serviceTags = RangerServiceTagsCache.getInstance().getServiceTags(serviceName, serviceId, tagStore); + } catch (Exception e) { + LOG.error("Could not get cached service-tags, continue to use old ones..", e); + } + + if (serviceTags != null) { + Long enrichedServiceTagsVersion = getServiceTagsVersion(); + + if (enrichedServiceTagsVersion == null || !enrichedServiceTagsVersion.equals(serviceTags.getTagVersion())) { + synchronized(this) { + enrichedServiceTagsVersion = getServiceTagsVersion(); + + if (enrichedServiceTagsVersion == null || !enrichedServiceTagsVersion.equals(serviceTags.getTagVersion())) { + setServiceTags(serviceTags); + } + } + } + } + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("RangerAdminTagEnricher={serviceName=").append(serviceName).append(", "); + sb.append("serviceId=").append(serviceId).append("}"); + return sb.toString(); + } +} diff --git a/security-admin/src/main/java/org/apache/ranger/common/RangerServicePoliciesCache.java b/security-admin/src/main/java/org/apache/ranger/common/RangerServicePoliciesCache.java index 7d1f28c4a5..0d5689a034 100644 --- a/security-admin/src/main/java/org/apache/ranger/common/RangerServicePoliciesCache.java +++ b/security-admin/src/main/java/org/apache/ranger/common/RangerServicePoliciesCache.java @@ -42,7 +42,7 @@ public class RangerServicePoliciesCache { private static final int MAX_WAIT_TIME_FOR_UPDATE = 10; - private static volatile RangerServicePoliciesCache sInstance = null; + public static volatile RangerServicePoliciesCache sInstance = null; private final boolean useServicePoliciesCache; private final int waitTimeInSeconds; @@ -270,7 +270,7 @@ private void pruneUnusedPolicyAttributes(List policies) { policy.setCreateTime(null); policy.setUpdatedBy(null); policy.setUpdateTime(null); - policy.setGuid(null); + // policy.setGuid(null); /* this is used by import policy */ // policy.setName(null); /* this is used by GUI in policy list page */ // policy.setDescription(null); /* this is used by export policy */ policy.setResourceSignature(null); diff --git a/security-admin/src/main/java/org/apache/ranger/common/ServiceUtil.java b/security-admin/src/main/java/org/apache/ranger/common/ServiceUtil.java index 6864c5a54a..c3c39dc82f 100644 --- a/security-admin/src/main/java/org/apache/ranger/common/ServiceUtil.java +++ b/security-admin/src/main/java/org/apache/ranger/common/ServiceUtil.java @@ -1563,5 +1563,59 @@ private Integer getAssetType(RangerService service, String serviceName) { return assetType; } + + public List getMatchingPoliciesForResource(HttpServletRequest request, + List policyLists) { + List policies = new ArrayList(); + if (request != null) { + String resource = request.getParameter(SearchFilter.POL_RESOURCE); + String serviceType = request.getParameter(SearchFilter.SERVICE_TYPE); + if (!StringUtil.isEmpty(resource) && !StringUtil.isEmpty(serviceType)) { + List resourceList = null; + Map rangerPolicyResourceMap = null; + RangerPolicy.RangerPolicyResource rangerPolicyResource = null; + for (RangerPolicy rangerPolicy : policyLists) { + if (rangerPolicy != null) { + rangerPolicyResourceMap = rangerPolicy.getResources(); + if (rangerPolicyResourceMap != null) { + if (rangerPolicyResourceMap.containsKey("path")) { + rangerPolicyResource = rangerPolicyResourceMap.get("path"); + if (rangerPolicyResource != null) { + resourceList = rangerPolicyResource.getValues(); + if (CollectionUtils.isNotEmpty(resourceList) && resourceList.size() == 1) { + String resourcePath = resourceList.get(0); + if (!StringUtil.isEmpty(resourcePath)) { + if (resourcePath.equals(resource) + || resourcePath.startsWith(resource + "/")) { + policies.add(rangerPolicy); + } + } + } + } + } else if (rangerPolicyResourceMap.containsKey("database")) { + rangerPolicyResource = rangerPolicyResourceMap.get("database"); + if (rangerPolicyResource != null) { + resourceList = rangerPolicyResource.getValues(); + if (CollectionUtils.isNotEmpty(resourceList) && resourceList.size() == 1) { + String resourcePath = resourceList.get(0); + if (!StringUtil.isEmpty(resourcePath)) { + if (resourcePath.equals(resource)) { + policies.add(rangerPolicy); + } + } + } + } + } + } + } + } + policyLists.clear(); + if (CollectionUtils.isNotEmpty(policies)) { + policyLists.addAll(policies); + } + } + } + return policyLists; + } } diff --git a/security-admin/src/main/java/org/apache/ranger/common/db/BaseDao.java b/security-admin/src/main/java/org/apache/ranger/common/db/BaseDao.java index c2832ea931..51c0de56c2 100644 --- a/security-admin/src/main/java/org/apache/ranger/common/db/BaseDao.java +++ b/security-admin/src/main/java/org/apache/ranger/common/db/BaseDao.java @@ -250,6 +250,25 @@ public void updateUserIDReference(String paramName,long oldID) { } } + public boolean deletePolicyIDReference(String paramName,long oldID) { + Table table = tClass.getAnnotation(Table.class); + if(table != null) { + String tableName = table.name(); + String query = "delete from " + tableName + " where " +paramName+"=" + oldID; + if (logger.isDebugEnabled()) { + logger.debug("Delete Query:" + query); + } + int count=getEntityManager().createNativeQuery(query).executeUpdate(); + getEntityManager().flush(); + if(count>0){ + return true; + } + }else{ + logger.warn("Required annotation `Table` not found"); + } + return false; + } + public String getDBVersion(){ String dbVersion="Not Available"; String query ="SELECT 1"; diff --git a/security-admin/src/main/java/org/apache/ranger/common/db/RangerTransactionSynchronizationAdapter.java b/security-admin/src/main/java/org/apache/ranger/common/db/RangerTransactionSynchronizationAdapter.java index 2a62fb4087..536ca29e7e 100644 --- a/security-admin/src/main/java/org/apache/ranger/common/db/RangerTransactionSynchronizationAdapter.java +++ b/security-admin/src/main/java/org/apache/ranger/common/db/RangerTransactionSynchronizationAdapter.java @@ -22,6 +22,7 @@ import java.util.ArrayList; import java.util.List; +import org.apache.commons.collections.CollectionUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -46,6 +47,7 @@ public class RangerTransactionSynchronizationAdapter extends TransactionSynchron private static final Log LOG = LogFactory.getLog(RangerTransactionSynchronizationAdapter.class); private static final ThreadLocal> RUNNABLES = new ThreadLocal>(); + private static final ThreadLocal> RUNNABLES_AFTER_COMMIT = new ThreadLocal>(); public void executeOnTransactionCompletion(Runnable runnable) { if (LOG.isDebugEnabled()) { @@ -64,7 +66,7 @@ public void executeOnTransactionCompletion(Runnable runnable) { TransactionSynchronizationAdapter */ - if (!TransactionSynchronizationManager.isSynchronizationActive()) { + if (!registerSynchronization()) { LOG.info("Transaction synchronization is NOT ACTIVE. Executing right now runnable {" + runnable + "}"); runnable.run(); return; @@ -73,9 +75,36 @@ public void executeOnTransactionCompletion(Runnable runnable) { if (threadRunnables == null) { threadRunnables = new ArrayList(); RUNNABLES.set(threadRunnables); - // Register a new transaction synchronization for the current thread. - // TransactionSynchronizationManage will call afterCompletion() when current transaction completes. - TransactionSynchronizationManager.registerSynchronization(this); + } + threadRunnables.add(runnable); + } + + public void executeOnTransactionCommit(Runnable runnable) { + if (LOG.isDebugEnabled()) { + LOG.debug("Submitting new runnable {" + runnable + "} to run after transaction is committed"); + } + + /* + From TransactionSynchronizationManager documentation: + TransactionSynchronizationManager is a central helper that manages resources and transaction synchronizations per thread. + Resource management code should only register synchronizations when this manager is active, + which can be checked via isSynchronizationActive(); it should perform immediate resource cleanup else. + If transaction synchronization isn't active, there is either no current transaction, + or the transaction manager doesn't support transaction synchronization. + + Note: Synchronization is an Interface for transaction synchronization callbacks which is implemented by + TransactionSynchronizationAdapter + */ + + if (!registerSynchronization()) { + LOG.info("Transaction synchronization is NOT ACTIVE. Executing right now runnable {" + runnable + "}"); + runnable.run(); + return; + } + List threadRunnables = RUNNABLES_AFTER_COMMIT.get(); + if (threadRunnables == null) { + threadRunnables = new ArrayList(); + RUNNABLES_AFTER_COMMIT.set(threadRunnables); } threadRunnables.add(runnable); } @@ -83,48 +112,93 @@ public void executeOnTransactionCompletion(Runnable runnable) { @Override public void afterCompletion(int status) { if (LOG.isDebugEnabled()) { - LOG.debug("Transaction completed with status {" + (status == STATUS_COMMITTED ? "COMMITTED" : "ROLLED_BACK") + "}"); + LOG.debug("==> RangerTransactionSynchronizationAdapter.afterCompletion(status=" + (status == STATUS_COMMITTED ? "COMMITTED" : "ROLLED_BACK") + ")"); } - /* Thread runnables are expected to be executed only when the status is STATUS_ROLLED_BACK. Currently, executeOnTransactionCompletion() - * is called only for those changes that are going to be rolled-back by TransactionSynchronizationManager - such - * as when the operation returns HttpServletResponse.SC_NOT_MODIFIED status. - */ - //if (status == STATUS_ROLLED_BACK) { - final List threadRunnables = RUNNABLES.get(); - if (LOG.isDebugEnabled()) { - LOG.debug("Transaction completed, executing {" + threadRunnables.size() + "} runnables"); + + List allRunnables = null; + + if (status == STATUS_COMMITTED) { + final List postCommitRunnables = RUNNABLES_AFTER_COMMIT.get(); + if (CollectionUtils.isNotEmpty(postCommitRunnables)) { + allRunnables = postCommitRunnables; } - if (threadRunnables != null) { - try { - //Create new transaction - TransactionTemplate txTemplate = new TransactionTemplate(txManager); - txTemplate.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRES_NEW); - - txTemplate.execute(new TransactionCallback() { - public Object doInTransaction(TransactionStatus status) { - for (Runnable runnable : threadRunnables) { - if (LOG.isDebugEnabled()) { - LOG.debug("Executing runnable {" + runnable + "}"); - } - try { - runnable.run(); - } catch (RuntimeException e) { - LOG.error("Failed to execute runnable " + runnable, e); - break; - } - } + } - return null; - } - }); - } catch (Exception e) { - LOG.error("Failed to commit TransactionService transaction", e); - LOG.error("Ignoring..."); - } + final List postCompletionRunnables = RUNNABLES.get(); + + if (CollectionUtils.isNotEmpty(postCompletionRunnables)) { + if (allRunnables == null) { + allRunnables = postCompletionRunnables; + } else { + allRunnables.addAll(postCompletionRunnables); } + } - //} + runRunnables(allRunnables); + + RUNNABLES_AFTER_COMMIT.remove(); RUNNABLES.remove(); + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerTransactionSynchronizationAdapter.afterCompletion(status=" + (status == STATUS_COMMITTED ? "COMMITTED" : "ROLLED_BACK") + ")"); + } } + private boolean registerSynchronization() { + final boolean ret = TransactionSynchronizationManager.isSynchronizationActive(); + if (ret) { + List threadRunnablesOnCompletion = RUNNABLES.get(); + List threadRunnablesOnCommit = RUNNABLES_AFTER_COMMIT.get(); + if (threadRunnablesOnCompletion == null && threadRunnablesOnCommit == null) { + TransactionSynchronizationManager.registerSynchronization(this); + } + } + return ret; + } + + private void runRunnables(final List runnables) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerTransactionSynchronizationAdapter.runRunnables()"); + } + + if (runnables != null) { + if (LOG.isDebugEnabled()) { + LOG.debug("Executing {" + runnables.size() + "} runnables"); + } + try { + //Create new transaction + TransactionTemplate txTemplate = new TransactionTemplate(txManager); + txTemplate.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRES_NEW); + + txTemplate.execute(new TransactionCallback() { + public Object doInTransaction(TransactionStatus status) { + for (Runnable runnable : runnables) { + if (LOG.isDebugEnabled()) { + LOG.debug("Executing runnable {" + runnable + "}"); + } + try { + runnable.run(); + } catch (RuntimeException e) { + LOG.error("Failed to execute runnable " + runnable, e); + break; + } + } + + return null; + } + }); + } catch (Exception e) { + LOG.error("Failed to commit TransactionService transaction", e); + LOG.error("Ignoring..."); + } + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("No runnables to execute"); + } + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerTransactionSynchronizationAdapter.runRunnables()"); + } + } } \ No newline at end of file diff --git a/security-admin/src/main/java/org/apache/ranger/db/RangerDaoManager.java b/security-admin/src/main/java/org/apache/ranger/db/RangerDaoManager.java index e3b878b3ec..80cb4be158 100644 --- a/security-admin/src/main/java/org/apache/ranger/db/RangerDaoManager.java +++ b/security-admin/src/main/java/org/apache/ranger/db/RangerDaoManager.java @@ -28,6 +28,7 @@ import org.apache.ranger.common.RangerConstants; import org.apache.ranger.common.StringUtil; import org.apache.ranger.common.db.BaseDao; +import org.apache.ranger.common.db.RangerTransactionSynchronizationAdapter; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; @@ -44,6 +45,9 @@ public class RangerDaoManager extends RangerDaoManagerBase { @Autowired StringUtil stringUtil; + @Autowired + RangerTransactionSynchronizationAdapter transactionSynchronizationAdapter; + @Override public EntityManager getEntityManager() { return em; @@ -68,15 +72,19 @@ public StringUtil getStringUtil() { return stringUtil; } - /* - * (non-Javadoc) - */ - @Override - public BaseDao getDaoForClassType(int classType) { - if (classType == RangerConstants.CLASS_TYPE_NONE) { - return null; - } - return super.getDaoForClassType(classType); + /** + * (non-Javadoc) + */ + @Override + public BaseDao getDaoForClassType(int classType) { + if (classType == RangerConstants.CLASS_TYPE_NONE) { + return null; + } + return super.getDaoForClassType(classType); + } + + public RangerTransactionSynchronizationAdapter getRangerTransactionSynchronizationAdapter() { + return transactionSynchronizationAdapter; } } diff --git a/security-admin/src/main/java/org/apache/ranger/db/RangerDaoManagerBase.java b/security-admin/src/main/java/org/apache/ranger/db/RangerDaoManagerBase.java index d718441ee9..1886aae529 100644 --- a/security-admin/src/main/java/org/apache/ranger/db/RangerDaoManagerBase.java +++ b/security-admin/src/main/java/org/apache/ranger/db/RangerDaoManagerBase.java @@ -24,19 +24,13 @@ */ import javax.persistence.EntityManager; - import org.apache.log4j.Logger; import org.apache.ranger.common.AppConstants; -import org.apache.ranger.common.RESTErrorUtil; import org.apache.ranger.common.db.BaseDao; -import org.springframework.beans.factory.annotation.Autowired; - public abstract class RangerDaoManagerBase { private static final Logger logger = Logger.getLogger(RangerDaoManagerBase.class); - @Autowired - protected RESTErrorUtil restErrorUtil; abstract public EntityManager getEntityManager(); public RangerDaoManagerBase() { @@ -366,6 +360,24 @@ public BaseDao getDaoForClassName(String className) { if (className.equals("XXPluginInfo")) { return getXXPluginInfo(); } + if (className.equals("XXPolicyRefCondition")) { + return getXXPolicyRefCondition(); + } + if (className.equals("XXPolicyRefGroup")) { + return getXXPolicyRefGroup(); + } + if (className.equals("XXPolicyRefDataMaskType")) { + return getXXPolicyRefDataMaskType(); + } + if (className.equals("XXPolicyRefResource")) { + return getXXPolicyRefResource(); + } + if (className.equals("XXPolicyRefUser")) { + return getXXPolicyRefUser(); + } + if (className.equals("XXPolicyRefAccessType")) { + return getXXPolicyRefAccessType(); + } logger.error("No DaoManager found for className=" + className, new Throwable()); return null; } @@ -590,5 +602,29 @@ public XXServiceVersionInfoDao getXXServiceVersionInfo() { public XXPluginInfoDao getXXPluginInfo() { return new XXPluginInfoDao(this); } + + public XXPolicyRefConditionDao getXXPolicyRefCondition() { + return new XXPolicyRefConditionDao(this); + } + + public XXPolicyRefGroupDao getXXPolicyRefGroup() { + return new XXPolicyRefGroupDao(this); + } + + public XXPolicyRefDataMaskTypeDao getXXPolicyRefDataMaskType() { + return new XXPolicyRefDataMaskTypeDao(this); + } + + public XXPolicyRefResourceDao getXXPolicyRefResource() { + return new XXPolicyRefResourceDao(this); + } + + public XXPolicyRefUserDao getXXPolicyRefUser() { + return new XXPolicyRefUserDao(this); + } + + public XXPolicyRefAccessTypeDao getXXPolicyRefAccessType() { + return new XXPolicyRefAccessTypeDao(this); + } } diff --git a/security-admin/src/main/java/org/apache/ranger/db/XXAccessTypeDefDao.java b/security-admin/src/main/java/org/apache/ranger/db/XXAccessTypeDefDao.java index 8f3a506871..8db657f698 100644 --- a/security-admin/src/main/java/org/apache/ranger/db/XXAccessTypeDefDao.java +++ b/security-admin/src/main/java/org/apache/ranger/db/XXAccessTypeDefDao.java @@ -58,5 +58,4 @@ public XXAccessTypeDef findByNameAndServiceId(String name, Long serviceId) { return null; } } - } diff --git a/security-admin/src/main/java/org/apache/ranger/db/XXDataMaskTypeDefDao.java b/security-admin/src/main/java/org/apache/ranger/db/XXDataMaskTypeDefDao.java index f6e1aff0b4..7e43db497a 100644 --- a/security-admin/src/main/java/org/apache/ranger/db/XXDataMaskTypeDefDao.java +++ b/security-admin/src/main/java/org/apache/ranger/db/XXDataMaskTypeDefDao.java @@ -58,5 +58,4 @@ public XXDataMaskTypeDef findByNameAndServiceId(String name, Long serviceId) { return null; } } - } diff --git a/security-admin/src/main/java/org/apache/ranger/db/XXGroupDao.java b/security-admin/src/main/java/org/apache/ranger/db/XXGroupDao.java index 19e2e11f36..acc8700f73 100644 --- a/security-admin/src/main/java/org/apache/ranger/db/XXGroupDao.java +++ b/security-admin/src/main/java/org/apache/ranger/db/XXGroupDao.java @@ -23,8 +23,6 @@ import java.util.ArrayList; import java.util.List; -import javax.persistence.NoResultException; - import org.apache.ranger.common.db.BaseDao; import org.apache.ranger.entity.XXGroup; @@ -66,19 +64,5 @@ public XXGroup findByGroupName(String groupName) { } return null; } - - @SuppressWarnings("unchecked") - public List findByPolicyItemId(Long polItemId) { - if (polItemId == null) { - return null; - } - try { - return getEntityManager() - .createNamedQuery("XXGroup.findByPolicyItemId") - .setParameter("polItemId", polItemId).getResultList(); - } catch (NoResultException e) { - return null; - } - } } diff --git a/security-admin/src/main/java/org/apache/ranger/db/XXGroupUserDao.java b/security-admin/src/main/java/org/apache/ranger/db/XXGroupUserDao.java index 1a76d27db4..c8c105de47 100644 --- a/security-admin/src/main/java/org/apache/ranger/db/XXGroupUserDao.java +++ b/security-admin/src/main/java/org/apache/ranger/db/XXGroupUserDao.java @@ -27,6 +27,7 @@ import javax.persistence.NoResultException; +import org.apache.commons.lang.StringUtils; import org.apache.log4j.Logger; import org.apache.ranger.common.db.BaseDao; import org.apache.ranger.entity.XXGroupUser; @@ -114,4 +115,21 @@ public List findByGroupId(Long groupId) { } } + public XXGroupUser findByGroupNameAndUserId(String groupName, Long userId) { + if (StringUtils.isNotBlank(groupName) && userId != null) { + try { + return getEntityManager() + .createNamedQuery("XXGroupUser.findByGroupNameAndUserId", XXGroupUser.class) + .setParameter("userId", userId) + .setParameter("groupName", groupName) + .getSingleResult(); + } catch (NoResultException e) { + logger.debug(e.getMessage()); + } + } else { + logger.debug("userId and/or groupId not provided."); + return new XXGroupUser(); + } + return null; + } } diff --git a/security-admin/src/main/java/org/apache/ranger/db/XXPolicyConditionDefDao.java b/security-admin/src/main/java/org/apache/ranger/db/XXPolicyConditionDefDao.java index 21afcac891..e2048aed18 100644 --- a/security-admin/src/main/java/org/apache/ranger/db/XXPolicyConditionDefDao.java +++ b/security-admin/src/main/java/org/apache/ranger/db/XXPolicyConditionDefDao.java @@ -59,33 +59,5 @@ public XXPolicyConditionDef findByServiceDefIdAndName(Long serviceDefId, String return null; } } - - public List findByPolicyItemId(Long polItemId) { - if(polItemId == null) { - return new ArrayList(); - } - try { - return getEntityManager() - .createNamedQuery("XXPolicyConditionDef.findByPolicyItemId", tClass) - .setParameter("polItemId", polItemId).getResultList(); - } catch (NoResultException e) { - return new ArrayList(); - } - } - - public XXPolicyConditionDef findByPolicyItemIdAndName(Long polItemId, String name) { - if(polItemId == null || name == null) { - return null; - } - try { - return getEntityManager() - .createNamedQuery("XXPolicyConditionDef.findByPolicyItemIdAndName", tClass) - .setParameter("polItemId", polItemId) - .setParameter("name", name).getSingleResult(); - } catch (NoResultException e) { - return null; - } - } - } diff --git a/security-admin/src/main/java/org/apache/ranger/db/XXPolicyItemAccessDao.java b/security-admin/src/main/java/org/apache/ranger/db/XXPolicyItemAccessDao.java index de37e10cde..aeafc8ac1f 100644 --- a/security-admin/src/main/java/org/apache/ranger/db/XXPolicyItemAccessDao.java +++ b/security-admin/src/main/java/org/apache/ranger/db/XXPolicyItemAccessDao.java @@ -30,19 +30,6 @@ public class XXPolicyItemAccessDao extends BaseDao { public XXPolicyItemAccessDao(RangerDaoManagerBase daoManager) { super(daoManager); } - - public List findByPolicyItemId(Long polItemId) { - if(polItemId == null) { - return new ArrayList(); - } - try { - return getEntityManager() - .createNamedQuery("XXPolicyItemAccess.findByPolicyItemId", tClass) - .setParameter("polItemId", polItemId).getResultList(); - } catch (NoResultException e) { - return new ArrayList(); - } - } public List findByPolicyId(Long policyId) { if(policyId == null) { @@ -70,16 +57,4 @@ public List findByServiceId(Long serviceId) { } } - public List findByType(Long type) { - if (type == null) { - return new ArrayList(); - } - try { - return getEntityManager().createNamedQuery("XXPolicyItemAccess.findByType", tClass) - .setParameter("type", type).getResultList(); - } catch (NoResultException e) { - return new ArrayList(); - } - } - } diff --git a/security-admin/src/main/java/org/apache/ranger/db/XXPolicyItemConditionDao.java b/security-admin/src/main/java/org/apache/ranger/db/XXPolicyItemConditionDao.java index 11596ef4d1..4b974a7738 100644 --- a/security-admin/src/main/java/org/apache/ranger/db/XXPolicyItemConditionDao.java +++ b/security-admin/src/main/java/org/apache/ranger/db/XXPolicyItemConditionDao.java @@ -30,19 +30,6 @@ public class XXPolicyItemConditionDao extends BaseDao { public XXPolicyItemConditionDao(RangerDaoManagerBase daoManager) { super(daoManager); } - - public List findByPolicyItemId(Long polItemId) { - if(polItemId == null) { - return new ArrayList(); - } - try { - return getEntityManager() - .createNamedQuery("XXPolicyItemCondition.findByPolicyItemId", tClass) - .setParameter("polItemId", polItemId).getResultList(); - } catch (NoResultException e) { - return new ArrayList(); - } - } public List findByPolicyId(Long policyId) { if(policyId == null) { @@ -70,31 +57,4 @@ public List findByServiceId(Long serviceId) { } } - public List findByPolicyItemAndDefId(Long polItemId, - Long polCondDefId) { - if(polItemId == null || polCondDefId == null) { - return new ArrayList(); - } - try { - return getEntityManager() - .createNamedQuery("XXPolicyItemCondition.findByPolicyItemAndDefId", tClass) - .setParameter("polItemId", polItemId) - .setParameter("polCondDefId", polCondDefId).getResultList(); - } catch (NoResultException e) { - return new ArrayList(); - } - } - - public List findByPolicyConditionDefId(Long polCondDefId) { - if (polCondDefId == null) { - return new ArrayList(); - } - try { - return getEntityManager().createNamedQuery("XXPolicyItemCondition.findByPolicyConditionDefId", tClass) - .setParameter("polCondDefId", polCondDefId).getResultList(); - } catch (NoResultException e) { - return new ArrayList(); - } - } - } diff --git a/security-admin/src/main/java/org/apache/ranger/db/XXPolicyItemDataMaskInfoDao.java b/security-admin/src/main/java/org/apache/ranger/db/XXPolicyItemDataMaskInfoDao.java index a8418c6261..7deda623a1 100644 --- a/security-admin/src/main/java/org/apache/ranger/db/XXPolicyItemDataMaskInfoDao.java +++ b/security-admin/src/main/java/org/apache/ranger/db/XXPolicyItemDataMaskInfoDao.java @@ -30,19 +30,6 @@ public class XXPolicyItemDataMaskInfoDao extends BaseDao findByPolicyItemId(Long polItemId) { - if(polItemId == null) { - return new ArrayList(); - } - try { - return getEntityManager() - .createNamedQuery("XXPolicyItemDataMaskInfo.findByPolicyItemId", tClass) - .setParameter("polItemId", polItemId).getResultList(); - } catch (NoResultException e) { - return new ArrayList(); - } - } public List findByPolicyId(Long policyId) { if(policyId == null) { @@ -70,16 +57,4 @@ public List findByServiceId(Long serviceId) { } } - public List findByType(Long type) { - if (type == null) { - return new ArrayList(); - } - try { - return getEntityManager().createNamedQuery("XXPolicyItemDataMaskInfo.findByType", tClass) - .setParameter("type", type).getResultList(); - } catch (NoResultException e) { - return new ArrayList(); - } - } - } diff --git a/security-admin/src/main/java/org/apache/ranger/db/XXPolicyItemGroupPermDao.java b/security-admin/src/main/java/org/apache/ranger/db/XXPolicyItemGroupPermDao.java index 8c05699b25..aa4a3497ed 100644 --- a/security-admin/src/main/java/org/apache/ranger/db/XXPolicyItemGroupPermDao.java +++ b/security-admin/src/main/java/org/apache/ranger/db/XXPolicyItemGroupPermDao.java @@ -31,19 +31,6 @@ public XXPolicyItemGroupPermDao(RangerDaoManagerBase daoManager) { super(daoManager); } - public List findByPolicyItemId(Long polItemId) { - if(polItemId == null) { - return new ArrayList(); - } - try { - return getEntityManager() - .createNamedQuery("XXPolicyItemGroupPerm.findByPolicyItemId", tClass) - .setParameter("polItemId", polItemId).getResultList(); - } catch (NoResultException e) { - return new ArrayList(); - } - } - public List findByPolicyId(Long policyId) { if(policyId == null) { return new ArrayList(); diff --git a/security-admin/src/main/java/org/apache/ranger/db/XXPolicyItemRowFilterInfoDao.java b/security-admin/src/main/java/org/apache/ranger/db/XXPolicyItemRowFilterInfoDao.java index 4618e7dc76..ff889e7a33 100644 --- a/security-admin/src/main/java/org/apache/ranger/db/XXPolicyItemRowFilterInfoDao.java +++ b/security-admin/src/main/java/org/apache/ranger/db/XXPolicyItemRowFilterInfoDao.java @@ -29,19 +29,6 @@ public class XXPolicyItemRowFilterInfoDao extends BaseDao findByPolicyItemId(Long polItemId) { - if(polItemId == null) { - return new ArrayList(); - } - try { - return getEntityManager() - .createNamedQuery("XXPolicyItemRowFilterInfo.findByPolicyItemId", tClass) - .setParameter("polItemId", polItemId).getResultList(); - } catch (NoResultException e) { - return new ArrayList(); - } - } public List findByPolicyId(Long policyId) { if(policyId == null) { diff --git a/security-admin/src/main/java/org/apache/ranger/db/XXPolicyItemUserPermDao.java b/security-admin/src/main/java/org/apache/ranger/db/XXPolicyItemUserPermDao.java index 40a0da13b2..66f156630c 100644 --- a/security-admin/src/main/java/org/apache/ranger/db/XXPolicyItemUserPermDao.java +++ b/security-admin/src/main/java/org/apache/ranger/db/XXPolicyItemUserPermDao.java @@ -31,19 +31,6 @@ public XXPolicyItemUserPermDao(RangerDaoManagerBase daoManager) { super(daoManager); } - public List findByPolicyItemId(Long polItemId) { - if(polItemId == null) { - return new ArrayList(); - } - try { - return getEntityManager() - .createNamedQuery("XXPolicyItemUserPerm.findByPolicyItemId", tClass) - .setParameter("polItemId", polItemId).getResultList(); - } catch (NoResultException e) { - return new ArrayList(); - } - } - public List findByPolicyId(Long policyId) { if(policyId == null) { return new ArrayList(); diff --git a/security-admin/src/main/java/org/apache/ranger/db/XXPolicyRefAccessTypeDao.java b/security-admin/src/main/java/org/apache/ranger/db/XXPolicyRefAccessTypeDao.java new file mode 100644 index 0000000000..1ef01bb28c --- /dev/null +++ b/security-admin/src/main/java/org/apache/ranger/db/XXPolicyRefAccessTypeDao.java @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.db; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import javax.persistence.NoResultException; + +import org.apache.ranger.biz.RangerPolicyRetriever; +import org.apache.ranger.common.db.BaseDao; +import org.apache.ranger.entity.XXPolicyRefAccessType; +import org.springframework.stereotype.Service; + +@Service +public class XXPolicyRefAccessTypeDao extends BaseDao { + + public XXPolicyRefAccessTypeDao(RangerDaoManagerBase daoManager) { + super(daoManager); + } + + public List findByPolicyId(Long polId) { + if(polId == null) { + return Collections.EMPTY_LIST; + } + try { + return getEntityManager() + .createNamedQuery("XXPolicyRefAccessType.findByPolicyId", tClass) + .setParameter("policyId", polId).getResultList(); + } catch (NoResultException e) { + return Collections.EMPTY_LIST; + } + } + + public List findByAccessTypeDefId(Long accessTypeDefId) { + if (accessTypeDefId == null) { + return Collections.EMPTY_LIST; + } + try { + return getEntityManager().createNamedQuery("XXPolicyRefAccessType.findByAccessTypeDefId", tClass) + .setParameter("accessDefId", accessTypeDefId) + .getResultList(); + } catch (NoResultException e) { + return Collections.EMPTY_LIST; + } + } + + @SuppressWarnings("unchecked") + public List findUpdatedAccessNamesByPolicy(Long policyId) { + List ret = new ArrayList<>(); + if (policyId != null) { + List rows = (List) getEntityManager() + .createNamedQuery("XXPolicyRefAccessType.findUpdatedAccessNamesByPolicy") + .setParameter("policy", policyId) + .getResultList(); + if (rows != null) { + for (Object[] row : rows) { + ret.add(new RangerPolicyRetriever.PolicyTextNameMap((Long)row[0], (String)row[1], (String)row[2])); + } + } + } + return ret; + } + + @SuppressWarnings("unchecked") + public List findUpdatedAccessNamesByService(Long serviceId) { + List ret = new ArrayList<>(); + if (serviceId != null) { + List rows = (List) getEntityManager() + .createNamedQuery("XXPolicyRefAccessType.findUpdatedAccessNamesByService") + .setParameter("service", serviceId) + .getResultList(); + if (rows != null) { + for (Object[] row : rows) { + ret.add(new RangerPolicyRetriever.PolicyTextNameMap((Long)row[0], (String)row[1], (String)row[2])); + } + } + } + return ret; + } + +} diff --git a/security-admin/src/main/java/org/apache/ranger/db/XXPolicyRefConditionDao.java b/security-admin/src/main/java/org/apache/ranger/db/XXPolicyRefConditionDao.java new file mode 100644 index 0000000000..2c04ab4834 --- /dev/null +++ b/security-admin/src/main/java/org/apache/ranger/db/XXPolicyRefConditionDao.java @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.db; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import javax.persistence.NoResultException; + +import org.apache.ranger.biz.RangerPolicyRetriever; +import org.apache.ranger.common.db.BaseDao; +import org.apache.ranger.entity.XXPolicyRefCondition; +import org.springframework.stereotype.Service; + +@Service +public class XXPolicyRefConditionDao extends BaseDao { + + public XXPolicyRefConditionDao(RangerDaoManagerBase daoManager) { + super(daoManager); + } + + public List findByPolicyId(Long polId) { + if(polId == null) { + return Collections.EMPTY_LIST; + } + try { + return getEntityManager() + .createNamedQuery("XXPolicyRefCondition.findByPolicyId", tClass) + .setParameter("policyId", polId).getResultList(); + } catch (NoResultException e) { + return Collections.EMPTY_LIST; + } + } + public List findByConditionName(String conditionName) { + if (conditionName == null) { + return Collections.EMPTY_LIST; + } + try { + return getEntityManager().createNamedQuery("XXPolicyRefCondition.findByConditionName", tClass) + .setParameter("conditionName", conditionName).getResultList(); + } catch (NoResultException e) { + return Collections.EMPTY_LIST; + } + } + + public List findByConditionDefId(Long conditionDefId) { + if (conditionDefId == null) { + return Collections.EMPTY_LIST; + } + try { + return getEntityManager().createNamedQuery("XXPolicyRefCondition.findByConditionDefId", tClass) + .setParameter("conditionDefId", conditionDefId) + .getResultList(); + } catch (NoResultException e) { + return Collections.EMPTY_LIST; + } + } + + @SuppressWarnings("unchecked") + public List findUpdatedConditionNamesByPolicy(Long policyId) { + List ret = new ArrayList<>(); + if (policyId != null) { + List rows = (List) getEntityManager() + .createNamedQuery("XXPolicyRefCondition.findUpdatedConditionNamesByPolicy") + .setParameter("policy", policyId) + .getResultList(); + if (rows != null) { + for (Object[] row : rows) { + ret.add(new RangerPolicyRetriever.PolicyTextNameMap((Long)row[0], (String)row[1], (String)row[2])); + } + } + } + return ret; + } + + @SuppressWarnings("unchecked") + public List findUpdatedConditionNamesByService(Long serviceId) { + List ret = new ArrayList<>(); + if (serviceId != null) { + List rows = (List) getEntityManager() + .createNamedQuery("XXPolicyRefCondition.findUpdatedConditionNamesByService") + .setParameter("service", serviceId) + .getResultList(); + if (rows != null) { + for (Object[] row : rows) { + ret.add(new RangerPolicyRetriever.PolicyTextNameMap((Long)row[0], (String)row[1], (String)row[2])); + } + } + } + return ret; + } + +} diff --git a/security-admin/src/main/java/org/apache/ranger/db/XXPolicyRefDataMaskTypeDao.java b/security-admin/src/main/java/org/apache/ranger/db/XXPolicyRefDataMaskTypeDao.java new file mode 100644 index 0000000000..258e3b0bab --- /dev/null +++ b/security-admin/src/main/java/org/apache/ranger/db/XXPolicyRefDataMaskTypeDao.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.db; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import javax.persistence.NoResultException; + +import org.apache.ranger.biz.RangerPolicyRetriever; +import org.apache.ranger.common.db.BaseDao; +import org.apache.ranger.entity.XXPolicyRefDataMaskType; +import org.springframework.stereotype.Service; + +@Service +public class XXPolicyRefDataMaskTypeDao extends BaseDao{ + + public XXPolicyRefDataMaskTypeDao(RangerDaoManagerBase daoManager) { + super(daoManager); + } + + public List findByPolicyId(Long policyId) { + if(policyId == null) { + return Collections.EMPTY_LIST; + } + try { + return getEntityManager() + .createNamedQuery("XXPolicyRefDataMaskType.findByPolicyId", tClass) + .setParameter("policyId", policyId).getResultList(); + } catch (NoResultException e) { + return Collections.EMPTY_LIST; + } + } + + @SuppressWarnings("unchecked") + public List findUpdatedDataMaskNamesByPolicy(Long policyId) { + List ret = new ArrayList<>(); + if (policyId != null) { + List rows = (List) getEntityManager() + .createNamedQuery("XXPolicyRefDataMaskType.findUpdatedDataMaskNamesByPolicy") + .setParameter("policy", policyId) + .getResultList(); + if (rows != null) { + for (Object[] row : rows) { + ret.add(new RangerPolicyRetriever.PolicyTextNameMap((Long)row[0], (String)row[1], (String)row[2])); + } + } + } + return ret; + } + + @SuppressWarnings("unchecked") + public List findUpdatedDataMaskNamesByService(Long serviceId) { + List ret = new ArrayList<>(); + if (serviceId != null) { + List rows = (List) getEntityManager() + .createNamedQuery("XXPolicyRefDataMaskType.findUpdatedDataMaskNamesByService") + .setParameter("service", serviceId) + .getResultList(); + if (rows != null) { + for (Object[] row : rows) { + ret.add(new RangerPolicyRetriever.PolicyTextNameMap((Long)row[0], (String)row[1], (String)row[2])); + } + } + } + return ret; + } +} diff --git a/security-admin/src/main/java/org/apache/ranger/db/XXPolicyRefGroupDao.java b/security-admin/src/main/java/org/apache/ranger/db/XXPolicyRefGroupDao.java new file mode 100644 index 0000000000..08829d4f78 --- /dev/null +++ b/security-admin/src/main/java/org/apache/ranger/db/XXPolicyRefGroupDao.java @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.db; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import javax.persistence.NoResultException; + +import org.apache.ranger.biz.RangerPolicyRetriever; +import org.apache.ranger.common.db.BaseDao; +import org.apache.ranger.entity.XXPolicyRefGroup; +import org.springframework.stereotype.Service; + +@Service +public class XXPolicyRefGroupDao extends BaseDao{ + + + public XXPolicyRefGroupDao(RangerDaoManagerBase daoManager) { + super(daoManager); + } + + public List findByPolicyId(Long policyId) { + if(policyId == null) { + return Collections.EMPTY_LIST; + } + try { + return getEntityManager() + .createNamedQuery("XXPolicyRefGroup.findByPolicyId", tClass) + .setParameter("policyId", policyId).getResultList(); + } catch (NoResultException e) { + return Collections.EMPTY_LIST; + } + } + public List findByGroupName(String groupName) { + if (groupName == null) { + return Collections.EMPTY_LIST; + } + try { + return getEntityManager().createNamedQuery("XXPolicyRefGroup.findByGroupName", tClass) + .setParameter("groupName", groupName).getResultList(); + } catch (NoResultException e) { + return Collections.EMPTY_LIST; + } + } + + @SuppressWarnings("unchecked") + public List findUpdatedGroupNamesByPolicy(Long policyId) { + List ret = new ArrayList<>(); + if (policyId != null) { + List rows = (List) getEntityManager() + .createNamedQuery("XXPolicyRefGroup.findUpdatedGroupNamesByPolicy") + .setParameter("policy", policyId) + .getResultList(); + if (rows != null) { + for (Object[] row : rows) { + ret.add(new RangerPolicyRetriever.PolicyTextNameMap((Long)row[0], (String)row[1], (String)row[2])); + } + } + } + return ret; + } + + @SuppressWarnings("unchecked") + public List findUpdatedGroupNamesByService(Long serviceId) { + List ret = new ArrayList<>(); + if (serviceId != null) { + List rows = (List) getEntityManager() + .createNamedQuery("XXPolicyRefGroup.findUpdatedGroupNamesByService") + .setParameter("service", serviceId) + .getResultList(); + if (rows != null) { + for (Object[] row : rows) { + ret.add(new RangerPolicyRetriever.PolicyTextNameMap((Long)row[0], (String)row[1], (String)row[2])); + } + } + } + return ret; + } + +} diff --git a/security-admin/src/main/java/org/apache/ranger/db/XXPolicyRefResourceDao.java b/security-admin/src/main/java/org/apache/ranger/db/XXPolicyRefResourceDao.java new file mode 100644 index 0000000000..e259ee8646 --- /dev/null +++ b/security-admin/src/main/java/org/apache/ranger/db/XXPolicyRefResourceDao.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.db; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import javax.persistence.NoResultException; + +import org.apache.ranger.biz.RangerPolicyRetriever; +import org.apache.ranger.common.db.BaseDao; +import org.apache.ranger.entity.XXPolicyRefResource; +import org.springframework.stereotype.Service; + +@Service +public class XXPolicyRefResourceDao extends BaseDao{ + + public XXPolicyRefResourceDao(RangerDaoManagerBase daoManager) { + super(daoManager); + } + + public List findByPolicyId(Long policyId) { + if(policyId == null) { + return Collections.EMPTY_LIST; + } + try { + return getEntityManager() + .createNamedQuery("XXPolicyRefResource.findByPolicyId", tClass) + .setParameter("policyId", policyId).getResultList(); + } catch (NoResultException e) { + return Collections.EMPTY_LIST; + } + } + + public List findByResourceDefID(Long resourceDefId) { + if (resourceDefId == null) { + return Collections.EMPTY_LIST; + } + try { + return getEntityManager().createNamedQuery("XXPolicyRefResource.findByResourceDefId", tClass) + .setParameter("resourceDefId", resourceDefId).getResultList(); + } catch (NoResultException e) { + return Collections.EMPTY_LIST; + } + } + + @SuppressWarnings("unchecked") + public List findUpdatedResourceNamesByPolicy(Long policyId) { + List ret = new ArrayList<>(); + if (policyId != null) { + List rows = (List) getEntityManager() + .createNamedQuery("XXPolicyRefResource.findUpdatedResourceNamesByPolicy") + .setParameter("policy", policyId) + .getResultList(); + if (rows != null) { + for (Object[] row : rows) { + ret.add(new RangerPolicyRetriever.PolicyTextNameMap((Long)row[0], (String)row[1], (String)row[2])); + } + } + } + return ret; + } + + @SuppressWarnings("unchecked") + public List findUpdatedResourceNamesByService(Long serviceId) { + List ret = new ArrayList<>(); + if (serviceId != null) { + List rows = (List) getEntityManager() + .createNamedQuery("XXPolicyRefResource.findUpdatedResourceNamesByService") + .setParameter("service", serviceId) + .getResultList(); + if (rows != null) { + for (Object[] row : rows) { + ret.add(new RangerPolicyRetriever.PolicyTextNameMap((Long)row[0], (String)row[1], (String)row[2])); + } + } + } + return ret; + } +} diff --git a/security-admin/src/main/java/org/apache/ranger/db/XXPolicyRefUserDao.java b/security-admin/src/main/java/org/apache/ranger/db/XXPolicyRefUserDao.java new file mode 100644 index 0000000000..f7b6131c6a --- /dev/null +++ b/security-admin/src/main/java/org/apache/ranger/db/XXPolicyRefUserDao.java @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.db; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import javax.persistence.NoResultException; + +import org.apache.ranger.biz.RangerPolicyRetriever; +import org.apache.ranger.common.db.BaseDao; +import org.apache.ranger.entity.XXPolicyRefUser; +import org.springframework.stereotype.Service; + +@Service +public class XXPolicyRefUserDao extends BaseDao{ + + + public XXPolicyRefUserDao(RangerDaoManagerBase daoManager) { + super(daoManager); + } + + public List findByPolicyId(Long policyId) { + if(policyId == null) { + return Collections.EMPTY_LIST; + } + try { + return getEntityManager() + .createNamedQuery("XXPolicyRefUser.findByPolicyId", tClass) + .setParameter("policyId", policyId).getResultList(); + } catch (NoResultException e) { + return Collections.EMPTY_LIST; + } + } + public List findByUserName(String userName) { + if (userName == null) { + return Collections.EMPTY_LIST; + } + try { + return getEntityManager().createNamedQuery("XXPolicyRefUser.findByUserName", tClass) + .setParameter("userName", userName).getResultList(); + } catch (NoResultException e) { + return Collections.EMPTY_LIST; + } + } + + public List findByUserId(String userID) { + if (userID == null) { + return Collections.EMPTY_LIST; + } + try { + return getEntityManager().createNamedQuery("XXPolicyRefUser.findByUserId", tClass) + .setParameter("userID", userID).getResultList(); + } catch (NoResultException e) { + return Collections.EMPTY_LIST; + } + } + + @SuppressWarnings("unchecked") + public List findUpdatedUserNamesByPolicy(Long policyId) { + List ret = new ArrayList<>(); + if (policyId != null) { + List rows = (List) getEntityManager() + .createNamedQuery("XXPolicyRefUser.findUpdatedUserNamesByPolicy") + .setParameter("policy", policyId) + .getResultList(); + if (rows != null) { + for (Object[] row : rows) { + ret.add(new RangerPolicyRetriever.PolicyTextNameMap((Long)row[0], (String)row[1], (String)row[2])); + } + } + } + return ret; + } + + @SuppressWarnings("unchecked") + public List findUpdatedUserNamesByService(Long serviceId) { + List ret = new ArrayList<>(); + if (serviceId != null) { + List rows = (List) getEntityManager() + .createNamedQuery("XXPolicyRefUser.findUpdatedUserNamesByService") + .setParameter("service", serviceId) + .getResultList(); + if (rows != null) { + for (Object[] row : rows) { + ret.add(new RangerPolicyRetriever.PolicyTextNameMap((Long)row[0], (String)row[1], (String)row[2])); + } + } + } + return ret; + } + +} diff --git a/security-admin/src/main/java/org/apache/ranger/db/XXPolicyResourceDao.java b/security-admin/src/main/java/org/apache/ranger/db/XXPolicyResourceDao.java index 4b04b9664d..c34e44e94f 100644 --- a/security-admin/src/main/java/org/apache/ranger/db/XXPolicyResourceDao.java +++ b/security-admin/src/main/java/org/apache/ranger/db/XXPolicyResourceDao.java @@ -30,20 +30,6 @@ public class XXPolicyResourceDao extends BaseDao { public XXPolicyResourceDao(RangerDaoManagerBase daoManager) { super(daoManager); } - - public XXPolicyResource findByResDefIdAndPolicyId(Long resDefId, Long polId) { - if(resDefId == null || polId == null) { - return null; - } - try { - return getEntityManager() - .createNamedQuery("XXPolicyResource.findByResDefIdAndPolicyId", tClass) - .setParameter("resDefId", resDefId).setParameter("polId", polId) - .getSingleResult(); - } catch (NoResultException e) { - return null; - } - } public List findByPolicyId(Long policyId) { if(policyId == null) { diff --git a/security-admin/src/main/java/org/apache/ranger/db/XXPolicyResourceMapDao.java b/security-admin/src/main/java/org/apache/ranger/db/XXPolicyResourceMapDao.java index 70657370af..edda109f1f 100644 --- a/security-admin/src/main/java/org/apache/ranger/db/XXPolicyResourceMapDao.java +++ b/security-admin/src/main/java/org/apache/ranger/db/XXPolicyResourceMapDao.java @@ -30,19 +30,6 @@ public class XXPolicyResourceMapDao extends BaseDao { public XXPolicyResourceMapDao(RangerDaoManagerBase daoManager) { super(daoManager); } - - public List findByPolicyResId(Long polResId) { - if(polResId == null) { - return new ArrayList(); - } - try { - return getEntityManager() - .createNamedQuery("XXPolicyResourceMap.findByPolicyResId", tClass) - .setParameter("polResId", polResId).getResultList(); - } catch (NoResultException e) { - return new ArrayList(); - } - } public List findByPolicyId(Long policyId) { if(policyId == null) { diff --git a/security-admin/src/main/java/org/apache/ranger/db/XXResourceDefDao.java b/security-admin/src/main/java/org/apache/ranger/db/XXResourceDefDao.java index aee56e833e..646e61d897 100644 --- a/security-admin/src/main/java/org/apache/ranger/db/XXResourceDefDao.java +++ b/security-admin/src/main/java/org/apache/ranger/db/XXResourceDefDao.java @@ -97,5 +97,4 @@ public List findByParentResId(Long parentId) { return new ArrayList(); } } - } diff --git a/security-admin/src/main/java/org/apache/ranger/db/XXServiceConfigMapDao.java b/security-admin/src/main/java/org/apache/ranger/db/XXServiceConfigMapDao.java index 9f97b6073c..9559161a21 100644 --- a/security-admin/src/main/java/org/apache/ranger/db/XXServiceConfigMapDao.java +++ b/security-admin/src/main/java/org/apache/ranger/db/XXServiceConfigMapDao.java @@ -60,4 +60,18 @@ public XXServiceConfigMap findByServiceAndConfigKey(Long serviceId, } } + public XXServiceConfigMap findByServiceNameAndConfigKey(String serviceName, String configKey) { + if(serviceName == null || configKey == null) { + return null; + } + try { + return getEntityManager() + .createNamedQuery("XXServiceConfigMap.findByServiceNameAndConfigKey", tClass) + .setParameter("name", serviceName) + .setParameter("configKey", configKey).getSingleResult(); + } catch (NoResultException e) { + return null; + } + } + } diff --git a/security-admin/src/main/java/org/apache/ranger/db/XXServiceResourceDao.java b/security-admin/src/main/java/org/apache/ranger/db/XXServiceResourceDao.java index 98599921f0..7c25d61f16 100644 --- a/security-admin/src/main/java/org/apache/ranger/db/XXServiceResourceDao.java +++ b/security-admin/src/main/java/org/apache/ranger/db/XXServiceResourceDao.java @@ -24,6 +24,7 @@ import javax.persistence.NoResultException; +import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang.StringUtils; import org.apache.ranger.authorization.utils.StringUtil; import org.apache.ranger.common.db.BaseDao; @@ -73,14 +74,43 @@ public XXServiceResource findByServiceAndResourceSignature(Long serviceId, Strin } public List findTaggedResourcesInServiceId(Long serviceId) { + List ret = new ArrayList<>(); + if (serviceId != null) { + List rows = null; + try { + rows = getEntityManager().createNamedQuery("XXServiceResource.findTaggedResourcesInServiceId", Object[].class) + .setParameter("serviceId", serviceId).getResultList(); + } catch (NoResultException e) { + // Nothing + } + if (CollectionUtils.isNotEmpty(rows)) { + for (Object[] row : rows) { + XXServiceResource xxServiceResource = new XXServiceResource(); + xxServiceResource.setId((Long) row[0]); + xxServiceResource.setGuid((String) row[1]); + xxServiceResource.setVersion((Long) row[2]); + xxServiceResource.setIsEnabled((Boolean) row[3]); + xxServiceResource.setResourceSignature((String) row[4]); + xxServiceResource.setServiceId((Long) row[5]); + xxServiceResource.setServiceResourceElements((String) row[6]); + xxServiceResource.setTags((String) row[7]); + + ret.add(xxServiceResource); + } + } + } + return ret; + } + + public long countTaggedResourcesInServiceId(Long serviceId) { if (serviceId == null) { - return new ArrayList(); + return -1; } try { - return getEntityManager().createNamedQuery("XXServiceResource.findTaggedResourcesInServiceId", tClass) - .setParameter("serviceId", serviceId).getResultList(); + return getEntityManager().createNamedQuery("XXServiceResource.countTaggedResourcesInServiceId", Long.class) + .setParameter("serviceId", serviceId).getSingleResult(); } catch (NoResultException e) { - return new ArrayList(); + return -1; } } diff --git a/security-admin/src/main/java/org/apache/ranger/db/XXServiceResourceElementDao.java b/security-admin/src/main/java/org/apache/ranger/db/XXServiceResourceElementDao.java index 72fe2140b5..22a0e01527 100644 --- a/security-admin/src/main/java/org/apache/ranger/db/XXServiceResourceElementDao.java +++ b/security-admin/src/main/java/org/apache/ranger/db/XXServiceResourceElementDao.java @@ -72,16 +72,4 @@ public List findTaggedResourcesInServiceId(Long servic } } - public List findForServicePlugin(Long serviceId) { - if (serviceId == null) { - return new ArrayList(); - } - try { - return getEntityManager().createNamedQuery("XXServiceResourceElement.findForServicePlugin", tClass) - .setParameter("serviceId", serviceId) - .getResultList(); - } catch (NoResultException e) { - return new ArrayList(); - } - } } diff --git a/security-admin/src/main/java/org/apache/ranger/db/XXServiceResourceElementValueDao.java b/security-admin/src/main/java/org/apache/ranger/db/XXServiceResourceElementValueDao.java index e726d047c0..3170edd992 100644 --- a/security-admin/src/main/java/org/apache/ranger/db/XXServiceResourceElementValueDao.java +++ b/security-admin/src/main/java/org/apache/ranger/db/XXServiceResourceElementValueDao.java @@ -84,19 +84,6 @@ public List findTaggedResourcesInServiceId(Long s } } - @SuppressWarnings("unchecked") - public List findForServicePlugin(Long serviceId) { - if (serviceId == null) { - return new ArrayList(); - } - try { - return getEntityManager().createNamedQuery("XXServiceResourceElementValue.findForServicePlugin") - .setParameter("serviceId", serviceId).getResultList(); - } catch (NoResultException e) { - return new ArrayList(); - } - } - @SuppressWarnings("unchecked") public List findByResourceId(Long resourceId) { if (resourceId == null) { diff --git a/security-admin/src/main/java/org/apache/ranger/db/XXServiceVersionInfoDao.java b/security-admin/src/main/java/org/apache/ranger/db/XXServiceVersionInfoDao.java index 0098bff233..8e4376d630 100644 --- a/security-admin/src/main/java/org/apache/ranger/db/XXServiceVersionInfoDao.java +++ b/security-admin/src/main/java/org/apache/ranger/db/XXServiceVersionInfoDao.java @@ -23,6 +23,7 @@ import javax.persistence.NoResultException; import org.apache.commons.collections.CollectionUtils; +import org.apache.ranger.biz.ServiceDBStore; import org.apache.ranger.common.db.BaseDao; import org.apache.ranger.entity.XXServiceVersionInfo; @@ -30,6 +31,7 @@ */ public class XXServiceVersionInfoDao extends BaseDao { + /** * Default Constructor */ @@ -121,14 +123,14 @@ private void updateTagVersionAndTagUpdateTime(List service } for(XXServiceVersionInfo serviceVersionInfo : serviceVersionInfos) { - Long currentTagVersion = serviceVersionInfo.getTagVersion(); + final RangerDaoManager finaldaoManager = daoManager; + final Long finalServiceId = serviceVersionInfo.getServiceId(); + final ServiceDBStore.VERSION_TYPE versionType = ServiceDBStore.VERSION_TYPE.TAG_VERSION; - if(currentTagVersion == null) { - currentTagVersion = Long.valueOf(0); - } + Runnable serviceVersionUpdater = new ServiceDBStore.ServiceVersionUpdater(finaldaoManager, finalServiceId, versionType); - serviceVersionInfo.setTagVersion(currentTagVersion + 1); - serviceVersionInfo.setTagUpdateTime(updateTime); + daoManager.getRangerTransactionSynchronizationAdapter().executeOnTransactionCommit(serviceVersionUpdater); } + } } diff --git a/security-admin/src/main/java/org/apache/ranger/db/XXTagAttributeDao.java b/security-admin/src/main/java/org/apache/ranger/db/XXTagAttributeDao.java index e361b33bd3..a24e3dbbad 100644 --- a/security-admin/src/main/java/org/apache/ranger/db/XXTagAttributeDao.java +++ b/security-admin/src/main/java/org/apache/ranger/db/XXTagAttributeDao.java @@ -57,18 +57,6 @@ public List findByServiceId(Long serviceId) { } } - public List findForServicePlugin(Long serviceId) { - if (serviceId == null) { - return new ArrayList(); - } - try { - return getEntityManager().createNamedQuery("XXTagAttribute.findForServicePlugin", tClass) - .setParameter("serviceId", serviceId).getResultList(); - } catch (NoResultException e) { - return new ArrayList(); - } - } - public List findByServiceIdAndOwner(Long serviceId, Short owner) { if (serviceId == null) { return new ArrayList(); diff --git a/security-admin/src/main/java/org/apache/ranger/db/XXTagAttributeDefDao.java b/security-admin/src/main/java/org/apache/ranger/db/XXTagAttributeDefDao.java index 145399f1cf..294c22287b 100644 --- a/security-admin/src/main/java/org/apache/ranger/db/XXTagAttributeDefDao.java +++ b/security-admin/src/main/java/org/apache/ranger/db/XXTagAttributeDefDao.java @@ -57,18 +57,6 @@ public List findByServiceId(Long serviceId) { } } - public List findForServicePlugin(Long serviceId) { - if (serviceId == null) { - return new ArrayList(); - } - try { - return getEntityManager().createNamedQuery("XXTagAttributeDef.findForServicePlugin", tClass) - .setParameter("serviceId", serviceId).getResultList(); - } catch (NoResultException e) { - return new ArrayList(); - } - } - public List findByResourceId(Long resourceId) { if (resourceId == null) { return new ArrayList(); diff --git a/security-admin/src/main/java/org/apache/ranger/db/XXTagDao.java b/security-admin/src/main/java/org/apache/ranger/db/XXTagDao.java index 77428dbf9e..199a155a8f 100644 --- a/security-admin/src/main/java/org/apache/ranger/db/XXTagDao.java +++ b/security-admin/src/main/java/org/apache/ranger/db/XXTagDao.java @@ -124,19 +124,6 @@ public List findByServiceId(Long serviceId) { } } - public List findForServicePlugin(Long serviceId) { - if (serviceId == null) { - return new ArrayList(); - } - - try { - return getEntityManager().createNamedQuery("XXTag.findForServicePlugin", tClass) - .setParameter("serviceId", serviceId).getResultList(); - } catch (NoResultException e) { - return new ArrayList(); - } - } - public List findByServiceIdAndOwner(Long serviceId, Short owner) { if (serviceId == null) { return new ArrayList(); diff --git a/security-admin/src/main/java/org/apache/ranger/db/XXTagDefDao.java b/security-admin/src/main/java/org/apache/ranger/db/XXTagDefDao.java index dfd2fcb5d0..877344b2d5 100644 --- a/security-admin/src/main/java/org/apache/ranger/db/XXTagDefDao.java +++ b/security-admin/src/main/java/org/apache/ranger/db/XXTagDefDao.java @@ -24,6 +24,7 @@ import javax.persistence.NoResultException; +import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang.StringUtils; import org.apache.ranger.common.db.BaseDao; import org.apache.ranger.entity.XXTagDef; @@ -60,31 +61,33 @@ public XXTagDef findByName(String name) { } } - public List findByServiceId(Long serviceId) { - if (serviceId == null) { - return new ArrayList(); - } - - try { - return getEntityManager().createNamedQuery("XXTagDef.findByServiceId", tClass) - .setParameter("serviceId", serviceId).getResultList(); - } catch (NoResultException e) { - return new ArrayList(); - } - } - - public List findForServicePlugin(Long serviceId) { - if (serviceId == null) { - return new ArrayList(); - } - - try { - return getEntityManager().createNamedQuery("XXTagDef.findForServicePlugin", tClass) - .setParameter("serviceId", serviceId).getResultList(); - } catch (NoResultException e) { - return new ArrayList(); - } - } + public List findByServiceId(Long serviceId) { + List ret = new ArrayList<>(); + if (serviceId != null) { + List rows = null; + try { + rows = getEntityManager().createNamedQuery("XXTagDef.findByServiceId", Object[].class) + .setParameter("serviceId", serviceId).getResultList(); + } catch (NoResultException e) { + // Nothing + } + if (CollectionUtils.isNotEmpty(rows)) { + for (Object[] row : rows) { + XXTagDef xxTagDef = new XXTagDef(); + xxTagDef.setId((Long) row[0]); + xxTagDef.setGuid((String) row[1]); + xxTagDef.setVersion((Long) row[2]); + xxTagDef.setIsEnabled((Boolean) row[3]); + xxTagDef.setName((String) row[4]); + xxTagDef.setSource((String) row[5]); + xxTagDef.setTagAttrDefs((String) row[6]); + + ret.add(xxTagDef); + } + } + } + return ret; + } public List getAllNames() { try { diff --git a/security-admin/src/main/java/org/apache/ranger/db/XXTagResourceMapDao.java b/security-admin/src/main/java/org/apache/ranger/db/XXTagResourceMapDao.java index bbcd54618a..c16cad0b95 100644 --- a/security-admin/src/main/java/org/apache/ranger/db/XXTagResourceMapDao.java +++ b/security-admin/src/main/java/org/apache/ranger/db/XXTagResourceMapDao.java @@ -141,15 +141,4 @@ public List findByServiceId(Long serviceId) { } } - public List findForServicePlugin(Long serviceId) { - if (serviceId == null) { - return new ArrayList(); - } - try { - return getEntityManager().createNamedQuery("XXTagResourceMap.findForServicePlugin", tClass) - .setParameter("serviceId", serviceId).getResultList(); - } catch (NoResultException e) { - return new ArrayList(); - } - } } diff --git a/security-admin/src/main/java/org/apache/ranger/db/XXUserDao.java b/security-admin/src/main/java/org/apache/ranger/db/XXUserDao.java index 7af1bf96a3..4d0d8e4acc 100644 --- a/security-admin/src/main/java/org/apache/ranger/db/XXUserDao.java +++ b/security-admin/src/main/java/org/apache/ranger/db/XXUserDao.java @@ -17,10 +17,7 @@ * under the License. */ - package org.apache.ranger.db; - - -import java.util.List; +package org.apache.ranger.db; import javax.persistence.NoResultException; @@ -51,20 +48,6 @@ public XXUser findByUserName(String name) { return null; } - @SuppressWarnings("unchecked") - public List findByPolicyItemId(Long polItemId) { - if (polItemId == null) { - return null; - } - try { - return getEntityManager() - .createNamedQuery("XXUser.findByPolicyItemId") - .setParameter("polItemId", polItemId).getResultList(); - } catch (NoResultException e) { - return null; - } - } - public XXUser findByPortalUserId(Long portalUserId) { if (portalUserId == null) { return null; @@ -76,4 +59,5 @@ public XXUser findByPortalUserId(Long portalUserId) { return null; } } + } diff --git a/security-admin/src/main/java/org/apache/ranger/entity/XXDBBase.java b/security-admin/src/main/java/org/apache/ranger/entity/XXDBBase.java index 8405eb3e4c..4816b02f90 100644 --- a/security-admin/src/main/java/org/apache/ranger/entity/XXDBBase.java +++ b/security-admin/src/main/java/org/apache/ranger/entity/XXDBBase.java @@ -25,6 +25,7 @@ */ import java.util.Date; +import java.util.Objects; import javax.persistence.Column; import javax.persistence.EntityListeners; @@ -205,6 +206,11 @@ public String toString( ) { return str; } + @Override + public int hashCode() { + return Objects.hash(createTime, updateTime, addedByUserId, updatedByUserId); + } + /** * Checks for all attributes except referenced db objects * @return true if all attributes match diff --git a/security-admin/src/main/java/org/apache/ranger/entity/XXPolicyBase.java b/security-admin/src/main/java/org/apache/ranger/entity/XXPolicyBase.java index 69d28bb54d..6c2503234f 100644 --- a/security-admin/src/main/java/org/apache/ranger/entity/XXPolicyBase.java +++ b/security-admin/src/main/java/org/apache/ranger/entity/XXPolicyBase.java @@ -110,6 +110,10 @@ public abstract class XXPolicyBase extends XXDBBase { @Column(name = "is_audit_enabled") protected boolean isAuditEnabled; + + @Column(name = "policy_text") + protected String policyText; + /** * @return the gUID */ @@ -267,7 +271,15 @@ public void setPolicyType(Integer policyType) { this.policyType = policyType; } - /* + public void setPolicyText(String policyText) { + this.policyText = policyText; + } + + public String getPolicyText() { + return this.policyText; + } + + /* * (non-Javadoc) * * @see java.lang.Object#equals(java.lang.Object) diff --git a/security-admin/src/main/java/org/apache/ranger/entity/XXPolicyRefAccessType.java b/security-admin/src/main/java/org/apache/ranger/entity/XXPolicyRefAccessType.java new file mode 100644 index 0000000000..6af8f99f4b --- /dev/null +++ b/security-admin/src/main/java/org/apache/ranger/entity/XXPolicyRefAccessType.java @@ -0,0 +1,191 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ranger.entity; + +import javax.persistence.*; +import javax.xml.bind.annotation.XmlRootElement; +import java.util.Objects; + +@Entity +@Cacheable +@XmlRootElement +@Table(name = "x_policy_ref_access_type") +public class XXPolicyRefAccessType extends XXDBBase implements + java.io.Serializable { + private static final long serialVersionUID = 1L; + /** + * id of the XXPolicyRefAccessType + *
    + *
+ * + */ + @Id + @SequenceGenerator(name = "x_policy_ref_access_type_SEQ", sequenceName = "x_policy_ref_access_type_SEQ", allocationSize = 1) + @GeneratedValue(strategy = GenerationType.AUTO, generator = "x_policy_ref_access_type_SEQ") + @Column(name = "id") + protected Long id; + + /** + * policyId of the XXPolicyRefAccessType + *
    + *
+ * + */ + @Column(name = "policy_id") + protected Long policyId; + + /** + * accessDefId of the XXPolicyRefAccessType + *
    + *
+ * + */ + @Column(name = "access_def_id") + protected Long accessDefId; + + /** + * accessTypeName of the XXPolicyRefAccessType + *
    + *
+ * + */ + @Column(name = "access_type_name") + protected String accessTypeName; + + /** + * This method sets the value to the member attribute id . You + * cannot set null to the attribute. + * + * @param id + * Value to set member attribute id + */ + public void setId(Long id) { + this.id = id; + } + + /** + * Returns the value for the member attribute id + * + * @return Date - value of member attribute id . + */ + public Long getId() { + return this.id; + } + + /** + * This method sets the value to the member attribute policyId . + * You cannot set null to the attribute. + * + * @param policyId + * Value to set member attribute policyId + */ + public void setPolicyId(Long policyId) { + this.policyId = policyId; + } + + /** + * Returns the value for the member attribute policyId + * + * @return Date - value of member attribute policyId . + */ + public Long getPolicyId() { + return this.policyId; + } + + /** + * This method sets the value to the member attribute accessDefId . + * You cannot set null to the attribute. + * + * @param accessDefId + * Value to set member attribute accessDefId + */ + public void setAccessDefId(Long accessDefId) { + this.accessDefId = accessDefId; + } + + /** + * Returns the value for the member attribute accessDefId + * + * @return Date - value of member attribute accessDefId . + */ + public Long getAccessDefId() { + return accessDefId; + } + + /** + * This method sets the value to the member attribute accessTypeName . + * You cannot set null to the attribute. + * + * @param accessTypeName + * Value to set member attribute accessTypeName + */ + public void setAccessTypeName(String accessTypeName) { + this.accessTypeName = accessTypeName; + } + + /** + * Returns the value for the member attribute accessTypeName + * + * @return Date - value of member attribute accessTypeName . + */ + public String getAccessTypeName() { + return accessTypeName; + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), id, policyId, accessDefId, accessTypeName); + } + + /* + * (non-Javadoc) + * + * @see java.lang.Object#equals(java.lang.Object) + */ + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + + if (getClass() != obj.getClass()) { + return false; + } + + XXPolicyRefAccessType other = (XXPolicyRefAccessType) obj; + + return super.equals(obj) && + Objects.equals(id, other.id) && + Objects.equals(policyId, other.policyId) && + Objects.equals(accessDefId, other.accessDefId) && + Objects.equals(accessTypeName, other.accessTypeName); + } + + /* (non-Javadoc) + * @see java.lang.Object#toString() + */ + @Override + public String toString() { + return "XXPolicyRefAccessType [" + super.toString() + " id=" + id + ", policyId=" + policyId + ", accessDefId=" + + accessDefId + ", accessTypeName=" + accessTypeName + "]"; + } + + + +} \ No newline at end of file diff --git a/security-admin/src/main/java/org/apache/ranger/entity/XXPolicyRefCondition.java b/security-admin/src/main/java/org/apache/ranger/entity/XXPolicyRefCondition.java new file mode 100644 index 0000000000..4f4409d6af --- /dev/null +++ b/security-admin/src/main/java/org/apache/ranger/entity/XXPolicyRefCondition.java @@ -0,0 +1,191 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ranger.entity; + +import javax.persistence.*; +import javax.xml.bind.annotation.XmlRootElement; +import java.util.Objects; + +@Entity +@Cacheable +@XmlRootElement +@Table(name = "x_policy_ref_condition") +public class XXPolicyRefCondition extends XXDBBase implements + java.io.Serializable { + private static final long serialVersionUID = 1L; + /** + * id of the XXPolicyRefCondition + *
    + *
+ * + */ + @Id + @SequenceGenerator(name = "x_policy_ref_condition_SEQ", sequenceName = "x_policy_ref_condition_SEQ", allocationSize = 1) + @GeneratedValue(strategy = GenerationType.AUTO, generator = "x_policy_ref_condition_SEQ") + @Column(name = "id") + protected Long id; + + /** + * policyId of the XXPolicyRefCondition + *
    + *
+ * + */ + @Column(name = "policy_id") + protected Long policyId; + + /** + * conditionDefId of the XXPolicyRefCondition + *
    + *
+ * + */ + @Column(name = "condition_def_id") + protected Long conditionDefId; + + /** + * conditionName of the XXPolicyRefCondition + *
    + *
+ * + */ + @Column(name = "condition_name") + protected String conditionName; + + /** + * This method sets the value to the member attribute id . You + * cannot set null to the attribute. + * + * @param id + * Value to set member attribute id + */ + public void setId(Long id) { + this.id = id; + } + + /** + * Returns the value for the member attribute id + * + * @return Date - value of member attribute id . + */ + public Long getId() { + return this.id; + } + + /** + * This method sets the value to the member attribute policyId . + * You cannot set null to the attribute. + * + * @param policyId + * Value to set member attribute policyId + */ + public void setPolicyId(Long policyId) { + this.policyId = policyId; + } + + /** + * Returns the value for the member attribute policyId + * + * @return Date - value of member attribute policyId . + */ + public Long getPolicyId() { + return this.policyId; + } + + /** + * This method sets the value to the member attribute conditionDefId . + * You cannot set null to the attribute. + * + * @param conditionDefId + * Value to set member attribute conditionDefId + */ + public void setConditionDefId(Long conditionDefId) { + this.conditionDefId = conditionDefId; + } + + /** + * Returns the value for the member attribute conditionDefId + * + * @return Date - value of member attribute conditionDefId . + */ + public Long getConditionDefId() { + return conditionDefId; + } + + /** + * This method sets the value to the member attribute conditionName . + * You cannot set null to the attribute. + * + * @param conditionName + * Value to set member attribute conditionName + */ + public void setConditionName(String conditionName) { + this.conditionName = conditionName; + } + + /** + * Returns the value for the member attribute conditionName + * + * @return Date - value of member attribute conditionName . + */ + public String getConditionName() { + return conditionName; + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), id, policyId, conditionDefId, conditionName); + } + + /* + * (non-Javadoc) + * + * @see java.lang.Object#equals(java.lang.Object) + */ + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + + if (getClass() != obj.getClass()) { + return false; + } + + XXPolicyRefCondition other = (XXPolicyRefCondition) obj; + + return super.equals(obj) && + Objects.equals(id, other.id) && + Objects.equals(policyId, other.policyId) && + Objects.equals(conditionDefId, other.conditionDefId) && + Objects.equals(conditionName, other.conditionName); + } + + /* (non-Javadoc) + * @see java.lang.Object#toString() + */ + @Override + public String toString() { + return "XXPolicyRefCondition [" + super.toString() + " id=" + id + ", policyId=" + policyId + ", conditionDefId=" + + conditionDefId + ", conditionName=" + conditionName + "]"; + } + + + +} \ No newline at end of file diff --git a/security-admin/src/main/java/org/apache/ranger/entity/XXPolicyRefDataMaskType.java b/security-admin/src/main/java/org/apache/ranger/entity/XXPolicyRefDataMaskType.java new file mode 100644 index 0000000000..cb926740e9 --- /dev/null +++ b/security-admin/src/main/java/org/apache/ranger/entity/XXPolicyRefDataMaskType.java @@ -0,0 +1,192 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ranger.entity; + +import javax.persistence.*; +import javax.xml.bind.annotation.XmlRootElement; +import java.util.Objects; + +@Entity +@Cacheable +@XmlRootElement +@Table(name = "x_policy_ref_datamask_type") +public class XXPolicyRefDataMaskType extends XXDBBase implements + java.io.Serializable { + private static final long serialVersionUID = 1L; + /** + * id of the XXPolicyRefDataMaskType + *
    + *
+ * + */ + @Id + @SequenceGenerator(name = "x_policy_ref_datamask_type_SEQ", sequenceName = "x_policy_ref_datamask_type_SEQ", allocationSize = 1) + @GeneratedValue(strategy = GenerationType.AUTO, generator = "x_policy_ref_datamask_type_SEQ") + @Column(name = "id") + protected Long id; + + /** + * policyId of the XXPolicyRefDataMaskType + *
    + *
+ * + */ + @Column(name = "policy_id") + protected Long policyId; + + /** + * DatamaskDefId of the XXPolicyRefDataMaskType + *
    + *
+ * + */ + @Column(name = "datamask_def_id") + protected Long dataMaskDefId; + + /** + * dataMaskTypeName of the XXPolicyRefDataMaskType + *
    + *
+ * + */ + @Column(name = "datamask_type_name") + protected String dataMaskTypeName; + + /** + * This method sets the value to the member attribute id . You + * cannot set null to the attribute. + * + * @param id + * Value to set member attribute id + */ + public void setId(Long id) { + this.id = id; + } + + /** + * Returns the value for the member attribute id + * + * @return Date - value of member attribute id . + */ + public Long getId() { + return this.id; + } + + + /** + * This method sets the value to the member attribute policyId . + * You cannot set null to the attribute. + * + * @param policyId + * Value to set member attribute policyId + */ + public void setPolicyId(Long policyId) { + this.policyId = policyId; + } + + /** + * Returns the value for the member attribute policyId + * + * @return Date - value of member attribute policyId . + */ + public Long getPolicyId() { + return this.policyId; + } + + /** + * This method sets the value to the member attribute dataMaskDefId . + * You cannot set null to the attribute. + * + * @param dataMaskDefId + * Value to set member attribute dataMaskDefId + */ + public void setDataMaskDefId(Long dataMaskDefId) { + this.dataMaskDefId = dataMaskDefId; + } + + /** + * Returns the value for the member attribute dataMaskDefId + * + * @return Date - value of member attribute dataMaskDefId . + */ + public Long getDataMaskDefId() { + return dataMaskDefId; + } + + /** + * This method sets the value to the member attribute dataMaskTypeName . + * You cannot set null to the attribute. + * + * @param dataMaskTypeName + * Value to set member attribute dataMaskTypeName + */ + public void setDataMaskTypeName(String dataMaskTypeName) { + this.dataMaskTypeName = dataMaskTypeName; + } + + /** + * Returns the value for the member attribute dataMaskTypeName + * + * @return Date - value of member attribute dataMaskTypeName . + */ + public String getDataMaskTypeName() { + return dataMaskTypeName; + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), id, policyId, dataMaskDefId, dataMaskTypeName); + } + + /* + * (non-Javadoc) + * + * @see java.lang.Object#equals(java.lang.Object) + */ + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + + if (getClass() != obj.getClass()) { + return false; + } + + XXPolicyRefDataMaskType other = (XXPolicyRefDataMaskType) obj; + + return super.equals(obj) && + Objects.equals(id, other.id) && + Objects.equals(policyId, other.policyId) && + Objects.equals(dataMaskDefId, other.dataMaskDefId) && + Objects.equals(dataMaskTypeName, other.dataMaskTypeName); + } + + /* (non-Javadoc) + * @see java.lang.Object#toString() + */ + @Override + public String toString() { + return "XXPolicyRefDataMaskType [" + super.toString() + " id=" + id + ", policyId=" + policyId + ", dataMaskDefId=" + + dataMaskDefId + ", dataMaskTypeName=" + dataMaskTypeName + "]"; + } + + + +} \ No newline at end of file diff --git a/security-admin/src/main/java/org/apache/ranger/entity/XXPolicyRefGroup.java b/security-admin/src/main/java/org/apache/ranger/entity/XXPolicyRefGroup.java new file mode 100644 index 0000000000..32a1b9f24c --- /dev/null +++ b/security-admin/src/main/java/org/apache/ranger/entity/XXPolicyRefGroup.java @@ -0,0 +1,206 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.entity; + +import java.io.Serializable; +import java.util.Objects; + +import javax.persistence.Cacheable; +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; +import javax.persistence.Id; +import javax.persistence.SequenceGenerator; +import javax.persistence.Table; +import javax.xml.bind.annotation.XmlRootElement; + + +/** + * The persistent class for the x_policy_ref_group database table. + * + */ +@Entity +@Cacheable +@XmlRootElement +@Table(name="x_policy_ref_group") +public class XXPolicyRefGroup extends XXDBBase implements Serializable { + + private static final long serialVersionUID = 1L; + /** + * id of the XXPolicyRefGroup + *
    + *
+ * + */ + @Id + @SequenceGenerator(name = "x_policy_ref_group_SEQ", sequenceName = "x_policy_ref_group_SEQ", allocationSize = 1) + @GeneratedValue(strategy = GenerationType.AUTO, generator = "x_policy_ref_group_SEQ") + @Column(name = "id") + protected Long id; + + /** + * policyId of the XXPolicyRefGroup + *
    + *
+ * + */ + @Column(name = "policy_id") + protected Long policyId; + + /** + * groupId of the XXPolicyRefGroup + *
    + *
+ * + */ + @Column(name = "group_id") + protected Long groupId; + + /** + * groupName of the XXPolicyRefGroup + *
    + *
+ * + */ + @Column(name = "group_name") + protected String groupName; + + /** + * This method sets the value to the member attribute id . You + * cannot set null to the attribute. + * + * @param id + * Value to set member attribute id + */ + public void setId(Long id) { + this.id = id; + } + + /** + * Returns the value for the member attribute id + * + * @return Date - value of member attribute id . + */ + public Long getId() { + return this.id; + } + + /** + * This method sets the value to the member attribute policyId . + * You cannot set null to the attribute. + * + * @param policyId + * Value to set member attribute policyId + */ + public void setPolicyId(Long policyId) { + this.policyId = policyId; + } + + /** + * Returns the value for the member attribute policyId + * + * @return Date - value of member attribute policyId . + */ + public Long getPolicyId() { + return this.policyId; + } + + /** + * This method sets the value to the member attribute groupId . + * You cannot set null to the attribute. + * + * @param groupId + * Value to set member attribute groupId + */ + public void setGroupId(Long groupId) { + this.groupId = groupId; + } + + /** + * Returns the value for the member attribute groupId + * + * @return Date - value of member attribute groupId . + */ + public Long getGroupId() { + return groupId; + } + + /** + * This method sets the value to the member attribute groupName . + * You cannot set null to the attribute. + * + * @param groupName + * Value to set member attribute groupName + */ + public void setGroupName(String groupName) { + this.groupName = groupName; + } + + /** + * Returns the value for the member attribute groupName + * + * @return Date - value of member attribute groupName . + */ + public String getGroupName() { + return groupName; + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), id, policyId, groupId, groupName); + } + + /* + * (non-Javadoc) + * + * @see java.lang.Object#equals(java.lang.Object) + */ + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + + if (getClass() != obj.getClass()) { + return false; + } + + XXPolicyRefGroup other = (XXPolicyRefGroup) obj; + + return super.equals(obj) && + Objects.equals(id, other.id) && + Objects.equals(policyId, other.policyId) && + Objects.equals(groupId, other.groupId) && + Objects.equals(groupName, other.groupName); + } + + /* + * (non-Javadoc) + * + * @see java.lang.Object#toString() + */ + @Override + public String toString() { + return "XXPolicyRefGroup [" + super.toString() + " id=" + id + ", policyId=" + policyId + ", groupId=" + groupId + + ", groupName=" + groupName + "]"; + } + +} \ No newline at end of file diff --git a/security-admin/src/main/java/org/apache/ranger/entity/XXPolicyRefResource.java b/security-admin/src/main/java/org/apache/ranger/entity/XXPolicyRefResource.java new file mode 100644 index 0000000000..1150646210 --- /dev/null +++ b/security-admin/src/main/java/org/apache/ranger/entity/XXPolicyRefResource.java @@ -0,0 +1,191 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ranger.entity; + +import javax.persistence.*; +import javax.xml.bind.annotation.XmlRootElement; +import java.util.Objects; + +@Entity +@Cacheable +@XmlRootElement +@Table(name = "x_policy_ref_resource") +public class XXPolicyRefResource extends XXDBBase implements + java.io.Serializable { + private static final long serialVersionUID = 1L; + /** + * id of the XXPolicyRefResource + *
    + *
+ * + */ + @Id + @SequenceGenerator(name = "x_policy_ref_resource_SEQ", sequenceName = "x_policy_ref_resource_SEQ", allocationSize = 1) + @GeneratedValue(strategy = GenerationType.AUTO, generator = "x_policy_ref_resource_SEQ") + @Column(name = "id") + protected Long id; + + /** + * policyId of the XXPolicyRefResource + *
    + *
+ * + */ + @Column(name = "policy_id") + protected Long policyId; + + /** + * resourceDefId of the XXPolicyRefResource + *
    + *
+ * + */ + @Column(name = "resource_def_id") + protected Long resourceDefId; + + /** + * resource_name of the XXPolicyRefResource + *
    + *
+ * + */ + @Column(name = "resource_name") + protected String resourceName; + + /** + * This method sets the value to the member attribute id . You + * cannot set null to the attribute. + * + * @param id + * Value to set member attribute id + */ + public void setId(Long id) { + this.id = id; + } + + /** + * Returns the value for the member attribute id + * + * @return Date - value of member attribute id . + */ + public Long getId() { + return this.id; + } + + /** + * This method sets the value to the member attribute policyId . + * You cannot set null to the attribute. + * + * @param policyId + * Value to set member attribute policyId + */ + public void setPolicyId(Long policyId) { + this.policyId = policyId; + } + + /** + * Returns the value for the member attribute policyId + * + * @return Date - value of member attribute policyId . + */ + public Long getPolicyId() { + return this.policyId; + } + + /** + * This method sets the value to the member attribute resourceDefId . + * You cannot set null to the attribute. + * + * @param resourceDefId + * Value to set member attribute resourceDefId + */ + public void setResourceDefId(Long resourceDefId) { + this.resourceDefId = resourceDefId; + } + + /** + * Returns the value for the member attribute resourceDefId + * + * @return Date - value of member attribute resourceDefId . + */ + public Long getResourceDefId() { + return resourceDefId; + } + + /** + * This method sets the value to the member attribute resource_name . + * You cannot set null to the attribute. + * + * @param resourceName + * Value to set member attribute resource_name + */ + public void setResourceName(String resourceName) { + this.resourceName = resourceName; + } + + /** + * Returns the value for the member attribute resourceName + * + * @return Date - value of member attribute resourceName . + */ + public String getResourceName() { + return resourceName; + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), id, policyId, resourceDefId, resourceName); + } + + /* + * (non-Javadoc) + * + * @see java.lang.Object#equals(java.lang.Object) + */ + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + + if (getClass() != obj.getClass()) { + return false; + } + + XXPolicyRefResource other = (XXPolicyRefResource) obj; + + return super.equals(obj) && + Objects.equals(id, other.id) && + Objects.equals(policyId, other.policyId) && + Objects.equals(resourceDefId, other.resourceDefId) && + Objects.equals(resourceName, other.resourceName); + } + + /* (non-Javadoc) + * @see java.lang.Object#toString() + */ + @Override + public String toString() { + return "XXPolicyRefResource [" + super.toString() + " id=" + id + ", policyId=" + policyId + ", resourceDefId=" + + resourceDefId + ", resource_name=" + resourceName + "]"; + } + + + +} \ No newline at end of file diff --git a/security-admin/src/main/java/org/apache/ranger/entity/XXPolicyRefUser.java b/security-admin/src/main/java/org/apache/ranger/entity/XXPolicyRefUser.java new file mode 100644 index 0000000000..8dfb928336 --- /dev/null +++ b/security-admin/src/main/java/org/apache/ranger/entity/XXPolicyRefUser.java @@ -0,0 +1,191 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ranger.entity; + +import javax.persistence.*; +import javax.xml.bind.annotation.XmlRootElement; +import java.util.Objects; + +@Entity +@Cacheable +@XmlRootElement +@Table(name = "x_policy_ref_user") +public class XXPolicyRefUser extends XXDBBase implements + java.io.Serializable { + private static final long serialVersionUID = 1L; + /** + * id of the XXPolicyRefUser + *
    + *
+ * + */ + @Id + @SequenceGenerator(name = "x_policy_ref_user_SEQ", sequenceName = "x_policy_ref_user_SEQ", allocationSize = 1) + @GeneratedValue(strategy = GenerationType.AUTO, generator = "x_policy_ref_user_SEQ") + @Column(name = "id") + protected Long id; + + /** + * policyId of the XXPolicyRefUser + *
    + *
+ * + */ + @Column(name = "policy_id") + protected Long policyId; + + /** + * userId of the XXPolicyRefUser + *
    + *
+ * + */ + @Column(name = "user_id") + protected Long userId; + + /** + * userName of the XXPolicyRefUser + *
    + *
+ * + */ + @Column(name = "user_name") + protected String userName; + + /** + * This method sets the value to the member attribute id . You + * cannot set null to the attribute. + * + * @param id + * Value to set member attribute id + */ + public void setId(Long id) { + this.id = id; + } + + /** + * Returns the value for the member attribute id + * + * @return Date - value of member attribute id . + */ + public Long getId() { + return this.id; + } + + /** + * This method sets the value to the member attribute policyId . + * You cannot set null to the attribute. + * + * @param policyId + * Value to set member attribute policyId + */ + public void setPolicyId(Long policyId) { + this.policyId = policyId; + } + + /** + * Returns the value for the member attribute policyId + * + * @return Date - value of member attribute policyId . + */ + public Long getPolicyId() { + return this.policyId; + } + + /** + * This method sets the value to the member attribute userId . + * You cannot set null to the attribute. + * + * @param userId + * Value to set member attribute userId + */ + public void setUserId(Long userId) { + this.userId = userId; + } + + /** + * Returns the value for the member attribute userId + * + * @return Date - value of member attribute userId . + */ + public Long getUserId() { + return userId; + } + + /** + * This method sets the value to the member attribute userName . + * You cannot set null to the attribute. + * + * @param userName + * Value to set member attribute userName + */ + public void setUserName(String userName) { + this.userName = userName; + } + + /** + * Returns the value for the member attribute userName + * + * @return Date - value of member attribute userName . + */ + public String getUserName() { + return userName; + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), id, policyId, userId, userName); + } + + /* + * (non-Javadoc) + * + * @see java.lang.Object#equals(java.lang.Object) + */ + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + + if (getClass() != obj.getClass()) { + return false; + } + + XXPolicyRefUser other = (XXPolicyRefUser) obj; + + return super.equals(obj) && + Objects.equals(id, other.id) && + Objects.equals(policyId, other.policyId) && + Objects.equals(userId, other.userId) && + Objects.equals(userName, other.userName); + } + + /* (non-Javadoc) + * @see java.lang.Object#toString() + */ + @Override + public String toString() { + return "XXPolicyRefUser [" + super.toString() + " id=" + id + ", policyId=" + policyId + ", userId=" + + userId + ", userName=" + userName + "]"; + } + + + +} \ No newline at end of file diff --git a/security-admin/src/main/java/org/apache/ranger/entity/XXServiceResource.java b/security-admin/src/main/java/org/apache/ranger/entity/XXServiceResource.java index 961627a3c3..c784830ef3 100644 --- a/security-admin/src/main/java/org/apache/ranger/entity/XXServiceResource.java +++ b/security-admin/src/main/java/org/apache/ranger/entity/XXServiceResource.java @@ -63,6 +63,12 @@ public class XXServiceResource extends XXDBBase implements Serializable { @Column(name = "service_id") protected Long serviceId; + @Column(name = "service_resource_elements_text") + protected String serviceResourceElements; + + @Column(name = "tags_text") + protected String tags; + @Override public void setId(Long id) { this.id = id; @@ -148,6 +154,16 @@ public void setIsEnabled(Boolean isEnabled) { this.isEnabled = isEnabled; } + public String getServiceResourceElements() { return serviceResourceElements; } + + public void setServiceResourceElements(String serviceResourceElements) { + this.serviceResourceElements = serviceResourceElements; + } + + public String getTags() { return tags; } + + public void setTags(String tags) { this.tags = tags; } + @Override public int getMyClassType() { return AppConstants.CLASS_TYPE_XA_SERVICE_RESOURCE; @@ -168,6 +184,8 @@ public int hashCode() { result = prime * result + ((isEnabled == null) ? 0 : isEnabled.hashCode()); result = prime * result + ((resourceSignature == null) ? 0 : resourceSignature.hashCode()); result = prime * result + ((serviceId == null) ? 0 : serviceId.hashCode()); + result = prime * result + ((serviceResourceElements == null) ? 0 : serviceResourceElements.hashCode()); + result = prime * result + ((tags == null) ? 0 : tags.hashCode()); return result; } @@ -215,6 +233,16 @@ public boolean equals(Object obj) { return false; } else if (!version.equals(other.version)) return false; + if (serviceResourceElements == null) { + if (other.serviceResourceElements != null) + return false; + } else if (!serviceResourceElements.equals(other.serviceResourceElements)) + return false; + if (tags == null) { + if (other.tags != null) + return false; + } else if (!tags.equals(other.tags)) + return false; return true; } @@ -239,6 +267,8 @@ public StringBuilder toString(StringBuilder sb) { sb.append("isEnabled={").append(isEnabled).append("} "); sb.append("resourceSignature={").append(resourceSignature).append("} "); sb.append("serviceId={").append(serviceId).append("} "); + sb.append("serviceResourceElements={").append(serviceResourceElements).append("} "); + sb.append("tags={").append(tags).append("} "); sb.append(" }"); return sb; diff --git a/security-admin/src/main/java/org/apache/ranger/entity/XXTag.java b/security-admin/src/main/java/org/apache/ranger/entity/XXTag.java index 9155385ec2..432119c97c 100644 --- a/security-admin/src/main/java/org/apache/ranger/entity/XXTag.java +++ b/security-admin/src/main/java/org/apache/ranger/entity/XXTag.java @@ -60,6 +60,9 @@ public class XXTag extends XXDBBase implements Serializable { @Column(name = "owned_by") protected Short owner; + @Column(name = "tag_attrs_text") + protected String tagAttrs; + @Override public void setId(Long id) { this.id = id; @@ -118,7 +121,11 @@ public void setType(Long type) { public Short getOwner() { return owner; } public void setOwner(Short owner) { this.owner = owner; } - @Override + public String getTagAttrs() { return tagAttrs; } + + public void setTagAttrs(String tagAttrs) { this.tagAttrs = tagAttrs; } + + @Override public int getMyClassType() { return AppConstants.CLASS_TYPE_XA_TAG; } @@ -137,6 +144,7 @@ public int hashCode() { result = prime * result + ((id == null) ? 0 : id.hashCode()); result = prime * result + ((type == null) ? 0 : type.hashCode()); result = prime * result + ((owner == null) ? 0 : owner.hashCode()); + result = prime * result + ((tagAttrs == null) ? 0 : tagAttrs.hashCode()); return result; } @@ -179,6 +187,11 @@ public boolean equals(Object obj) { return false; } else if (!owner.equals(other.owner)) return false; + if (tagAttrs == null) { + if (other.tagAttrs != null) + return false; + } else if (!tagAttrs.equals(other.tagAttrs)) + return false; return true; } @@ -201,6 +214,7 @@ public StringBuilder toString(StringBuilder sb) { sb.append("guid={").append(guid).append("} "); sb.append("type={").append(type).append("} "); sb.append("owned_by={").append(owner).append("} "); + sb.append("tagAttrs={").append(tagAttrs).append("} "); sb.append(" }"); return sb; diff --git a/security-admin/src/main/java/org/apache/ranger/entity/XXTagDef.java b/security-admin/src/main/java/org/apache/ranger/entity/XXTagDef.java index 818908ba81..88a7633ea4 100644 --- a/security-admin/src/main/java/org/apache/ranger/entity/XXTagDef.java +++ b/security-admin/src/main/java/org/apache/ranger/entity/XXTagDef.java @@ -63,6 +63,9 @@ public class XXTagDef extends XXDBBase implements Serializable { @Column(name = "source") protected String source; + @Column(name = "tag_attrs_def_text") + protected String tagAttrDefs; + /** * @return the guid */ @@ -138,6 +141,10 @@ public void setSource(String source) { this.source = source; } + public String getTagAttrDefs() { return tagAttrDefs; } + + public void setTagAttrDefs(String tagAttrDefs) { this.tagAttrDefs = tagAttrDefs; } + @Override public void setId(Long id) { this.id = id; @@ -168,6 +175,7 @@ public int hashCode() { result = prime * result + ((name == null) ? 0 : name.hashCode()); result = prime * result + ((source == null) ? 0 : source.hashCode()); result = prime * result + ((version == null) ? 0 : version.hashCode()); + result = prime * result + ((tagAttrDefs == null) ? 0 : tagAttrDefs.hashCode()); return result; } @@ -215,6 +223,11 @@ public boolean equals(Object obj) { return false; } else if (!version.equals(other.version)) return false; + if (tagAttrDefs == null) { + if (other.tagAttrDefs != null) + return false; + } else if (!tagAttrDefs.equals(other.tagAttrDefs)) + return false; return true; } @@ -239,6 +252,7 @@ public StringBuilder toString(StringBuilder sb) { sb.append("isEnabled={").append(isEnabled).append("} "); sb.append("source={").append(source).append("} "); sb.append("name={").append(name).append("} "); + sb.append("tagAttrDefs={").append(tagAttrDefs).append("} "); sb.append(" }"); return sb; diff --git a/security-admin/src/main/java/org/apache/ranger/patch/PatchForNifiResourceUpdateExclude_J10011.java b/security-admin/src/main/java/org/apache/ranger/patch/PatchForNifiResourceUpdateExclude_J10011.java new file mode 100644 index 0000000000..0025eb89ba --- /dev/null +++ b/security-admin/src/main/java/org/apache/ranger/patch/PatchForNifiResourceUpdateExclude_J10011.java @@ -0,0 +1,149 @@ + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ranger.patch; + +import java.util.List; +import org.apache.commons.collections.CollectionUtils; +import org.apache.log4j.Logger; +import org.apache.ranger.biz.ServiceDBStore; +import org.apache.ranger.common.JSONUtil; +import org.apache.ranger.common.RangerValidatorFactory; +import org.apache.ranger.common.StringUtil; +import org.apache.ranger.db.RangerDaoManager; +import org.apache.ranger.entity.XXPolicyResource; +import org.apache.ranger.entity.XXResourceDef; +import org.apache.ranger.plugin.model.RangerPolicy; +import org.apache.ranger.plugin.model.RangerServiceDef; +import org.apache.ranger.plugin.model.RangerServiceDef.RangerResourceDef; +import org.apache.ranger.plugin.model.validation.RangerServiceDefValidator; +import org.apache.ranger.plugin.model.validation.RangerValidator.Action; +import org.apache.ranger.plugin.store.EmbeddedServiceDefsUtil; +import org.apache.ranger.service.RangerPolicyService; +import org.apache.ranger.util.CLIUtil; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +/** + * Disables the Nifi plugin's exclude toggle in Ranger UI. + * After running this patch user wont be able to add exclude resource policies in NIFI. + * + */ +@Component +public class PatchForNifiResourceUpdateExclude_J10011 extends BaseLoader { + private static final Logger logger = Logger.getLogger(PatchForNifiResourceUpdateExclude_J10011.class); + @Autowired + RangerDaoManager daoMgr; + + @Autowired + ServiceDBStore svcDBStore; + + @Autowired + JSONUtil jsonUtil; + + @Autowired + StringUtil stringUtil; + + @Autowired + RangerValidatorFactory validatorFactory; + + @Autowired + ServiceDBStore svcStore; + + @Autowired + RangerPolicyService policyService; + + public static void main(String[] args) { + logger.info("main()"); + try { + PatchForNifiResourceUpdateExclude_J10011 loader = (PatchForNifiResourceUpdateExclude_J10011) CLIUtil.getBean(PatchForNifiResourceUpdateExclude_J10011.class); + loader.init(); + while (loader.isMoreToProcess()) { + loader.load(); + } + logger.info("Load complete. Exiting!!!"); + System.exit(0); + } catch (Exception e) { + logger.error("Error loading", e); + System.exit(1); + } + } + + @Override + public void init() throws Exception { + // Do Nothing + } + + @Override + public void execLoad() { + logger.info("==> PatchForNifiResourceUpdateExclude.execLoad()"); + try { + updateNifiServiceDef(); + } catch (Exception e) { + logger.error("Error whille updateNifiServiceDef()data.", e); + } + logger.info("<== PatchForNifiResourceUpdateExclude.execLoad()"); + } + + @Override + public void printStats() { + logger.info("updateNifiServiceDef data "); + } + + private void updateNifiServiceDef(){ + RangerServiceDef ret = null; + RangerServiceDef dbNifiServiceDef = null; + try { + dbNifiServiceDef = svcDBStore.getServiceDefByName(EmbeddedServiceDefsUtil.EMBEDDED_SERVICEDEF_NIFI_NAME); + if (dbNifiServiceDef != null) { + List rRDefList = null; + rRDefList = dbNifiServiceDef.getResources(); + if (CollectionUtils.isNotEmpty(rRDefList)) { + for (RangerResourceDef rRDef : rRDefList) { + + if (rRDef.getExcludesSupported()) { + rRDef.setExcludesSupported(false); + } + + XXResourceDef sdf=daoMgr.getXXResourceDef().findByNameAndServiceDefId(rRDef.getName(), dbNifiServiceDef.getId()); + long ResourceDefId=sdf.getId(); + List RangerPolicyResourceList=daoMgr.getXXPolicyResource().findByResDefId(ResourceDefId); + if (CollectionUtils.isNotEmpty(RangerPolicyResourceList)){ + for(XXPolicyResource RangerPolicyResource : RangerPolicyResourceList){ + if(RangerPolicyResource.getIsexcludes()){ + RangerPolicy rPolicy=svcDBStore.getPolicy(RangerPolicyResource.getPolicyid()); + rPolicy.setIsEnabled(false); + svcStore.updatePolicy(rPolicy); + } + } + } + } + } + RangerServiceDefValidator validator = validatorFactory.getServiceDefValidator(svcStore); + validator.validate(dbNifiServiceDef, Action.UPDATE); + ret = svcStore.updateServiceDef(dbNifiServiceDef); + } + if (ret == null) { + logger.error("Error while updating " + EmbeddedServiceDefsUtil.EMBEDDED_SERVICEDEF_NIFI_NAME+ "service-def"); + } + } catch (Exception e) { + logger.error("Error while updating " + EmbeddedServiceDefsUtil.EMBEDDED_SERVICEDEF_NIFI_NAME + "service-def", e); + } + } + +} diff --git a/security-admin/src/main/java/org/apache/ranger/patch/PatchForTagServiceDefUpdate_J10008.java b/security-admin/src/main/java/org/apache/ranger/patch/PatchForTagServiceDefUpdate_J10008.java new file mode 100644 index 0000000000..918fe1ecfc --- /dev/null +++ b/security-admin/src/main/java/org/apache/ranger/patch/PatchForTagServiceDefUpdate_J10008.java @@ -0,0 +1,202 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ranger.patch; + +import org.apache.commons.lang.StringUtils; +import org.apache.log4j.Logger; +import org.apache.ranger.biz.RangerBizUtil; +import org.apache.ranger.biz.ServiceDBStore; +import org.apache.ranger.common.JSONUtil; +import org.apache.ranger.common.RangerValidatorFactory; +import org.apache.ranger.common.StringUtil; +import org.apache.ranger.db.RangerDaoManager; +import org.apache.ranger.plugin.model.RangerServiceDef; +import org.apache.ranger.plugin.model.validation.RangerServiceDefValidator; +import org.apache.ranger.plugin.model.validation.RangerValidator.Action; +import org.apache.ranger.plugin.store.EmbeddedServiceDefsUtil; +import org.apache.ranger.service.RangerPolicyService; +import org.apache.ranger.service.XPermMapService; +import org.apache.ranger.service.XPolicyService; +import org.apache.ranger.util.CLIUtil; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +import org.apache.ranger.entity.XXServiceDef; +import java.util.List; +import java.util.Map; + +@Component +public class PatchForTagServiceDefUpdate_J10008 extends BaseLoader { + private static final Logger logger = Logger.getLogger(PatchForTagServiceDefUpdate_J10008.class); + public static final String SERVICEDBSTORE_SERVICEDEFBYNAME_TAG_NAME = "tag"; + public static final String SCRIPT_POLICY_CONDITION_NAME = "expression"; + + @Autowired + RangerDaoManager daoMgr; + + @Autowired + ServiceDBStore svcDBStore; + + @Autowired + JSONUtil jsonUtil; + + @Autowired + RangerPolicyService policyService; + + @Autowired + StringUtil stringUtil; + + @Autowired + XPolicyService xPolService; + + @Autowired + XPermMapService xPermMapService; + + @Autowired + RangerBizUtil bizUtil; + + @Autowired + RangerValidatorFactory validatorFactory; + + @Autowired + ServiceDBStore svcStore; + + public static void main(String[] args) { + logger.info("main()"); + try { + PatchForTagServiceDefUpdate_J10008 loader = (PatchForTagServiceDefUpdate_J10008) CLIUtil.getBean(PatchForTagServiceDefUpdate_J10008.class); + loader.init(); + while (loader.isMoreToProcess()) { + loader.load(); + } + logger.info("Load complete. Exiting!!!"); + System.exit(0); + } catch (Exception e) { + logger.error("Error loading", e); + System.exit(1); + } + } + + @Override + public void init() throws Exception { + // Do Nothing + } + + @Override + public void execLoad() { + logger.info("==> PatchForTagServiceDefUpdate.execLoad()"); + try { + updateTagServiceDef(); + } catch (Exception e) { + logger.error("Error whille updateTagServiceDef()data.", e); + } + logger.info("<== PatchForTagServiceDefUpdate.execLoad()"); + } + + @Override + public void printStats() { + logger.info("PatchForTagServiceDefUpdate data "); + } + + private void updateTagServiceDef(){ + RangerServiceDef embeddedTagServiceDef = null; + RangerServiceDef dbTagServiceDef = null; + List embeddedTagPolicyConditionDefs = null; + XXServiceDef xXServiceDefObj = null; + try{ + embeddedTagServiceDef=EmbeddedServiceDefsUtil.instance().getEmbeddedServiceDef(SERVICEDBSTORE_SERVICEDEFBYNAME_TAG_NAME); + if(embeddedTagServiceDef!=null){ + embeddedTagPolicyConditionDefs = embeddedTagServiceDef.getPolicyConditions(); + if (embeddedTagPolicyConditionDefs == null) { + logger.error("Policy Conditions are empyt in tag service def json"); + return; + } + + if (checkScriptPolicyCondPresent(embeddedTagPolicyConditionDefs) == false) { + logger.error(SCRIPT_POLICY_CONDITION_NAME + "policy condition not found!!"); + return; + } + + xXServiceDefObj = daoMgr.getXXServiceDef().findByName(SERVICEDBSTORE_SERVICEDEFBYNAME_TAG_NAME); + if (xXServiceDefObj == null) { + logger.error("Service def for " + SERVICEDBSTORE_SERVICEDEFBYNAME_TAG_NAME + " is not found!!"); + return; + } + + Map serviceDefOptionsPreUpdate=null; + String jsonStrPreUpdate=null; + jsonStrPreUpdate=xXServiceDefObj.getDefOptions(); + if (!StringUtils.isEmpty(jsonStrPreUpdate)) { + serviceDefOptionsPreUpdate=jsonUtil.jsonToMap(jsonStrPreUpdate); + } + xXServiceDefObj=null; + dbTagServiceDef=svcDBStore.getServiceDefByName(SERVICEDBSTORE_SERVICEDEFBYNAME_TAG_NAME); + + if(dbTagServiceDef!=null){ + dbTagServiceDef.setPolicyConditions(embeddedTagPolicyConditionDefs); + RangerServiceDefValidator validator = validatorFactory.getServiceDefValidator(svcStore); + validator.validate(dbTagServiceDef, Action.UPDATE); + + svcStore.updateServiceDef(dbTagServiceDef); + + xXServiceDefObj = daoMgr.getXXServiceDef().findByName(SERVICEDBSTORE_SERVICEDEFBYNAME_TAG_NAME); + if(xXServiceDefObj!=null) { + String jsonStrPostUpdate=xXServiceDefObj.getDefOptions(); + Map serviceDefOptionsPostUpdate = null; + if (!StringUtils.isEmpty(jsonStrPostUpdate)) { + serviceDefOptionsPostUpdate =jsonUtil.jsonToMap(jsonStrPostUpdate); + } + if (serviceDefOptionsPostUpdate != null && serviceDefOptionsPostUpdate.containsKey(RangerServiceDef.OPTION_ENABLE_DENY_AND_EXCEPTIONS_IN_POLICIES)) { + if(serviceDefOptionsPreUpdate == null || !serviceDefOptionsPreUpdate.containsKey(RangerServiceDef.OPTION_ENABLE_DENY_AND_EXCEPTIONS_IN_POLICIES)) { + String preUpdateValue = serviceDefOptionsPreUpdate == null ? null : serviceDefOptionsPreUpdate.get(RangerServiceDef.OPTION_ENABLE_DENY_AND_EXCEPTIONS_IN_POLICIES); + if (preUpdateValue == null) { + serviceDefOptionsPostUpdate.remove(RangerServiceDef.OPTION_ENABLE_DENY_AND_EXCEPTIONS_IN_POLICIES); + } else { + serviceDefOptionsPostUpdate.put(RangerServiceDef.OPTION_ENABLE_DENY_AND_EXCEPTIONS_IN_POLICIES, preUpdateValue); + } + xXServiceDefObj.setDefOptions(mapToJsonString(serviceDefOptionsPostUpdate)); + daoMgr.getXXServiceDef().update(xXServiceDefObj); + } + } + } + } + } + }catch(Exception e) + { + logger.error("Error while updating "+SERVICEDBSTORE_SERVICEDEFBYNAME_TAG_NAME+"service-def", e); + } + } + + private boolean checkScriptPolicyCondPresent(List policyCondDefs) { + boolean ret = false; + for(RangerServiceDef.RangerPolicyConditionDef policyCondDef : policyCondDefs) { + if ( SCRIPT_POLICY_CONDITION_NAME.equals(policyCondDef.getName()) ) { + ret = true ; + break; + } + } + return ret; + } + + private String mapToJsonString(Map map) throws Exception{ + String ret = null; + if(map != null) { + ret = jsonUtil.readMapToString(map); + } + return ret; + } +} diff --git a/security-admin/src/main/java/org/apache/ranger/patch/PatchForUpdatingPolicyJson_J10019.java b/security-admin/src/main/java/org/apache/ranger/patch/PatchForUpdatingPolicyJson_J10019.java new file mode 100644 index 0000000000..0f251de9dd --- /dev/null +++ b/security-admin/src/main/java/org/apache/ranger/patch/PatchForUpdatingPolicyJson_J10019.java @@ -0,0 +1,1174 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ranger.patch; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.ListIterator; +import java.util.Map; +import java.util.Set; + +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.MapUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.log4j.Logger; +import org.apache.ranger.authorization.utils.JsonUtils; +import org.apache.ranger.authorization.utils.StringUtil; +import org.apache.ranger.biz.PolicyRefUpdater; +import org.apache.ranger.biz.ServiceDBStore; +import org.apache.ranger.db.RangerDaoManager; +import org.apache.ranger.db.XXGroupDao; +import org.apache.ranger.db.XXPolicyDao; +import org.apache.ranger.db.XXPolicyRefAccessTypeDao; +import org.apache.ranger.db.XXPolicyRefConditionDao; +import org.apache.ranger.db.XXPolicyRefDataMaskTypeDao; +import org.apache.ranger.db.XXPolicyRefGroupDao; +import org.apache.ranger.db.XXPolicyRefResourceDao; +import org.apache.ranger.db.XXPolicyRefUserDao; +import org.apache.ranger.db.XXUserDao; +import org.apache.ranger.entity.XXAccessTypeDef; +import org.apache.ranger.entity.XXDataMaskTypeDef; +import org.apache.ranger.entity.XXGroup; +import org.apache.ranger.entity.XXPolicy; +import org.apache.ranger.entity.XXPolicyConditionDef; +import org.apache.ranger.entity.XXPolicyItem; +import org.apache.ranger.entity.XXPolicyItemAccess; +import org.apache.ranger.entity.XXPolicyItemCondition; +import org.apache.ranger.entity.XXPolicyItemDataMaskInfo; +import org.apache.ranger.entity.XXPolicyItemGroupPerm; +import org.apache.ranger.entity.XXPolicyItemRowFilterInfo; +import org.apache.ranger.entity.XXPolicyItemUserPerm; +import org.apache.ranger.entity.XXPolicyRefAccessType; +import org.apache.ranger.entity.XXPolicyRefCondition; +import org.apache.ranger.entity.XXPolicyRefDataMaskType; +import org.apache.ranger.entity.XXPolicyRefGroup; +import org.apache.ranger.entity.XXPolicyRefResource; +import org.apache.ranger.entity.XXPolicyRefUser; +import org.apache.ranger.entity.XXPolicyResource; +import org.apache.ranger.entity.XXPolicyResourceMap; +import org.apache.ranger.entity.XXPortalUser; +import org.apache.ranger.entity.XXResourceDef; +import org.apache.ranger.entity.XXService; +import org.apache.ranger.entity.XXServiceDef; +import org.apache.ranger.entity.XXUser; +import org.apache.ranger.plugin.model.RangerPolicy; +import org.apache.ranger.plugin.model.RangerPolicy.RangerDataMaskPolicyItem; +import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyItem; +import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyItemAccess; +import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyItemCondition; +import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyItemDataMaskInfo; +import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyItemRowFilterInfo; +import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyResource; +import org.apache.ranger.plugin.model.RangerPolicy.RangerRowFilterPolicyItem; +import org.apache.ranger.plugin.model.RangerService; +import org.apache.ranger.plugin.policyevaluator.RangerPolicyItemEvaluator; +import org.apache.ranger.plugin.util.RangerPerfTracer; +import org.apache.ranger.plugin.util.SearchFilter; +import org.apache.ranger.util.CLIUtil; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.stereotype.Component; +import org.springframework.transaction.PlatformTransactionManager; +import org.springframework.transaction.TransactionStatus; +import org.springframework.transaction.support.TransactionCallback; +import org.springframework.transaction.support.TransactionTemplate; + +/** + * Consolidates Ranger policy details into a JSON string and stores it into a + * column in x_policy table After running this patch Ranger policy can be + * completely read/saved into x_policy table and some related Ref tables (which + * maintain ID->String mapping for each policy). + * + */ +@Component +public class PatchForUpdatingPolicyJson_J10019 extends BaseLoader { + private static final Logger logger = Logger.getLogger(PatchForUpdatingPolicyJson_J10019.class); + + @Autowired + RangerDaoManager daoMgr; + + @Autowired + ServiceDBStore svcStore; + + @Autowired + @Qualifier(value = "transactionManager") + PlatformTransactionManager txManager; + + @Autowired + PolicyRefUpdater policyRefUpdater; + + private final Map groupIdMap = new HashMap<>(); + private final Map userIdMap = new HashMap<>(); + private final Map> resourceNameIdMap = new HashMap<>(); + private final Map> accessTypeIdMap = new HashMap<>(); + private final Map> conditionNameIdMap = new HashMap<>(); + private final Map> dataMaskTypeIdMap = new HashMap<>(); + + public static void main(String[] args) { + logger.info("main()"); + try { + PatchForUpdatingPolicyJson_J10019 loader = (PatchForUpdatingPolicyJson_J10019) CLIUtil.getBean(PatchForUpdatingPolicyJson_J10019.class); + + loader.init(); + + while (loader.isMoreToProcess()) { + loader.load(); + } + + logger.info("Load complete. Exiting!!!"); + + System.exit(0); + } catch (Exception e) { + logger.error("Error loading", e); + System.exit(1); + } + } + + @Override + public void init() throws Exception { + // Do Nothing + } + + @Override + public void execLoad() { + logger.info("==> PatchForUpdatingPolicyJson.execLoad()"); + + try { + updateRangerPolicyTableWithPolicyJson(); + } catch (Exception e) { + logger.error("Error while updateRangerPolicyTableWithPolicyJson()", e); + System.exit(1); + } + + logger.info("<== PatchForUpdatingPolicyJson.execLoad()"); + } + + @Override + public void printStats() { + logger.info("updateRangerPolicyTableWithPolicyJson data "); + } + + private void updateRangerPolicyTableWithPolicyJson() throws Exception { + logger.info("==> updateRangerPolicyTableWithPolicyJson() "); + + List allServices = svcStore.getServices(new SearchFilter()); + + if (CollectionUtils.isNotEmpty(allServices)) { + for (RangerService service : allServices) { + XXService dbService = daoMgr.getXXService().getById(service.getId()); + + logger.info("==> Port Policies of service(name=" + dbService.getName() + ")"); + + RangerPolicyRetriever policyRetriever = new RangerPolicyRetriever(daoMgr, txManager); + + List policies = policyRetriever.getServicePolicies(dbService); + + if (CollectionUtils.isNotEmpty(policies)) { + TransactionTemplate txTemplate = new TransactionTemplate(txManager); + + for (RangerPolicy policy : policies) { + XXPolicy xPolicy = daoMgr.getXXPolicy().getById(policy.getId()); + if (xPolicy != null && StringUtil.isEmpty(xPolicy.getPolicyText())) { + + PolicyUpdaterThread updaterThread = new PolicyUpdaterThread(txTemplate, service, policy); + updaterThread.setDaemon(true); + updaterThread.start(); + updaterThread.join(); + + String errorMsg = updaterThread.getErrorMsg(); + if (StringUtils.isNotEmpty(errorMsg)) { + throw new Exception(errorMsg); + } + } + } + } + } + } + + logger.info("<== updateRangerPolicyTableWithPolicyJson() "); + } + + private class PolicyUpdaterThread extends Thread { + final TransactionTemplate txTemplate; + final RangerService service; + final RangerPolicy policy; + String errorMsg; + + PolicyUpdaterThread(TransactionTemplate txTemplate, final RangerService service, final RangerPolicy policy) { + this.txTemplate = txTemplate; + this.service = service; + this.policy = policy; + this.errorMsg = null; + } + + public String getErrorMsg() { + return errorMsg; + } + + @Override + public void run() { + errorMsg = txTemplate.execute(new TransactionCallback() { + @Override + public String doInTransaction(TransactionStatus status) { + String ret = null; + try { + policyRefUpdater.cleanupRefTables(policy); + portPolicy(service.getType(), policy); + } catch (Throwable e) { + logger.error("PortPolicy failed for policy:[" + policy + "]", e); + ret = e.toString(); + } + return ret; + } + }); + } + } + + private void portPolicy(String serviceType, RangerPolicy policy) throws Exception { + logger.info("==> portPolicy(id=" + policy.getId() + ")"); + + String policyText = JsonUtils.objectToJson(policy); + + if (StringUtils.isEmpty(policyText)) { + throw new Exception("Failed to convert policy to json string. Policy: [id=" + policy.getId() + "; name=" + policy.getName() + "; serviceType=" + serviceType + "]"); + } + + XXPolicyDao policyDao = daoMgr.getXXPolicy(); + XXPolicy dbBean = policyDao.getById(policy.getId()); + + dbBean.setPolicyText(policyText); + + policyDao.update(dbBean); + + try { + Set accesses = new HashSet<>(); + Set users = new HashSet<>(); + Set groups = new HashSet<>(); + Set conditions = new HashSet<>(); + Set dataMasks = new HashSet<>(); + + buildLists(policy.getPolicyItems(), accesses, conditions, users, groups); + buildLists(policy.getDenyPolicyItems(), accesses, conditions, users, groups); + buildLists(policy.getAllowExceptions(), accesses, conditions, users, groups); + buildLists(policy.getDenyExceptions(), accesses, conditions, users, groups); + buildLists(policy.getDataMaskPolicyItems(), accesses, conditions, users, groups); + buildLists(policy.getRowFilterPolicyItems(), accesses, conditions, users, groups); + + buildList(policy.getDataMaskPolicyItems(), dataMasks); + + addResourceDefRef(serviceType, policy); + addUserNameRef(policy.getId(), users); + addGroupNameRef(policy.getId(), groups); + addAccessDefRef(serviceType, policy.getId(), accesses); + addPolicyConditionDefRef(serviceType, policy.getId(), conditions); + addDataMaskDefRef(serviceType, policy.getId(), dataMasks); + } catch (Exception e) { + logger.error("portPoliry(id=" + policy.getId() +") failed!!"); + logger.error("Offending policy:" + policyText); + throw e; + } + + logger.info("<== portPolicy(id=" + policy.getId() + ")"); + } + + private void addResourceDefRef(String serviceType, RangerPolicy policy) throws Exception { + logger.info("==> addResourceDefRef(id=" + policy.getId() + ")"); + + Map serviceDefResourceNameIDMap = resourceNameIdMap.get(serviceType); + + if (serviceDefResourceNameIDMap == null) { + serviceDefResourceNameIDMap = new HashMap<>(); + + resourceNameIdMap.put(serviceType, serviceDefResourceNameIDMap); + + XXServiceDef dbServiceDef = daoMgr.getXXServiceDef().findByName(serviceType); + + for (XXResourceDef resourceDef : daoMgr.getXXResourceDef().findByServiceDefId(dbServiceDef.getId())) { + serviceDefResourceNameIDMap.put(resourceDef.getName(), resourceDef.getId()); + } + } + + Map policyResources = policy.getResources(); + + if (MapUtils.isNotEmpty(policyResources)) { + XXPolicyRefResourceDao policyRefResourceDao = daoMgr.getXXPolicyRefResource(); + Set resourceNames = policyResources.keySet(); + + for (String resourceName : resourceNames) { + Long resourceDefId = serviceDefResourceNameIDMap.get(resourceName); + + if (resourceDefId == null) { + throw new Exception(resourceName + ": unknown resource in policy [id=" + policy.getId() + "; name=" + policy.getName() + "; serviceType=" + serviceType + "]. Known resources: " + serviceDefResourceNameIDMap.keySet()); + } + + // insert policy-id, resourceDefId, resourceName into Ref table + XXPolicyRefResource policyRefResource = new XXPolicyRefResource(); + + policyRefResource.setPolicyId(policy.getId()); + policyRefResource.setResourceDefId(resourceDefId); + policyRefResource.setResourceName(resourceName); + + policyRefResourceDao.create(policyRefResource); + } + } + + logger.info("<== addResourceDefRef(id=" + policy.getId() + ")"); + } + + private void addUserNameRef(Long policyId, Set users) throws Exception { + logger.info("==> addUserNameRef(id=" + policyId + ")"); + + XXPolicyRefUserDao policyRefUserDao = daoMgr.getXXPolicyRefUser(); + XXUserDao userDao = daoMgr.getXXUser(); + + // insert policy-id, userName into Ref table + for (String user : users) { + Long userId = userIdMap.get(user); + + if (userId == null) { + XXUser userObject = userDao.findByUserName(user); + + if (userObject == null) { + throw new Exception(user + ": unknown user in policy [id=" + policyId + "]"); + } + + userId = userObject.getId(); + + userIdMap.put(user, userId); + } + + XXPolicyRefUser policyRefUser = new XXPolicyRefUser(); + + policyRefUser.setPolicyId(policyId); + policyRefUser.setUserName(user); + policyRefUser.setUserId(userId); + + policyRefUserDao.create(policyRefUser); + } + + logger.info("<== addUserNameRef(id=" + policyId + ")"); + } + + private void addGroupNameRef(Long policyId, Set groups) throws Exception { + logger.info("==> addGroupNameRef(id=" + policyId + ")"); + + // insert policy-id, groupName into Ref table + XXPolicyRefGroupDao policyRefGroupDao = daoMgr.getXXPolicyRefGroup(); + XXGroupDao groupDao = daoMgr.getXXGroup(); + + for (String group : groups) { + Long groupId = groupIdMap.get(group); + + if (groupId == null) { + XXGroup groupObject = groupDao.findByGroupName(group); + + if (groupObject == null) { + throw new Exception(group + ": unknown group in policy [id=" + policyId + "]"); + } + + groupId = groupObject.getId(); + + groupIdMap.put(group, groupId); + } + + XXPolicyRefGroup policyRefGroup = new XXPolicyRefGroup(); + + policyRefGroup.setPolicyId(policyId); + policyRefGroup.setGroupName(group); + policyRefGroup.setGroupId(groupId); + + policyRefGroupDao.create(policyRefGroup); + } + + logger.info("<== addGroupNameRef(id=" + policyId + ")"); + + } + + private void addAccessDefRef(String serviceType, Long policyId, Set accesses) throws Exception { + logger.info("==> addAccessDefRef(id=" + policyId + ")"); + // insert policy-id, accessName into Ref table + + Map serviceDefAccessTypeIDMap = accessTypeIdMap.get(serviceType); + + if (serviceDefAccessTypeIDMap == null) { + serviceDefAccessTypeIDMap = new HashMap<>(); + + accessTypeIdMap.put(serviceType, serviceDefAccessTypeIDMap); + + XXServiceDef dbServiceDef = daoMgr.getXXServiceDef().findByName(serviceType); + + for (XXAccessTypeDef accessTypeDef : daoMgr.getXXAccessTypeDef().findByServiceDefId(dbServiceDef.getId())) { + serviceDefAccessTypeIDMap.put(accessTypeDef.getName(), accessTypeDef.getId()); + } + } + + XXPolicyRefAccessTypeDao policyRefAccessTypeDao = daoMgr.getXXPolicyRefAccessType(); + + for (String access : accesses) { + Long accessTypeDefId = serviceDefAccessTypeIDMap.get(access); + + if (accessTypeDefId == null) { + throw new Exception(access + ": unknown accessType in policy [id=" + policyId + "; serviceType=" + serviceType + "]. Known accessTypes: " + serviceDefAccessTypeIDMap.keySet()); + } + + XXPolicyRefAccessType policyRefAccessType = new XXPolicyRefAccessType(); + + policyRefAccessType.setPolicyId(policyId); + policyRefAccessType.setAccessTypeName(access); + policyRefAccessType.setAccessDefId(accessTypeDefId); + + policyRefAccessTypeDao.create(policyRefAccessType); + } + + logger.info("<== addAccessDefRef(id=" + policyId + ")"); + } + + private void addPolicyConditionDefRef(String serviceType, Long policyId, Set conditions) throws Exception { + logger.info("==> addPolicyConditionDefRef(id=" + policyId + ")"); + // insert policy-id, conditionName into Ref table + + Map serviceDefConditionNameIDMap = conditionNameIdMap.get(serviceType); + + if (serviceDefConditionNameIDMap == null) { + serviceDefConditionNameIDMap = new HashMap<>(); + + conditionNameIdMap.put(serviceType, serviceDefConditionNameIDMap); + + XXServiceDef dbServiceDef = daoMgr.getXXServiceDef().findByName(serviceType); + + for (XXPolicyConditionDef conditionDef : daoMgr.getXXPolicyConditionDef().findByServiceDefId(dbServiceDef.getId())) { + serviceDefConditionNameIDMap.put(conditionDef.getName(), conditionDef.getId()); + } + } + + XXPolicyRefConditionDao policyRefConditionDao = daoMgr.getXXPolicyRefCondition(); + + for (String condition : conditions) { + Long conditionDefId = serviceDefConditionNameIDMap.get(condition); + + if (conditionDefId == null) { + throw new Exception(condition + ": unknown condition in policy [id=" + policyId + "; serviceType=" + serviceType + "]. Known conditions are: " + serviceDefConditionNameIDMap.keySet()); + } + + XXPolicyRefCondition policyRefCondition = new XXPolicyRefCondition(); + + policyRefCondition.setPolicyId(policyId); + policyRefCondition.setConditionName(condition); + policyRefCondition.setConditionDefId(conditionDefId); + + policyRefConditionDao.create(policyRefCondition); + } + + logger.info("<== addPolicyConditionDefRef(id=" + policyId + ")"); + } + + private void addDataMaskDefRef(String serviceType, Long policyId, Set datamasks) throws Exception { + logger.info("==> addDataMaskDefRef(id=" + policyId + ")"); + + // insert policy-id, datamaskName into Ref table + + Map serviceDefDataMaskTypeIDMap = dataMaskTypeIdMap.get(serviceType); + + if (serviceDefDataMaskTypeIDMap == null) { + serviceDefDataMaskTypeIDMap = new HashMap<>(); + + dataMaskTypeIdMap.put(serviceType, serviceDefDataMaskTypeIDMap); + + XXServiceDef dbServiceDef = daoMgr.getXXServiceDef().findByName(serviceType); + + for (XXDataMaskTypeDef dataMaskTypeDef : daoMgr.getXXDataMaskTypeDef().findByServiceDefId(dbServiceDef.getId())) { + serviceDefDataMaskTypeIDMap.put(dataMaskTypeDef.getName(), dataMaskTypeDef.getId()); + } + } + + XXPolicyRefDataMaskTypeDao policyRefDataMaskTypeDao = daoMgr.getXXPolicyRefDataMaskType(); + + for (String datamask : datamasks) { + Long dataMaskTypeId = serviceDefDataMaskTypeIDMap.get(datamask); + + if (dataMaskTypeId == null) { + throw new Exception(datamask + ": unknown dataMaskType in policy [id=" + policyId + "; serviceType=" + serviceType + "]. Known dataMaskTypes " + serviceDefDataMaskTypeIDMap.keySet()); + } + + XXPolicyRefDataMaskType policyRefDataMaskType = new XXPolicyRefDataMaskType(); + + policyRefDataMaskType.setPolicyId(policyId); + policyRefDataMaskType.setDataMaskTypeName(datamask); + policyRefDataMaskType.setDataMaskDefId(dataMaskTypeId); + + policyRefDataMaskTypeDao.create(policyRefDataMaskType); + } + + logger.info("<== addDataMaskDefRef(id=" + policyId + ")"); + + } + + private void buildLists(List policyItems, Set accesses, Set conditions, Set users, Set groups) { + for (RangerPolicyItem item : policyItems) { + for (RangerPolicyItemAccess policyAccess : item.getAccesses()) { + accesses.add(policyAccess.getType()); + } + + for (RangerPolicyItemCondition policyCondition : item.getConditions()) { + conditions.add(policyCondition.getType()); + } + + users.addAll(item.getUsers()); + groups.addAll(item.getGroups()); + } + } + + private void buildList(List dataMaskPolicyItems, Set dataMasks) { + for (RangerDataMaskPolicyItem datMaskPolicyItem : dataMaskPolicyItems) { + dataMasks.add(datMaskPolicyItem.getDataMaskInfo().getDataMaskType()); + } + } + + static private class RangerPolicyRetriever { + static final Log LOG = LogFactory.getLog(RangerPolicyRetriever.class); + static final Log PERF_LOG = RangerPerfTracer.getPerfLogger("db.RangerPolicyRetriever"); + + private final RangerDaoManager daoMgr; + private final LookupCache lookupCache = new LookupCache(); + + private final PlatformTransactionManager txManager; + private final TransactionTemplate txTemplate; + + RangerPolicyRetriever(RangerDaoManager daoMgr, PlatformTransactionManager txManager) { + this.daoMgr = daoMgr; + this.txManager = txManager; + + if (this.txManager != null) { + this.txTemplate = new TransactionTemplate(this.txManager); + + this.txTemplate.setReadOnly(true); + } else { + this.txTemplate = null; + } + } + + private class PolicyLoaderThread extends Thread { + final TransactionTemplate txTemplate; + final XXService xService; + List policies; + + PolicyLoaderThread(TransactionTemplate txTemplate, final XXService xService) { + this.txTemplate = txTemplate; + this.xService = xService; + } + + public List getPolicies() { + return policies; + } + + @Override + public void run() { + txTemplate.setReadOnly(true); + policies = txTemplate.execute(new TransactionCallback>() { + @Override + public List doInTransaction(TransactionStatus status) { + RetrieverContext ctx = new RetrieverContext(xService); + return ctx.getAllPolicies(); + } + }); + } + } + + public List getServicePolicies(final XXService xService) throws InterruptedException { + String serviceName = xService == null ? null : xService.getName(); + Long serviceId = xService == null ? null : xService.getId(); + + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerPolicyRetriever.getServicePolicies(serviceName=" + serviceName + ", serviceId=" + serviceId + ")"); + } + + List ret = null; + RangerPerfTracer perf = null; + + if (RangerPerfTracer.isPerfTraceEnabled(PERF_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_LOG, "RangerPolicyRetriever.getServicePolicies(serviceName=" + serviceName + ",serviceId=" + serviceId + ")"); + } + + if (xService != null) { + if (txTemplate == null) { + if (LOG.isDebugEnabled()) { + LOG.debug("Transaction Manager is null; Retrieving policies in the existing transaction"); + } + + RetrieverContext ctx = new RetrieverContext(xService); + + ret = ctx.getAllPolicies(); + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("Retrieving policies in a new, read-only transaction"); + } + + PolicyLoaderThread t = new PolicyLoaderThread(txTemplate, xService); + t.start(); + t.join(); + ret = t.getPolicies(); + } + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("RangerPolicyRetriever.getServicePolicies(xService=" + xService + "): invalid parameter"); + } + } + + RangerPerfTracer.log(perf); + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerPolicyRetriever.getServicePolicies(serviceName=" + serviceName + ", serviceId=" + serviceId + "): policyCount=" + (ret == null ? 0 : ret.size())); + } + + return ret; + } + + class LookupCache { + final Map userNames = new HashMap(); + final Map userScreenNames = new HashMap(); + final Map groupNames = new HashMap(); + final Map accessTypes = new HashMap(); + final Map conditions = new HashMap(); + final Map resourceDefs = new HashMap(); + final Map dataMasks = new HashMap(); + + String getUserName(Long userId) { + String ret = null; + + if (userId != null) { + ret = userNames.get(userId); + + if (ret == null) { + XXUser user = daoMgr.getXXUser().getById(userId); + + if (user != null) { + ret = user.getName(); // Name is `loginId` + + userNames.put(userId, ret); + } + } + } + + return ret; + } + + String getUserScreenName(Long userId) { + String ret = null; + + if (userId != null) { + ret = userScreenNames.get(userId); + + if (ret == null) { + XXPortalUser user = daoMgr.getXXPortalUser().getById(userId); + + if (user != null) { + ret = user.getPublicScreenName(); + + if (StringUtil.isEmpty(ret)) { + ret = user.getFirstName(); + + if (StringUtil.isEmpty(ret)) { + ret = user.getLoginId(); + } else { + if (!StringUtil.isEmpty(user.getLastName())) { + ret += (" " + user.getLastName()); + } + } + } + + if (ret != null) { + userScreenNames.put(userId, ret); + } + } + } + } + + return ret; + } + + String getGroupName(Long groupId) { + String ret = null; + + if (groupId != null) { + ret = groupNames.get(groupId); + + if (ret == null) { + XXGroup group = daoMgr.getXXGroup().getById(groupId); + + if (group != null) { + ret = group.getName(); + + groupNames.put(groupId, ret); + } + } + } + + return ret; + } + + String getAccessType(Long accessTypeId) { + String ret = null; + + if (accessTypeId != null) { + ret = accessTypes.get(accessTypeId); + + if (ret == null) { + XXAccessTypeDef xAccessType = daoMgr.getXXAccessTypeDef().getById(accessTypeId); + + if (xAccessType != null) { + ret = xAccessType.getName(); + + accessTypes.put(accessTypeId, ret); + } else { + LOG.warn("getAccessType(): Canot find name for accessTypeId " + accessTypeId + ". This will cause Ranger policy migration to fail. Please check if all service-defs are migrated correctly!"); + } + } + } + + return ret; + } + + String getConditionType(Long conditionDefId) { + String ret = null; + + if (conditionDefId != null) { + ret = conditions.get(conditionDefId); + + if (ret == null) { + XXPolicyConditionDef xPolicyConditionDef = daoMgr.getXXPolicyConditionDef() + .getById(conditionDefId); + + if (xPolicyConditionDef != null) { + ret = xPolicyConditionDef.getName(); + + conditions.put(conditionDefId, ret); + } + } + } + + return ret; + } + + String getResourceName(Long resourceDefId) { + String ret = null; + + if (resourceDefId != null) { + ret = resourceDefs.get(resourceDefId); + + if (ret == null) { + XXResourceDef xResourceDef = daoMgr.getXXResourceDef().getById(resourceDefId); + + if (xResourceDef != null) { + ret = xResourceDef.getName(); + + resourceDefs.put(resourceDefId, ret); + } + } + } + + return ret; + } + + String getDataMaskName(Long dataMaskDefId) { + String ret = null; + + if (dataMaskDefId != null) { + ret = dataMasks.get(dataMaskDefId); + + if (ret == null) { + XXDataMaskTypeDef xDataMaskDef = daoMgr.getXXDataMaskTypeDef().getById(dataMaskDefId); + + if (xDataMaskDef != null) { + ret = xDataMaskDef.getName(); + + dataMasks.put(dataMaskDefId, ret); + } + } + } + + return ret; + } + } + + static List asList(XXPolicy policy) { + List ret = new ArrayList(); + + if (policy != null) { + ret.add(policy); + } + + return ret; + } + + class RetrieverContext { + final XXService service; + final ListIterator iterPolicy; + final ListIterator iterResources; + final ListIterator iterResourceMaps; + final ListIterator iterPolicyItems; + final ListIterator iterUserPerms; + final ListIterator iterGroupPerms; + final ListIterator iterAccesses; + final ListIterator iterConditions; + final ListIterator iterDataMaskInfos; + final ListIterator iterRowFilterInfos; + + RetrieverContext(XXService xService) { + Long serviceId = xService == null ? null : xService.getId(); + List xPolicies = daoMgr.getXXPolicy().findByServiceId(serviceId); + + this.service = xService; + this.iterPolicy = xPolicies.listIterator(); + + List xResources = daoMgr.getXXPolicyResource().findByServiceId(serviceId); + List xResourceMaps = daoMgr.getXXPolicyResourceMap().findByServiceId(serviceId); + List xPolicyItems = daoMgr.getXXPolicyItem().findByServiceId(serviceId); + List xUserPerms = daoMgr.getXXPolicyItemUserPerm().findByServiceId(serviceId); + List xGroupPerms = daoMgr.getXXPolicyItemGroupPerm().findByServiceId(serviceId); + List xAccesses = daoMgr.getXXPolicyItemAccess().findByServiceId(serviceId); + List xConditions = daoMgr.getXXPolicyItemCondition().findByServiceId(serviceId); + List xDataMaskInfos = daoMgr.getXXPolicyItemDataMaskInfo().findByServiceId(serviceId); + List xRowFilterInfos = daoMgr.getXXPolicyItemRowFilterInfo().findByServiceId(serviceId); + + this.iterResources = xResources.listIterator(); + this.iterResourceMaps = xResourceMaps.listIterator(); + this.iterPolicyItems = xPolicyItems.listIterator(); + this.iterUserPerms = xUserPerms.listIterator(); + this.iterGroupPerms = xGroupPerms.listIterator(); + this.iterAccesses = xAccesses.listIterator(); + this.iterConditions = xConditions.listIterator(); + this.iterDataMaskInfos = xDataMaskInfos.listIterator(); + this.iterRowFilterInfos = xRowFilterInfos.listIterator(); + } + + RetrieverContext(XXPolicy xPolicy, XXService xService) { + Long policyId = xPolicy == null ? null : xPolicy.getId(); + List xPolicies = asList(xPolicy); + + this.service = xService; + this.iterPolicy = xPolicies.listIterator(); + + List xResources = daoMgr.getXXPolicyResource().findByPolicyId(policyId); + List xResourceMaps = daoMgr.getXXPolicyResourceMap().findByPolicyId(policyId); + List xPolicyItems = daoMgr.getXXPolicyItem().findByPolicyId(policyId); + List xUserPerms = daoMgr.getXXPolicyItemUserPerm().findByPolicyId(policyId); + List xGroupPerms = daoMgr.getXXPolicyItemGroupPerm().findByPolicyId(policyId); + List xAccesses = daoMgr.getXXPolicyItemAccess().findByPolicyId(policyId); + List xConditions = daoMgr.getXXPolicyItemCondition().findByPolicyId(policyId); + List xDataMaskInfos = daoMgr.getXXPolicyItemDataMaskInfo().findByPolicyId(policyId); + List xRowFilterInfos = daoMgr.getXXPolicyItemRowFilterInfo().findByPolicyId(policyId); + + this.iterResources = xResources.listIterator(); + this.iterResourceMaps = xResourceMaps.listIterator(); + this.iterPolicyItems = xPolicyItems.listIterator(); + this.iterUserPerms = xUserPerms.listIterator(); + this.iterGroupPerms = xGroupPerms.listIterator(); + this.iterAccesses = xAccesses.listIterator(); + this.iterConditions = xConditions.listIterator(); + this.iterDataMaskInfos = xDataMaskInfos.listIterator(); + this.iterRowFilterInfos = xRowFilterInfos.listIterator(); + } + + RangerPolicy getNextPolicy() { + RangerPolicy ret = null; + + if (iterPolicy.hasNext()) { + XXPolicy xPolicy = iterPolicy.next(); + + if (xPolicy != null) { + ret = new RangerPolicy(); + + ret.setId(xPolicy.getId()); + ret.setGuid(xPolicy.getGuid()); + ret.setIsEnabled(xPolicy.getIsEnabled()); + ret.setCreatedBy(lookupCache.getUserScreenName(xPolicy.getAddedByUserId())); + ret.setUpdatedBy(lookupCache.getUserScreenName(xPolicy.getUpdatedByUserId())); + ret.setCreateTime(xPolicy.getCreateTime()); + ret.setUpdateTime(xPolicy.getUpdateTime()); + ret.setVersion(xPolicy.getVersion()); + ret.setService(service == null ? null : service.getName()); + ret.setName(StringUtils.trim(xPolicy.getName())); + ret.setPolicyType(xPolicy.getPolicyType() == null ? RangerPolicy.POLICY_TYPE_ACCESS : xPolicy.getPolicyType()); + ret.setDescription(xPolicy.getDescription()); + ret.setResourceSignature(xPolicy.getResourceSignature()); + ret.setIsAuditEnabled(xPolicy.getIsAuditEnabled()); + + getResource(ret); + getPolicyItems(ret); + } + } + + return ret; + } + + List getAllPolicies() { + List ret = new ArrayList(); + + while (iterPolicy.hasNext()) { + RangerPolicy policy = getNextPolicy(); + + if (policy != null) { + ret.add(policy); + } + } + + if (!hasProcessedAll()) { + LOG.warn("getAllPolicies(): perhaps one or more policies got updated during retrieval. Falling back to secondary method"); + + ret = getAllPoliciesBySecondary(); + } + + return ret; + } + + List getAllPoliciesBySecondary() { + List ret = null; + + if (service != null) { + List xPolicies = daoMgr.getXXPolicy().findByServiceId(service.getId()); + + if (CollectionUtils.isNotEmpty(xPolicies)) { + ret = new ArrayList(xPolicies.size()); + + for (XXPolicy xPolicy : xPolicies) { + RetrieverContext ctx = new RetrieverContext(xPolicy, service); + + RangerPolicy policy = ctx.getNextPolicy(); + + if (policy != null) { + ret.add(policy); + } + } + } + } + + return ret; + } + + private boolean hasProcessedAll() { + boolean moreToProcess = iterPolicy.hasNext() || iterResources.hasNext() || iterResourceMaps.hasNext() + || iterPolicyItems.hasNext() || iterUserPerms.hasNext() || iterGroupPerms.hasNext() + || iterAccesses.hasNext() || iterConditions.hasNext() || iterDataMaskInfos.hasNext() + || iterRowFilterInfos.hasNext(); + + return !moreToProcess; + } + + private void getResource(RangerPolicy policy) { + while (iterResources.hasNext()) { + XXPolicyResource xResource = iterResources.next(); + + if (xResource.getPolicyid().equals(policy.getId())) { + RangerPolicyResource resource = new RangerPolicyResource(); + + resource.setIsExcludes(xResource.getIsexcludes()); + resource.setIsRecursive(xResource.getIsrecursive()); + + while (iterResourceMaps.hasNext()) { + XXPolicyResourceMap xResourceMap = iterResourceMaps.next(); + + if (xResourceMap.getResourceid().equals(xResource.getId())) { + resource.getValues().add(xResourceMap.getValue()); + } else { + if (iterResourceMaps.hasPrevious()) { + iterResourceMaps.previous(); + } + + break; + } + } + + policy.getResources().put(lookupCache.getResourceName(xResource.getResdefid()), resource); + } else if (xResource.getPolicyid().compareTo(policy.getId()) > 0) { + if (iterResources.hasPrevious()) { + iterResources.previous(); + } + + break; + } + } + } + + private void getPolicyItems(RangerPolicy policy) { + while (iterPolicyItems.hasNext()) { + XXPolicyItem xPolicyItem = iterPolicyItems.next(); + + if (xPolicyItem.getPolicyid().equals(policy.getId())) { + final RangerPolicyItem policyItem; + final RangerDataMaskPolicyItem dataMaskPolicyItem; + final RangerRowFilterPolicyItem rowFilterPolicyItem; + + if (xPolicyItem.getItemType() == RangerPolicyItemEvaluator.POLICY_ITEM_TYPE_DATAMASK) { + dataMaskPolicyItem = new RangerDataMaskPolicyItem(); + rowFilterPolicyItem = null; + policyItem = dataMaskPolicyItem; + } else if (xPolicyItem.getItemType() == RangerPolicyItemEvaluator.POLICY_ITEM_TYPE_ROWFILTER) { + dataMaskPolicyItem = null; + rowFilterPolicyItem = new RangerRowFilterPolicyItem(); + policyItem = rowFilterPolicyItem; + } else { + dataMaskPolicyItem = null; + rowFilterPolicyItem = null; + policyItem = new RangerPolicyItem(); + } + + while (iterAccesses.hasNext()) { + XXPolicyItemAccess xAccess = iterAccesses.next(); + + if (xAccess.getPolicyitemid().equals(xPolicyItem.getId())) { + policyItem.getAccesses().add(new RangerPolicyItemAccess(lookupCache.getAccessType(xAccess.getType()), xAccess.getIsallowed())); + } else { + if (iterAccesses.hasPrevious()) { + iterAccesses.previous(); + } + + break; + } + } + + while (iterUserPerms.hasNext()) { + XXPolicyItemUserPerm xUserPerm = iterUserPerms.next(); + + if (xUserPerm.getPolicyitemid().equals(xPolicyItem.getId())) { + String userName = lookupCache.getUserName(xUserPerm.getUserid()); + + if (userName != null) { + policyItem.getUsers().add(userName); + } + } else { + if (iterUserPerms.hasPrevious()) { + iterUserPerms.previous(); + } + + break; + } + } + + while (iterGroupPerms.hasNext()) { + XXPolicyItemGroupPerm xGroupPerm = iterGroupPerms.next(); + + if (xGroupPerm.getPolicyitemid().equals(xPolicyItem.getId())) { + String groupName = lookupCache.getGroupName(xGroupPerm.getGroupid()); + + if (groupName != null) { + policyItem.getGroups().add(groupName); + } + } else { + if (iterGroupPerms.hasPrevious()) { + iterGroupPerms.previous(); + } + + break; + } + } + + RangerPolicyItemCondition condition = null; + Long prevConditionType = null; + + while (iterConditions.hasNext()) { + XXPolicyItemCondition xCondition = iterConditions.next(); + + if (xCondition.getPolicyitemid().equals(xPolicyItem.getId())) { + if (!xCondition.getType().equals(prevConditionType)) { + condition = new RangerPolicyItemCondition(); + + condition.setType(lookupCache.getConditionType(xCondition.getType())); + condition.getValues().add(xCondition.getValue()); + + policyItem.getConditions().add(condition); + + prevConditionType = xCondition.getType(); + } else { + condition.getValues().add(xCondition.getValue()); + } + } else { + if (iterConditions.hasPrevious()) { + iterConditions.previous(); + } + + break; + } + } + + policyItem.setDelegateAdmin(xPolicyItem.getDelegateAdmin()); + + if (dataMaskPolicyItem != null) { + while (iterDataMaskInfos.hasNext()) { + XXPolicyItemDataMaskInfo xDataMaskInfo = iterDataMaskInfos.next(); + + if (xDataMaskInfo.getPolicyItemId().equals(xPolicyItem.getId())) { + dataMaskPolicyItem.setDataMaskInfo(new RangerPolicyItemDataMaskInfo(lookupCache.getDataMaskName(xDataMaskInfo.getType()), xDataMaskInfo.getConditionExpr(), xDataMaskInfo.getValueExpr())); + } else { + if (iterDataMaskInfos.hasPrevious()) { + iterDataMaskInfos.previous(); + } + + break; + } + } + } + + if (rowFilterPolicyItem != null) { + while (iterRowFilterInfos.hasNext()) { + XXPolicyItemRowFilterInfo xRowFilterInfo = iterRowFilterInfos.next(); + + if (xRowFilterInfo.getPolicyItemId().equals(xPolicyItem.getId())) { + rowFilterPolicyItem.setRowFilterInfo(new RangerPolicyItemRowFilterInfo(xRowFilterInfo.getFilterExpr())); + } else { + if (iterRowFilterInfos.hasPrevious()) { + iterRowFilterInfos.previous(); + } + + break; + } + } + } + + int itemType = xPolicyItem.getItemType() == null ? RangerPolicyItemEvaluator.POLICY_ITEM_TYPE_ALLOW : xPolicyItem.getItemType(); + + if (itemType == RangerPolicyItemEvaluator.POLICY_ITEM_TYPE_ALLOW) { + policy.getPolicyItems().add(policyItem); + } else if (itemType == RangerPolicyItemEvaluator.POLICY_ITEM_TYPE_DENY) { + policy.getDenyPolicyItems().add(policyItem); + } else if (itemType == RangerPolicyItemEvaluator.POLICY_ITEM_TYPE_ALLOW_EXCEPTIONS) { + policy.getAllowExceptions().add(policyItem); + } else if (itemType == RangerPolicyItemEvaluator.POLICY_ITEM_TYPE_DENY_EXCEPTIONS) { + policy.getDenyExceptions().add(policyItem); + } else if (itemType == RangerPolicyItemEvaluator.POLICY_ITEM_TYPE_DATAMASK) { + policy.getDataMaskPolicyItems().add(dataMaskPolicyItem); + } else if (itemType == RangerPolicyItemEvaluator.POLICY_ITEM_TYPE_ROWFILTER) { + policy.getRowFilterPolicyItems().add(rowFilterPolicyItem); + } else { // unknown itemType + LOG.warn("RangerPolicyRetriever.getPolicy(policyId=" + policy.getId() + "): ignoring unknown policyItemType " + itemType); + } + } else if (xPolicyItem.getPolicyid().compareTo(policy.getId()) > 0) { + if (iterPolicyItems.hasPrevious()) { + iterPolicyItems.previous(); + } + + break; + } + } + } + } + } +} diff --git a/security-admin/src/main/java/org/apache/ranger/patch/PatchForUpdatingTagsJson_J10020.java b/security-admin/src/main/java/org/apache/ranger/patch/PatchForUpdatingTagsJson_J10020.java new file mode 100644 index 0000000000..f76864e1dd --- /dev/null +++ b/security-admin/src/main/java/org/apache/ranger/patch/PatchForUpdatingTagsJson_J10020.java @@ -0,0 +1,873 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Consolidates Ranger policy details into a JSON string and stores it into a + * column in x_policy table After running this patch Ranger policy can be + * completely read/saved into x_policy table and some related Ref tables (which + * maintain ID->String mapping for each policy). + * + */ + +package org.apache.ranger.patch; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.ListIterator; +import java.util.Map; + +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.MapUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.log4j.Logger; +import org.apache.ranger.authorization.utils.StringUtil; +import org.apache.ranger.biz.ServiceDBStore; +import org.apache.ranger.biz.TagDBStore; +import org.apache.ranger.db.RangerDaoManager; +import org.apache.ranger.db.XXServiceResourceDao; +import org.apache.ranger.db.XXTagDao; +import org.apache.ranger.db.XXTagDefDao; +import org.apache.ranger.entity.XXPortalUser; +import org.apache.ranger.entity.XXResourceDef; +import org.apache.ranger.entity.XXService; +import org.apache.ranger.entity.XXServiceResource; +import org.apache.ranger.entity.XXServiceResourceElement; +import org.apache.ranger.entity.XXServiceResourceElementValue; +import org.apache.ranger.entity.XXTag; +import org.apache.ranger.entity.XXTagAttribute; +import org.apache.ranger.entity.XXTagAttributeDef; +import org.apache.ranger.entity.XXTagDef; +import org.apache.ranger.plugin.model.RangerPolicy; +import org.apache.ranger.plugin.model.RangerService; +import org.apache.ranger.plugin.model.RangerServiceResource; +import org.apache.ranger.plugin.model.RangerTag; +import org.apache.ranger.plugin.model.RangerTagDef; +import org.apache.ranger.plugin.util.SearchFilter; +import org.apache.ranger.service.RangerServiceResourceService; +import org.apache.ranger.service.RangerTagDefService; +import org.apache.ranger.service.RangerTagService; +import org.apache.ranger.util.CLIUtil; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.stereotype.Component; +import org.springframework.transaction.PlatformTransactionManager; +import org.springframework.transaction.TransactionStatus; +import org.springframework.transaction.support.TransactionCallback; +import org.springframework.transaction.support.TransactionTemplate; + +@Component +public class PatchForUpdatingTagsJson_J10020 extends BaseLoader { + + private static final Logger logger = Logger.getLogger(PatchForUpdatingTagsJson_J10020.class); + + @Autowired + RangerDaoManager daoMgr; + + @Autowired + ServiceDBStore svcStore; + + @Autowired + TagDBStore tagStore; + + @Autowired + @Qualifier(value = "transactionManager") + PlatformTransactionManager txManager; + + @Autowired + RangerTagDefService tagDefService; + + @Autowired + RangerTagService tagService; + + @Autowired + RangerServiceResourceService serviceResourceService; + + public static void main(String[] args) { + logger.info("main()"); + try { + PatchForUpdatingTagsJson_J10020 loader = (PatchForUpdatingTagsJson_J10020) CLIUtil + .getBean(PatchForUpdatingTagsJson_J10020.class); + + loader.init(); + + while (loader.isMoreToProcess()) { + loader.load(); + } + + logger.info("Load complete. Exiting!!!"); + + System.exit(0); + } catch (Exception e) { + logger.error("Error loading", e); + System.exit(1); + } + } + + @Override + public void init() throws Exception { + // Do Nothing + } + + @Override + public void execLoad() { + logger.info("==> PatchForUpdatingTagsJson.execLoad()"); + + try { + updateRangerTagsTablesWithTagsJson(); + } catch (Exception e) { + logger.error("Error while UpdateRangerTagsTablesWithTagsJson()", e); + System.exit(1); + } + + logger.info("<== PatchForUpdatingTagsJson.execLoad()"); + } + + @Override + public void printStats() { + logger.info("Update Ranger Tags Tables with Json data "); + } + + private void updateRangerTagsTablesWithTagsJson() throws Exception { + logger.info("==> updateRangerTagsTablesWithTagsJson() "); + + List allServices = svcStore.getServices(new SearchFilter()); + + if (CollectionUtils.isNotEmpty(allServices)) { + TransactionTemplate txTemplate = new TransactionTemplate(txManager); + + for (RangerService service : allServices) { + XXService dbService = daoMgr.getXXService().getById(service.getId()); + RangerTagDBRetriever tagsRetriever = new RangerTagDBRetriever(daoMgr, txManager, dbService); + Map tagDefs = tagsRetriever.getTagDefs(); + Map tags = tagsRetriever.getTags(); + List serviceResources = tagsRetriever.getServiceResources(); + + XXTagDefDao tagDefDao = daoMgr.getXXTagDef(); + XXTagDao tagDao = daoMgr.getXXTag(); + XXServiceResourceDao serviceResourceDao = daoMgr.getXXServiceResource(); + + if (MapUtils.isNotEmpty(tagDefs)) { + logger.info("==> Port " + tagDefs.size() + " Tag Definitions for service(name=" + dbService.getName() + ")"); + + for (Map.Entry entry : tagDefs.entrySet()) { + RangerTagDef tagDef = entry.getValue(); + XXTagDef xTagDef = tagDefDao.getById(tagDef.getId()); + + if (xTagDef != null && StringUtils.isEmpty(xTagDef.getTagAttrDefs())) { + + TagsUpdaterThread updaterThread = new TagsUpdaterThread(txTemplate, null, null, tagDef); + String errorMsg = runThread(updaterThread); + if (StringUtils.isNotEmpty(errorMsg)) { + throw new Exception(errorMsg); + } + } + } + } + + if (MapUtils.isNotEmpty(tags)) { + logger.info("==> Port " + tags.size() + " Tags for service(name=" + dbService.getName() + ")"); + + for (Map.Entry entry : tags.entrySet()) { + RangerTag tag = entry.getValue(); + XXTag xTag = tagDao.getById(tag.getId()); + + if (xTag != null && StringUtils.isEmpty(xTag.getTagAttrs())) { + TagsUpdaterThread updaterThread = new TagsUpdaterThread(txTemplate, null, tag, null); + String errorMsg = runThread(updaterThread); + if (StringUtils.isNotEmpty(errorMsg)) { + throw new Exception(errorMsg); + } + } + } + } + + if (CollectionUtils.isNotEmpty(serviceResources)) { + logger.info("==> Port " + serviceResources.size() + " Service Resources for service(name=" + dbService.getName() + ")"); + + for (RangerServiceResource serviceResource : serviceResources) { + + XXServiceResource xServiceResource = serviceResourceDao.getById(serviceResource.getId()); + + if (xServiceResource != null && StringUtils.isEmpty(xServiceResource.getServiceResourceElements())) { + TagsUpdaterThread updaterThread = new TagsUpdaterThread(txTemplate, serviceResource, null, null); + String errorMsg = runThread(updaterThread); + if (StringUtils.isNotEmpty(errorMsg)) { + throw new Exception(errorMsg); + } + } + } + } + } + } + + logger.info("<== updateRangerTagsTablesWithTagsJson() "); + } + + private String runThread(TagsUpdaterThread updaterThread) throws Exception { + updaterThread.setDaemon(true); + updaterThread.start(); + updaterThread.join(); + return updaterThread.getErrorMsg(); + } + + private class TagsUpdaterThread extends Thread { + final TransactionTemplate txTemplate; + final RangerServiceResource serviceResource; + final RangerTag tag; + final RangerTagDef tagDef; + String errorMsg; + + TagsUpdaterThread(TransactionTemplate txTemplate, final RangerServiceResource serviceResource, final RangerTag tag, final RangerTagDef tagDef) { + this.txTemplate = txTemplate; + this.serviceResource = serviceResource; + this.tag = tag; + this.tagDef = tagDef; + this.errorMsg = null; + } + + public String getErrorMsg() { + return errorMsg; + } + + @Override + public void run() { + errorMsg = txTemplate.execute(new TransactionCallback() { + @Override + public String doInTransaction(TransactionStatus status) { + String ret = null; + try { + if (serviceResource != null) { + portServiceResource(serviceResource); + } + if (tag != null) { + portTag(tag); + } + if (tagDef != null) { + portTagDef(tagDef); + } + } catch (Throwable e) { + logger.error("Port failed :[serviceResource=" + serviceResource + ", tag=" + tag + ", tagDef=" + tagDef +"]", e); + ret = e.toString(); + } + return ret; + } + }); + } + } + private void portTagDef(RangerTagDef tagDef) { + tagDefService.update(tagDef); + } + + private void portTag(RangerTag tag) { + tagService.update(tag); + } + + private void portServiceResource(RangerServiceResource serviceResource) throws Exception { + serviceResourceService.update(serviceResource); + tagStore.refreshServiceResource(serviceResource.getId()); + } + + private class RangerTagDBRetriever { + final Log LOG = LogFactory.getLog(RangerTagDBRetriever.class); + + private final RangerDaoManager daoMgr; + private final XXService xService; + private final RangerTagDBRetriever.LookupCache lookupCache; + private final PlatformTransactionManager txManager; + private final TransactionTemplate txTemplate; + private List serviceResources; + private Map tagDefs; + private Map tags; + + RangerTagDBRetriever(final RangerDaoManager daoMgr, final PlatformTransactionManager txManager, final XXService xService) throws InterruptedException { + this.daoMgr = daoMgr; + this.xService = xService; + this.lookupCache = new RangerTagDBRetriever.LookupCache(); + this.txManager = txManager; + + if (this.txManager != null) { + this.txTemplate = new TransactionTemplate(this.txManager); + this.txTemplate.setReadOnly(true); + } else { + this.txTemplate = null; + } + + if (this.daoMgr != null && this.xService != null) { + if (this.txTemplate == null) { + if (LOG.isDebugEnabled()) { + LOG.debug("Load Tags in the same thread and using an existing transaction"); + } + + if (!initializeTagCache(xService)) { + LOG.error("Failed to get tags for service:[" + xService.getName() + "] in the same thread and using an existing transaction"); + } + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("Load Tags in a separate thread and using a new transaction"); + } + + RangerTagDBRetriever.TagLoaderThread t = new RangerTagDBRetriever.TagLoaderThread(txTemplate, xService); + t.setDaemon(true); + t.start(); + t.join(); + } + + } + } + + List getServiceResources() { + return serviceResources; + } + + Map getTagDefs() { + return tagDefs; + } + + Map getTags() { + return tags; + } + + private boolean initializeTagCache(XXService xService) { + boolean ret; + RangerTagDBRetriever.TagRetrieverServiceResourceContext serviceResourceContext = new RangerTagDBRetriever.TagRetrieverServiceResourceContext(xService); + RangerTagDBRetriever.TagRetrieverTagDefContext tagDefContext = new RangerTagDBRetriever.TagRetrieverTagDefContext(xService); + RangerTagDBRetriever.TagRetrieverTagContext tagContext = new RangerTagDBRetriever.TagRetrieverTagContext(xService); + + serviceResources = serviceResourceContext.getAllServiceResources(); + tagDefs = tagDefContext.getAllTagDefs(); + tags = tagContext.getAllTags(); + + ret = true; + return ret; + } + + private List asList(T obj) { + List ret = new ArrayList<>(); + + if (obj != null) { + ret.add(obj); + } + + return ret; + } + + private class LookupCache { + final Map userScreenNames = new HashMap<>(); + final Map resourceDefs = new HashMap<>(); + + String getUserScreenName(Long userId) { + String ret = null; + + if (userId != null) { + ret = userScreenNames.get(userId); + + if (ret == null) { + XXPortalUser user = daoMgr.getXXPortalUser().getById(userId); + + if (user != null) { + ret = user.getPublicScreenName(); + + if (StringUtil.isEmpty(ret)) { + ret = user.getFirstName(); + + if (StringUtil.isEmpty(ret)) { + ret = user.getLoginId(); + } else { + if (!StringUtil.isEmpty(user.getLastName())) { + ret += (" " + user.getLastName()); + } + } + } + + if (ret != null) { + userScreenNames.put(userId, ret); + } + } + } + } + + return ret; + } + + String getResourceName(Long resourceDefId) { + String ret = null; + + if (resourceDefId != null) { + ret = resourceDefs.get(resourceDefId); + + if (ret == null) { + XXResourceDef xResourceDef = daoMgr.getXXResourceDef().getById(resourceDefId); + + if (xResourceDef != null) { + ret = xResourceDef.getName(); + + resourceDefs.put(resourceDefId, ret); + } + } + } + + return ret; + } + } + + private class TagLoaderThread extends Thread { + final TransactionTemplate txTemplate; + final XXService xService; + + TagLoaderThread(TransactionTemplate txTemplate, final XXService xService) { + this.txTemplate = txTemplate; + this.xService = xService; + } + + @Override + public void run() { + txTemplate.setReadOnly(true); + Boolean result = txTemplate.execute(new TransactionCallback() { + @Override + public Boolean doInTransaction(TransactionStatus status) { + boolean ret = initializeTagCache(xService); + + if (!ret) { + status.setRollbackOnly(); + LOG.error("Failed to get tags for service:[" + xService.getName() + "] in a new transaction"); + } + return ret; + } + }); + + if (LOG.isDebugEnabled()) { + LOG.debug("transaction result:[" + result +"]"); + } + } + } + + private class TagRetrieverServiceResourceContext { + final XXService service; + final ListIterator iterServiceResource; + final ListIterator iterServiceResourceElement; + final ListIterator iterServiceResourceElementValue; + + TagRetrieverServiceResourceContext(XXService xService) { + Long serviceId = xService == null ? null : xService.getId(); + List xServiceResources = daoMgr.getXXServiceResource().findByServiceId(serviceId); + List xServiceResourceElements = daoMgr.getXXServiceResourceElement().findTaggedResourcesInServiceId(serviceId); + List xServiceResourceElementValues = daoMgr.getXXServiceResourceElementValue().findTaggedResourcesInServiceId(serviceId); + + this.service = xService; + this.iterServiceResource = xServiceResources.listIterator(); + this.iterServiceResourceElement = xServiceResourceElements.listIterator(); + this.iterServiceResourceElementValue = xServiceResourceElementValues.listIterator(); + + } + + TagRetrieverServiceResourceContext(XXServiceResource xServiceResource, XXService xService) { + Long resourceId = xServiceResource == null ? null : xServiceResource.getId(); + List xServiceResources = asList(xServiceResource); + List xServiceResourceElements = daoMgr.getXXServiceResourceElement().findByResourceId(resourceId); + List xServiceResourceElementValues = daoMgr.getXXServiceResourceElementValue().findByResourceId(resourceId); + + this.service = xService; + this.iterServiceResource = xServiceResources.listIterator(); + this.iterServiceResourceElement = xServiceResourceElements.listIterator(); + this.iterServiceResourceElementValue = xServiceResourceElementValues.listIterator(); + } + + List getAllServiceResources() { + List ret = new ArrayList<>(); + + while (iterServiceResource.hasNext()) { + RangerServiceResource serviceResource = getNextServiceResource(); + + if (serviceResource != null) { + ret.add(serviceResource); + } + } + + if (!hasProcessedAll()) { + LOG.warn("getAllServiceResources(): perhaps one or more serviceResources got updated during retrieval. Using fallback ... "); + + ret = getServiceResourcesBySecondary(); + } + + return ret; + } + + RangerServiceResource getNextServiceResource() { + RangerServiceResource ret = null; + + if (iterServiceResource.hasNext()) { + XXServiceResource xServiceResource = iterServiceResource.next(); + + if (xServiceResource != null) { + ret = new RangerServiceResource(); + + ret.setId(xServiceResource.getId()); + ret.setGuid(xServiceResource.getGuid()); + ret.setIsEnabled(xServiceResource.getIsEnabled()); + ret.setCreatedBy(lookupCache.getUserScreenName(xServiceResource.getAddedByUserId())); + ret.setUpdatedBy(lookupCache.getUserScreenName(xServiceResource.getUpdatedByUserId())); + ret.setCreateTime(xServiceResource.getCreateTime()); + ret.setUpdateTime(xServiceResource.getUpdateTime()); + ret.setVersion(xServiceResource.getVersion()); + ret.setResourceSignature(xServiceResource.getResourceSignature()); + ret.setServiceName(xService.getName()); + + getServiceResourceElements(ret); + } + } + + return ret; + } + + void getServiceResourceElements(RangerServiceResource serviceResource) { + while (iterServiceResourceElement.hasNext()) { + XXServiceResourceElement xServiceResourceElement = iterServiceResourceElement.next(); + + if (xServiceResourceElement.getResourceId().equals(serviceResource.getId())) { + RangerPolicy.RangerPolicyResource resource = new RangerPolicy.RangerPolicyResource(); + + resource.setIsExcludes(xServiceResourceElement.getIsExcludes()); + resource.setIsRecursive(xServiceResourceElement.getIsRecursive()); + + while (iterServiceResourceElementValue.hasNext()) { + XXServiceResourceElementValue xServiceResourceElementValue = iterServiceResourceElementValue.next(); + + if (xServiceResourceElementValue.getResElementId().equals(xServiceResourceElement.getId())) { + resource.getValues().add(xServiceResourceElementValue.getValue()); + } else { + if (iterServiceResourceElementValue.hasPrevious()) { + iterServiceResourceElementValue.previous(); + } + + break; + } + } + + serviceResource.getResourceElements().put(lookupCache.getResourceName(xServiceResourceElement.getResDefId()), resource); + } else if (xServiceResourceElement.getResourceId().compareTo(serviceResource.getId()) > 0) { + if (iterServiceResourceElement.hasPrevious()) { + iterServiceResourceElement.previous(); + } + + break; + } + } + } + + boolean hasProcessedAll() { + boolean moreToProcess = iterServiceResource.hasNext() + || iterServiceResourceElement.hasNext() + || iterServiceResourceElementValue.hasNext(); + + return !moreToProcess; + } + + List getServiceResourcesBySecondary() { + List ret = null; + + if (service != null) { + List xServiceResources = daoMgr.getXXServiceResource().findTaggedResourcesInServiceId(service.getId()); + + if (CollectionUtils.isNotEmpty(xServiceResources)) { + ret = new ArrayList<>(xServiceResources.size()); + + for (XXServiceResource xServiceResource : xServiceResources) { + RangerTagDBRetriever.TagRetrieverServiceResourceContext ctx = new RangerTagDBRetriever.TagRetrieverServiceResourceContext(xServiceResource, service); + + RangerServiceResource serviceResource = ctx.getNextServiceResource(); + + if (serviceResource != null) { + ret.add(serviceResource); + } + } + } + } + return ret; + } + } + + private class TagRetrieverTagDefContext { + final XXService service; + final ListIterator iterTagDef; + final ListIterator iterTagAttributeDef; + + + TagRetrieverTagDefContext(XXService xService) { + Long serviceId = xService == null ? null : xService.getId(); + List xTagDefs = daoMgr.getXXTagDef().findByServiceId(serviceId); + List xTagAttributeDefs = daoMgr.getXXTagAttributeDef().findByServiceId(serviceId); + + this.service = xService; + this.iterTagDef = xTagDefs.listIterator(); + this.iterTagAttributeDef = xTagAttributeDefs.listIterator(); + } + + TagRetrieverTagDefContext(XXTagDef xTagDef, XXService xService) { + Long tagDefId = xTagDef == null ? null : xTagDef.getId(); + List xTagDefs = asList(xTagDef); + List xTagAttributeDefs = daoMgr.getXXTagAttributeDef().findByTagDefId(tagDefId); + + this.service = xService; + this.iterTagDef = xTagDefs.listIterator(); + this.iterTagAttributeDef = xTagAttributeDefs.listIterator(); + } + + Map getAllTagDefs() { + Map ret = new HashMap<>(); + + while (iterTagDef.hasNext()) { + RangerTagDef tagDef = getNextTagDef(); + + if (tagDef != null) { + ret.put(tagDef.getId(), tagDef); + } + } + + if (!hasProcessedAllTagDefs()) { + LOG.warn("getAllTagDefs(): perhaps one or more tag-definitions got updated during retrieval. Using fallback ... "); + + ret = getTagDefsBySecondary(); + } + + return ret; + } + + RangerTagDef getNextTagDef() { + RangerTagDef ret = null; + + if (iterTagDef.hasNext()) { + XXTagDef xTagDef = iterTagDef.next(); + + if (xTagDef != null) { + ret = new RangerTagDef(); + + ret.setId(xTagDef.getId()); + ret.setGuid(xTagDef.getGuid()); + ret.setIsEnabled(xTagDef.getIsEnabled()); + ret.setCreatedBy(lookupCache.getUserScreenName(xTagDef.getAddedByUserId())); + ret.setUpdatedBy(lookupCache.getUserScreenName(xTagDef.getUpdatedByUserId())); + ret.setCreateTime(xTagDef.getCreateTime()); + ret.setUpdateTime(xTagDef.getUpdateTime()); + ret.setVersion(xTagDef.getVersion()); + ret.setName(xTagDef.getName()); + ret.setSource(xTagDef.getSource()); + + getTagAttributeDefs(ret); + } + } + + return ret; + } + + void getTagAttributeDefs(RangerTagDef tagDef) { + while (iterTagAttributeDef.hasNext()) { + XXTagAttributeDef xTagAttributeDef = iterTagAttributeDef.next(); + + if (xTagAttributeDef.getTagDefId().equals(tagDef.getId())) { + RangerTagDef.RangerTagAttributeDef tagAttributeDef = new RangerTagDef.RangerTagAttributeDef(); + + tagAttributeDef.setName(xTagAttributeDef.getName()); + tagAttributeDef.setType(xTagAttributeDef.getType()); + + tagDef.getAttributeDefs().add(tagAttributeDef); + } else if (xTagAttributeDef.getTagDefId().compareTo(tagDef.getId()) > 0) { + if (iterTagAttributeDef.hasPrevious()) { + iterTagAttributeDef.previous(); + } + break; + } + } + } + + boolean hasProcessedAllTagDefs() { + boolean moreToProcess = iterTagAttributeDef.hasNext(); + + return !moreToProcess; + } + + Map getTagDefsBySecondary() { + Map ret = null; + + if (service != null) { + List xTagDefs = daoMgr.getXXTagDef().findByServiceId(service.getId()); + + if (CollectionUtils.isNotEmpty(xTagDefs)) { + ret = new HashMap<>(xTagDefs.size()); + + for (XXTagDef xTagDef : xTagDefs) { + TagRetrieverTagDefContext ctx = new TagRetrieverTagDefContext(xTagDef, service); + + RangerTagDef tagDef = ctx.getNextTagDef(); + + if (tagDef != null) { + ret.put(tagDef.getId(), tagDef); + } + } + } + } + return ret; + } + } + + private class TagRetrieverTagContext { + final XXService service; + final ListIterator iterTag; + final ListIterator iterTagAttribute; + + + TagRetrieverTagContext(XXService xService) { + Long serviceId = xService == null ? null : xService.getId(); + List xTags = daoMgr.getXXTag().findByServiceId(serviceId); + List xTagAttributes = daoMgr.getXXTagAttribute().findByServiceId(serviceId); + + this.service = xService; + this.iterTag = xTags.listIterator(); + this.iterTagAttribute = xTagAttributes.listIterator(); + + } + + TagRetrieverTagContext(XXTag xTag, XXService xService) { + Long tagId = xTag == null ? null : xTag.getId(); + List xTags = asList(xTag); + List xTagAttributes = daoMgr.getXXTagAttribute().findByTagId(tagId); + + this.service = xService; + this.iterTag = xTags.listIterator(); + this.iterTagAttribute = xTagAttributes.listIterator(); + } + + + Map getAllTags() { + Map ret = new HashMap<>(); + + while (iterTag.hasNext()) { + RangerTag tag = getNextTag(); + + if (tag != null) { + ret.put(tag.getId(), tag); + } + } + + if (!hasProcessedAllTags()) { + LOG.warn("getAllTags(): perhaps one or more tags got updated during retrieval. Using fallback ... "); + + ret = getTagsBySecondary(); + } + + return ret; + } + + RangerTag getNextTag() { + RangerTag ret = null; + + if (iterTag.hasNext()) { + XXTag xTag = iterTag.next(); + + if (xTag != null) { + ret = new RangerTag(); + + ret.setId(xTag.getId()); + ret.setGuid(xTag.getGuid()); + ret.setOwner(xTag.getOwner()); + ret.setCreatedBy(lookupCache.getUserScreenName(xTag.getAddedByUserId())); + ret.setUpdatedBy(lookupCache.getUserScreenName(xTag.getUpdatedByUserId())); + ret.setCreateTime(xTag.getCreateTime()); + ret.setUpdateTime(xTag.getUpdateTime()); + ret.setVersion(xTag.getVersion()); + + /* + Map mapOfOptions = JsonUtils.jsonToMapStringString(xTag.getOptions()); + + if (MapUtils.isNotEmpty(mapOfOptions)) { + String validityPeriodsStr = mapOfOptions.get(RangerTag.OPTION_TAG_VALIDITY_PERIODS); + + if (StringUtils.isNotEmpty(validityPeriodsStr)) { + List validityPeriods = JsonUtils.jsonToRangerValiditySchedule(validityPeriodsStr); + + ret.setValidityPeriods(validityPeriods); + } + } + */ + + Map tagDefs = getTagDefs(); + if (tagDefs != null) { + RangerTagDef tagDef = tagDefs.get(xTag.getType()); + if (tagDef != null) { + ret.setType(tagDef.getName()); + } + } + + getTagAttributes(ret); + } + } + + return ret; + } + + void getTagAttributes(RangerTag tag) { + while (iterTagAttribute.hasNext()) { + XXTagAttribute xTagAttribute = iterTagAttribute.next(); + + if (xTagAttribute.getTagId().equals(tag.getId())) { + String attributeName = xTagAttribute.getName(); + String attributeValue = xTagAttribute.getValue(); + + tag.getAttributes().put(attributeName, attributeValue); + } else if (xTagAttribute.getTagId().compareTo(tag.getId()) > 0) { + if (iterTagAttribute.hasPrevious()) { + iterTagAttribute.previous(); + } + break; + } + } + } + + boolean hasProcessedAllTags() { + boolean moreToProcess = iterTagAttribute.hasNext(); + + return !moreToProcess; + } + + Map getTagsBySecondary() { + Map ret = null; + + if (service != null) { + List xTags = daoMgr.getXXTag().findByServiceId(service.getId()); + + if (CollectionUtils.isNotEmpty(xTags)) { + ret = new HashMap<>(xTags.size()); + + for (XXTag xTag : xTags) { + TagRetrieverTagContext ctx = new TagRetrieverTagContext(xTag, service); + + RangerTag tag = ctx.getNextTag(); + + if (tag != null) { + ret.put(tag.getId(), tag); + } + } + } + } + return ret; + } + } + } +} + diff --git a/security-admin/src/main/java/org/apache/ranger/patch/cliutil/RoleBasedUserSearchUtil.java b/security-admin/src/main/java/org/apache/ranger/patch/cliutil/RoleBasedUserSearchUtil.java new file mode 100644 index 0000000000..d3a28f7660 --- /dev/null +++ b/security-admin/src/main/java/org/apache/ranger/patch/cliutil/RoleBasedUserSearchUtil.java @@ -0,0 +1,270 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.patch.cliutil; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; + +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.MapUtils; +import org.apache.log4j.Logger; +import org.apache.ranger.biz.UserMgr; +import org.apache.ranger.biz.XUserMgr; +import org.apache.ranger.common.RangerConstants; +import org.apache.ranger.db.RangerDaoManager; +import org.apache.ranger.entity.XXPortalUser; +import org.apache.commons.lang.StringUtils; +import org.apache.ranger.patch.BaseLoader; +import org.apache.ranger.service.XUserService; +import org.apache.ranger.util.CLIUtil; +import org.apache.ranger.view.VXUser; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +@Component +public class RoleBasedUserSearchUtil extends BaseLoader { + + private static final Logger logger = Logger + .getLogger(RoleBasedUserSearchUtil.class); + @Autowired + XUserService xUserService; + + @Autowired + RangerDaoManager daoMgr; + + @Autowired + UserMgr userMgr; + + @Autowired + XUserMgr xUserMgr; + + public static Boolean checkRole = true; + public static String userLoginId = ""; + public static String currentPassword = ""; + public static String userRole = ""; + + public static void main(String[] args) { + logger.info("RoleBaseUserSearchUtil : main()"); + try { + RoleBasedUserSearchUtil loader = (RoleBasedUserSearchUtil) CLIUtil.getBean(RoleBasedUserSearchUtil.class); + loader.init(); + if (args.length == 3 || args.length == 2) { + userLoginId = args[0]; + currentPassword = args[1]; + if (args.length == 3) { + userRole = args[2]; + List roles = new ArrayList(); + roles.add(RangerConstants.ROLE_USER); + roles.add(RangerConstants.ROLE_SYS_ADMIN); + roles.add(RangerConstants.ROLE_KEY_ADMIN); + if (!StringUtils.isBlank(userRole)) { + userRole = userRole.toUpperCase(); + if (!roles.contains(userRole)) { + System.out.println("Invalid UserRole. Exiting!!!"); + logger.info("Invalid UserRole. Exiting!!!"); + System.exit(1); + } else { + checkRole = false; + } + } + } + if (StringUtils.isBlank(userLoginId)) { + System.out.println("Invalid login ID. Exiting!!!"); + logger.info("Invalid login ID. Exiting!!!"); + System.exit(1); + } + if (StringUtils.isBlank(currentPassword)) { + System.out.println("Invalid current password. Exiting!!!"); + logger.info("Invalid current password. Exiting!!!"); + System.exit(1); + } + while (loader.isMoreToProcess()) { + loader.load(); + } + logger.info("Load complete. Exiting!!!"); + System.exit(0); + } else { + System.out.println("RoleBaseUserSearchUtil: Incorrect Arguments \n Usage: \n "); + logger.error("RoleBaseUserSearchUtil: Incorrect Arguments \n Usage: \n "); + System.exit(1); + } + } catch (Exception e) { + logger.error("Error loading", e); + System.exit(1); + } + } + + @Override + public void init() throws Exception { + logger.info("==> RoleBaseUserSearchUtil.init()"); + } + + @Override + public void printStats() { + } + + @Override + public void execLoad() { + logger.info("==> RoleBaseUserSearchUtil.execLoad()"); + validateUserAndFetchUserList(); + logger.info("<== RoleBaseUserSearchUtil.execLoad()"); + } + + public void getUsersBasedOnRole(List userRoleList) { + try { + if (!CollectionUtils.isEmpty(userRoleList) && userRoleList != null) { + Map roleSysAdminMap = new HashMap(); + Map roleKeyAdminMap = new HashMap(); + Map roleUserMap = new HashMap(); + for (String userRole : userRoleList) { + List listXXPortalUser = daoMgr.getXXPortalUser().findByRole(userRole); + if (listXXPortalUser != null && !CollectionUtils.isEmpty(listXXPortalUser)) { + if (userRole.equalsIgnoreCase(RangerConstants.ROLE_SYS_ADMIN)) { + for (XXPortalUser xXPortalUser : listXXPortalUser) { + roleSysAdminMap.put(xXPortalUser.getLoginId(),userRole); + } + } else if (userRole.equalsIgnoreCase(RangerConstants.ROLE_KEY_ADMIN)) { + for (XXPortalUser xXPortalUser : listXXPortalUser) { + roleKeyAdminMap.put(xXPortalUser.getLoginId(),userRole); + } + } else if (userRole.equalsIgnoreCase(RangerConstants.ROLE_USER)) { + for (XXPortalUser xXPortalUser : listXXPortalUser) { + roleUserMap.put(xXPortalUser.getLoginId(),userRole); + } + } + } + } + if (MapUtils.isEmpty( roleSysAdminMap) && MapUtils.isEmpty(roleKeyAdminMap) && MapUtils.isEmpty(roleUserMap)) { + System.out.println("users with given user role are not there"); + logger.error("users with given user role are not there"); + System.exit(1); + } else { + if (!MapUtils.isEmpty(roleSysAdminMap)) { + for(Entry entry : roleSysAdminMap.entrySet()){ + System.out.println(entry.getValue() + " : " + entry.getKey()); + } + } + if (!MapUtils.isEmpty(roleKeyAdminMap)) { + for(Entry entry : roleKeyAdminMap.entrySet()){ + System.out.println(entry.getValue() + " : " + entry.getKey()); + } + } + if (!MapUtils.isEmpty(roleUserMap)) { + for(Entry entry : roleUserMap.entrySet()){ + System.out.println(entry.getValue() + " : " + entry.getKey()); + } + } + if (userRoleList.contains(RangerConstants.ROLE_SYS_ADMIN)) { + System.out.println("ROLE_SYS_ADMIN Total Count : " + roleSysAdminMap.size()); + } + if (userRoleList.contains(RangerConstants.ROLE_KEY_ADMIN)) { + System.out.println("ROLE_KEY_ADMIN Total Count : " + roleKeyAdminMap.size()); + } + if (userRoleList.contains(RangerConstants.ROLE_USER)) { + System.out.println("ROLE_USER Total Count : " + roleUserMap.size()); + } + int total = roleSysAdminMap.size() + roleKeyAdminMap.size() + roleUserMap.size(); + System.out.println("Total Count : " + total); + } + } + + } catch (Exception e) { + logger.error("Error getting User's List with the mentioned role: "+ e.getMessage()); + } + } + + public void validateUserAndFetchUserList() { + userLoginId = userLoginId.toLowerCase(); + XXPortalUser xxPortalUser = daoMgr.getXXPortalUser().findByLoginId(userLoginId); + Boolean isUserAuthorized = false; + if (xxPortalUser != null) { + String dbPassword = xxPortalUser.getPassword(); + String currentEncryptedPassword = null; + try { + currentEncryptedPassword = userMgr.encrypt(userLoginId,currentPassword); + if (currentEncryptedPassword != null && currentEncryptedPassword.equals(dbPassword)) { + VXUser vxUser = xUserService.getXUserByUserName(xxPortalUser.getLoginId()); + if (vxUser != null) { + List existingRole = (List) vxUser.getUserRoleList(); + List permissionList = daoMgr.getXXModuleDef().findAccessibleModulesByUserId(xxPortalUser.getId(), vxUser.getId()); + if (permissionList != null && permissionList.contains(RangerConstants.MODULE_USER_GROUPS) && !CollectionUtils.isEmpty(existingRole) && !StringUtils.isBlank(existingRole.get(0))) { + List userRoleList = new ArrayList(); + if (existingRole.get(0).equalsIgnoreCase(RangerConstants.ROLE_USER)) { + userRoleList.add(RangerConstants.ROLE_USER); + if (checkRole) { + getUsersBasedOnRole(userRoleList); + } else if (existingRole.get(0).equalsIgnoreCase(userRole) || userRole.equalsIgnoreCase(RangerConstants.ROLE_USER)) { + getUsersBasedOnRole(userRoleList); + } else { + isUserAuthorized = true; + } + } else if (existingRole.get(0).equalsIgnoreCase(RangerConstants.ROLE_SYS_ADMIN)) { + if (checkRole) { + userRoleList.add(RangerConstants.ROLE_SYS_ADMIN); + userRoleList.add(RangerConstants.ROLE_USER); + getUsersBasedOnRole(userRoleList); + } else if (existingRole.get(0).equalsIgnoreCase(userRole) || userRole.equalsIgnoreCase(RangerConstants.ROLE_USER)) { + userRoleList.add(userRole); + getUsersBasedOnRole(userRoleList); + } else { + isUserAuthorized = true; + } + } else if (existingRole.get(0).equalsIgnoreCase(RangerConstants.ROLE_KEY_ADMIN) || userRole.equalsIgnoreCase(RangerConstants.ROLE_USER)) { + if (checkRole) { + userRoleList.add(RangerConstants.ROLE_KEY_ADMIN); + userRoleList.add(RangerConstants.ROLE_USER); + getUsersBasedOnRole(userRoleList); + } else if (existingRole.get(0).equalsIgnoreCase(userRole) || userRole.equalsIgnoreCase(RangerConstants.ROLE_USER)) { + userRoleList.add(userRole); + getUsersBasedOnRole(userRoleList); + } else { + isUserAuthorized = true; + } + } + if (isUserAuthorized == true) { + System.out.println("user is not authorized to fetch this list"); + logger.error("user is not authorized to fetch this list"); + System.exit(1); + } + } else { + System.out.println("user permission denied"); + logger.error("user permission denied"); + System.exit(1); + } + } + } else { + System.out.println("Invalid user password"); + logger.error("Invalid user password"); + System.exit(1); + } + } catch (Exception e) { + logger.error("Getting User's List with the mentioned role failure. Detail: \n",e); + System.exit(1); + } + } else { + System.out.println("User does not exist in DB!!"); + logger.error("User does not exist in DB"); + System.exit(1); + } + } +} \ No newline at end of file diff --git a/security-admin/src/main/java/org/apache/ranger/rest/AssetREST.java b/security-admin/src/main/java/org/apache/ranger/rest/AssetREST.java index 54226d97c4..3c274e3faa 100644 --- a/security-admin/src/main/java/org/apache/ranger/rest/AssetREST.java +++ b/security-admin/src/main/java/org/apache/ranger/rest/AssetREST.java @@ -190,7 +190,7 @@ public VXAsset updateXAsset(VXAsset vXAsset) { RangerService service = serviceUtil.toRangerService(vXAsset); - RangerService updatedService = serviceREST.updateService(service); + RangerService updatedService = serviceREST.updateService(service, null); VXAsset ret = serviceUtil.toVXAsset(updatedService); diff --git a/security-admin/src/main/java/org/apache/ranger/rest/PublicAPIs.java b/security-admin/src/main/java/org/apache/ranger/rest/PublicAPIs.java index d3c22d7226..7818eb591e 100644 --- a/security-admin/src/main/java/org/apache/ranger/rest/PublicAPIs.java +++ b/security-admin/src/main/java/org/apache/ranger/rest/PublicAPIs.java @@ -144,7 +144,7 @@ public VXRepository updateRepository(VXRepository vXRepository, RangerService service = serviceUtil.toRangerService(vXAsset); service.setVersion(existing.getVersion()); - RangerService updatedService = serviceREST.updateService(service); + RangerService updatedService = serviceREST.updateService(service, null); VXAsset retvXAsset = serviceUtil.toVXAsset(updatedService); diff --git a/security-admin/src/main/java/org/apache/ranger/rest/PublicAPIsv2.java b/security-admin/src/main/java/org/apache/ranger/rest/PublicAPIsv2.java index dbb34bdff5..0281c94a5e 100644 --- a/security-admin/src/main/java/org/apache/ranger/rest/PublicAPIsv2.java +++ b/security-admin/src/main/java/org/apache/ranger/rest/PublicAPIsv2.java @@ -209,7 +209,8 @@ public RangerService createService(RangerService service) { @Path("/api/service/{id}") @PreAuthorize("@rangerPreAuthSecurityHandler.isAPISpnegoAccessible()") @Produces({ "application/json", "application/xml" }) - public RangerService updateService(RangerService service, @PathParam("id") Long id) { + public RangerService updateService(RangerService service, @PathParam("id") Long id, + @Context HttpServletRequest request) { // if service.id is specified, it should be same as the param 'id' if(service.getId() == null) { service.setId(id); @@ -217,7 +218,7 @@ public RangerService updateService(RangerService service, @PathParam("id") Long throw restErrorUtil.createRESTException(HttpServletResponse.SC_BAD_REQUEST , "service id mismatch", true); } - return serviceREST.updateService(service); + return serviceREST.updateService(service, request); } @@ -226,7 +227,8 @@ public RangerService updateService(RangerService service, @PathParam("id") Long @PreAuthorize("@rangerPreAuthSecurityHandler.isAPISpnegoAccessible()") @Produces({ "application/json", "application/xml" }) public RangerService updateServiceByName(RangerService service, - @PathParam("name") String name) { + @PathParam("name") String name, + @Context HttpServletRequest request) { // ignore service.id - if specified. Retrieve using the given name and use id from the retrieved object RangerService existingService = getServiceByName(name); service.setId(existingService.getId()); @@ -237,7 +239,7 @@ public RangerService updateServiceByName(RangerService service, service.setName(existingService.getName()); } - return serviceREST.updateService(service); + return serviceREST.updateService(service, request); } /* @@ -339,6 +341,15 @@ public List searchPolicies(@PathParam("servicename") String servic return serviceREST.getServicePoliciesByName(serviceName, request).getPolicies(); } + @GET + @Path("/api/policies/{serviceDefName}/for-resource/") + @Produces({ "application/json", "application/xml" }) + public List getPoliciesForResource(@PathParam("serviceDefName") String serviceDefName, + @DefaultValue("") @QueryParam("serviceName") String serviceName, + @Context HttpServletRequest request) { + return serviceREST.getPoliciesForResource(serviceDefName, serviceName, request); + } + @POST @Path("/api/policy/") @Produces({ "application/json", "application/xml" }) diff --git a/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java b/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java index c33d044e39..f12da80d7b 100644 --- a/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java +++ b/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java @@ -32,6 +32,7 @@ import java.util.Set; import java.util.TreeMap; +import javax.annotation.PostConstruct; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import javax.ws.rs.Consumes; @@ -51,6 +52,7 @@ import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.MapUtils; import org.apache.commons.io.IOUtils; +import org.apache.commons.lang.ArrayUtils; import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -69,6 +71,7 @@ import org.apache.ranger.common.MessageEnums; import org.apache.ranger.common.PropertiesUtil; import org.apache.ranger.common.RESTErrorUtil; +import org.apache.ranger.common.RangerConstants; import org.apache.ranger.common.RangerSearchUtil; import org.apache.ranger.common.RangerValidatorFactory; import org.apache.ranger.common.ServiceUtil; @@ -86,19 +89,20 @@ import org.apache.ranger.plugin.model.RangerService; import org.apache.ranger.plugin.model.RangerServiceDef; import org.apache.ranger.plugin.model.validation.RangerPolicyValidator; +import org.apache.ranger.plugin.model.validation.RangerServiceDefHelper; import org.apache.ranger.plugin.model.validation.RangerServiceDefValidator; import org.apache.ranger.plugin.model.validation.RangerServiceValidator; import org.apache.ranger.plugin.model.validation.RangerValidator.Action; import org.apache.ranger.plugin.policyengine.RangerAccessResource; import org.apache.ranger.plugin.policyengine.RangerAccessResourceImpl; import org.apache.ranger.plugin.policyengine.RangerPolicyEngine; -import org.apache.ranger.plugin.policyengine.RangerPolicyEngineCache; +import org.apache.ranger.plugin.policyengine.RangerPolicyEngineCacheForEngineOptions; import org.apache.ranger.plugin.policyengine.RangerPolicyEngineImpl; import org.apache.ranger.plugin.policyengine.RangerPolicyEngineOptions; -import org.apache.ranger.plugin.policyevaluator.RangerPolicyEvaluator; import org.apache.ranger.plugin.service.ResourceLookupContext; -import org.apache.ranger.plugin.store.PList; import org.apache.ranger.plugin.store.EmbeddedServiceDefsUtil; +import org.apache.ranger.plugin.store.PList; +import org.apache.ranger.plugin.store.ServiceStore; import org.apache.ranger.plugin.util.GrantRevokeRequest; import org.apache.ranger.plugin.util.RangerAccessRequestUtil; import org.apache.ranger.plugin.util.RangerPerfTracer; @@ -110,6 +114,7 @@ import org.apache.ranger.service.RangerPolicyService; import org.apache.ranger.service.RangerServiceDefService; import org.apache.ranger.service.RangerServiceService; +import org.apache.ranger.service.XUserService; import org.apache.ranger.view.RangerExportPolicyList; import org.apache.ranger.view.RangerPluginInfoList; import org.apache.ranger.view.RangerPolicyList; @@ -117,6 +122,7 @@ import org.apache.ranger.view.RangerServiceList; import org.apache.ranger.view.VXResponse; import org.apache.ranger.view.VXString; +import org.apache.ranger.view.VXUser; import org.codehaus.jackson.map.ObjectMapper; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Scope; @@ -200,10 +206,25 @@ public class ServiceREST { @Autowired JSONUtil jsonUtil; + + @Autowired + XUserService xUserService; + + private RangerPolicyEngineOptions delegateAdminOptions; + private RangerPolicyEngineOptions policySearchAdminOptions; + private RangerPolicyEngineOptions defaultAdminOptions; public ServiceREST() { } + @PostConstruct + public void initStore() { + tagStore.setServiceStore(svcStore); + delegateAdminOptions = getDelegatedAdminPolicyEngineOptions(); + policySearchAdminOptions = getPolicySearchRangerAdminPolicyEngineOptions(); + defaultAdminOptions = getDefaultRangerAdminPolicyEngineOptions(); + } + @POST @Path("/definitions") @Produces({ "application/json", "application/xml" }) @@ -304,7 +325,9 @@ public void deleteServiceDef(@PathParam("id") Long id, @Context HttpServletReque bizUtil.hasAdminPermissions("Service-Def"); XXServiceDef xServiceDef = daoManager.getXXServiceDef().getById(id); - bizUtil.hasKMSPermissions("Service-Def", xServiceDef.getImplclassname()); + if (xServiceDef != null) { + bizUtil.hasKMSPermissions("Service-Def", xServiceDef.getImplclassname()); + } String forceDeleteStr = request.getParameter("forceDelete"); boolean forceDelete = false; @@ -472,7 +495,130 @@ public RangerServiceDefList getServiceDefs(@Context HttpServletRequest request) } return ret; } - + + @GET + @Path("/policies/{serviceDefName}/for-resource") + @Produces({ "application/json", "application/xml" }) + public List getPoliciesForResource(@PathParam("serviceDefName") String serviceDefName, + @DefaultValue("") @QueryParam("serviceName") String serviceName, + @Context HttpServletRequest request) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> ServiceREST.getPoliciesForResource(service-type=" + serviceDefName + ", service-name=" + serviceName + ")"); + } + + List ret = new ArrayList<>(); + + List services = new ArrayList<>(); + Map resource = new HashMap<>(); + + String validationMessage = validateResourcePoliciesRequest(serviceDefName, serviceName, request, services, resource); + + if (StringUtils.isNotEmpty(validationMessage)) { + LOG.error("Invalid request: [" + validationMessage + "]"); + throw restErrorUtil.createRESTException(validationMessage, + MessageEnums.INVALID_INPUT_DATA); + } else { + RangerService service = services.get(0); + if (LOG.isDebugEnabled()) { + LOG.debug("getServicePolicies with service-name=" + service.getName()); + } + + RangerPolicyEngine engine = null; + + try { + engine = getPolicySearchPolicyEngine(service.getName()); + } catch (Exception e) { + LOG.error("Cannot initialize Policy-Engine", e); + throw restErrorUtil.createRESTException("Cannot initialize Policy Engine", + MessageEnums.ERROR_SYSTEM); + } + + if (engine != null) { + ret = engine.getMatchingPolicies(new RangerAccessResourceImpl(resource)); + } + + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== ServiceREST.getPoliciesForResource(service-type=" + serviceDefName + ", service-name=" + serviceName + ") : " + ret.toString()); + } + return ret; + } + + private String validateResourcePoliciesRequest(String serviceDefName, String serviceName, HttpServletRequest request, List services, Map resource) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> ServiceREST.validatePoliciesForResourceRequest(service-type=" + serviceDefName + ", service-name=" + serviceName + ")"); + } + final String ret; + + if (MapUtils.isNotEmpty(request.getParameterMap())) { + for (Map.Entry e : request.getParameterMap().entrySet()) { + String name = e.getKey(); + String[] values = e.getValue(); + + if (!StringUtils.isEmpty(name) && !ArrayUtils.isEmpty(values) + && name.startsWith(SearchFilter.RESOURCE_PREFIX)) { + resource.put(name.substring(SearchFilter.RESOURCE_PREFIX.length()), values[0]); + } + } + } + if (MapUtils.isEmpty(resource)) { + ret = "No resource specified"; + } else { + RangerServiceDef serviceDef = null; + try { + serviceDef = svcStore.getServiceDefByName(serviceDefName); + } catch (Exception e) { + LOG.error("Invalid service-type:[" + serviceDefName + "]", e); + } + if (serviceDef == null) { + ret = "Invalid service-type:[" + serviceDefName + "]"; + } else { + Set resourceDefNames = resource.keySet(); + RangerServiceDefHelper serviceDefHelper = new RangerServiceDefHelper(serviceDef); + Set> resourceHierarchies = serviceDefHelper.getResourceHierarchies(RangerPolicy.POLICY_TYPE_ACCESS, resourceDefNames); + if (CollectionUtils.isEmpty(resourceHierarchies)) { + ret = "Invalid resource specified: resource-names:" + resourceDefNames +" are not part of any valid resource hierarchy for service-type:[" + serviceDefName + "]"; + } else { + if (StringUtils.isNotBlank(serviceName)) { + RangerService service = null; + try { + service = svcStore.getServiceByName(serviceName); + } catch (Exception e) { + LOG.error("Invalid service-name:[" + serviceName + "]"); + } + if (service == null || !StringUtils.equals(service.getType(), serviceDefName)) { + ret = "Invalid service-name:[" + serviceName + "] or service-name is not of service-type:[" + serviceDefName + "]"; + } else { + services.add(service); + ret = StringUtils.EMPTY; + } + } else { + SearchFilter filter = new SearchFilter(); + filter.setParam(SearchFilter.SERVICE_TYPE, serviceDefName); + List serviceList = null; + try { + serviceList = svcStore.getServices(filter); + } catch (Exception e) { + LOG.error("Cannot find service of service-type:[" + serviceDefName + "]"); + } + if (CollectionUtils.isEmpty(serviceList) || serviceList.size() != 1) { + ret = "Either 0 or more than 1 services found for service-type :[" + serviceDefName + "]"; + } else { + services.add(serviceList.get(0)); + ret = StringUtils.EMPTY; + } + } + } + } + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== ServiceREST.validatePoliciesForResourceRequest(service-type=" + serviceDefName + ", service-name=" + serviceName + ") : " + ret); + } + return ret; + } + @POST @Path("/services") @Produces({ "application/json", "application/xml" }) @@ -486,12 +632,17 @@ public RangerService createService(RangerService service) { RangerPerfTracer perf = null; try { + if(RangerPerfTracer.isPerfTraceEnabled(PERF_LOG)) { perf = RangerPerfTracer.getPerfTracer(PERF_LOG, "ServiceREST.createService(serviceName=" + service.getName() + ")"); } RangerServiceValidator validator = validatorFactory.getServiceValidator(svcStore); validator.validate(service, Action.CREATE); + if(!StringUtils.isEmpty(service.getName().trim())){ + service.setName(service.getName().trim()); + } + UserSessionBase session = ContextUtil.getCurrentUserSession(); XXServiceDef xxServiceDef = daoManager.getXXServiceDef().findByName(service.getType()); if(session != null && !session.isSpnegoEnabled()){ @@ -533,7 +684,8 @@ public RangerService createService(RangerService service) { @Path("/services/{id}") @Produces({ "application/json", "application/xml" }) @PreAuthorize("@rangerPreAuthSecurityHandler.isAPIAccessible(\"" + RangerAPIList.UPDATE_SERVICE + "\")") - public RangerService updateService(RangerService service) { + public RangerService updateService(RangerService service, + @Context HttpServletRequest request) { if(LOG.isDebugEnabled()) { LOG.debug("==> ServiceREST.updateService(): " + service); } @@ -542,12 +694,17 @@ public RangerService updateService(RangerService service) { RangerPerfTracer perf = null; try { + if(RangerPerfTracer.isPerfTraceEnabled(PERF_LOG)) { perf = RangerPerfTracer.getPerfTracer(PERF_LOG, "ServiceREST.updateService(serviceName=" + service.getName() + ")"); } RangerServiceValidator validator = validatorFactory.getServiceValidator(svcStore); validator.validate(service, Action.UPDATE); + if(!StringUtils.isEmpty(service.getName().trim())){ + service.setName(service.getName().trim()); + } + bizUtil.hasAdminPermissions("Services"); // TODO: As of now we are allowing SYS_ADMIN to create all the @@ -556,7 +713,9 @@ public RangerService updateService(RangerService service) { XXServiceDef xxServiceDef = daoManager.getXXServiceDef().findByName(service.getType()); bizUtil.hasKMSPermissions("Service", xxServiceDef.getImplclassname()); - ret = svcStore.updateService(service); + Map options = getOptions(request); + + ret = svcStore.updateService(service, options); } catch(WebApplicationException excp) { throw excp; } catch(Throwable excp) { @@ -654,6 +813,19 @@ public RangerService getService(@PathParam("id") Long id) { perf = RangerPerfTracer.getPerfTracer(PERF_LOG, "ServiceREST.getService(serviceId=" + id + ")"); } ret = svcStore.getService(id); + if (ret != null) { + UserSessionBase userSession = ContextUtil.getCurrentUserSession(); + if (userSession != null && userSession.getLoginId() != null) { + VXUser loggedInVXUser = xUserService.getXUserByUserName(userSession.getLoginId()); + if (loggedInVXUser != null) { + if (loggedInVXUser.getUserRoleList().size() == 1 + && loggedInVXUser.getUserRoleList().contains(RangerConstants.ROLE_USER)) { + + ret = hideCriticalServiceDetailsForRoleUser(ret); + } + } + } + } } catch(WebApplicationException excp) { throw excp; } catch(Throwable excp) { @@ -692,6 +864,19 @@ public RangerService getServiceByName(@PathParam("name") String name) { perf = RangerPerfTracer.getPerfTracer(PERF_LOG, "ServiceREST.getService(serviceName=" + name + ")"); } ret = svcStore.getServiceByName(name); + if (ret != null) { + UserSessionBase userSession = ContextUtil.getCurrentUserSession(); + if (userSession != null && userSession.getLoginId() != null) { + VXUser loggedInVXUser = xUserService.getXUserByUserName(userSession.getLoginId()); + if (loggedInVXUser != null) { + if (loggedInVXUser.getUserRoleList().size() == 1 + && loggedInVXUser.getUserRoleList().contains(RangerConstants.ROLE_USER)) { + + ret = hideCriticalServiceDetailsForRoleUser(ret); + } + } + } + } } catch(WebApplicationException excp) { throw excp; } catch(Throwable excp) { @@ -734,8 +919,29 @@ public RangerServiceList getServices(@Context HttpServletRequest request) { perf = RangerPerfTracer.getPerfTracer(PERF_LOG, "ServiceREST.getServices()"); } paginatedSvcs = svcStore.getPaginatedServices(filter); + if (paginatedSvcs != null && !paginatedSvcs.getList().isEmpty()) { + UserSessionBase userSession = ContextUtil.getCurrentUserSession(); + if (userSession != null && userSession.getLoginId() != null) { + VXUser loggedInVXUser = xUserService.getXUserByUserName(userSession.getLoginId()); + if (loggedInVXUser != null) { + if (loggedInVXUser.getUserRoleList().size() == 1 + && loggedInVXUser.getUserRoleList().contains(RangerConstants.ROLE_USER)) { + + List updateServiceList = new ArrayList(); + for (RangerService rangerService : paginatedSvcs.getList()) { + if (rangerService != null) { + updateServiceList.add(hideCriticalServiceDetailsForRoleUser(rangerService)); + } + } - if(paginatedSvcs != null) { + if (updateServiceList != null && !updateServiceList.isEmpty()) { + paginatedSvcs.setList(updateServiceList); + } + } + } + } + } + if (paginatedSvcs != null) { ret = new RangerServiceList(); ret.setServices(paginatedSvcs.getList()); @@ -1274,6 +1480,12 @@ public RangerPolicy createPolicy(RangerPolicy policy, @Context HttpServletReques String policyName = request.getParameter(PARAM_POLICY_NAME); String updateIfExists = request.getParameter(PARAM_UPDATE_IF_EXISTS); + if (serviceName == null && policyName == null && updateIfExists != null + && updateIfExists.equalsIgnoreCase("true")) { + serviceName = (String) request.getAttribute(PARAM_SERVICE_NAME); + policyName = (String) request.getAttribute(PARAM_POLICY_NAME); + } + if(StringUtils.isNotEmpty(serviceName)) { policy.setService(serviceName); } @@ -1282,7 +1494,7 @@ public RangerPolicy createPolicy(RangerPolicy policy, @Context HttpServletReques policy.setName(StringUtils.trim(policyName)); } - if(Boolean.valueOf(updateIfExists)) { + if (updateIfExists != null && Boolean.valueOf(updateIfExists)) { RangerPolicy existingPolicy = null; try { if(StringUtils.isNotEmpty(policy.getGuid())) { @@ -1294,6 +1506,7 @@ public RangerPolicy createPolicy(RangerPolicy policy, @Context HttpServletReques } if(existingPolicy != null) { + policy.setId(existingPolicy.getId()); ret = updatePolicy(policy); } } catch(Exception excp) { @@ -1745,6 +1958,7 @@ public void importPoliciesFromFile( XXTrxLog xxTrxLog = new XXTrxLog(); xxTrxLog.setAction("IMPORT START"); xxTrxLog.setObjectClassType(AppConstants.CLASS_TYPE_RANGER_POLICY); + xxTrxLog.setPreviousValue("IMPORT START"); trxLogList.add(xxTrxLog); bizUtil.createTrxLog(trxLogList); @@ -1872,15 +2086,35 @@ public void importPoliciesFromFile( } } } - if (isOverride){ + String updateIfExists = request.getParameter(PARAM_UPDATE_IF_EXISTS); + String polResource = request.getParameter(SearchFilter.POL_RESOURCE); + if (updateIfExists == null || updateIfExists.isEmpty()) { + updateIfExists = "false"; + } else if (updateIfExists.equalsIgnoreCase("true")) { + isOverride = false; + } + + if (isOverride && "false".equalsIgnoreCase(updateIfExists) && StringUtils.isEmpty(polResource)) { if (LOG.isDebugEnabled()) { LOG.debug("Deleting Policy from provided services in servicesMapJson file..."); } - if (CollectionUtils.isNotEmpty(sourceServices) && CollectionUtils.isNotEmpty(destinationServices)){ - deletePoliciesProvidedInServiceMap(sourceServices, - destinationServices, null); + if (CollectionUtils.isNotEmpty(sourceServices) + && CollectionUtils.isNotEmpty(destinationServices)) { + deletePoliciesProvidedInServiceMap(sourceServices, destinationServices, null); } } + if ("true".equalsIgnoreCase(updateIfExists) && StringUtils.isNotEmpty(polResource)) { + if (LOG.isDebugEnabled()) { + LOG.debug( + "Deleting Policy from provided services in servicesMapJson file for specific resource..."); + } + if (CollectionUtils.isNotEmpty(sourceServices) + && CollectionUtils.isNotEmpty(destinationServices)) { + deletePoliciesForResource(sourceServices, destinationServices, polResource, request, + policies); + } + } + if (policies != null && !CollectionUtils.sizeIsEmpty(policies)){ for (RangerPolicy policyInJson: policies){ if (policyInJson != null){ @@ -1896,7 +2130,11 @@ public void importPoliciesFromFile( for (String service : serviceNameList) { if (StringUtils.isNotEmpty(service.trim()) && StringUtils.isNotEmpty(policy.getService().trim())){ if (policy.getService().trim().equalsIgnoreCase(service.trim())) { - createPolicy(policy, null); + if (updateIfExists != null && !updateIfExists.isEmpty()) { + request.setAttribute(PARAM_SERVICE_NAME, policy.getService()); + request.setAttribute(PARAM_POLICY_NAME, policy.getName()); + } + createPolicy(policy, request); totalPolicyCreate = totalPolicyCreate + 1; if (LOG.isDebugEnabled()) { LOG.debug("Policy " + policy.getName() + " created successfully."); @@ -1908,8 +2146,12 @@ public void importPoliciesFromFile( throw restErrorUtil.createRESTException("Service Name or Policy Name is not provided!!"); } } - }else{ - createPolicy(policy, null); + } else { + if (updateIfExists != null && !updateIfExists.isEmpty()) { + request.setAttribute(PARAM_SERVICE_NAME, policy.getService()); + request.setAttribute(PARAM_POLICY_NAME, policy.getName()); + } + createPolicy(policy, request); totalPolicyCreate = totalPolicyCreate + 1; if (LOG.isDebugEnabled()) { LOG.debug("Policy " + policy.getName() + " created successfully."); @@ -2054,6 +2296,10 @@ private List getAllFilteredPolicyList(SearchFilter filter, } } } + if (StringUtils.isNotEmpty(request.getParameter("resourceMatch")) + && "full".equalsIgnoreCase(request.getParameter("resourceMatch"))) { + policyLists = serviceUtil.getMatchingPoliciesForResource(request, policyLists); + } Map orderedPolicies = new TreeMap(); if (!CollectionUtils.isEmpty(policyLists)) { @@ -2076,6 +2322,7 @@ private void deletePoliciesProvidedInServiceMap( int totalDeletedPilicies = 0; if (CollectionUtils.isNotEmpty(sourceServices) && CollectionUtils.isNotEmpty(destinationServices)) { + RangerPolicyValidator validator = validatorFactory.getPolicyValidator(svcStore); for (int i = 0; i < sourceServices.size(); i++) { if (!destinationServices.get(i).isEmpty()) { RangerPolicyList servicePolicies = null; @@ -2085,12 +2332,76 @@ private void deletePoliciesProvidedInServiceMap( if (CollectionUtils.isNotEmpty(rangerPolicyList)) { for (RangerPolicy rangerPolicy : rangerPolicyList) { if (rangerPolicy != null) { - if (rangerPolicy.getId() != null){ - deletePolicy(rangerPolicy.getId()); + try { + validator.validate(rangerPolicy.getId(), Action.DELETE); + ensureAdminAccess(rangerPolicy.getService(), rangerPolicy.getResources()); + svcStore.deletePolicy(rangerPolicy.getId()); + totalDeletedPilicies = totalDeletedPilicies + 1; if (LOG.isDebugEnabled()) { LOG.debug("Policy " + rangerPolicy.getName() + " deleted successfully." ); + LOG.debug("TotalDeletedPilicies: " +totalDeletedPilicies); + } + } catch(Throwable excp) { + LOG.error("deletePolicy(" + rangerPolicy.getId() + ") failed", excp); + } + } + } + } + } + } + } + } + if (LOG.isDebugEnabled()) { + LOG.debug("Total Deleted Policy : " + totalDeletedPilicies); + } + } + + private void deletePoliciesForResource(List sourceServices, List destinationServices, + String resource, HttpServletRequest request, List exportPolicies) { + int totalDeletedPilicies = 0; + if (CollectionUtils.isNotEmpty(sourceServices) && CollectionUtils.isNotEmpty(destinationServices)) { + Set exportedPolicyNames = new HashSet(); + if (CollectionUtils.isNotEmpty(exportPolicies)) { + for (RangerPolicy rangerPolicy : exportPolicies) { + if (rangerPolicy != null) { + exportedPolicyNames.add(rangerPolicy.getName()); + } + } + } + for (int i = 0; i < sourceServices.size(); i++) { + if (!destinationServices.get(i).isEmpty()) { + RangerPolicyList servicePolicies = null; + servicePolicies = getServicePoliciesByName(destinationServices.get(i), request); + if (servicePolicies != null) { + List rangerPolicyList = servicePolicies.getPolicies(); + if (CollectionUtils.isNotEmpty(rangerPolicyList)) { + for (RangerPolicy rangerPolicy : rangerPolicyList) { + if (rangerPolicy != null) { + Map rangerPolicyResourceMap = rangerPolicy + .getResources(); + if (rangerPolicyResourceMap != null) { + RangerPolicy.RangerPolicyResource rangerPolicyResource = null; + if (rangerPolicyResourceMap.containsKey("path")) { + rangerPolicyResource = rangerPolicyResourceMap.get("path"); + } else if (rangerPolicyResourceMap.containsKey("database")) { + rangerPolicyResource = rangerPolicyResourceMap.get("database"); + } + if (rangerPolicyResource != null) { + if (CollectionUtils.isNotEmpty(rangerPolicyResource.getValues()) + && rangerPolicyResource.getValues().size() > 1) { + continue; + } + } + } + if (rangerPolicy.getId() != null) { + if (!exportedPolicyNames.contains(rangerPolicy.getName())) { + deletePolicy(rangerPolicy.getId()); + if (LOG.isDebugEnabled()) { + LOG.debug( + "Policy " + rangerPolicy.getName() + " deleted successfully."); + } + totalDeletedPilicies = totalDeletedPilicies + 1; } - totalDeletedPilicies = totalDeletedPilicies + 1; } } } @@ -2104,6 +2415,7 @@ private void deletePoliciesProvidedInServiceMap( } } + public List getPolicies(SearchFilter filter) { if(LOG.isDebugEnabled()) { LOG.debug("==> ServiceREST.getPolicies(filter)"); @@ -2739,7 +3051,8 @@ private List applyAdminAccessFilter(List policies) { List listToFilter = entry.getValue(); if (CollectionUtils.isNotEmpty(listToFilter)) { - if (isAdmin || isKeyAdmin) { + boolean isServiceAdminUser=isAdmin || svcStore.isServiceAdminUser(serviceName, userName); + if (isAdmin || isKeyAdmin || isServiceAdminUser) { XXService xService = daoManager.getXXService().findByName(serviceName); Long serviceDefId = xService.getType(); boolean isKmsService = serviceDefId.equals(EmbeddedServiceDefsUtil.instance().getKmsServiceDefId()); @@ -2748,10 +3061,12 @@ private List applyAdminAccessFilter(List policies) { if (!isKmsService) { ret.addAll(listToFilter); } - } else { // isKeyAdmin + } else if (isKeyAdmin) { if (isKmsService) { ret.addAll(listToFilter); } + } else if (isServiceAdminUser) { + ret.addAll(listToFilter); } continue; @@ -2785,16 +3100,11 @@ void ensureAdminAccess(String serviceName, Map res boolean isKeyAdmin = bizUtil.isKeyAdmin(); String userName = bizUtil.getCurrentUserLoginId(); - if(!isAdmin && !isKeyAdmin) { - boolean isAllowed = false; + boolean isSvcAdmin = isAdmin || svcStore.isServiceAdminUser(serviceName, userName); - RangerPolicyEngine policyEngine = getDelegatedAdminPolicyEngine(serviceName); - - if (policyEngine != null) { - Set userGroups = userMgr.getGroupsForUser(userName); - - isAllowed = hasAdminAccess(serviceName, userName, userGroups, resources); - } + if(!isAdmin && !isKeyAdmin && !isSvcAdmin) { + Set userGroups = userMgr.getGroupsForUser(userName); + boolean isAllowed = hasAdminAccess(serviceName, userName, userGroups, resources); if (!isAllowed) { throw restErrorUtil.createRESTException(HttpServletResponse.SC_UNAUTHORIZED, @@ -2806,13 +3116,13 @@ void ensureAdminAccess(String serviceName, Map res XXServiceDef xServiceDef = daoManager.getXXServiceDef().getById(xService.getType()); if (isAdmin) { - if (xServiceDef.getImplclassname().equals(EmbeddedServiceDefsUtil.KMS_IMPL_CLASS_NAME)) { + if (EmbeddedServiceDefsUtil.KMS_IMPL_CLASS_NAME.equals(xServiceDef.getImplclassname())) { throw restErrorUtil.createRESTException( "KMS Policies/Services/Service-Defs are not accessible for user '" + userName + "'.", MessageEnums.OPER_NO_PERMISSION); } } else if (isKeyAdmin) { - if (!xServiceDef.getImplclassname().equals(EmbeddedServiceDefsUtil.KMS_IMPL_CLASS_NAME)) { + if (!EmbeddedServiceDefsUtil.KMS_IMPL_CLASS_NAME.equals(xServiceDef.getImplclassname())) { throw restErrorUtil.createRESTException( "Only KMS Policies/Services/Service-Defs are accessible for user '" + userName + "'.", MessageEnums.OPER_NO_PERMISSION); @@ -2821,6 +3131,34 @@ void ensureAdminAccess(String serviceName, Map res } } + private RangerPolicyEngineOptions getDelegatedAdminPolicyEngineOptions() { + RangerPolicyEngineOptions opts = new RangerPolicyEngineOptions(); + + final String propertyPrefix = "ranger.admin"; + + opts.configureDelegateAdmin(RangerConfiguration.getInstance(), propertyPrefix); + + return opts; + } + + private RangerPolicyEngineOptions getPolicySearchRangerAdminPolicyEngineOptions() { + RangerPolicyEngineOptions opts = new RangerPolicyEngineOptions(); + + final String propertyPrefix = "ranger.admin"; + + opts.configureRangerAdminForPolicySearch(RangerConfiguration.getInstance(), propertyPrefix); + return opts; + } + + private RangerPolicyEngineOptions getDefaultRangerAdminPolicyEngineOptions() { + RangerPolicyEngineOptions opts = new RangerPolicyEngineOptions(); + + final String propertyPrefix = "ranger.admin"; + + opts.configureDefaultRangerAdmin(RangerConfiguration.getInstance(), propertyPrefix); + return opts; + } + private boolean hasAdminAccess(String serviceName, String userName, Set userGroups, Map resources) { boolean isAllowed = false; @@ -2846,40 +3184,18 @@ private boolean hasAdminAccess(String serviceName, String userName, Set } private RangerPolicyEngine getDelegatedAdminPolicyEngine(String serviceName) { - if(RangerPolicyEngineCache.getInstance().getPolicyEngineOptions() == null) { - RangerPolicyEngineOptions options = new RangerPolicyEngineOptions(); - - String propertyPrefix = "ranger.admin"; - - options.evaluatorType = RangerPolicyEvaluator.EVALUATOR_TYPE_OPTIMIZED; - options.cacheAuditResults = RangerConfiguration.getInstance().getBoolean(propertyPrefix + ".policyengine.option.cache.audit.results", false); - options.disableContextEnrichers = RangerConfiguration.getInstance().getBoolean(propertyPrefix + ".policyengine.option.disable.context.enrichers", true); - options.disableCustomConditions = RangerConfiguration.getInstance().getBoolean(propertyPrefix + ".policyengine.option.disable.custom.conditions", true); - options.evaluateDelegateAdminOnly = RangerConfiguration.getInstance().getBoolean(propertyPrefix + ".policyengine.option.evaluate.delegateadmin.only", true); - - RangerPolicyEngineCache.getInstance().setPolicyEngineOptions(options); - } - - RangerPolicyEngine ret = RangerPolicyEngineCache.getInstance().getPolicyEngine(serviceName, svcStore); + return RangerPolicyEngineCacheForEngineOptions.getInstance().getPolicyEngine(serviceName, svcStore, delegateAdminOptions); + } - return ret; + private RangerPolicyEngine getPolicySearchPolicyEngine(String serviceName) throws Exception { + return RangerPolicyEngineCacheForEngineOptions.getInstance().getPolicyEngine(serviceName, svcStore, policySearchAdminOptions); } private RangerPolicyEngine getPolicyEngine(String serviceName) throws Exception { - RangerPolicyEngineOptions options = new RangerPolicyEngineOptions(); - - String propertyPrefix = "ranger.admin"; - - options.evaluatorType = RangerPolicyEvaluator.EVALUATOR_TYPE_OPTIMIZED; - options.cacheAuditResults = RangerConfiguration.getInstance().getBoolean(propertyPrefix + ".policyengine.option.cache.audit.results", false); - options.disableContextEnrichers = RangerConfiguration.getInstance().getBoolean(propertyPrefix + ".policyengine.option.disable.context.enrichers", true); - options.disableCustomConditions = RangerConfiguration.getInstance().getBoolean(propertyPrefix + ".policyengine.option.disable.custom.conditions", true); - options.evaluateDelegateAdminOnly = false; - options.disableTrieLookupPrefilter = RangerConfiguration.getInstance().getBoolean(propertyPrefix + ".policyengine.option.disable.trie.lookup.prefilter", false); ServicePolicies policies = svcStore.getServicePoliciesIfUpdated(serviceName, -1L); - RangerPolicyEngine ret = new RangerPolicyEngineImpl("ranger-admin", policies, options); + RangerPolicyEngine ret = new RangerPolicyEngineImpl("ranger-admin", policies, defaultAdminOptions); return ret; } @@ -3055,4 +3371,34 @@ private void validateGrantRevokeRequest(GrantRevokeRequest request){ } } } + + private Map getOptions(HttpServletRequest request) { + Map ret = null; + if (request != null) { + String isForceRenameOption = request.getParameter(ServiceStore.OPTION_FORCE_RENAME); + if (StringUtils.isNotBlank(isForceRenameOption)) { + ret = new HashMap(); + ret.put(ServiceStore.OPTION_FORCE_RENAME, Boolean.valueOf(isForceRenameOption)); + } + } + return ret; + } + + private RangerService hideCriticalServiceDetailsForRoleUser(RangerService rangerService) { + RangerService ret = rangerService; + + ret.setConfigs(null); + ret.setDescription(null); + ret.setCreatedBy(null); + ret.setUpdatedBy(null); + ret.setCreateTime(null); + ret.setUpdateTime(null); + ret.setPolicyVersion(null); + ret.setPolicyUpdateTime(null); + ret.setTagVersion(null); + ret.setTagUpdateTime(null); + ret.setVersion(null); + + return ret; + } } diff --git a/security-admin/src/main/java/org/apache/ranger/rest/XUserREST.java b/security-admin/src/main/java/org/apache/ranger/rest/XUserREST.java index 6e9161ee62..d06827deba 100644 --- a/security-admin/src/main/java/org/apache/ranger/rest/XUserREST.java +++ b/security-admin/src/main/java/org/apache/ranger/rest/XUserREST.java @@ -17,9 +17,11 @@ * under the License. */ - package org.apache.ranger.rest; +package org.apache.ranger.rest; +import java.util.ArrayList; import java.util.HashMap; +import java.util.List; import javax.servlet.http.HttpServletRequest; import javax.ws.rs.DELETE; @@ -31,15 +33,20 @@ import javax.ws.rs.Produces; import javax.ws.rs.core.Context; +import org.apache.log4j.Logger; +import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang.StringUtils; import org.apache.ranger.biz.RangerBizUtil; import org.apache.ranger.biz.SessionMgr; import org.apache.ranger.biz.XUserMgr; +import org.apache.ranger.common.ContextUtil; import org.apache.ranger.common.MessageEnums; import org.apache.ranger.common.RESTErrorUtil; +import org.apache.ranger.common.RangerConstants; import org.apache.ranger.common.SearchCriteria; import org.apache.ranger.common.SearchUtil; import org.apache.ranger.common.StringUtil; +import org.apache.ranger.common.UserSessionBase; import org.apache.ranger.common.annotation.RangerAnnotationClassName; import org.apache.ranger.common.annotation.RangerAnnotationJSMgrName; import org.apache.ranger.db.RangerDaoManager; @@ -146,6 +153,11 @@ public class XUserREST { @Autowired XResourceService xResourceService; + @Autowired + StringUtil stringUtil; + + static final Logger logger = Logger.getLogger(XUserMgr.class); + // Handle XGroup @GET @Path("/groups/{id}") @@ -238,8 +250,7 @@ public VXGroupList searchXGroups(@Context HttpServletRequest request) { request, xGroupService.sortFields); searchUtil.extractString(request, searchCriteria, "name", "group name", null); searchUtil.extractInt(request, searchCriteria, "isVisible", "Group Visibility"); - searchUtil.extractString(request, searchCriteria, "groupSource", "group source", null); -// searchUtil.extractInt(request, searchCriteria, "groupSource", "group source"); + searchUtil.extractInt(request, searchCriteria, "groupSource", "group source"); return xUserMgr.searchXGroups(searchCriteria); } @@ -346,18 +357,52 @@ public void deleteXUser(@PathParam("id") Long id, @Produces({ "application/xml", "application/json" }) @PreAuthorize("@rangerPreAuthSecurityHandler.isAPIAccessible(\"" + RangerAPIList.SEARCH_X_USERS + "\")") public VXUserList searchXUsers(@Context HttpServletRequest request) { + String UserRoleParamName = RangerConstants.ROLE_USER; SearchCriteria searchCriteria = searchUtil.extractCommonCriterias( request, xUserService.sortFields); - + String userName = null; + if (request.getUserPrincipal() != null){ + userName = request.getUserPrincipal().getName(); + } searchUtil.extractString(request, searchCriteria, "name", "User name",null); searchUtil.extractString(request, searchCriteria, "emailAddress", "Email Address", null); searchUtil.extractInt(request, searchCriteria, "userSource", "User Source"); searchUtil.extractInt(request, searchCriteria, "isVisible", "User Visibility"); searchUtil.extractInt(request, searchCriteria, "status", "User Status"); - searchUtil.extractStringList(request, searchCriteria, "userRoleList", "User Role List", "userRoleList", null, + List userRolesList = searchUtil.extractStringList(request, searchCriteria, "userRoleList", "User Role List", "userRoleList", null, null); searchUtil.extractString(request, searchCriteria, "userRole", "UserRole", null); + if (CollectionUtils.isNotEmpty(userRolesList) && CollectionUtils.size(userRolesList) == 1 && userRolesList.get(0).equalsIgnoreCase(UserRoleParamName)) { + if (!(searchCriteria.getParamList().containsKey("name"))) { + searchCriteria.addParam("name", userName); + } + else if ((searchCriteria.getParamList().containsKey("name")) && userName!= null && userName.contains((String) searchCriteria.getParamList().get("name"))) { + searchCriteria.addParam("name", userName); + } + } + + UserSessionBase userSession = ContextUtil.getCurrentUserSession(); + if (userSession != null && userSession.getLoginId() != null) { + VXUser loggedInVXUser = xUserService.getXUserByUserName(userSession + .getLoginId()); + if (loggedInVXUser != null) { + if (loggedInVXUser.getUserRoleList().size() == 1 + && loggedInVXUser.getUserRoleList().contains( + RangerConstants.ROLE_USER)) { + logger.info("Logged-In user having user role will be able to fetch his own user details."); + if (!searchCriteria.getParamList().containsKey("name")) { + searchCriteria.addParam("name", loggedInVXUser.getName()); + }else if(searchCriteria.getParamList().containsKey("name") + && !stringUtil.isEmpty(searchCriteria.getParamValue("name").toString()) + && !searchCriteria.getParamValue("name").toString().equalsIgnoreCase(loggedInVXUser.getName())){ + throw restErrorUtil.create403RESTException("Logged-In user is not allowed to access requested user data."); + } + + } + } + } + return xUserMgr.searchXUsers(searchCriteria); } @@ -1026,6 +1071,7 @@ public VXStringList getUserRolesByName(@PathParam("userName") String userName) { return vXStringList; } + @DELETE @Path("/secure/users/delete") @Produces({ "application/xml", "application/json" }) @@ -1046,6 +1092,7 @@ public void deleteUsersByUserName(@Context HttpServletRequest request,VXStringLi } } + @DELETE @Path("/secure/groups/delete") @Produces({ "application/xml", "application/json" }) @@ -1066,4 +1113,125 @@ public void deleteGroupsByGroupName( } } } + + @DELETE + @Path("/secure/users/{userName}") + @Produces({ "application/xml", "application/json" }) + @PreAuthorize("hasRole('ROLE_SYS_ADMIN')") + public void deleteSingleUserByUserName(@Context HttpServletRequest request, @PathParam("userName") String userName) { + String forceDeleteStr = request.getParameter("forceDelete"); + boolean forceDelete = false; + if (StringUtils.isNotEmpty(forceDeleteStr) && "true".equalsIgnoreCase(forceDeleteStr)) { + forceDelete = true; + } + + if (StringUtils.isNotEmpty(userName)) { + VXUser vxUser = xUserService.getXUserByUserName(userName); + xUserMgr.deleteXUser(vxUser.getId(), forceDelete); + } + } + + @DELETE + @Path("/secure/groups/{groupName}") + @Produces({ "application/xml", "application/json" }) + @PreAuthorize("hasRole('ROLE_SYS_ADMIN')") + public void deleteSingleGroupByGroupName(@Context HttpServletRequest request, @PathParam("groupName") String groupName) { + String forceDeleteStr = request.getParameter("forceDelete"); + boolean forceDelete = false; + if (StringUtils.isNotEmpty(forceDeleteStr) && "true".equalsIgnoreCase(forceDeleteStr)) { + forceDelete = true; + } + if (StringUtils.isNotEmpty(groupName)) { + VXGroup vxGroup = xGroupService.getGroupByGroupName(groupName.trim()); + xUserMgr.deleteXGroup(vxGroup.getId(), forceDelete); + } + } + + @DELETE + @Path("/secure/users/id/{userId}") + @Produces({ "application/xml", "application/json" }) + @PreAuthorize("hasRole('ROLE_SYS_ADMIN')") + public void deleteSingleUserByUserId(@Context HttpServletRequest request, @PathParam("userId") Long userId) { + String forceDeleteStr = request.getParameter("forceDelete"); + boolean forceDelete = false; + if (StringUtils.isNotEmpty(forceDeleteStr) && "true".equalsIgnoreCase(forceDeleteStr)) { + forceDelete = true; + } + if (userId != null) { + xUserMgr.deleteXUser(userId, forceDelete); + } + } + + @DELETE + @Path("/secure/groups/id/{groupId}") + @Produces({ "application/xml", "application/json" }) + @PreAuthorize("hasRole('ROLE_SYS_ADMIN')") + public void deleteSingleGroupByGroupId(@Context HttpServletRequest request, @PathParam("groupId") Long groupId) { + String forceDeleteStr = request.getParameter("forceDelete"); + boolean forceDelete = false; + if (StringUtils.isNotEmpty(forceDeleteStr) && "true".equalsIgnoreCase(forceDeleteStr)) { + forceDelete = true; + } + if (groupId != null) { + xUserMgr.deleteXGroup(groupId, forceDelete); + } + } + + @GET + @Path("/lookup/users") + @Produces({ "application/xml", "application/json" }) + @PreAuthorize("@rangerPreAuthSecurityHandler.isAPIAccessible(\"" + RangerAPIList.GET_USERS_LOOKUP + "\")") + public VXStringList getUsersLookup(@Context HttpServletRequest request) { + SearchCriteria searchCriteria = searchUtil.extractCommonCriterias(request, xUserService.sortFields); + VXStringList ret = new VXStringList(); + List vXList = new ArrayList<>(); + searchUtil.extractString(request, searchCriteria, "name", "User name", null); + searchUtil.extractInt(request, searchCriteria, "isVisible", "User Visibility"); + try { + VXUserList vXUserList = xUserMgr.searchXUsers(searchCriteria); + VXString VXString = null; + for (VXUser vxUser : vXUserList.getList()) { + VXString = new VXString(); + VXString.setValue(vxUser.getName()); + vXList.add(VXString); + } + ret.setVXStrings(vXList); + ret.setPageSize(vXUserList.getPageSize()); + ret.setTotalCount(vXUserList.getTotalCount()); + ret.setSortType(vXUserList.getSortType()); + ret.setSortBy(vXUserList.getSortBy()); + } catch (Throwable excp) { + throw restErrorUtil.createRESTException(excp.getMessage()); + } + return ret; + } + + @GET + @Path("/lookup/groups") + @Produces({ "application/xml", "application/json" }) + @PreAuthorize("@rangerPreAuthSecurityHandler.isAPIAccessible(\"" + RangerAPIList.GET_GROUPS_LOOKUP + "\")") + public VXStringList getGroupsLookup(@Context HttpServletRequest request) { + VXStringList ret = new VXStringList(); + SearchCriteria searchCriteria = searchUtil.extractCommonCriterias(request, xGroupService.sortFields); + List vXList = new ArrayList<>(); + searchUtil.extractString(request, searchCriteria, "name", "group name", null); + searchUtil.extractInt(request, searchCriteria, "isVisible", "Group Visibility"); + try { + VXGroupList vXGroupList = xUserMgr.lookupXGroups(searchCriteria); + for (VXGroup vxGroup : vXGroupList.getList()) { + VXString VXString = new VXString(); + VXString.setValue(vxGroup.getName()); + vXList.add(VXString); + } + ret.setVXStrings(vXList); + ret.setPageSize(vXGroupList.getPageSize()); + ret.setTotalCount(vXGroupList.getTotalCount()); + ret.setSortType(vXGroupList.getSortType()); + ret.setSortBy(vXGroupList.getSortBy()); + } catch (Throwable excp) { + throw restErrorUtil.createRESTException(excp.getMessage()); + } + return ret; + } + } diff --git a/security-admin/src/main/java/org/apache/ranger/security/context/RangerAPIList.java b/security-admin/src/main/java/org/apache/ranger/security/context/RangerAPIList.java index 460c7fda20..da45338708 100644 --- a/security-admin/src/main/java/org/apache/ranger/security/context/RangerAPIList.java +++ b/security-admin/src/main/java/org/apache/ranger/security/context/RangerAPIList.java @@ -137,6 +137,8 @@ public class RangerAPIList { public static final String MODIFY_GROUPS_VISIBILITY = "XUserREST.modifyGroupsVisibility"; public static final String DELETE_X_GROUP = "XUserREST.deleteXGroup"; public static final String SEARCH_X_GROUPS = "XUserREST.searchXGroups"; + public static final String GET_USERS_LOOKUP = "XUserREST.getUsersLookup"; + public static final String GET_GROUPS_LOOKUP = "XUserREST.getGroupsLookup"; public static final String COUNT_X_GROUPS = "XUserREST.countXGroups"; public static final String GET_X_USER = "XUserREST.getXUser"; public static final String SECURE_GET_X_USER = "XUserREST.secureGetXUser"; diff --git a/security-admin/src/main/java/org/apache/ranger/security/web/filter/RangerKRBAuthenticationFilter.java b/security-admin/src/main/java/org/apache/ranger/security/web/filter/RangerKRBAuthenticationFilter.java index c8d22aa45f..02020f2a9e 100644 --- a/security-admin/src/main/java/org/apache/ranger/security/web/filter/RangerKRBAuthenticationFilter.java +++ b/security-admin/src/main/java/org/apache/ranger/security/web/filter/RangerKRBAuthenticationFilter.java @@ -32,6 +32,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.Collections; import javax.servlet.Filter; import javax.servlet.FilterChain; @@ -53,6 +54,13 @@ import javax.servlet.http.HttpServletResponse; import org.apache.commons.collections.iterators.IteratorEnumeration; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.SaslRpcServer; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.authentication.server.AuthenticationToken; +import org.apache.hadoop.security.authorize.AuthorizationException; +import org.apache.hadoop.security.authorize.ProxyUsers; +import org.apache.hadoop.util.HttpExceptionUtils; import org.apache.ranger.biz.UserMgr; import org.apache.ranger.common.PropertiesUtil; import org.apache.ranger.common.RESTErrorUtil; @@ -98,6 +106,8 @@ public class RangerKRBAuthenticationFilter extends RangerKrbFilter { static final String RANGER_AUTH_TYPE = "hadoop.security.authentication"; static final String AUTH_COOKIE_NAME = "hadoop.auth"; static final String HOST_NAME = "ranger.service.host"; + static final String ALLOW_TRUSTED_PROXY = "ranger.authentication.allow.trustedproxy"; + static final String PROXY_PREFIX = "ranger.proxyuser."; private static final String KERBEROS_TYPE = "kerberos"; private static final String S_USER = "suser"; @@ -119,6 +129,7 @@ public void init(FilterConfig conf) throws ServletException { params.put(TOKEN_VALID_PARAM, PropertiesUtil.getProperty(TOKEN_VALID,"30")); params.put(COOKIE_DOMAIN_PARAM, PropertiesUtil.getProperty(COOKIE_DOMAIN, PropertiesUtil.getProperty(HOST_NAME, "localhost"))); params.put(COOKIE_PATH_PARAM, PropertiesUtil.getProperty(COOKIE_PATH, "/")); + params.put(ALLOW_TRUSTED_PROXY, PropertiesUtil.getProperty(ALLOW_TRUSTED_PROXY, "false")); try { params.put(PRINCIPAL_PARAM, SecureClientLogin.getPrincipal(PropertiesUtil.getProperty(PRINCIPAL,""), PropertiesUtil.getProperty(HOST_NAME))); } catch (IOException ignored) { @@ -153,6 +164,20 @@ public String getFilterName() { } }; super.init(myConf); + Configuration conf1 = this.getProxyuserConfiguration(); + ProxyUsers.refreshSuperUserGroupsConfiguration(conf1, PROXY_PREFIX); + } + + protected Configuration getProxyuserConfiguration() { + Configuration conf = new Configuration(false); + Map propertiesMap = PropertiesUtil.getPropertiesMap(); + for (String key : propertiesMap.keySet()) { + if (!key.startsWith(PROXY_PREFIX)) { + continue; + } + conf.set(key, propertiesMap.get(key)); + } + return conf; } @Override @@ -162,6 +187,7 @@ protected void doFilter(FilterChain filterChain, String authType = PropertiesUtil.getProperty(RANGER_AUTH_TYPE); String userName = null; boolean checkCookie = response.containsHeader("Set-Cookie"); + boolean allowTrustedProxy = PropertiesUtil.getBooleanProperty(ALLOW_TRUSTED_PROXY, false); if(checkCookie){ Collection authUserName = response.getHeaders("Set-Cookie"); if(authUserName != null){ @@ -200,24 +226,98 @@ protected void doFilter(FilterChain filterChain, userName = sessionUserName; } + if(LOG.isDebugEnabled()) { + LOG.debug("Remote user from request = " + request.getRemoteUser()); + } + if((isSpnegoEnable(authType) && (!StringUtils.isEmpty(userName)))){ Authentication existingAuth = SecurityContextHolder.getContext().getAuthentication(); if(existingAuth == null || !existingAuth.isAuthenticated()){ //--------------------------- To Create Ranger Session -------------------------------------- String rangerLdapDefaultRole = PropertiesUtil.getProperty("ranger.ldap.default.role", "ROLE_USER"); - //if we get the userName from the token then log into ranger using the same user - final List grantedAuths = new ArrayList<>(); - grantedAuths.add(new SimpleGrantedAuthority(rangerLdapDefaultRole)); - final UserDetails principal = new User(userName, "",grantedAuths); - final Authentication finalAuthentication = new UsernamePasswordAuthenticationToken(principal, "", grantedAuths); - WebAuthenticationDetails webDetails = new WebAuthenticationDetails(request); - ((AbstractAuthenticationToken) finalAuthentication).setDetails(webDetails); - RangerAuthenticationProvider authenticationProvider = new RangerAuthenticationProvider(); - Authentication authentication = authenticationProvider.authenticate(finalAuthentication); - authentication = getGrantedAuthority(authentication); - SecurityContextHolder.getContext().setAuthentication(authentication); - request.setAttribute("spnegoEnabled", true); - LOG.info("Logged into Ranger as = "+userName); + if(LOG.isDebugEnabled()) { + LOG.debug("Http headers: " + Collections.list(request.getHeaderNames()).toString()); + } + String doAsUser = request.getParameter("doAs"); + + if (allowTrustedProxy && doAsUser != null && !doAsUser.isEmpty()) { + if(LOG.isDebugEnabled()) { + LOG.debug("userPrincipal from request = " + request.getUserPrincipal() + " request paramerters = " + request.getParameterMap().keySet()); + } + AuthenticationToken authToken = (AuthenticationToken)request.getUserPrincipal(); + if(authToken != null && authToken != AuthenticationToken.ANONYMOUS) { + if(LOG.isDebugEnabled()) { + LOG.debug("remote user from authtoken = " + authToken.getUserName()); + } + UserGroupInformation ugi = UserGroupInformation.createRemoteUser(authToken.getUserName(), SaslRpcServer.AuthMethod.KERBEROS); + if(ugi != null) { + ugi = UserGroupInformation.createProxyUser(doAsUser, ugi); + if(LOG.isDebugEnabled()) { + LOG.debug("Real user from UGI = " + ugi.getRealUser().getShortUserName()); + } + + try { + ProxyUsers.authorize(ugi, request.getRemoteAddr()); + } catch (AuthorizationException ex) { + HttpExceptionUtils.createServletExceptionResponse(response, 403, ex); + if(LOG.isDebugEnabled()) { + LOG.debug("Authentication exception: " + ex.getMessage(), ex); + } else { + LOG.warn("Authentication exception: " + ex.getMessage()); + } + return; + } + final List grantedAuths = new ArrayList<>(); + grantedAuths.add(new SimpleGrantedAuthority(rangerLdapDefaultRole)); + final UserDetails principal = new User(doAsUser, "", grantedAuths); + Authentication authentication = new UsernamePasswordAuthenticationToken(principal, "", grantedAuths); + WebAuthenticationDetails webDetails = new WebAuthenticationDetails(request); + ((AbstractAuthenticationToken) authentication).setDetails(webDetails); + authentication = getGrantedAuthority(authentication); + SecurityContextHolder.getContext().setAuthentication(authentication); + request.setAttribute("spnegoEnabled", true); + LOG.info("Logged into Ranger as doAsUser = " + doAsUser + ", by authenticatedUser=" + authToken.getUserName()); + } + + } + + }else { + //if we get the userName from the token then log into ranger using the same user + final List grantedAuths = new ArrayList<>(); + grantedAuths.add(new SimpleGrantedAuthority(rangerLdapDefaultRole)); + final UserDetails principal = new User(userName, "", grantedAuths); + final Authentication finalAuthentication = new UsernamePasswordAuthenticationToken(principal, "", grantedAuths); + WebAuthenticationDetails webDetails = new WebAuthenticationDetails(request); + ((AbstractAuthenticationToken) finalAuthentication).setDetails(webDetails); + RangerAuthenticationProvider authenticationProvider = new RangerAuthenticationProvider(); + Authentication authentication = authenticationProvider.authenticate(finalAuthentication); + authentication = getGrantedAuthority(authentication); + if (authentication != null && authentication.isAuthenticated()) { + if (request.getParameterMap().containsKey("doAs")) { + if (!response.isCommitted()) { + if (LOG.isDebugEnabled()) { + LOG.debug("Request contains unsupported parameter, doAs."); + } + request.setAttribute("spnegoenabled", false); + response.sendError(HttpServletResponse.SC_FORBIDDEN, "Missing authentication token."); + } + } + if (request.getParameterMap().containsKey("user.name")) { + if (!response.isCommitted()) { + if (LOG.isDebugEnabled()) { + LOG.debug("Request contains an unsupported parameter user.name"); + } + request.setAttribute("spnegoenabled", false); + response.sendError(HttpServletResponse.SC_FORBIDDEN, "Missing authentication token."); + } else { + LOG.info("Response seems to be already committed for user.name."); + } + } + } + SecurityContextHolder.getContext().setAuthentication(authentication); + request.setAttribute("spnegoEnabled", true); + LOG.info("Logged into Ranger as = " + userName); + } filterChain.doFilter(request, response); }else{ try{ @@ -236,9 +336,9 @@ public void doFilter(ServletRequest request, ServletResponse response, FilterChain filterChain) throws IOException, ServletException { String authtype = PropertiesUtil.getProperty(RANGER_AUTH_TYPE); HttpServletRequest httpRequest = (HttpServletRequest)request; - if(isSpnegoEnable(authtype)){ + Authentication existingAuth = SecurityContextHolder.getContext().getAuthentication(); + if(isSpnegoEnable(authtype) && (existingAuth == null || !existingAuth.isAuthenticated())){ KerberosName.setRules(PropertiesUtil.getProperty(NAME_RULES, "DEFAULT")); - Authentication existingAuth = SecurityContextHolder.getContext().getAuthentication(); String userName = null; Cookie[] cookie = httpRequest.getCookies(); if(cookie != null){ @@ -261,8 +361,8 @@ public void doFilter(ServletRequest request, ServletResponse response, userName = cname.substring(ustr+2, andStr); } } - } - } + } + } } if((existingAuth == null || !existingAuth.isAuthenticated()) && (!StringUtils.isEmpty(userName))){ //--------------------------- To Create Ranger Session -------------------------------------- diff --git a/security-admin/src/main/java/org/apache/ranger/security/web/filter/RangerSSOAuthenticationFilter.java b/security-admin/src/main/java/org/apache/ranger/security/web/filter/RangerSSOAuthenticationFilter.java index 2f0b3c16b9..1843d698e5 100644 --- a/security-admin/src/main/java/org/apache/ranger/security/web/filter/RangerSSOAuthenticationFilter.java +++ b/security-admin/src/main/java/org/apache/ranger/security/web/filter/RangerSSOAuthenticationFilter.java @@ -52,8 +52,10 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Date; +import java.util.Enumeration; import java.util.List; +import org.apache.commons.lang.StringUtils; import org.apache.ranger.biz.UserMgr; import org.apache.ranger.common.PropertiesUtil; import org.apache.ranger.common.UserSessionBase; @@ -78,6 +80,7 @@ public class RangerSSOAuthenticationFilter implements Filter { public static final String JWT_ORIGINAL_URL_QUERY_PARAM_DEFAULT = "originalUrl"; public static final String LOCAL_LOGIN_URL = "locallogin"; public static final String DEFAULT_BROWSER_USERAGENT = "ranger.default.browser-useragents"; + public static final String PROXY_RANGER_URL_PATH = "/ranger"; private SSOAuthenticationProperties jwtProperties; @@ -116,6 +119,9 @@ public void init(FilterConfig filterConfig) throws ServletException { public void doFilter(ServletRequest servletRequest, ServletResponse servletResponse, FilterChain filterChain)throws IOException, ServletException { HttpServletRequest httpRequest = (HttpServletRequest)servletRequest; + + String xForwardedURL = constructForwardableURL(httpRequest); + if (httpRequest.getRequestedSessionId() != null && !httpRequest.isRequestedSessionIdValid()){ synchronized(httpRequest.getServletContext()){ if(httpRequest.getServletContext().getAttribute(httpRequest.getRequestedSessionId()) != null && httpRequest.getServletContext().getAttribute(httpRequest.getRequestedSessionId()).toString().equals("locallogin")){ @@ -178,7 +184,7 @@ public void doFilter(ServletRequest servletRequest, ServletResponse servletRespo // if the token is not valid then redirect to knox sso else { if (isWebUserAgent(userAgent)) { - String ssourl = constructLoginURL(httpRequest); + String ssourl = constructLoginURL(httpRequest, xForwardedURL); if (LOG.isDebugEnabled()) { LOG.debug("SSO URL = " + ssourl); } @@ -194,7 +200,7 @@ public void doFilter(ServletRequest servletRequest, ServletResponse servletRespo // if the jwt token is not available then redirect it to knox sso else { if (isWebUserAgent(userAgent)) { - String ssourl = constructLoginURL(httpRequest); + String ssourl = constructLoginURL(httpRequest, xForwardedURL); if (LOG.isDebugEnabled()) { LOG.debug("SSO URL = " + ssourl); } @@ -222,6 +228,45 @@ public void doFilter(ServletRequest servletRequest, ServletResponse servletRespo } } + private String constructForwardableURL(HttpServletRequest httpRequest){ + String xForwardedProto = ""; + String xForwardedHost = ""; + String xForwardedContext = ""; + Enumeration names = httpRequest.getHeaderNames(); + while (names.hasMoreElements()) { + String name = (String) names.nextElement(); + Enumeration values = httpRequest.getHeaders(name); + String value = ""; + if (values != null) { + while (values.hasMoreElements()) { + value = (String) values.nextElement(); + } + } + if (StringUtils.trimToNull(name) != null && StringUtils.trimToNull(value) != null) { + if (name.equalsIgnoreCase("x-forwarded-proto")) { + xForwardedProto = value; + } else if (name.equalsIgnoreCase("x-forwarded-host")) { + xForwardedHost = value; + } else if (name.equalsIgnoreCase("x-forwarded-context")) { + xForwardedContext = value; + } + } + } + if (xForwardedHost.contains(",")) { + if(LOG.isDebugEnabled()) { + LOG.debug("xForwardedHost value is " + xForwardedHost + " it contains multiple hosts, selecting the first host."); + } + xForwardedHost = xForwardedHost.split(",")[0].trim(); + } + String xForwardedURL = ""; + if (StringUtils.trimToNull(xForwardedProto) != null && StringUtils.trimToNull(xForwardedHost) != null && StringUtils.trimToNull(xForwardedContext) != null) { + xForwardedURL = xForwardedProto + "://" + xForwardedHost + + xForwardedContext + PROXY_RANGER_URL_PATH + + httpRequest.getRequestURI(); + } + return xForwardedURL; + } + private Authentication getGrantedAuthority(Authentication authentication) { UsernamePasswordAuthenticationToken result=null; if(authentication!=null && authentication.isAuthenticated()){ @@ -326,12 +371,17 @@ protected String getJWTFromCookie(HttpServletRequest req) { * for getting the original request URL * @return url to use as login url for redirect */ - protected String constructLoginURL(HttpServletRequest request) { + protected String constructLoginURL(HttpServletRequest request, String xForwardedURL) { String delimiter = "?"; if (authenticationProviderUrl.contains("?")) { delimiter = "&"; } - String loginURL = authenticationProviderUrl + delimiter + originalUrlQueryParam + "=" + request.getRequestURL().append(getOriginalQueryString(request)); + String loginURL = authenticationProviderUrl + delimiter + originalUrlQueryParam + "="; + if (StringUtils.trimToNull(xForwardedURL) != null) { + loginURL += xForwardedURL + getOriginalQueryString(request); + } else { + loginURL += request.getRequestURL().append(getOriginalQueryString(request)); + } return loginURL; } diff --git a/security-admin/src/main/java/org/apache/ranger/security/web/filter/RangerSecurityContextFormationFilter.java b/security-admin/src/main/java/org/apache/ranger/security/web/filter/RangerSecurityContextFormationFilter.java index 721dd44023..dc1e106906 100644 --- a/security-admin/src/main/java/org/apache/ranger/security/web/filter/RangerSecurityContextFormationFilter.java +++ b/security-admin/src/main/java/org/apache/ranger/security/web/filter/RangerSecurityContextFormationFilter.java @@ -139,6 +139,9 @@ public void doFilter(ServletRequest request, ServletResponse response, } HttpServletResponse res = (HttpServletResponse)response; res.setHeader("X-Frame-Options", "DENY" ); + res.setHeader("X-Content-Type-Options", "nosniff"); + res.setHeader("X-XSS-Protection", "1; mode=block"); + res.setHeader("Strict-Transport-Security", "max-age=31536000; includeSubDomains"); chain.doFilter(request, res); } finally { diff --git a/security-admin/src/main/java/org/apache/ranger/service/RangerAuditFields.java b/security-admin/src/main/java/org/apache/ranger/service/RangerAuditFields.java index 7223f109c9..79097cc779 100644 --- a/security-admin/src/main/java/org/apache/ranger/service/RangerAuditFields.java +++ b/security-admin/src/main/java/org/apache/ranger/service/RangerAuditFields.java @@ -17,15 +17,13 @@ package org.apache.ranger.service; -import org.apache.ranger.common.ContextUtil; -import org.apache.ranger.common.DateUtil; import org.apache.ranger.entity.XXDBBase; import org.springframework.stereotype.Component; @Component public class RangerAuditFields { - public T populateAuditFields(T xObj, T parentObj) { + public T populateAuditFields(T xObj, PARENT parentObj) { xObj.setCreateTime(parentObj.getCreateTime()); xObj.setUpdateTime(parentObj.getUpdateTime()); xObj.setAddedByUserId(parentObj.getAddedByUserId()); @@ -33,12 +31,4 @@ public T populateAuditFields(T xObj, T parentObj) { return xObj; } - public T populateAuditFieldsForCreate(T xObj) { - xObj.setCreateTime(DateUtil.getUTCDate()); - xObj.setUpdateTime(DateUtil.getUTCDate()); - xObj.setAddedByUserId(ContextUtil.getCurrentUserId()); - xObj.setUpdatedByUserId(ContextUtil.getCurrentUserId()); - return xObj; - } - } diff --git a/security-admin/src/main/java/org/apache/ranger/service/RangerDataHistService.java b/security-admin/src/main/java/org/apache/ranger/service/RangerDataHistService.java index e94bad44a5..a96d648a91 100644 --- a/security-admin/src/main/java/org/apache/ranger/service/RangerDataHistService.java +++ b/security-admin/src/main/java/org/apache/ranger/service/RangerDataHistService.java @@ -26,6 +26,8 @@ import org.apache.ranger.common.RESTErrorUtil; import org.apache.ranger.db.RangerDaoManager; import org.apache.ranger.entity.XXDataHist; +import org.apache.ranger.entity.XXService; +import org.apache.ranger.entity.XXServiceDef; import org.apache.ranger.plugin.model.RangerBaseModelObject; import org.apache.ranger.plugin.model.RangerPolicy; import org.apache.ranger.plugin.model.RangerService; @@ -92,6 +94,14 @@ public void createObjectDataHistory(RangerBaseModelObject baseModelObj, String a RangerPolicy policy = (RangerPolicy) baseModelObj; objectName = policy.getName(); classType = AppConstants.CLASS_TYPE_RANGER_POLICY; + XXService xXService = daoMgr.getXXService().findByName(policy.getService()); + XXServiceDef xxServiceDef = null; + if(xXService != null){ + xxServiceDef = daoMgr.getXXServiceDef().getById(xXService.getType()); + } + if(xxServiceDef != null){ + policy.setServiceType(xxServiceDef.getName()); + } content = writeObjectAsString(policy); } diff --git a/security-admin/src/main/java/org/apache/ranger/service/RangerPolicyService.java b/security-admin/src/main/java/org/apache/ranger/service/RangerPolicyService.java index ecefc4bba5..cfcb56e3be 100644 --- a/security-admin/src/main/java/org/apache/ranger/service/RangerPolicyService.java +++ b/security-admin/src/main/java/org/apache/ranger/service/RangerPolicyService.java @@ -128,10 +128,10 @@ public RangerPolicy getPopulatedViewObject(XXPolicy xPolicy) { } public List getTransactionLog(RangerPolicy vPolicy, int action) { - return getTransactionLog(vPolicy, null, action); + return getTransactionLog(vPolicy, null, null, action); } - public List getTransactionLog(RangerPolicy vObj, XXPolicy mObj, int action) { + public List getTransactionLog(RangerPolicy vObj, XXPolicy mObj, RangerPolicy oldPolicy, int action) { if (vObj == null || action == 0 || (action == OPERATION_UPDATE_CONTEXT && mObj == null)) { return null; } @@ -147,7 +147,7 @@ public List getTransactionLog(RangerPolicy vObj, XXPolicy mObj, int ac if (!trxLogAttrs.containsKey(field.getName())) { continue; } - XXTrxLog xTrxLog = processFieldToCreateTrxLog(field, objectName, nameField, vObj, mObj, action); + XXTrxLog xTrxLog = processFieldToCreateTrxLog(field, objectName, nameField, vObj, mObj, oldPolicy, action); if (xTrxLog != null) { trxLogList.add(xTrxLog); } @@ -156,8 +156,8 @@ public List getTransactionLog(RangerPolicy vObj, XXPolicy mObj, int ac Field[] superClassFields = vObj.getClass().getSuperclass() .getDeclaredFields(); for (Field field : superClassFields) { - if (field.getName().equalsIgnoreCase("isEnabled")) { - XXTrxLog xTrx = processFieldToCreateTrxLog(field, objectName, nameField, vObj, mObj, action); + if ("isEnabled".equalsIgnoreCase(field.getName())) { + XXTrxLog xTrx = processFieldToCreateTrxLog(field, objectName, nameField, vObj, mObj, oldPolicy, action); if (xTrx != null) { trxLogList.add(xTrx); } @@ -174,7 +174,7 @@ public List getTransactionLog(RangerPolicy vObj, XXPolicy mObj, int ac } private XXTrxLog processFieldToCreateTrxLog(Field field, String objectName, - Field nameField, RangerPolicy vObj, XXPolicy mObj, int action) { + Field nameField, RangerPolicy vObj, XXPolicy mObj, RangerPolicy oldPolicy, int action) { String actionString = ""; @@ -228,10 +228,10 @@ private XXTrxLog processFieldToCreateTrxLog(Field field, String objectName, } else if (fieldName.equalsIgnoreCase(ROWFILTER_POLICY_ITEM_CLASS_FIELD_NAME)) { value = processRowFilterPolicyItemForTrxLog(field.get(vObj)); } else if (fieldName.equalsIgnoreCase(IS_ENABLED_CLASS_FIELD_NAME)) { - value = String.valueOf(processIsEnabledClassFieldNameForTrxLog(field.get(vObj))); - - } - else { + value = processIsEnabledClassFieldNameForTrxLog(field.get(vObj)); + } else if (fieldName.equalsIgnoreCase(IS_AUDIT_ENABLED_CLASS_FIELD_NAME)) { + value = processIsAuditEnabledClassFieldNameForTrxLog(field.get(vObj)); + } else { value = "" + field.get(vObj); } @@ -260,8 +260,7 @@ private XXTrxLog processFieldToCreateTrxLog(Field field, String objectName, break; } } - RangerPolicy oldPolicy = populateViewBean(mObj); - if (fieldName.equalsIgnoreCase(POLICY_RESOURCE_CLASS_FIELD_NAME)) { + if (POLICY_RESOURCE_CLASS_FIELD_NAME.equalsIgnoreCase(fieldName)) { if (oldPolicy != null) { oldValue = processPolicyResourcesForTrxLog(oldPolicy.getResources()); } @@ -319,7 +318,11 @@ private XXTrxLog processFieldToCreateTrxLog(Field field, String objectName, } }else if (fieldName.equalsIgnoreCase(IS_ENABLED_CLASS_FIELD_NAME)) { if (oldPolicy != null) { - oldValue = String.valueOf(processIsEnabledClassFieldNameForTrxLog(oldPolicy.getIsEnabled())); + oldValue = processIsEnabledClassFieldNameForTrxLog(oldPolicy.getIsEnabled()); + } + } else if(fieldName.equalsIgnoreCase(IS_AUDIT_ENABLED_CLASS_FIELD_NAME)) { + if (oldPolicy != null) { + oldValue = processIsAuditEnabledClassFieldNameForTrxLog(oldPolicy.getIsAuditEnabled()); } } if (oldValue == null || oldValue.equalsIgnoreCase(value)) { @@ -578,6 +581,13 @@ private String processIsEnabledClassFieldNameForTrxLog(Object value) { return isEnabled; } + private String processIsAuditEnabledClassFieldNameForTrxLog(Object value) { + if(value == null) + return null; + String isAuditEnabled = String.valueOf(value); + return isAuditEnabled; + } + private boolean compareTwoDataMaskingPolicyItemList(String value, String oldValue) { if (value == null && oldValue == null) { return true; diff --git a/security-admin/src/main/java/org/apache/ranger/service/RangerPolicyServiceBase.java b/security-admin/src/main/java/org/apache/ranger/service/RangerPolicyServiceBase.java index 0195c72a76..09f15c0909 100644 --- a/security-admin/src/main/java/org/apache/ranger/service/RangerPolicyServiceBase.java +++ b/security-admin/src/main/java/org/apache/ranger/service/RangerPolicyServiceBase.java @@ -18,6 +18,7 @@ package org.apache.ranger.service; import org.apache.commons.lang.StringUtils; +import org.apache.ranger.authorization.utils.JsonUtils; import org.apache.ranger.common.GUIDUtil; import org.apache.ranger.common.MessageEnums; import org.apache.ranger.common.SearchField; @@ -49,20 +50,22 @@ public RangerPolicyServiceBase() { "XXService xSvc", "xSvc.id = obj.service")); searchFields .add(new SearchField(SearchFilter.IS_ENABLED, "obj.isEnabled", DATA_TYPE.BOOLEAN, SEARCH_TYPE.FULL)); - searchFields.add(new SearchField(SearchFilter.IS_RECURSIVE,"xPolRes.isRecursive",DATA_TYPE.BOOLEAN,SEARCH_TYPE.FULL, - "XXPolicyResource xPolRes","obj.id=xPolRes.policyId")); + //might need updation + /*searchFields.add(new SearchField(SearchFilter.IS_RECURSIVE,"xPolRes.isRecursive",DATA_TYPE.BOOLEAN,SEARCH_TYPE.FULL, + "XXPolicyResource xPolRes","obj.id=xPolRes.policyId"));*/ searchFields.add(new SearchField(SearchFilter.POLICY_ID, "obj.id", DATA_TYPE.INTEGER, SEARCH_TYPE.FULL)); searchFields.add(new SearchField(SearchFilter.POLICY_NAME, "obj.name", DATA_TYPE.STRING, SEARCH_TYPE.FULL)); searchFields.add(new SearchField(SearchFilter.GUID, "obj.guid", DATA_TYPE.STRING, SEARCH_TYPE.FULL)); searchFields.add(new SearchField(SearchFilter.USER, "xUser.name", DATA_TYPE.STRING, SEARCH_TYPE.FULL, - "XXUser xUser, XXPolicyItem xPolItem, XXPolicyItemUserPerm userPerm", "obj.id = xPolItem.policyId " - + "and userPerm.policyItemId = xPolItem.id and xUser.id = userPerm.userId")); + "XXUser xUser, XXPolicyRefUser refUser", "obj.id = refUser.policyId " + + "and xUser.id = refUser.userId")); searchFields.add(new SearchField(SearchFilter.GROUP, "xGrp.name", DATA_TYPE.STRING, SEARCH_TYPE.FULL, - "XXGroup xGrp, XXPolicyItem xPolItem, XXPolicyItemGroupPerm grpPerm", "obj.id = xPolItem.policyId " - + "and grpPerm.policyItemId = xPolItem.id and xGrp.id = grpPerm.groupId")); - searchFields.add(new SearchField(SearchFilter.POL_RESOURCE, "resMap.value", DATA_TYPE.STRING, + "XXGroup xGrp , XXPolicyRefGroup refGroup", "obj.id = refGroup.policyId " + + "and xGrp.id = refGroup.groupId")); + //might need updation + /*searchFields.add(new SearchField(SearchFilter.POL_RESOURCE, "resMap.value", DATA_TYPE.STRING, SEARCH_TYPE.PARTIAL, "XXPolicyResourceMap resMap, XXPolicyResource polRes", - "resMap.resourceId = polRes.id and polRes.policyId = obj.id")); + "resMap.resourceId = polRes.id and polRes.policyId = obj.id"));*/ searchFields.add(new SearchField(SearchFilter.POLICY_NAME_PARTIAL, "obj.name", DATA_TYPE.STRING, SEARCH_TYPE.PARTIAL)); searchFields.add(new SearchField(SearchFilter.POLICY_TYPE, "obj.policyType", DATA_TYPE.INTEGER, SEARCH_TYPE.FULL)); @@ -92,7 +95,7 @@ protected T mapViewToEntityBean(V vObj, T xObj, int OPERATION_CONTEXT) { xObj.setResourceSignature(vObj.getResourceSignature()); xObj.setIsAuditEnabled(vObj.getIsAuditEnabled()); xObj.setIsEnabled(vObj.getIsEnabled()); - + xObj.setPolicyText(JsonUtils.objectToJson(vObj)); return xObj; } diff --git a/security-admin/src/main/java/org/apache/ranger/service/RangerServiceDefService.java b/security-admin/src/main/java/org/apache/ranger/service/RangerServiceDefService.java index 53b12d83a0..1e385a0246 100644 --- a/security-admin/src/main/java/org/apache/ranger/service/RangerServiceDefService.java +++ b/security-admin/src/main/java/org/apache/ranger/service/RangerServiceDefService.java @@ -19,9 +19,13 @@ import java.util.ArrayList; import java.util.List; +import java.util.Map; +import org.apache.commons.lang.StringUtils; +import org.apache.ranger.authorization.hadoop.config.RangerConfiguration; import org.apache.ranger.entity.XXServiceDef; import org.apache.ranger.plugin.model.RangerServiceDef; +import org.apache.ranger.plugin.store.EmbeddedServiceDefsUtil; import org.springframework.context.annotation.Scope; import org.springframework.stereotype.Service; @@ -50,7 +54,20 @@ protected XXServiceDef mapViewToEntityBean(RangerServiceDef vObj, XXServiceDef x @Override protected RangerServiceDef mapEntityToViewBean(RangerServiceDef vObj, XXServiceDef xObj) { - return super.mapEntityToViewBean(vObj, xObj); + RangerServiceDef ret = super.mapEntityToViewBean(vObj, xObj); + + Map serviceDefOptions = ret.getOptions(); + + if (serviceDefOptions.get(RangerServiceDef.OPTION_ENABLE_DENY_AND_EXCEPTIONS_IN_POLICIES) == null) { + boolean enableDenyAndExceptionsInPoliciesHiddenOption = RangerConfiguration.getInstance().getBoolean("ranger.servicedef.enableDenyAndExceptionsInPolicies", true); + if (enableDenyAndExceptionsInPoliciesHiddenOption || StringUtils.equalsIgnoreCase(ret.getName(), EmbeddedServiceDefsUtil.EMBEDDED_SERVICEDEF_TAG_NAME)) { + serviceDefOptions.put(RangerServiceDef.OPTION_ENABLE_DENY_AND_EXCEPTIONS_IN_POLICIES, "true"); + } else { + serviceDefOptions.put(RangerServiceDef.OPTION_ENABLE_DENY_AND_EXCEPTIONS_IN_POLICIES, "false"); + } + ret.setOptions(serviceDefOptions); + } + return ret; } public List getAllServiceDefs() { diff --git a/security-admin/src/main/java/org/apache/ranger/service/RangerServiceResourceService.java b/security-admin/src/main/java/org/apache/ranger/service/RangerServiceResourceService.java index d7256802a6..f0cb8f4d20 100644 --- a/security-admin/src/main/java/org/apache/ranger/service/RangerServiceResourceService.java +++ b/security-admin/src/main/java/org/apache/ranger/service/RangerServiceResourceService.java @@ -19,113 +19,167 @@ package org.apache.ranger.service; +import java.lang.reflect.Type; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import com.google.gson.reflect.TypeToken; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.MapUtils; import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.ranger.authorization.utils.JsonUtils; +import org.apache.ranger.biz.RangerTagDBRetriever; import org.apache.ranger.common.SearchField; import org.apache.ranger.common.SearchField.DATA_TYPE; import org.apache.ranger.common.SearchField.SEARCH_TYPE; import org.apache.ranger.entity.XXServiceResource; +import org.apache.ranger.plugin.model.RangerPolicy; import org.apache.ranger.plugin.model.RangerServiceResource; +import org.apache.ranger.plugin.model.RangerTag; import org.apache.ranger.plugin.util.SearchFilter; import org.springframework.stereotype.Service; @Service public class RangerServiceResourceService extends RangerServiceResourceServiceBase { - private boolean serviceUpdateNeeded = true; + private static final Log LOG = LogFactory.getLog(RangerServiceResourceService.class); - public RangerServiceResourceService() { - searchFields.add(new SearchField(SearchFilter.TAG_RESOURCE_ID, "obj.id", DATA_TYPE.INTEGER, SEARCH_TYPE.FULL)); - searchFields.add(new SearchField(SearchFilter.TAG_SERVICE_ID, "obj.serviceId", DATA_TYPE.INTEGER, SEARCH_TYPE.FULL)); - searchFields.add(new SearchField(SearchFilter.TAG_RESOURCE_SIGNATURE, "obj.resourceSignature", DATA_TYPE.STRING, SEARCH_TYPE.FULL)); - } + private boolean serviceUpdateNeeded = true; - @Override - protected void validateForCreate(RangerServiceResource vObj) { + public static final Type subsumedDataType = new TypeToken>() {}.getType(); + public static final Type duplicatedDataType = new TypeToken>() {}.getType(); - } + public RangerServiceResourceService() { + searchFields.add(new SearchField(SearchFilter.TAG_RESOURCE_ID, "obj.id", DATA_TYPE.INTEGER, SEARCH_TYPE.FULL)); + searchFields.add(new SearchField(SearchFilter.TAG_SERVICE_ID, "obj.serviceId", DATA_TYPE.INTEGER, SEARCH_TYPE.FULL)); + searchFields.add(new SearchField(SearchFilter.TAG_RESOURCE_SIGNATURE, "obj.resourceSignature", DATA_TYPE.STRING, SEARCH_TYPE.FULL)); + } - @Override - protected void validateForUpdate(RangerServiceResource vObj, XXServiceResource entityObj) { - if (StringUtils.equals(entityObj.getGuid(), vObj.getGuid()) && - StringUtils.equals(entityObj.getResourceSignature(), vObj.getResourceSignature())) { - serviceUpdateNeeded = false; - } else { - serviceUpdateNeeded = true; - } - } + @Override + protected void validateForCreate(RangerServiceResource vObj) { - @Override - public RangerServiceResource postUpdate(XXServiceResource resource) { - RangerServiceResource ret = super.postUpdate(resource); + } - if (serviceUpdateNeeded) { - daoMgr.getXXServiceVersionInfo().updateServiceVersionInfoForServiceResourceUpdate(resource.getId(), resource.getUpdateTime()); - } + @Override + protected void validateForUpdate(RangerServiceResource vObj, XXServiceResource entityObj) { + if (StringUtils.equals(entityObj.getGuid(), vObj.getGuid()) && + StringUtils.equals(entityObj.getResourceSignature(), vObj.getResourceSignature())) { + serviceUpdateNeeded = false; + } else { + serviceUpdateNeeded = true; + } + } - return ret; - } + @Override + public RangerServiceResource postUpdate(XXServiceResource resource) { + RangerServiceResource ret = super.postUpdate(resource); - public RangerServiceResource getPopulatedViewObject(XXServiceResource xObj) { - return populateViewBean(xObj); - } + if (serviceUpdateNeeded) { + daoMgr.getXXServiceVersionInfo().updateServiceVersionInfoForServiceResourceUpdate(resource.getId(), resource.getUpdateTime()); + } - public RangerServiceResource getServiceResourceByGuid(String guid) { - RangerServiceResource ret = null; + return ret; + } - XXServiceResource xxServiceResource = daoMgr.getXXServiceResource().findByGuid(guid); - - if(xxServiceResource != null) { - ret = populateViewBean(xxServiceResource); - } + public RangerServiceResource getPopulatedViewObject(XXServiceResource xObj) { + return populateViewBean(xObj); + } - return ret; - } + public RangerServiceResource getServiceResourceByGuid(String guid) { + RangerServiceResource ret = null; - public List getByServiceId(Long serviceId) { - List ret = new ArrayList(); + XXServiceResource xxServiceResource = daoMgr.getXXServiceResource().findByGuid(guid); - List xxServiceResources = daoMgr.getXXServiceResource().findByServiceId(serviceId); + if (xxServiceResource != null) { + ret = populateViewBean(xxServiceResource); + } - if(CollectionUtils.isNotEmpty(xxServiceResources)) { - for(XXServiceResource xxServiceResource : xxServiceResources) { - RangerServiceResource serviceResource = populateViewBean(xxServiceResource); + return ret; + } - ret.add(serviceResource); - } - } + public List getByServiceId(Long serviceId) { + List ret = new ArrayList(); - return ret; - } + List xxServiceResources = daoMgr.getXXServiceResource().findByServiceId(serviceId); - public RangerServiceResource getByServiceAndResourceSignature(Long serviceId, String resourceSignature) { - RangerServiceResource ret = null; + if (CollectionUtils.isNotEmpty(xxServiceResources)) { + for (XXServiceResource xxServiceResource : xxServiceResources) { + RangerServiceResource serviceResource = populateViewBean(xxServiceResource); - XXServiceResource xxServiceResource = daoMgr.getXXServiceResource().findByServiceAndResourceSignature(serviceId, resourceSignature); - - if(xxServiceResource != null) { - ret = populateViewBean(xxServiceResource); - } + ret.add(serviceResource); + } + } - return ret; - } + return ret; + } - public List getTaggedResourcesInServiceId(Long serviceId) { - List ret = new ArrayList(); + public RangerServiceResource getByServiceAndResourceSignature(Long serviceId, String resourceSignature) { + RangerServiceResource ret = null; - List xxServiceResources = daoMgr.getXXServiceResource().findByServiceId(serviceId); - - if(CollectionUtils.isNotEmpty(xxServiceResources)) { - for(XXServiceResource xxServiceResource : xxServiceResources) { - RangerServiceResource serviceResource = populateViewBean(xxServiceResource); + XXServiceResource xxServiceResource = daoMgr.getXXServiceResource().findByServiceAndResourceSignature(serviceId, resourceSignature); - ret.add(serviceResource); - } - } + if (xxServiceResource != null) { + ret = populateViewBean(xxServiceResource); + } - return ret; - } + return ret; + } + + public List getTaggedResourcesInServiceId(Long serviceId) { + List ret = new ArrayList(); + + List xxServiceResources = daoMgr.getXXServiceResource().findByServiceId(serviceId); + + if (CollectionUtils.isNotEmpty(xxServiceResources)) { + for (XXServiceResource xxServiceResource : xxServiceResources) { + RangerServiceResource serviceResource = populateViewBean(xxServiceResource); + + ret.add(serviceResource); + } + } + + return ret; + } + + @Override + protected XXServiceResource mapViewToEntityBean(RangerServiceResource serviceResource, XXServiceResource xxServiceResource, int operationContext) { + XXServiceResource ret = super.mapViewToEntityBean(serviceResource, xxServiceResource, operationContext); + if (MapUtils.isNotEmpty(serviceResource.getResourceElements())) { + String serviceResourceElements = JsonUtils.mapToJson(serviceResource.getResourceElements()); + if (StringUtils.isNotEmpty(serviceResourceElements)) { + ret.setServiceResourceElements(serviceResourceElements); + } else { + LOG.info("Empty string representing serviceResourceElements in [" + ret + "]!!"); + } + } + + return ret; + } + + @Override + protected RangerServiceResource mapEntityToViewBean(RangerServiceResource serviceResource, XXServiceResource xxServiceResource) { + RangerServiceResource ret = super.mapEntityToViewBean(serviceResource, xxServiceResource); + if (StringUtils.isNotEmpty(xxServiceResource.getServiceResourceElements())) { + Map serviceResourceElements = + RangerTagDBRetriever.gsonBuilder.fromJson(xxServiceResource.getServiceResourceElements(), RangerServiceResourceService.subsumedDataType); + if (MapUtils.isNotEmpty(serviceResourceElements)) { + ret.setResourceElements(serviceResourceElements); + } else { + LOG.info("Empty serviceResourceElement in [" + ret + "]!!"); + } + } else { + LOG.info("Empty string representing serviceResourceElements in [" + xxServiceResource + "]!!"); + } + + return ret; + } + + @Override + Map getServiceResourceElements(XXServiceResource xxServiceResource) { + return new HashMap<>(); + } } diff --git a/security-admin/src/main/java/org/apache/ranger/service/RangerServiceResourceServiceBase.java b/security-admin/src/main/java/org/apache/ranger/service/RangerServiceResourceServiceBase.java index 6af682a811..329d3eeecb 100644 --- a/security-admin/src/main/java/org/apache/ranger/service/RangerServiceResourceServiceBase.java +++ b/security-admin/src/main/java/org/apache/ranger/service/RangerServiceResourceServiceBase.java @@ -73,26 +73,31 @@ protected V mapEntityToViewBean(V vObj, T xObj) { vObj.setServiceName(xService.getName()); - List resElementList = daoMgr.getXXServiceResourceElement().findByResourceId(xObj.getId()); - Map resourceElements = new HashMap(); + Map resourceElements = getServiceResourceElements(xObj); - for (XXServiceResourceElement resElement : resElementList) { - List resValueMapList = daoMgr.getXXServiceResourceElementValue().findValuesByResElementId(resElement.getId()); + vObj.setResourceElements(resourceElements); - XXResourceDef xResDef = daoMgr.getXXResourceDef().getById(resElement.getResDefId()); + return vObj; + } - RangerPolicyResource policyRes = new RangerPolicyResource(); - policyRes.setIsExcludes(resElement.getIsExcludes()); - policyRes.setIsRecursive(resElement.getIsRecursive()); - policyRes.setValues(resValueMapList); + Map getServiceResourceElements(T xObj) { + List resElementList = daoMgr.getXXServiceResourceElement().findByResourceId(xObj.getId()); + Map resourceElements = new HashMap(); - resourceElements.put(xResDef.getName(), policyRes); - } + for (XXServiceResourceElement resElement : resElementList) { + List resValueMapList = daoMgr.getXXServiceResourceElementValue().findValuesByResElementId(resElement.getId()); - vObj.setResourceElements(resourceElements); + XXResourceDef xResDef = daoMgr.getXXResourceDef().getById(resElement.getResDefId()); - return vObj; - } + RangerPolicyResource policyRes = new RangerPolicyResource(); + policyRes.setIsExcludes(resElement.getIsExcludes()); + policyRes.setIsRecursive(resElement.getIsRecursive()); + policyRes.setValues(resValueMapList); + + resourceElements.put(xResDef.getName(), policyRes); + } + return resourceElements; + } public PList searchServiceResources(SearchFilter searchFilter) { PList retList = new PList(); diff --git a/security-admin/src/main/java/org/apache/ranger/service/RangerServiceService.java b/security-admin/src/main/java/org/apache/ranger/service/RangerServiceService.java index ab44016816..30fc2131e6 100644 --- a/security-admin/src/main/java/org/apache/ranger/service/RangerServiceService.java +++ b/security-admin/src/main/java/org/apache/ranger/service/RangerServiceService.java @@ -25,6 +25,7 @@ import java.util.List; import java.util.Map; import java.util.Map.Entry; +import java.util.logging.Logger; import org.apache.commons.lang.StringUtils; import org.apache.ranger.biz.ServiceDBStore; @@ -35,7 +36,6 @@ import org.apache.ranger.common.view.VTrxLogAttr; import org.apache.ranger.db.XXServiceVersionInfoDao; import org.apache.ranger.entity.XXService; -import org.apache.ranger.entity.XXServiceBase; import org.apache.ranger.entity.XXServiceConfigMap; import org.apache.ranger.entity.XXServiceDef; import org.apache.ranger.entity.XXServiceVersionInfo; @@ -48,7 +48,7 @@ @Service @Scope("singleton") public class RangerServiceService extends RangerServiceServiceBase { - + private static final Logger LOG = Logger.getLogger(RangerServiceService.class.getName()); @Autowired JSONUtil jsonUtil; @@ -166,9 +166,9 @@ public List getTransactionLog(RangerService vObj, XXService mObj, int } } } catch (IllegalAccessException e) { - e.printStackTrace(); + LOG.info("Get transaction log failure." + e); } catch (NoSuchFieldException e) { - e.printStackTrace(); + LOG.info("Get transaction log failure." + e); } return trxLogList; } @@ -275,7 +275,7 @@ private XXTrxLog processFieldToCreateTrxLog(Field field, String objectName, xTrxLog.setNewValue(value); } } catch (IllegalArgumentException | IllegalAccessException e) { - e.printStackTrace(); + LOG.info("Process field to create trx log failure." + e); } xTrxLog.setAction(actionString); diff --git a/security-admin/src/main/java/org/apache/ranger/service/RangerTagDefService.java b/security-admin/src/main/java/org/apache/ranger/service/RangerTagDefService.java index 82eb252e6b..10c73f0d2e 100644 --- a/security-admin/src/main/java/org/apache/ranger/service/RangerTagDefService.java +++ b/security-admin/src/main/java/org/apache/ranger/service/RangerTagDefService.java @@ -23,6 +23,8 @@ import java.util.List; import org.apache.commons.collections.CollectionUtils; +import org.apache.ranger.authorization.utils.JsonUtils; +import org.apache.ranger.biz.RangerTagDBRetriever; import org.apache.ranger.common.SearchField; import org.apache.ranger.common.SearchField.DATA_TYPE; import org.apache.ranger.common.SearchField.SEARCH_TYPE; @@ -102,5 +104,27 @@ public List getTagDefsByServiceId(Long serviceId) { return ret; } + + @Override + protected RangerTagDef mapEntityToViewBean(RangerTagDef vObj, XXTagDef xObj) { + super.mapEntityToViewBean(vObj, xObj); + + List attributeDefs = RangerTagDBRetriever.gsonBuilder.fromJson(xObj.getTagAttrDefs(), RangerTagDBRetriever.subsumedDataType); + vObj.setAttributeDefs(attributeDefs); + + return vObj; + } + + @Override + protected XXTagDef mapViewToEntityBean(RangerTagDef vObj, XXTagDef xObj, int OPERATION_CONTEXT) { + super.mapViewToEntityBean(vObj, xObj, OPERATION_CONTEXT); + xObj.setTagAttrDefs(JsonUtils.listToJson(vObj.getAttributeDefs())); + return xObj; + } + + @Override + public List getAttributeDefForTagDef(XXTagDef xtagDef) { + return new ArrayList<>(); + } } diff --git a/security-admin/src/main/java/org/apache/ranger/service/RangerTagService.java b/security-admin/src/main/java/org/apache/ranger/service/RangerTagService.java index 28b9115fab..2fa883096c 100644 --- a/security-admin/src/main/java/org/apache/ranger/service/RangerTagService.java +++ b/security-admin/src/main/java/org/apache/ranger/service/RangerTagService.java @@ -19,13 +19,21 @@ package org.apache.ranger.service; +import java.lang.reflect.Type; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import com.google.gson.reflect.TypeToken; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.ranger.authorization.utils.JsonUtils; +import org.apache.ranger.biz.RangerTagDBRetriever; import org.apache.ranger.common.SearchField; import org.apache.ranger.common.SearchField.DATA_TYPE; import org.apache.ranger.common.SearchField.SEARCH_TYPE; +import org.apache.ranger.entity.XXServiceResource; import org.apache.ranger.entity.XXTag; import org.apache.ranger.plugin.model.RangerTag; import org.apache.ranger.plugin.util.SearchFilter; @@ -35,6 +43,8 @@ @Service public class RangerTagService extends RangerTagServiceBase { + public static final Type subsumedDataType = new TypeToken>() {}.getType(); + public RangerTagService() { searchFields.add(new SearchField(SearchFilter.TAG_ID, "obj.id", SearchField.DATA_TYPE.INTEGER, SearchField.SEARCH_TYPE.FULL)); searchFields.add(new SearchField(SearchFilter.TAG_DEF_ID, "obj.type", SearchField.DATA_TYPE.INTEGER, SearchField.SEARCH_TYPE.FULL)); @@ -95,13 +105,12 @@ public List getTagsByType(String name) { public List getTagsForResourceId(Long resourceId) { List ret = new ArrayList(); - List xxTags = daoMgr.getXXTag().findForResourceId(resourceId); - - if(CollectionUtils.isNotEmpty(xxTags)) { - for(XXTag xxTag : xxTags) { - RangerTag tag = populateViewBean(xxTag); + XXServiceResource serviceResourceEntity = daoMgr.getXXServiceResource().getById(resourceId); - ret.add(tag); + if (serviceResourceEntity != null) { + String tagsText = serviceResourceEntity.getTags(); + if (StringUtils.isNotEmpty(tagsText)) { + ret = RangerTagDBRetriever.gsonBuilder.fromJson(tagsText, RangerServiceResourceService.duplicatedDataType); } } @@ -111,13 +120,12 @@ public List getTagsForResourceId(Long resourceId) { public List getTagsForResourceGuid(String resourceGuid) { List ret = new ArrayList(); - List xxTags = daoMgr.getXXTag().findForResourceGuid(resourceGuid); - - if(CollectionUtils.isNotEmpty(xxTags)) { - for(XXTag xxTag : xxTags) { - RangerTag tag = populateViewBean(xxTag); + XXServiceResource serviceResourceEntity = daoMgr.getXXServiceResource().findByGuid(resourceGuid); - ret.add(tag); + if (serviceResourceEntity != null) { + String tagsText = serviceResourceEntity.getTags(); + if (StringUtils.isNotEmpty(tagsText)) { + ret = RangerTagDBRetriever.gsonBuilder.fromJson(tagsText, RangerServiceResourceService.duplicatedDataType); } } @@ -139,4 +147,25 @@ public List getTagsByServiceId(Long serviceId) { return ret; } + + @Override + protected RangerTag mapEntityToViewBean(RangerTag vObj, XXTag xObj) { + super.mapEntityToViewBean(vObj, xObj); + + Map attributes = RangerTagDBRetriever.gsonBuilder.fromJson(xObj.getTagAttrs(), RangerTagService.subsumedDataType); + vObj.setAttributes(attributes); + return vObj; + } + + @Override + protected XXTag mapViewToEntityBean(RangerTag vObj, XXTag xObj, int OPERATION_CONTEXT) { + super.mapViewToEntityBean(vObj, xObj, OPERATION_CONTEXT); + xObj.setTagAttrs(JsonUtils.mapToJson(vObj.getAttributes())); + return xObj; + } + + @Override + public Map getAttributesForTag(XXTag xTag) { + return new HashMap<>(); + } } \ No newline at end of file diff --git a/security-admin/src/main/java/org/apache/ranger/service/XGroupService.java b/security-admin/src/main/java/org/apache/ranger/service/XGroupService.java index e259eaeca2..fbb8a5c1f3 100644 --- a/security-admin/src/main/java/org/apache/ranger/service/XGroupService.java +++ b/security-admin/src/main/java/org/apache/ranger/service/XGroupService.java @@ -69,7 +69,7 @@ public XGroupService() { searchFields.add(new SearchField("name", "obj.name", SearchField.DATA_TYPE.STRING, SearchField.SEARCH_TYPE.PARTIAL)); searchFields.add(new SearchField("groupSource", "obj.groupSource", - SearchField.DATA_TYPE.STRING, SearchField.SEARCH_TYPE.FULL)); + SearchField.DATA_TYPE.INTEGER, SearchField.SEARCH_TYPE.FULL)); searchFields.add(new SearchField("isVisible", "obj.isVisible", SearchField.DATA_TYPE.INTEGER, SearchField.SEARCH_TYPE.FULL )); diff --git a/security-admin/src/main/java/org/apache/ranger/service/XGroupUserService.java b/security-admin/src/main/java/org/apache/ranger/service/XGroupUserService.java index d1901d9c2a..8df205a0d1 100644 --- a/security-admin/src/main/java/org/apache/ranger/service/XGroupUserService.java +++ b/security-admin/src/main/java/org/apache/ranger/service/XGroupUserService.java @@ -83,7 +83,13 @@ protected void validateForUpdate(VXGroupUser vObj, XXGroupUser mObj) { } public VXGroupUser createXGroupUserWithOutLogin(VXGroupUser vxGroupUser) { - XXGroupUser xxGroupUser = new XXGroupUser(); + boolean groupUserMappingExists = true; + XXGroupUser xxGroupUser = daoManager.getXXGroupUser().findByGroupNameAndUserId(vxGroupUser.getName(), vxGroupUser.getUserId()); + if (xxGroupUser == null) { + xxGroupUser = new XXGroupUser(); + groupUserMappingExists = false; + } + XXGroup xGroup = daoManager.getXXGroup().findByGroupName(vxGroupUser.getName()); vxGroupUser.setParentGroupId(xGroup.getId()); xxGroupUser = mapViewToEntityBean(vxGroupUser, xxGroupUser, 0); @@ -92,7 +98,11 @@ public VXGroupUser createXGroupUserWithOutLogin(VXGroupUser vxGroupUser) { xxGroupUser.setAddedByUserId(createdByUserId); xxGroupUser.setUpdatedByUserId(createdByUserId); } - xxGroupUser = getDao().create(xxGroupUser); + if (groupUserMappingExists) { + xxGroupUser = getDao().update(xxGroupUser); + } else { + xxGroupUser = getDao().create(xxGroupUser); + } vxGroupUser = postCreate(xxGroupUser); return vxGroupUser; } diff --git a/security-admin/src/main/java/org/apache/ranger/service/XTrxLogService.java b/security-admin/src/main/java/org/apache/ranger/service/XTrxLogService.java index def9699821..6c56eefd5f 100644 --- a/security-admin/src/main/java/org/apache/ranger/service/XTrxLogService.java +++ b/security-admin/src/main/java/org/apache/ranger/service/XTrxLogService.java @@ -34,14 +34,20 @@ import javax.persistence.metamodel.Metamodel; import javax.persistence.metamodel.SingularAttribute; +import org.apache.ranger.common.AppConstants; +import org.apache.ranger.common.ContextUtil; import org.apache.ranger.common.SearchCriteria; import org.apache.ranger.common.SearchField; import org.apache.ranger.common.SortField; import org.apache.ranger.common.SortField.SORT_ORDER; +import org.apache.ranger.common.UserSessionBase; import org.apache.ranger.db.RangerDaoManager; import org.apache.ranger.entity.XXPortalUser; +import org.apache.ranger.entity.XXPortalUserRole; +import org.apache.ranger.entity.XXServiceDef; import org.apache.ranger.entity.XXTrxLog; import org.apache.ranger.entity.view.VXXTrxLog; +import org.apache.ranger.plugin.store.EmbeddedServiceDefsUtil; import org.apache.ranger.view.VXTrxLog; import org.apache.ranger.view.VXTrxLogList; import org.springframework.beans.factory.annotation.Autowired; @@ -51,6 +57,8 @@ @Service @Scope("singleton") public class XTrxLogService extends XTrxLogServiceBase { + Long keyadminCount = 0L; + @Autowired RangerDaoManager rangerDaoManager; public XTrxLogService(){ @@ -182,9 +190,15 @@ public VXTrxLogList searchXTrxLogs(SearchCriteria searchCriteria) { int startIndex = searchCriteria.getStartIndex(); int pageSize = searchCriteria.getMaxRows(); List resultList = em.createQuery(selectCQ).setFirstResult(startIndex).setMaxResults(pageSize).getResultList(); - VXTrxLogList vxTrxLogList = new VXTrxLogList(); - vxTrxLogList.setStartIndex(startIndex); - vxTrxLogList.setPageSize(pageSize); + + int maxRowSize = Integer.MAX_VALUE; + int minRowSize = 0; + XXServiceDef xxServiceDef = rangerDaoManager.getXXServiceDef().findByName(EmbeddedServiceDefsUtil.EMBEDDED_SERVICEDEF_KMS_NAME); + UserSessionBase session = ContextUtil.getCurrentUserSession(); + if (session != null && session.isKeyAdmin()){ + resultList = em.createQuery(selectCQ).setFirstResult(minRowSize).setMaxResults(maxRowSize).getResultList(); + } + List trxLogList = new ArrayList(); XXPortalUser xXPortalUser=null; for(VXXTrxLog xTrxLog : resultList){ @@ -199,8 +213,58 @@ public VXTrxLogList searchXTrxLogs(SearchCriteria searchCriteria) { } trxLogList.add(trxLog); } + + List keyAdminTrxLogList = new ArrayList(); + if (session != null && session.isKeyAdmin() && xxServiceDef != null) { + List vXTrxLogs = new ArrayList(); + for (VXTrxLog xTrxLog : trxLogList) { + int parentObjectClassType = xTrxLog.getParentObjectClassType(); + Long parentObjectId = xTrxLog.getParentObjectId(); + if (parentObjectClassType == AppConstants.CLASS_TYPE_XA_SERVICE_DEF && parentObjectId.equals(xxServiceDef.getId())) { + vXTrxLogs.add(xTrxLog); + } else if (parentObjectClassType == AppConstants.CLASS_TYPE_XA_SERVICE && !(parentObjectId.equals(xxServiceDef.getId()))) { + for (VXTrxLog vxTrxLog : trxLogList) { + if (parentObjectClassType == vxTrxLog.getObjectClassType() + && parentObjectId.equals(vxTrxLog.getObjectId()) + && vxTrxLog.getParentObjectId().equals(xxServiceDef.getId())) { + vXTrxLogs.add(xTrxLog); + break; + } + } + } else if (xTrxLog.getObjectClassType() == AppConstants.CLASS_TYPE_XA_USER + || xTrxLog.getObjectClassType() == AppConstants.CLASS_TYPE_RANGER_POLICY + || xTrxLog.getObjectClassType() == AppConstants.HIST_OBJ_STATUS_UPDATED) { + XXPortalUser xxPortalUser = null; + if (xTrxLog.getUpdatedBy() != null) { + xxPortalUser = rangerDaoManager.getXXPortalUser().getById(Long.parseLong(xTrxLog.getUpdatedBy())); + } + if (xxPortalUser != null && xxPortalUser.getId() != null) { + List xxPortalUserRole = rangerDaoManager.getXXPortalUserRole().findByUserId(xxPortalUser.getId()); + if (xxPortalUserRole != null && xxPortalUserRole.get(0).getUserRole().equalsIgnoreCase("ROLE_KEY_ADMIN")) { + vXTrxLogs.add(xTrxLog); + } + } + } + } + keyadminCount = (long) vXTrxLogs.size(); + if(vXTrxLogs != null && !vXTrxLogs.isEmpty()){ + for (int k = startIndex; k <= pageSize; k++) { + if (k < vXTrxLogs.size()) { + keyAdminTrxLogList.add(vXTrxLogs.get(k)); + } + } + } + } + + VXTrxLogList vxTrxLogList = new VXTrxLogList(); + vxTrxLogList.setStartIndex(startIndex); + vxTrxLogList.setPageSize(pageSize); //vxTrxLogList.setTotalCount(count); - vxTrxLogList.setVXTrxLogs(trxLogList); + if (session != null && session.isKeyAdmin()) { + vxTrxLogList.setVXTrxLogs(keyAdminTrxLogList); + } else { + vxTrxLogList.setVXTrxLogs(trxLogList); + } return vxTrxLogList; } @@ -306,7 +370,11 @@ public Long searchXTrxLogsCount(SearchCriteria searchCriteria) { if(count == null) { count = 0L; } - } + } + UserSessionBase session = ContextUtil.getCurrentUserSession(); + if (session != null && session.isKeyAdmin()) { + count = keyadminCount; + } return count; } @@ -326,6 +394,7 @@ private VXTrxLog mapCustomViewToViewObj(VXXTrxLog vXXTrxLog){ } //We will have to get this from XXUser //vXTrxLog.setOwner(vXXTrxLog.getAddedByUserName()); + vXTrxLog.setParentObjectId(vXXTrxLog.getParentObjectId()); vXTrxLog.setParentObjectClassType(vXXTrxLog.getParentObjectClassType()); vXTrxLog.setParentObjectName(vXXTrxLog.getParentObjectName()); vXTrxLog.setObjectClassType(vXXTrxLog.getObjectClassType()); diff --git a/security-admin/src/main/java/org/apache/ranger/service/XUserService.java b/security-admin/src/main/java/org/apache/ranger/service/XUserService.java index 0d07982bbf..b2b06ff8c5 100644 --- a/security-admin/src/main/java/org/apache/ranger/service/XUserService.java +++ b/security-admin/src/main/java/org/apache/ranger/service/XUserService.java @@ -49,7 +49,7 @@ import org.springframework.context.annotation.Scope; import org.springframework.stereotype.Service; import org.springframework.util.CollectionUtils; - +import org.apache.ranger.common.RangerCommonEnums; @Service @Scope("singleton") public class XUserService extends XUserServiceBase { @@ -168,7 +168,10 @@ public VXUser createXUserWithOutLogin(VXUser vxUser) { xxUser = new XXUser(); userExists = false; } - + XXPortalUser xxPortalUser = daoManager.getXXPortalUser().findByLoginId(vxUser.getName()); + if (xxPortalUser != null && xxPortalUser.getUserSource() == RangerCommonEnums.USER_EXTERNAL) { + vxUser.setIsVisible(xxUser.getIsVisible()); + } xxUser = mapViewToEntityBean(vxUser, xxUser, 0); XXPortalUser xXPortalUser = daoManager.getXXPortalUser().getById(createdByUserId); if (xXPortalUser != null) { diff --git a/security-admin/src/main/java/org/apache/ranger/view/VXUser.java b/security-admin/src/main/java/org/apache/ranger/view/VXUser.java index ecfd1ac6fe..6e1d2996db 100644 --- a/security-admin/src/main/java/org/apache/ranger/view/VXUser.java +++ b/security-admin/src/main/java/org/apache/ranger/view/VXUser.java @@ -300,6 +300,7 @@ public String toString( ) { str += "isVisible={" + isVisible + "} "; str += "groupIdList={" + groupIdList + "} "; str += "groupNameList={" + groupNameList + "} "; + str += "roleList={" + userRoleList + "} "; str += "}"; return str; } diff --git a/security-admin/src/main/resources/META-INF/jpa_named_queries.xml b/security-admin/src/main/resources/META-INF/jpa_named_queries.xml index 68548a5da7..cacc43feb4 100644 --- a/security-admin/src/main/resources/META-INF/jpa_named_queries.xml +++ b/security-admin/src/main/resources/META-INF/jpa_named_queries.xml @@ -189,14 +189,14 @@ - - SELECT obj FROM XXTrxLog obj WHERE obj.transactionId = :transactionId + + SELECT obj FROM XXGroupUser obj WHERE obj.name=:groupName AND obj.userId=:userId - - select obj.name from XXUser obj, XXPolicyItemUserPerm polItemUser - where polItemUser.policyItemId = :polItemId and polItemUser.userId = obj.id + + SELECT obj FROM XXTrxLog obj WHERE obj.transactionId = :transactionId + @@ -204,12 +204,6 @@ obj.name = portalUser.loginId - - select obj.name from XXGroup obj, XXPolicyItemGroupPerm polItemGrp - where polItemGrp.policyItemId = :polItemId and polItemGrp.groupId = obj.id - - - select obj from XXPolicyItem obj @@ -338,18 +332,6 @@ select obj from XXPolicyConditionDef obj where obj.defId = :serviceDefId and obj.name = :name order by obj.order - - select obj from XXPolicyConditionDef obj, XXPolicyItemCondition xPolItemCond - where xPolItemCond.policyItemId = :polItemId and obj.id = xPolItemCond.type order by obj.order - - - - select obj from XXPolicyConditionDef obj, XXPolicyItemCondition xPolItemCond - where xPolItemCond.policyItemId = :polItemId and obj.name = :name - and obj.id = xPolItemCond.type order by obj.order - - - select obj from XXContextEnricherDef obj where obj.defId = :serviceDefId order by obj.order @@ -385,6 +367,11 @@ obj.serviceId = :serviceId and obj.configKey = :configKey + + select obj from XXServiceConfigMap obj, XXService xSvc where + xSvc.name = :name and xSvc.id=obj.serviceId and obj.configKey = :configKey + + select obj from XXService obj where obj.name = :name @@ -456,10 +443,6 @@ - - select obj from XXPolicyResource obj where - obj.policyId = :polId and obj.resDefId = :resDefId - select obj from XXPolicyResource obj @@ -478,11 +461,6 @@ select obj from XXPolicyResource obj where obj.resDefId = :resDefId - - - - select obj from XXPolicyResourceMap obj where obj.resourceId = :polResId order by obj.order - select obj from XXPolicyResourceMap obj, XXPolicyResource res @@ -501,13 +479,6 @@ - - select obj from XXPolicyItemAccess obj where obj.policyItemId = :polItemId order by obj.order - - - - select obj from XXPolicyItemAccess obj where obj.type = :type - select obj from XXPolicyItemAccess obj, XXPolicyItem item @@ -524,21 +495,214 @@ order by item.policyId, obj.policyItemId, obj.order - - - - select obj from XXPolicyItemCondition obj where obj.policyItemId = :polItemId order by obj.order + + + + select obj from XXPolicyRefAccessType obj where obj.policyId = :policyId + + + + select obj from XXPolicyRefAccessType obj where obj.accessDefId = :accessDefId + + + + + select obj from XXPolicyRefCondition obj where obj.policyId = :policyId + + + + select obj from XXPolicyRefCondition obj where obj.conditionName = :conditionName + + + + + select obj from XXPolicyRefGroup obj where obj.policyId = :policyId + + + + select obj from XXPolicyRefGroup obj where obj.groupName = :groupName + + + + + select obj from XXPolicyRefGroup obj where obj.groupId = :groupId and obj.policyId = :policyId + + + + select obj from XXPolicyRefCondition obj where obj.conditionDefId = :conditionDefId and obj.policyId = :policyId + + + select obj from XXPolicyRefCondition obj where obj.conditionDefId = :conditionDefId + + + + select distinct obj from XXUser obj, XXPolicyRefUser ref where ref.policyId = :policyId and ref.userId = obj.id + and ref.userName != obj.name + + + + select distinct obj from XXAccessTypeDef obj, XXPolicyRefAccessType ref where ref.policyId = :policyId and ref.accessDefId = obj.id + and ref.accessTypeName != obj.name + + + + select distinct obj from XXDataMaskTypeDef obj, XXPolicyRefDataMaskType ref where ref.policyId = :policyId and ref.dataMaskDefId = obj.id + and ref.dataMaskTypeName != obj.name + + + + select distinct obj from XXResourceDef obj, XXPolicyRefResource ref where ref.policyId = :policyId and ref.resourceDefId = obj.id + and ref.resourceName != obj.name + + + + select groupRef.policyId, groupRef.groupName, grp.name + from XXPolicyRefGroup groupRef, XXGroup grp + where groupRef.policyId = :policy + and groupRef.groupId = grp.id + and groupRef.groupName != grp.name + + + + + select groupRef.policyId, groupRef.groupName, grp.name + from XXPolicy policy, XXPolicyRefGroup groupRef, XXGroup grp + where policy.service = :service + and groupRef.policyId = policy.id + and groupRef.groupId = grp.id + and groupRef.groupName != grp.name + + + + + select userRef.policyId, userRef.userName, user.name + from XXPolicyRefUser userRef, XXUser user + where userRef.policyId = :policy + and userRef.userId = user.id + and userRef.userName != user.name + + + + + select userRef.policyId, userRef.userName, user.name + from XXPolicy policy, XXPolicyRefUser userRef, XXUser user + where policy.service = :service + and userRef.policyId = policy.id + and userRef.userId = user.id + and userRef.userName != user.name + + + + + select accessRef.policyId, accessRef.accessTypeName, accessDef.name + from XXPolicyRefAccessType accessRef, XXAccessTypeDef accessDef + where accessRef.policyId = :policy + and accessRef.accessDefId = accessDef.id + and accessRef.accessTypeName != accessDef.name + + + + + select accessRef.policyId, accessRef.accessTypeName, accessDef.name + from XXPolicy policy, XXPolicyRefAccessType accessRef, XXAccessTypeDef accessDef + where policy.service = :service + and accessRef.policyId = policy.id + and accessRef.accessDefId = accessDef.id + and accessRef.accessTypeName != accessDef.name + + + + + select resourceRef.policyId, resourceRef.resourceName, resourceDef.name + from XXPolicyRefResource resourceRef, XXResourceDef resourceDef + where resourceRef.policyId = :policy + and resourceRef.resourceDefId = resourceDef.id + and resourceRef.resourceName != resourceDef.name + + + + + select resourceRef.policyId, resourceRef.resourceName, resourceDef.name + from XXPolicy policy, XXPolicyRefResource resourceRef, XXResourceDef resourceDef + where policy.service = :service + and resourceRef.policyId = policy.id + and resourceRef.resourceDefId = resourceDef.id + and resourceRef.resourceName != resourceDef.name + - - select obj from XXPolicyItemCondition obj where - obj.policyItemId = :polItemId and obj.type = :polCondDefId order by obj.order + + select dataMaskRef.policyId, dataMaskRef.dataMaskTypeName, dMaskDef.name + from XXPolicyRefDataMaskType dataMaskRef, XXDataMaskTypeDef dMaskDef + where dataMaskRef.policyId = :policy + and dataMaskRef.dataMaskDefId = dMaskDef.id + and dataMaskRef.dataMaskTypeName != dMaskDef.name + + + + + select dataMaskRef.policyId, dataMaskRef.dataMaskTypeName, dMaskDef.name + from XXPolicy policy, XXPolicyRefDataMaskType dataMaskRef, XXDataMaskTypeDef dMaskDef + where policy.service = :service + and dataMaskRef.policyId = policy.id + and dataMaskRef.dataMaskDefId = dMaskDef.id + and dataMaskRef.dataMaskTypeName != dMaskDef.name + + + + + select conditionRef.policyId, conditionRef.conditionName, condDef.name + from XXPolicyRefCondition conditionRef, XXPolicyConditionDef condDef + where conditionRef.policyId = :policy + and conditionRef.conditionDefId = condDef.id + and conditionRef.conditionName != condDef.name + + + + + select conditionRef.policyId, conditionRef.conditionName, condDef.name + from XXPolicy policy, XXPolicyRefCondition conditionRef, XXPolicyConditionDef condDef + where policy.service = :service + and conditionRef.policyId = policy.id + and conditionRef.conditionDefId = condDef.id + and conditionRef.conditionName != condDef.name + + + + + + + select obj from XXPolicyRefDataMaskType obj where obj.policyId = :policyId + + + + select obj from XXPolicyRefDataMaskType obj where obj.dataMaskTypeName = :dataMaskTypeName + + + + + select obj from XXPolicyRefResource obj where obj.policyId = :policyId + + + + select obj from XXPolicyRefResource obj where obj.resourceDefId = :resourceDefId + + + + + select obj from XXPolicyRefUser obj where obj.policyId = :policyId - - select obj from XXPolicyItemCondition obj where obj.type = :polCondDefId + + select obj from XXPolicyRefUser obj where obj.userName = :userName + + select obj from XXPolicyRefUser obj where obj.userId = :userId + + + + select obj from XXPolicyItemCondition obj, XXPolicyItem item where obj.policyItemId = item.id @@ -556,9 +720,6 @@ - - select obj from XXPolicyItemGroupPerm obj where obj.policyItemId = :polItemId order by obj.order - select obj from XXPolicyItemGroupPerm obj, XXPolicyItem item @@ -577,9 +738,6 @@ - - select obj from XXPolicyItemUserPerm obj where obj.policyItemId = :polItemId order by obj.order - select obj from XXPolicyItemUserPerm obj, XXPolicyItem item @@ -598,9 +756,6 @@ - - select obj from XXPolicyItemDataMaskInfo obj where obj.policyItemId = :polItemId - select obj from XXPolicyItemDataMaskInfo obj, XXPolicyItem item @@ -618,14 +773,7 @@ - - select obj from XXPolicyItemDataMaskInfo obj where obj.type = :type - - - - select obj from XXPolicyItemRowFilterInfo obj where obj.policyItemId = :polItemId - select obj from XXPolicyItemRowFilterInfo obj, XXPolicyItem item @@ -800,20 +948,6 @@ - - - select obj from XXServiceResourceElement obj where obj.resourceId in - (select serviceRes.id from XXServiceResource serviceRes, XXService service where service.id = :serviceId and service.tagService is not null and serviceRes.serviceId = service.id and serviceRes.id in - (select tagResMap.resourceId from XXTagResourceMap tagResMap, XXTag tag, XXTagDef tagDef - where tagResMap.tagId = tag.id and tag.type = tagDef.id and tagDef.name in - (select policyResMap.value from XXPolicyResourceMap policyResMap, XXPolicyResource policyRes, XXPolicy policy - where policy.service = service.tagService and policy.isEnabled = TRUE and policyRes.policyId = policy.id and policyResMap.resourceId = policyRes.id) - ) - ) - order by obj.resourceId, obj.id - - - select obj from XXServiceResourceElement obj where obj.resourceId in (select serviceresource.id from XXServiceResource serviceresource where serviceresource.serviceId = :serviceId) @@ -838,20 +972,6 @@ - - - select obj from XXServiceResourceElementValue obj, XXServiceResourceElement serviceResElem where obj.resElementId = serviceResElem.id and - serviceResElem.resourceId in - (select serviceRes.id from XXServiceResource serviceRes, XXService service where service.id = :serviceId and service.tagService is not null and serviceRes.serviceId = service.id and serviceRes.id in - (select tagResMap.resourceId from XXTagResourceMap tagResMap, XXTag tag, XXTagDef tagDef - where tagResMap.tagId = tag.id and tag.type = tagDef.id and tagDef.name in - (select policyResMap.value from XXPolicyResourceMap policyResMap, XXPolicyResource policyRes, XXPolicy policy - where policy.service = service.tagService and policy.isEnabled = TRUE and policyRes.policyId = policy.id and policyResMap.resourceId = policyRes.id) - ) - ) - order by serviceResElem.resourceId, serviceResElem.id - - select obj from XXServiceResourceElementValue obj, XXServiceResourceElement resElem where obj.resElementId = resElem.id and resElem.resourceId in (select res.id from XXServiceResource res where res.serviceId = :serviceId) @@ -899,19 +1019,6 @@ order by obj.resourceId - - - select obj from XXTagResourceMap obj, XXService service, XXServiceResource serviceRes where service.id = :serviceId and service.tagService is not null - and obj.resourceId = serviceRes.id and serviceRes.serviceId = :serviceId - and obj.tagId in - (select tag.id from XXTag tag, XXTagDef tagDef where tag.type = tagDef.id and tagDef.name in - (select policyResMap.value from XXPolicyResourceMap policyResMap, XXPolicyResource policyRes, XXPolicy policy - where policy.service = service.tagService and policy.isEnabled = TRUE and policyRes.policyId = policy.id and policyResMap.resourceId = policyRes.id) - ) - order by obj.resourceId - - - select obj.tagId from XXTagResourceMap obj where obj.resourceId = :resourceId order by obj.id @@ -925,22 +1032,14 @@ select obj from XXTagDef obj where obj.name = :name - - - select obj from XXTagDef obj where obj.id in - (select tag.type from XXTag tag, XXTagResourceMap tagRes, XXServiceResource resource where tag.id = tagRes.tagId and tagRes.resourceId = resource.id and resource.serviceId = :serviceId) - order by obj.id - - - - select obj from XXTagDef obj, XXService service where service.id = :serviceId and service.tagService is not null and - obj.name in - (select policyResMap.value from XXPolicyResourceMap policyResMap, XXPolicyResource policyRes, XXPolicy policy - where policy.service = service.tagService and policyRes.policyId = policy.id and policy.isEnabled = TRUE and policyResMap.resourceId = policyRes.id) + + select obj.id, obj.guid, obj.version, obj.isEnabled, obj.name, obj.source, obj.tagAttrDefs from XXTagDef obj where obj.id in + (select tag.type from XXTag tag, XXTagResourceMap tagRes, XXServiceResource resource where tag.id = tagRes.tagId and tagRes.resourceId = resource.id and resource.serviceId = :serviceId) order by obj.id + select obj from XXTagDef obj where obj.id in (select tag.type from XXTag tag, XXTagResourceMap tagRes where tag.id = tagRes.tagId and tagRes.resourceId = :resourceId) @@ -962,16 +1061,6 @@ order by obj.tagDefId - - select obj from XXTagAttributeDef obj where obj.tagDefId in - (select tagDef.id from XXTagDef tagDef, XXService service where service.id = :serviceId and service.tagService is not null and - tagDef.name in - (select policyResMap.value from XXPolicyResourceMap policyResMap, XXPolicyResource policyRes, XXPolicy policy - where policy.service = service.tagService and policy.isEnabled = TRUE and policyRes.policyId = policy.id and policyResMap.resourceId = policyRes.id) - ) - order by obj.tagDefId - - select obj from XXTagAttributeDef obj where obj.tagDefId in (select tag.type from XXTag tag, XXTagResourceMap tagRes where tag.id = tagRes.tagId and tagRes.resourceId = :resourceId) @@ -1007,16 +1096,6 @@ - - - select obj from XXTag obj, XXTagDef tagDef, XXService service where service.id = :serviceId and service.tagService is not null and - obj.type = tagDef.id and tagDef.name in - (select policyResMap.value from XXPolicyResourceMap policyResMap, XXPolicyResource policyRes, XXPolicy policy - where policy.service = service.tagService and policy.isEnabled = TRUE and policyRes.policyId = policy.id and policyResMap.resourceId = policyRes.id) - order by obj.id - - - select obj from XXTag obj where obj.owner = :owner and obj.id in (select tagRes.tagId from XXTagResourceMap tagRes, XXServiceResource resource where tagRes.resourceId = resource.id and resource.serviceId = :serviceId) @@ -1051,18 +1130,6 @@ - - - select obj from XXTagAttribute obj where obj.tagId in - (select tag.id from XXTag tag, XXTagDef tagDef, XXService service where service.id = :serviceId and service.tagService is not null and - tag.type = tagDef.id and tagDef.name in - (select policyResMap.value from XXPolicyResourceMap policyResMap, XXPolicyResource policyRes, XXPolicy policy - where policy.service = service.tagService and policy.isEnabled = TRUE and policyRes.policyId = policy.id and policyResMap.resourceId = policyRes.id) - ) - order by obj.tagId - - - select obj from XXServiceResource obj where obj.guid = :guid @@ -1074,21 +1141,15 @@ - select obj from XXServiceResource obj where obj.serviceId = :serviceId and obj.id in - (select tagResMap.resourceId from XXTagResourceMap tagResMap) - order by obj.id + select obj.id, obj.guid, obj.version, obj.isEnabled, obj.resourceSignature, obj.serviceId, obj.serviceResourceElements, obj.tags from XXServiceResource obj where obj.serviceId = :serviceId and obj.id in + (select tagResMap.resourceId from XXTagResourceMap tagResMap) + order by obj.id - - - select obj from XXServiceResource obj, XXService service where service.id = :serviceId and service.tagService is not null and obj.serviceId = service.id and obj.id in - (select tagResMap.resourceId from XXTagResourceMap tagResMap, XXTag tag, XXTagDef tagDef - where tagResMap.tagId = tag.id and tag.type = tagDef.id and tagDef.name in - (select policyResMap.value from XXPolicyResourceMap policyResMap, XXPolicyResource policyRes, XXPolicy policy - where policy.service = service.tagService and policy.isEnabled = TRUE and policyRes.policyId = policy.id and policyResMap.resourceId = policyRes.id) - ) - order by obj.id + + select count(obj.id) from XXServiceResource obj where obj.serviceId = :serviceId and obj.id in + (select tagResMap.resourceId from XXTagResourceMap tagResMap) @@ -1121,12 +1182,12 @@ - select obj from XXPolicy obj, XXPolicyItem polItem,XXPolicyItemUserPerm polItemUserPerm where - obj.id = polItem.policyId and polItem.id = polItemUserPerm.policyItemId and polItemUserPerm.userId = :userId + select obj from XXPolicy obj, XXPolicyRefUser ref where + obj.id = ref.policyId and ref.userId = :userId - select obj from XXPolicy obj, XXPolicyItem polItem,XXPolicyItemGroupPerm polItemGroupPerm where - obj.id = polItem.policyId and polItem.id = polItemGroupPerm.policyItemId and polItemGroupPerm.groupId = :groupId + select obj from XXPolicy obj, XXPolicyRefGroup ref where + obj.id = ref.policyId and ref.groupId = :groupId select max(obj.id) from XXAccessAudit obj diff --git a/security-admin/src/main/resources/conf.dist/ranger-admin-default-site.xml b/security-admin/src/main/resources/conf.dist/ranger-admin-default-site.xml index 9dfc03df11..1e52a442cc 100644 --- a/security-admin/src/main/resources/conf.dist/ranger-admin-default-site.xml +++ b/security-admin/src/main/resources/conf.dist/ranger-admin-default-site.xml @@ -507,6 +507,10 @@ ranger.db.ssl.verifyServerCertificate false + + ranger.db.ssl.auth.type + 2-way + ranger.keystore.file diff --git a/security-admin/src/main/webapp/META-INF/applicationContext.xml b/security-admin/src/main/webapp/META-INF/applicationContext.xml index 79d20d3aee..aff0d6b6f9 100644 --- a/security-admin/src/main/webapp/META-INF/applicationContext.xml +++ b/security-admin/src/main/webapp/META-INF/applicationContext.xml @@ -170,8 +170,11 @@ http://www.springframework.org/schema/util/spring-util.xsd"> ${ranger.jpa.jdbc.idleconnectiontestperiod} - - + + 1 + + + ${ranger.jpa.jdbc.driver} diff --git a/security-admin/src/main/webapp/login.jsp b/security-admin/src/main/webapp/login.jsp index 39fb3274fa..b8f4cb982b 100644 --- a/security-admin/src/main/webapp/login.jsp +++ b/security-admin/src/main/webapp/login.jsp @@ -44,6 +44,11 @@ }); }; $(window).resize(updateBoxPosition); + var queryParams = JSON.parse('{"' + decodeURI((location.href.split('?')[1] || 'g=0').replace(/=/g, "\":\"")) + '"}'); + if(queryParams.sessionTimeout){ + window.alert('Session Timeout'); + location.replace("login.jsp"); + } setTimeout(updateBoxPosition, 50); }); diff --git a/security-admin/src/main/webapp/scripts/model_bases/VXGroupBase.js b/security-admin/src/main/webapp/scripts/model_bases/VXGroupBase.js index 96e0946704..fce4cf6f3a 100644 --- a/security-admin/src/main/webapp/scripts/model_bases/VXGroupBase.js +++ b/security-admin/src/main/webapp/scripts/model_bases/VXGroupBase.js @@ -88,15 +88,12 @@ define(function(require){ initialize: function() { this.modelName = 'VXGroupBase'; }, - deleteGroups : function(groupNameValues, options){ - var url = this.urlRoot + '/delete?forceDelete=true'; - - options = _.extend({ - data : JSON.stringify(groupNameValues), + deleteGroups : function(groupId, options){ + var url = this.urlRoot + '/id/' + groupId + '?forceDelete=true'; + options = _.extend({ contentType : 'application/json', dataType : 'json', - - }, options); + }, options); return this.constructor.nonCrudOperation.call(this, url, 'DELETE', options); }, diff --git a/security-admin/src/main/webapp/scripts/model_bases/VXUserBase.js b/security-admin/src/main/webapp/scripts/model_bases/VXUserBase.js index db6d511085..5f598746d5 100644 --- a/security-admin/src/main/webapp/scripts/model_bases/VXUserBase.js +++ b/security-admin/src/main/webapp/scripts/model_bases/VXUserBase.js @@ -42,10 +42,9 @@ define(function(require){ this.modelName = 'VXUserBase'; }, - deleteUsers : function(userNameValues,options){ - var url = this.urlRoot + '/delete?forceDelete=true'; + deleteUsers : function(userId,options){ + var url = this.urlRoot + '/id/' + userId +'?forceDelete=true'; options = _.extend({ - data : JSON.stringify(userNameValues), contentType : 'application/json', dataType : 'json', }, options); diff --git a/security-admin/src/main/webapp/scripts/models/RangerServiceDef.js b/security-admin/src/main/webapp/scripts/models/RangerServiceDef.js index 704f37818d..03bd15f357 100644 --- a/security-admin/src/main/webapp/scripts/models/RangerServiceDef.js +++ b/security-admin/src/main/webapp/scripts/models/RangerServiceDef.js @@ -58,7 +58,7 @@ define(function(require){ name : { type : 'Text', title : 'Service Name *', - validators : ['required',{type:'regexp',regexp:/^[a-zA-Z0-9\s_-]{1,512}$/,message :"Name should be less than 512 characters and special characters are not allowed."}], + validators : ['required',{type:'regexp',regexp:/^[a-zA-Z0-9_-][a-zA-Z0-9\s_-]{0,254}/,message :"Name should not start with space, it should be less than 256 characters and special characters are not allowed(except _ - and space)."}], }, description : { type : 'TextArea', @@ -107,6 +107,14 @@ define(function(require){ return {results : results}; } return {results : results}; + }, + transport : function (options) { + $.ajax(options).error(function(respones) { + XAUtils.defaultErrorHandler('error',respones); + this.success({ + resultSize : 0 + }); + }); } }, formatResult : function(result){ diff --git a/security-admin/src/main/webapp/scripts/models/XABaseModel.js b/security-admin/src/main/webapp/scripts/models/XABaseModel.js index 608321083f..cb0691c308 100644 --- a/security-admin/src/main/webapp/scripts/models/XABaseModel.js +++ b/security-admin/src/main/webapp/scripts/models/XABaseModel.js @@ -41,9 +41,9 @@ define(function(require){ }, bindErrorEvents :function(){ //Moved require inside fuctn expression due to ie issue - this.bind("error", function(e){ + this.bind("error", function(e, error){ var XAUtils = require('utils/XAUtils'); - XAUtils.defaultErrorHandler(undefined, e); + XAUtils.defaultErrorHandler(undefined, error, e); }); }, /** diff --git a/security-admin/src/main/webapp/scripts/modules/XAOverrides.js b/security-admin/src/main/webapp/scripts/modules/XAOverrides.js index 7d7a9d133d..12f351d7de 100644 --- a/security-admin/src/main/webapp/scripts/modules/XAOverrides.js +++ b/security-admin/src/main/webapp/scripts/modules/XAOverrides.js @@ -86,7 +86,7 @@ render: function () { this.$el.empty(); if(this.model.get(this.column.get("name")) != undefined){ - rawValue = (this.model.get(this.column.get("name"))); + var rawValue = (this.model.get(this.column.get("name"))); this.switchStatus = this.formatter.fromRaw(rawValue, this.model); } @@ -490,13 +490,13 @@ this.resourceOpts = {}; _.extend(this, _.pick(this.schema,'excludeSupport','recursiveSupport','resourceOpts','resourcesAtSameLevel','sameLevelOpts', 'initilializePathPlugin', 'validators','name','formView')); - //(edit mode)set values for sameLevel if first option is not selected - if(!_.isNull(this.value) && !_.isUndefined(this.value) + //(edit mode)set values for sameLevel if first option is not selected + if(!_.isNull(this.value) && !_.isUndefined(this.value) && !_.isUndefined(this.value.resourceType)){ - var def = _.findWhere(this.form.rangerServiceDefModel.get('resources'), {'name': this.value.resourceType }); - this.recursiveSupport = def.recursiveSupported; - this.excludeSupport = def.excludesSupported; - } + var def = _.findWhere(this.form.rangerServiceDefModel.get('resources'), {'name': this.value.resourceType }); + this.recursiveSupport = def.recursiveSupported; + this.excludeSupport = def.excludesSupported; + } this.template = this.getTemplate(); }, initializeElements : function() { @@ -574,7 +574,21 @@ this.value.isRecursive = _.isUndefined(this.value.isRecursive) ? true : this.value.isRecursive; isRecursive = this.value.isRecursive; } - } + this.$recursiveSupport.show(); + this.$recursiveSupport.removeClass('recursive-toggle-1 recursive-toggle-2'); + this.$recursiveSupport.addClass(this.excludeSupport ? 'recursive-toggle-2' : 'recursive-toggle-1') + this.$recursiveSupport.toggles({ + on: isRecursive, + text : {on : 'recursive', off : 'non-recursive' }, + width: 120, + }).on('toggle', function (e, active) { + that.value.isRecursive = active; + XAUtil.checkDirtyFieldForToggle($(e.currentTarget)) + }); + } else { + this.$recursiveSupport.hide(); + } + }, renderSameLevelResource : function() { var that = this, dirtyFieldValue = null; @@ -600,15 +614,15 @@ that.$el.parents('.control-group').attr('data-name', 'field-'+this.value); that.formView.trigger('policyForm:parentChildHideShow',true); if(!_.isUndefined(this.value) - && ( XAUtil.capitaliseFirstLetter(this.value) === XAEnums.ResourceType.RESOURCE_UDF.label) ){ + && ( XAUtil.capitaliseFirstLetter(this.value) === XAEnums.ResourceType.RESOURCE_UDF.label) ){ XAUtil.alertPopup({ msg :localization.tt('msg.udfPolicyViolation') }); } - //set flags for newly selected resource and re-render - var def = _.findWhere(that.form.rangerServiceDefModel.get('resources'), {'name': this.value}); - that.recursiveSupport = def.recursiveSupported; - if(that.recursiveSupport) that.value.isRecursive = true; - that.excludeSupport = def.excludesSupported; - that.renderToggles(); + //set flags for newly selected resource and re-render + var def = _.findWhere(that.form.rangerServiceDefModel.get('resources'), {'name': this.value}); + that.recursiveSupport = def.recursiveSupported; + if(that.recursiveSupport) that.value.isRecursive = true; + that.excludeSupport = def.excludesSupported; + that.renderToggles(); }); } @@ -667,16 +681,28 @@ } }, getTemplate : function() { - var optionsHtml="", selectTemplate = '',excludeSupportToggleDiv='', recursiveSupportToggleDiv=''; - this.preserveResourceValues = {}; - if(this.resourcesAtSameLevel){ - _.each(this.sameLevelOpts, function(option){ return optionsHtml += ""; },this); + var that = this , resourcesType , optionsHtml="" , selectTemplate = '', excludeSupportToggleDiv='', recursiveSupportToggleDiv='', klass = ''; + this.preserveResourceValues = {} ; + if(this.resourcesAtSameLevel){ + _.each(this.sameLevelOpts, function(option){ return optionsHtml += ""; },this); selectTemplate = ''; - } - excludeSupportToggleDiv = '
'; - return _.template(selectTemplate+''+ + } + excludeSupportToggleDiv = '
'; + _.each(this.form.rangerServiceDefModel.get('resources') , function(m){ + if(that.name === m.name){ + resourcesType = m.type ; + } + }) + if(resourcesType == "path"){ + klass = (!this.excludeSupport) ? "recursive-toggle-hdfs-1" : "recursive-toggle-hdfs-2"; + }else{ + klass = (!this.excludeSupport) ? "recursive-toggle-1" : "recursive-toggle-2"; + } + recursiveSupportToggleDiv = '
'; + + return _.template(selectTemplate+''+ excludeSupportToggleDiv+''+recursiveSupportToggleDiv); }, }); diff --git a/security-admin/src/main/webapp/scripts/modules/globalize/message/en.js b/security-admin/src/main/webapp/scripts/modules/globalize/message/en.js index 4bac746e5b..280844976d 100644 --- a/security-admin/src/main/webapp/scripts/modules/globalize/message/en.js +++ b/security-admin/src/main/webapp/scripts/modules/globalize/message/en.js @@ -36,7 +36,7 @@ if ( typeof require !== "undefined" && */ define(function(require) { 'use strict'; - Globalize = require( "globalize" ); + var Globalize = require( "globalize" ); Globalize.addCultureInfo( "en", { messages: { @@ -208,8 +208,6 @@ define(function(require) { selectTopologyName : 'Select Topology Name', selectServiceName : 'Select Service Name', topologyName : 'Topology Name', - serivceName : 'Service Name', - serivceType : 'Service Type', ipAddress : 'IP Address', isVisible : 'Visible', delegatedAdmin : 'Delegate Admin', @@ -242,7 +240,8 @@ define(function(require) { selectAndAddGroup : 'Select and Add Group', download : 'Download', lastUpdate : 'Last Update', - clusterName : 'Cluster Name' + clusterName : 'Cluster Name', + url : 'Hive url.' }, btn : { add : 'Add', @@ -396,7 +395,8 @@ define(function(require) { plsSelectUserToSetVisibility :' Please select user to set visibility or selected user is already visible/hidden.', plsSelectGroupToSetVisibility:' Please select group to set visibility or selected group is already visible/hidden.', activationTimeDelayMsg :'Policy activation time delayed by more than 1hr from last update time.', - pleaseSelectAccessTypeForTagMasking : 'Please select access type first to enable add masking options.' + pleaseSelectAccessTypeForTagMasking : 'Please select access type first to enable add masking options.', + addUserOrGroupForDelegateAdmin : 'Please select user/group for the selected permission(s)', }, plcHldr : { diff --git a/security-admin/src/main/webapp/scripts/utils/XAEnums.js b/security-admin/src/main/webapp/scripts/utils/XAEnums.js index 164e93994c..0e0958df36 100644 --- a/security-admin/src/main/webapp/scripts/utils/XAEnums.js +++ b/security-admin/src/main/webapp/scripts/utils/XAEnums.js @@ -23,7 +23,7 @@ define(function(require) { var $ = require('jquery'); var XAEnums = {}; - mergeParams = function(defaults, params) { + var mergeParams = function(defaults, params) { if (!params) { return defaults; } diff --git a/security-admin/src/main/webapp/scripts/utils/XAGlobals.js b/security-admin/src/main/webapp/scripts/utils/XAGlobals.js index d16e5deadf..d55ab4a1d2 100644 --- a/security-admin/src/main/webapp/scripts/utils/XAGlobals.js +++ b/security-admin/src/main/webapp/scripts/utils/XAGlobals.js @@ -73,5 +73,16 @@ define(function(require){ 'Permissions' : ['modulePermissionsAction','modulePermissionEditAction'] }, }; + XAGlobals.ActionType = { + Create : { value: 'create', label: 'Create' }, + Update : { value: 'update', label: 'Update' }, + Delete : { value: 'delete', label: 'Delete' }, + Password_Change : { value: 'password change', label: 'Password Change' }, + Export_Json : { value: 'EXPORT JSON', label: 'Export Json' }, + Export_Csv : { value: 'EXPORT CSV', label: 'Export Csv' }, + Export_Excel : { value: 'EXPORT EXCEL', label: 'Export Excel' }, + Import_End : { value: 'IMPORT END', label: 'Import End' }, + Import_Start : { value: 'IMPORT START', label: 'Import Start'} + }; return XAGlobals; }); diff --git a/security-admin/src/main/webapp/scripts/utils/XATemplateHelpers.js b/security-admin/src/main/webapp/scripts/utils/XATemplateHelpers.js index 785e2c2a8c..b64f9112a6 100644 --- a/security-admin/src/main/webapp/scripts/utils/XATemplateHelpers.js +++ b/security-admin/src/main/webapp/scripts/utils/XATemplateHelpers.js @@ -165,7 +165,7 @@ define(function ( require ){ return moment(Date(context)).format(f); }else{ return context; // moment plugin not available. return data as is. - }; + } }); return HHelpers; }); diff --git a/security-admin/src/main/webapp/scripts/utils/XAUtils.js b/security-admin/src/main/webapp/scripts/utils/XAUtils.js index e294353001..7f00911530 100644 --- a/security-admin/src/main/webapp/scripts/utils/XAUtils.js +++ b/security-admin/src/main/webapp/scripts/utils/XAUtils.js @@ -609,7 +609,9 @@ define(function(require) { XAUtils.defaultErrorHandler = function(model, error) { var App = require('App'); var vError = require('views/common/ErrorView'); - if(!_.isUndefined(model) && !_.isUndefined(model.modelName) && model.modelName == XAEnums.ClassTypes.CLASS_TYPE_XA_ACCESS_AUDIT.modelName){ + if(!_.isUndefined(model) && !_.isUndefined(model.modelName) + && model.modelName == XAEnums.ClassTypes.CLASS_TYPE_XA_ACCESS_AUDIT.modelName + && error.status !== 419){ return; } if (error.status == 404) { @@ -621,7 +623,7 @@ define(function(require) { status : error.status })); } else if (error.status == 419) { - window.location = 'login.jsp' + window.location = 'login.jsp?sessionTimeout=true'; } }; XAUtils.select2Focus = function(event) { @@ -1207,7 +1209,9 @@ define(function(require) { _.each(XAEnums.UserRoles,function(val, key){ if(SessionMgr.isKeyAdmin() && XAEnums.UserRoles.ROLE_SYS_ADMIN.value != val.value){ userRoleList.push(key) - }else if(!SessionMgr.isKeyAdmin() && XAEnums.UserRoles.ROLE_KEY_ADMIN.value != val.value){ + }else if(SessionMgr.isSystemAdmin() && XAEnums.UserRoles.ROLE_KEY_ADMIN.value != val.value){ + userRoleList.push(key) + }else if(SessionMgr.isUser() && XAEnums.UserRoles.ROLE_USER.value == val.value){ userRoleList.push(key) } }) @@ -1248,7 +1252,7 @@ define(function(require) { } return window.location.origin + window.location.pathname.substring(window.location.pathname - .indexOf('/', 2) + 1, 0); + .lastIndexOf('/') + 1, 0); }; XAUtils.isMaskingPolicy = function(type){ diff --git a/security-admin/src/main/webapp/scripts/views/DownloadServicePolicy.js b/security-admin/src/main/webapp/scripts/views/DownloadServicePolicy.js index fb10562c79..fc8edbbbfa 100644 --- a/security-admin/src/main/webapp/scripts/views/DownloadServicePolicy.js +++ b/security-admin/src/main/webapp/scripts/views/DownloadServicePolicy.js @@ -42,7 +42,6 @@ define(function(require){ }, ui:{ 'downloadReport' : '[data-id="downloadReport"]', - 'selectService' : '[data-id="selectService"]', 'servicesName' : '[data-id="servicesName"]', 'componentTypeSelected' : '[data-id="componentTypeSelected"]' }, @@ -51,8 +50,8 @@ define(function(require){ okClicked: function (modal) { var that = this, el = $(modal.currentTarget), - urls ='/service/plugins/policies/exportJson' - serviceName = this.ui.servicesName.val() + urls ='/service/plugins/policies/exportJson', + serviceName = this.ui.servicesName.val(); if (_.isEmpty(this.ui.componentTypeSelected.val())){ this.$el.find('.serviceValidationFile').show(); } @@ -80,6 +79,10 @@ define(function(require){ } }, + error : function(data,status,response){ + XAUtil.blockUI('unblock'); + XAUtil.defaultErrorHandler(status,data); + }, }); }, onRender: function() { @@ -121,7 +124,7 @@ define(function(require){ serviceSelect :function(e){ var options =this.serviceNames.map(function(m){ return { 'id' : m.name, 'text' : m.name}; }); var serviceTyp = options.map(function(m){return m.text}) - this.ui.servicesName.val(serviceTyp); + this.ui.servicesName.val(serviceTyp); this.ui.servicesName.select2({ multiple: true, closeOnSelect: true, diff --git a/security-admin/src/main/webapp/scripts/views/UploadServicePolicy.js b/security-admin/src/main/webapp/scripts/views/UploadServicePolicy.js index 338fffb964..62a1fcff24 100644 --- a/security-admin/src/main/webapp/scripts/views/UploadServicePolicy.js +++ b/security-admin/src/main/webapp/scripts/views/UploadServicePolicy.js @@ -207,11 +207,15 @@ define(function(require){ error : function(response,model){ XAUtil.blockUI('unblock'); if ( response && response.responseJSON && response.responseJSON.msgDesc){ - XAUtil.notifyError('Error', response.responseJSON.msgDesc); - } else { + if(response.status == '419'){ + XAUtil.defaultErrorHandler(model,response); + }else{ + XAUtil.notifyError('Error', response.responseJSON.msgDesc); + } + } else { XAUtil.notifyError('Error', 'File import failed.'); - } - } + } + } }); }, onAddClick : function(){ diff --git a/security-admin/src/main/webapp/scripts/views/common/AddGroup.js b/security-admin/src/main/webapp/scripts/views/common/AddGroup.js index 81fd9014d0..0952b898a3 100644 --- a/security-admin/src/main/webapp/scripts/views/common/AddGroup.js +++ b/security-admin/src/main/webapp/scripts/views/common/AddGroup.js @@ -93,8 +93,6 @@ define(function(require){ return; } that.checkDirtyFieldForGroup(values); - if(!_.isArray(values)) values=values.toString().split(','); - var valArr = []; if(!_.isUndefined($(that.el).find('.select2-container-multi')) && $(that.el).find('.select2-container-multi').length > 0){ diff --git a/security-admin/src/main/webapp/scripts/views/common/CustomSubgrid.js b/security-admin/src/main/webapp/scripts/views/common/CustomSubgrid.js index 0c7e52a899..ad89a1377e 100644 --- a/security-admin/src/main/webapp/scripts/views/common/CustomSubgrid.js +++ b/security-admin/src/main/webapp/scripts/views/common/CustomSubgrid.js @@ -152,7 +152,9 @@ define(function(require){ var labelName = this.column.attributes.label; $(this.el).html(""); if (this.state == "collasped"){ - $(this.el).parent().addClass("warning"); +// $(this.el).parent().addClass("warning"); +// Add warning class to sub grid table + $(this.el).addClass("warning"); this.state = "expanded"; this.subrow = new SubgridCustomRow({ columns: this.column.collection, @@ -164,7 +166,9 @@ define(function(require){ $(this.el).parent("tr").after(this.subrow.render().$el); } else { if( $(this.el).parent().siblings('.warning').length <= 1 ){ - $(this.el).parent().removeClass("warning") +// $(this.el).parent().removeClass("warning") +// Remove warning class from sub grid table + $(this.el).removeClass("warning") } this.state = "collasped"; this.subrow.remove(); diff --git a/security-admin/src/main/webapp/scripts/views/kms/KMSTableLayout.js b/security-admin/src/main/webapp/scripts/views/kms/KMSTableLayout.js index fbbee46d66..2749cea52e 100755 --- a/security-admin/src/main/webapp/scripts/views/kms/KMSTableLayout.js +++ b/security-admin/src/main/webapp/scripts/views/kms/KMSTableLayout.js @@ -266,6 +266,7 @@ define(function(require){ this.ui.selectServiceName.select2({ maximumSelectionSize : 1, closeOnSelect : true, + allowClear: true, width :'220px', placeholder : 'Please select KMS service', initSelection : function (element, callback) { @@ -284,6 +285,14 @@ define(function(require){ return { results : results }; } return { results : results }; + }, + transport: function (options) { + $.ajax(options).error(function(respones) { + XAUtil.defaultErrorHandler('error',respones); + this.success({ + resultSize : 0 + }); + }); } }, formatResult : function(result){ diff --git a/security-admin/src/main/webapp/scripts/views/permissions/ModulePermissionForm.js b/security-admin/src/main/webapp/scripts/views/permissions/ModulePermissionForm.js index aa4c332326..d9c522c47e 100644 --- a/security-admin/src/main/webapp/scripts/views/permissions/ModulePermissionForm.js +++ b/security-admin/src/main/webapp/scripts/views/permissions/ModulePermissionForm.js @@ -178,8 +178,9 @@ define(function(require) { return { results : results}; }, transport: function (options) { - $.ajax(options).error(function() { + $.ajax(options).error(function(respones) { console.log("ajax failed"); + XAUtil.defaultErrorHandler('error',respones); this.success({ resultSize : 0 }); diff --git a/security-admin/src/main/webapp/scripts/views/policies/PermissionList.js b/security-admin/src/main/webapp/scripts/views/policies/PermissionList.js index 067bf3b34c..742f0a77e6 100644 --- a/security-admin/src/main/webapp/scripts/views/policies/PermissionList.js +++ b/security-admin/src/main/webapp/scripts/views/policies/PermissionList.js @@ -185,7 +185,7 @@ define(function(require) { createDropDown :function($select, typeGroup){ var that = this, tags = [], placeholder = (typeGroup) ? 'Select Group' : 'Select User', - searchUrl = (typeGroup) ? "service/xusers/groups" : "service/xusers/users"; + searchUrl = (typeGroup) ? "service/xusers/lookup/groups" : "service/xusers/lookup/users"; if(this.model.has('editMode') && !_.isEmpty($select.val())){ var temp = this.model.attributes[ (typeGroup) ? 'groupName': 'userName']; _.each(temp , function(name){ @@ -211,11 +211,11 @@ define(function(require) { var results = [] , selectedVals = []; //Get selected values of groups/users dropdown selectedVals = that.getSelectedValues($select, typeGroup); - if(data.resultSize != "0"){ + if(data.totalCount != "0"){ if(typeGroup){ - results = data.vXGroups.map(function(m, i){ return {id : _.escape(m.name), text: _.escape(m.name) }; }); + results = data.vXStrings.map(function(m){ return {id : _.escape(m.value), text: _.escape(m.value) }; }); } else { - results = data.vXUsers.map(function(m, i){ return {id : _.escape(m.name), text: _.escape(m.name) }; }); + results = data.vXStrings.map(function(m){ return {id : _.escape(m.value), text: _.escape(m.value) }; }); } if(!_.isEmpty(selectedVals)){ results = XAUtil.filterResultByText(results, selectedVals); @@ -223,6 +223,14 @@ define(function(require) { return {results : results}; } return {results : results}; + }, + transport: function (options) { + $.ajax(options).error(function(respones) { + XAUtil.defaultErrorHandler('error',respones); + this.success({ + resultSize : 0 + }); + }); } }, formatResult : function(result){ @@ -239,12 +247,8 @@ define(function(require) { renderPerms :function(){ var that = this; this.perms = _.map(this.accessTypes,function(m){return {text:m.label, value:m.name};}); - this.perms.push({'value' : -1, 'text' : 'Select/Deselect All'}); - //set default access type 'select' for add new masking & row filter policies - if(this.perms.length == 2){ - if(!_.isUndefined(this.perms[0].value) && _.isEmpty(this.permsIds)){ - this.permsIds.push(this.perms[0].value); - } + if(this.perms.length > 1){ + this.perms.push({'value' : -1, 'text' : 'Select/Deselect All'}); } //create x-editable for permissions this.ui.addPerms.editable({ @@ -273,11 +277,6 @@ define(function(require) { return "" + obj.text + ""; } }); - var perms = [] - if(that.model.has('accesses')){ - perms = that.model.get('accesses'); - } - var items=[]; _.each(that.accessItems, function(item){ if($.inArray( item.type, values) >= 0){ @@ -310,10 +309,6 @@ define(function(require) { this.ui.addPerms.attr('title','Components Permissions') this.ui.delegatedAdmin.parent('td').hide(); this.perms = _.map(this.accessTypes,function(m){return {text:m.label, value:m.name};}); - //select defatult access type if single component exists - if(this.perms.length == 1 && this.permsIds.length >= 0){ - this.permsIds.push(this.perms[0].value) - } var select2optn = { width :'600px' }; if(XAUtil.isMaskingPolicy(this.rangerPolicyType)){ select2optn = {width :'600px' , maximumSelectionSize : 1 }; @@ -357,11 +352,6 @@ define(function(require) { return "" + id.substr(0,id.indexOf(":")).toUpperCase() + ""; } }); - var perms = [] - if(that.model.has('accesses')){ - perms = that.model.get('accesses'); - } - var items=[]; _.each(that.accessItems, function(item){ if($.inArray( item.type, values) >= 0){ @@ -485,7 +475,7 @@ define(function(require) { }); this.$el.find('input[data-id="maskTypeCustom"]').on('change', function(e){ if(!_.isUndefined(that.model.get('dataMaskInfo'))){ - that.model.get('dataMaskInfo').valueExpr = e.currentTarget.value; + that.model.get('dataMaskInfo').valueExpr = (e.currentTarget.value); } }).trigger('change'); if(!this.accessPermSetForTagMasking){ @@ -540,7 +530,7 @@ define(function(require) { emptytext : 'Add Conditions', value : this.conditions, display: function(value) { - var continue_ = false, i = 0; + var continue_ = false, i = 0, cond = []; if(!value) { $(this).empty(); return; @@ -555,19 +545,18 @@ define(function(require) { return ''; } //Add label for policy condition - var pcond = _.findWhere(that.multiLinecond, { 'name': name}) + var pcond = _.findWhere(that.multiLinecond, { 'name': name}); if(!_.isUndefined(pcond) && !_.isUndefined(pcond['evaluatorOptions']) && ! _.isUndefined(pcond['evaluatorOptions']["ui.isMultiline"]) && ! _.isUndefined(pcond['evaluatorOptions']['engineName'])){ - val = pcond['evaluatorOptions']['engineName'] + ' Condition' + cond.push({ 'type' : name, 'values' : !_.isArray(val) ? [val] : val }); + val = pcond['evaluatorOptions']['engineName'] + ' Condition'; + } else { + cond.push({ 'type' : name, 'values' : !_.isArray(val) ? val.split(',') : val }); } i++; - return ''+name+' : '+ _.escape(val) + ''; - }); - var cond = _.map(value, function(val, name) { - return {'type' : name, 'values' : !_.isArray(val) ? val.split(',') : val}; + return ''+name+' : '+ _.escape(val) + ''; }); - that.model.set('conditions', cond); $(this).html(html); that.ui.addConditionsSpan.find('i').attr('class', 'icon-pencil'); @@ -707,6 +696,7 @@ define(function(require) { $(this).siblings('[data-id="maskTypeCustom"]').css("display",""); }else{ $(this).siblings('[data-id="maskTypeCustom"]').css("display","none"); + $(this).siblings('[data-id="maskTypeCustom"]').val(" ") } $(this).html("" + obj.text + ""); @@ -723,7 +713,7 @@ define(function(require) { }); this.$el.find('input[data-id="maskTypeCustom"]').on('change', function(e){ if(!_.isUndefined(that.model.get('dataMaskInfo'))){ - that.model.get('dataMaskInfo').valueExpr = e.currentTarget.value; + that.model.get('dataMaskInfo').valueExpr = (e.currentTarget.value); } }).trigger('change'); }, diff --git a/security-admin/src/main/webapp/scripts/views/policies/RangerPolicyCreate.js b/security-admin/src/main/webapp/scripts/views/policies/RangerPolicyCreate.js index df13b7c243..1475dd97c1 100644 --- a/security-admin/src/main/webapp/scripts/views/policies/RangerPolicyCreate.js +++ b/security-admin/src/main/webapp/scripts/views/policies/RangerPolicyCreate.js @@ -163,8 +163,10 @@ define(function(require){ var userPerm = (validateObj1.userPerm || validateObj2.userPerm || validateObj3.userPerm || validateObj4.userPerm); var groupPerm = (validateObj1.groupPermSet || validateObj2.groupPermSet - || validateObj3.groupPermSet || validateObj4.groupPermSet) - if((!validateObj1.auditLoggin) && !(groupPerm || userPerm)){ + || validateObj3.groupPermSet || validateObj4.groupPermSet); + var delegatePerm = (validateObj1.delegateAdmin || validateObj2.delegateAdmin + || validateObj3.delegateAdmin || validateObj4.delegateAdmin); + if((!validateObj1.auditLoggin) && !(groupPerm || userPerm || delegatePerm )){ XAUtil.alertPopup({ msg :localization.tt('msg.yourAuditLogginIsOff') }); return; } @@ -177,6 +179,13 @@ define(function(require){ }, validatePolicyItem : function(validateObj){ var that = this, valid = false; + //DelegateAdmin checks + if((validateObj.groupSet || validateObj.userSet) && validateObj.delegateAdmin){ + return true; + }else if(validateObj.delegateAdmin && !(validateObj.groupSet || validateObj.userSet)) { + this.popupCallBack(localization.tt('msg.addUserOrGroupForDelegateAdmin'),validateObj); + return false; + } valid = (validateObj.groupSet && validateObj.permSet) || (validateObj.userSet && validateObj.userPerm); if(!valid){ if((!validateObj.groupSet && !validateObj.userSet) && (validateObj.condSet)) { diff --git a/security-admin/src/main/webapp/scripts/views/policies/RangerPolicyForm.js b/security-admin/src/main/webapp/scripts/views/policies/RangerPolicyForm.js index 9145b88020..e235cdacfa 100644 --- a/security-admin/src/main/webapp/scripts/views/policies/RangerPolicyForm.js +++ b/security-admin/src/main/webapp/scripts/views/policies/RangerPolicyForm.js @@ -118,22 +118,6 @@ define(function(require){ var attr1 = _.pick(_.result(this.model,'schemaBase'),basicSchema); var attr2 = _.pick(_.result(this.model,'schemaBase'),schemaNames); - var arr = {}; - - _.each(attrs,function(resourceObject,resourceName){ - if(resourceObject.hasOwnProperty('recursiveSupport')) { - if(resourceObject.recursiveSupport) { - var recursiveAttrSchema = _.pick(_.result(that.model,'schemaBase'),'recursive'); - if(!_.isUndefined(that.model.get('id'))) { - recursiveAttrSchema.recursive.switchOn=(that.model.get(resourceName)).isRecursive; - } - arr[resourceName] = resourceObject; - _.extend(arr,recursiveAttrSchema); - } - } - }); - _.extend(attrs,arr); - return _.extend(attr1,_.extend(attrs,attr2)); }, /** on render callback */ @@ -356,16 +340,6 @@ define(function(require){ var that = this, resources = []; var resources = {}; - //set 'isRecursive' attribute of resource object to value of field recursive - var recursiveValue = ''; - if(!_.isUndefined(this.model.get('recursive'))){ - recursiveValue = that.model.get('recursive'); - } - _.each(this.model.attributes,function(val) { - if(_.isObject(val) && !_.isUndefined(val.isRecursive)) { - val.isRecursive = recursiveValue; - } - });// 'isRecursive' attribute of model is updated //set sameLevel fieldAttr value with resource name _.each(this.model.attributes, function(val, key) { if(key.indexOf("sameLevel") >= 0 && !_.isNull(val)){ @@ -453,7 +427,6 @@ define(function(require){ var RangerPolicyItemAccessList = Backbone.Collection.extend(); var rangerPlcItemAccessList = new RangerPolicyItemAccessList(m.get('accesses')); policyItem.set('accesses', rangerPlcItemAccessList) - policyItemList.add(policyItem) } if(!_.isUndefined(m.get('dataMaskInfo'))){ policyItem.set("dataMaskInfo",m.get("dataMaskInfo")); @@ -461,6 +434,7 @@ define(function(require){ if(!_.isUndefined(m.get('rowFilterInfo'))){ policyItem.set("rowFilterInfo",m.get("rowFilterInfo")); } + policyItemList.add(policyItem); } @@ -681,16 +655,17 @@ define(function(require){ return JSON.stringify(context); }, formValidation : function(coll){ - var groupSet = false,permSet = false,groupPermSet = false, + var groupSet = false , permSet = false , groupPermSet = false , delegateAdmin = false , userSet=false, userPerm = false, userPermSet =false,breakFlag =false, condSet = false,customMaskSet = true; console.log('validation called..'); coll.each(function(m){ if(_.isEmpty(m.attributes)) return; - if(m.has('groupName') || m.has('userName') || m.has('accesses') ){ + if(m.has('groupName') || m.has('userName') || m.has('accesses') || m.has('delegateAdmin') ){ if(! breakFlag){ groupSet = m.has('groupName') ? true : false; userSet = m.has('userName') ? true : false; - permSet = m.has('accesses') ? true : false; + permSet = m.has('accesses') ? true : false; + delegateAdmin = m.has('delegateAdmin') ? m.get('delegateAdmin') : false; if(groupSet && permSet){ groupPermSet = true; userPermSet = false; @@ -698,7 +673,9 @@ define(function(require){ userPermSet = true; groupPermSet = false; }else{ - breakFlag=true; + if(!((userSet || groupSet) && delegateAdmin)){ + breakFlag=true; + } } } } @@ -718,7 +695,8 @@ define(function(require){ userSet : userSet, isUsers:userPermSet, auditLoggin : auditStatus, condSet : condSet, - customMaskSet : customMaskSet + customMaskSet : customMaskSet, + delegateAdmin : delegateAdmin, }; if(groupSet || userSet){ obj['permSet'] = groupSet ? permSet : false; diff --git a/security-admin/src/main/webapp/scripts/views/policies/RangerPolicyRO.js b/security-admin/src/main/webapp/scripts/views/policies/RangerPolicyRO.js index 0d8e3df2c7..d0a86836e5 100644 --- a/security-admin/src/main/webapp/scripts/views/policies/RangerPolicyRO.js +++ b/security-admin/src/main/webapp/scripts/views/policies/RangerPolicyRO.js @@ -35,8 +35,17 @@ define(function(require) { template: RangerPolicyROTmpl, templateHelpers: function() { + var isDelegatAdminChk; + if(this.policyDetails.serviceType !== XAEnums.ServiceType.SERVICE_TAG.label + && !XAUtils.isMaskingPolicy(this.policy.get('policyType')) + && !XAUtils.isRowFilterPolicy(this.policy.get('policyType'))) { + isDelegatAdminChk = true; + } else { + isDelegatAdminChk = false; + } return { PolicyDetails: this.policyDetails, + isDelegatAdmin: isDelegatAdminChk }; }, breadCrumbs: [], @@ -74,11 +83,24 @@ define(function(require) { this.policy.fetchByEventTime({ async: false, cache: false, - data : data + data : data, + error : function(error , response){ + if (response && response.status === 419 ) { + XAUtils.defaultErrorHandler(error , response); + } else { + XAUtils.showErrorMsg(response.responseJSON.msgDesc); + } + } }); }, initializePolicyDetailsObj : function(){ + // In this.policy service type is undefined then we take repotype. + if(_.isUndefined(this.policy.get('serviceType'))){ + this.serviceDef = this.serviceDefList.findWhere({'id' : this.repoType}) + }else{ + this.serviceDef = this.serviceDefList.findWhere({'name':this.policy.get('serviceType')}); + } var self = this; var details = this.policyDetails = {}; details.id = this.policy.get('id'); @@ -109,6 +131,18 @@ define(function(require) { perm.allowException = this.policy.get('allowExceptions'); perm.denyPolicyItems = this.policy.get('denyPolicyItems'); perm.denyExceptions = this.policy.get('denyExceptions'); + if(this.policy.get('dataMaskPolicyItems')){ + _.each(this.policy.get('dataMaskPolicyItems'), function(mask){ + var maskingInfo = _.find(self.serviceDef.get("dataMaskDef").maskTypes, function(m){ + return m.name == mask.dataMaskInfo.dataMaskType; + }); + if(maskingInfo){ + _.extend(mask.dataMaskInfo , _.pick(maskingInfo, 'label')); + } + }) + perm.maskingPolicy = this.policy.get('dataMaskPolicyItems'); + } + perm.rowLevelPolicy = this.policy.get('rowFilterPolicyItems'); details.createdBy = this.policy.get('createdBy'); details.createTime = Globalize.format(new Date(this.policy.get('createTime')), "MM/dd/yyyy hh:mm tt"); details.updatedBy = this.policy.get('updatedBy'); @@ -124,7 +158,9 @@ define(function(require) { var items = [{'itemName': 'policyItems',title : 'Allow Condition'}, {'itemName': 'allowExceptions',title : 'Exclude from Allow Conditions'}, {'itemName': 'denyPolicyItems',title : 'Deny Condition'}, - {'itemName': 'denyExceptions',title : 'Exclude from Deny Conditions'},] + {'itemName': 'denyExceptions',title : 'Exclude from Deny Conditions'}, + {'itemName': 'dataMaskPolicyItems',title : 'Masking Conditions'}, + {'itemName': 'rowFilterPolicyItems',title : 'Row Level Conditions'}] _.each(items, function(item){ if(!_.isUndefined(this.policy.get(item.itemName)) && !_.isEmpty(this.policy.get(item.itemName))){ this.policyDetails['policyItemsCond'].push({ title : item.title, headers : headers.header, policyItems : this.policy.get(item.itemName)}) @@ -145,9 +181,23 @@ define(function(require) { getPermHeaders : function(){ var permList = [], - policyCondition = false; - permList.unshift(localization.tt('lbl.delegatedAdmin')); - permList.unshift(localization.tt('lbl.permissions')); + policyCondition = false; + if(this.policyDetails.serviceType !== XAEnums.ServiceType.SERVICE_TAG.label + && !XAUtils.isMaskingPolicy(this.policy.get('policyType')) + && !XAUtils.isRowFilterPolicy(this.policy.get('policyType'))){ + permList.unshift(localization.tt('lbl.delegatedAdmin')); + } + if(XAUtils.isRowFilterPolicy(this.policy.get('policyType'))){ + permList.unshift(localization.tt('lbl.rowLevelFilter')); + } + if(XAUtils.isMaskingPolicy(this.policy.get('policyType'))){ + permList.unshift(localization.tt('lbl.selectMaskingOption')); + } + if(XAUtils.isRowFilterPolicy(this.policy.get('policyType')) || XAUtils.isMaskingPolicy(this.policy.get('policyType'))){ + permList.unshift(localization.tt('lbl.accessTypes')); + }else{ + permList.unshift(localization.tt('lbl.permissions')); + } if(!_.isEmpty(this.serviceDef.get('policyConditions'))){ permList.unshift(localization.tt('h.policyCondition')); policyCondition = true; @@ -177,6 +227,8 @@ define(function(require) { }, getPolicyByVersion : function(ver, e){ + //to support old policy log after updating that policy. + this.policy.set('serviceType',undefined); this.policy.fetchByVersion(ver, { cache : false, async : false diff --git a/security-admin/src/main/webapp/scripts/views/policies/RangerPolicyTableLayout.js b/security-admin/src/main/webapp/scripts/views/policies/RangerPolicyTableLayout.js index eb88686844..09e2e16699 100644 --- a/security-admin/src/main/webapp/scripts/views/policies/RangerPolicyTableLayout.js +++ b/security-admin/src/main/webapp/scripts/views/policies/RangerPolicyTableLayout.js @@ -354,7 +354,7 @@ define(function(require){ table:localization.tt('lbl.tableName') , tag : localization.tt('h.tagsMsg'), taxonomy:localization.tt('h.taxonomy') ,term: localization.tt('h.term') , topic:localization.tt('h.topic') ,topology:localization.tt('lbl.topologyName'), - type:localization.tt('h.type') ,udf:localization.tt('h.udf') , + type:localization.tt('h.type') ,udf:localization.tt('h.udf') , url:localization.tt('h.url') }; var serverRsrcAttrName = _.map(resourceSearchOpt,function(opt){ return { diff --git a/security-admin/src/main/webapp/scripts/views/policymanager/ServiceLayout.js b/security-admin/src/main/webapp/scripts/views/policymanager/ServiceLayout.js index c551887b6d..7aab1e2c33 100644 --- a/security-admin/src/main/webapp/scripts/views/policymanager/ServiceLayout.js +++ b/security-admin/src/main/webapp/scripts/views/policymanager/ServiceLayout.js @@ -128,7 +128,7 @@ define(function(require){ var el = $(e.currentTarget), serviceType = el.attr('data-servicetype'); if(serviceType){ var componentServices = this.services.where({'type' : serviceType }); - if(serviceType !== undefined && componentServices.length == 0 ){ + if(componentServices.length == 0 ){ XAUtil.alertBoxWithTimeSet(localization.tt('msg.noServiceToExport')); return; } @@ -176,7 +176,7 @@ define(function(require){ var el = $(e.currentTarget), serviceType = el.attr('data-servicetype'); if(serviceType){ var componentServices = this.services.where({'type' : serviceType }); - if(serviceType !== undefined && componentServices.length == 0 ){ + if(componentServices.length == 0 ){ XAUtil.alertBoxWithTimeSet(localization.tt('msg.noServiceToImport')); return; } @@ -231,7 +231,7 @@ define(function(require){ }, error :function(model, response) { XAUtil.blockUI('unblock'); - if(!_.isUndefined(response) && !_.isUndefined(response.responseJSON) && !_.isUndefined(response.responseJSON.msgDesc)){ + if(!_.isUndefined(response) && !_.isUndefined(response.responseJSON) && !_.isUndefined(response.responseJSON.msgDesc && response.status !='419')){ XAUtil.notifyError('Error', response.responseJSON.msgDesc); } } diff --git a/security-admin/src/main/webapp/scripts/views/reports/AuditLayout.js b/security-admin/src/main/webapp/scripts/views/reports/AuditLayout.js index 40f56dd5f4..f6c11b7e41 100644 --- a/security-admin/src/main/webapp/scripts/views/reports/AuditLayout.js +++ b/security-admin/src/main/webapp/scripts/views/reports/AuditLayout.js @@ -320,15 +320,25 @@ define(function(require) { switch (facet) { case 'Service Name': - var serviceList = new RangerServiceList(); + var serviceList = new RangerServiceList() , serviceNameVal = []; serviceList.setPageSize(100); serviceList.fetch().done(function(){ - callback(serviceList.map(function(model){return model.get('name');})); + serviceList.each(function(m){ + if(m.get('type') !== XAEnums.ServiceType.SERVICE_TAG.label){ + serviceNameVal.push(m.get('name')); + }; + }); + callback(serviceNameVal); }); break; case 'Service Type': - var serviceList = that.serviceDefList.map(function(serviceDef){ return {'label' : serviceDef.get('name').toUpperCase(), 'value' : serviceDef.get('name').toUpperCase()}; }) - callback(serviceList); + var serviveDefs = []; + that.serviceDefList.each(function(m){ + if(m.get('name').toUpperCase() != (XAEnums.ServiceType.SERVICE_TAG.label).toUpperCase()){ + serviveDefs.push({ 'label' : m.get('name').toUpperCase(), 'value' : m.get('name').toUpperCase() }); + } + }); + callback(serviveDefs); break; case 'Result': callback(XAUtils.hackForVSLabelValuePairs(XAEnums.AccessResult)); @@ -360,11 +370,11 @@ define(function(require) { var that = this; var searchOpt = ["Audit Type", "User", "Actions", "Session Id", "Start Date", "End Date"]; var serverAttrName = [{text : "Audit Type", label :"objectClassType",'multiple' : true, 'optionsArr' : XAUtils.enumToSelectLabelValuePairs(XAEnums.ClassTypes)}, - {text : "User", label :"owner"}, - {text : "Actions", label :"action"},{text : "Session Id", label :"sessionId"}, - {text : 'Start Date',label :'startDate'},{text : 'End Date',label :'endDate'} ]; + {text : "User", label :"owner"}, {text : "Session Id", label :"sessionId"}, + {text : 'Start Date',label :'startDate'},{text : 'End Date',label :'endDate'}, + {text : "Actions", label :"action",'multiple' : true, 'optionsArr' : XAUtils.enumToSelectLabelValuePairs(XAGlobals.ActionType)},]; - var auditList = [],query = ''; + var auditList = [],query = '', actionTypeList = []; _.each(XAEnums.ClassTypes, function(obj){ if((obj.value == XAEnums.ClassTypes.CLASS_TYPE_XA_ASSET.value) || (obj.value == XAEnums.ClassTypes.CLASS_TYPE_XA_RESOURCE.value) @@ -374,6 +384,11 @@ define(function(require) { || (obj.value == XAEnums.ClassTypes.CLASS_TYPE_XA_GROUP.value)) auditList.push({label :obj.label, value :obj.label+''}); }); + _.each(XAGlobals.ActionType, function(obj){ + if(obj.label){ + actionTypeList.push({label :obj.label, value :obj.label}) + } + }) if(!_.isUndefined(App.sessionId)){ App.vsHistory.admin = [] ; query = '"Session Id": "'+App.sessionId+'"'; @@ -394,7 +409,7 @@ define(function(require) { callback(auditList); break; case 'Actions': - callback(["Create","Update","Delete","Password Change","Export Json","Export Csv","Export Excel","Import End","Import Start"]); + callback(actionTypeList); break; case 'Start Date' : var endDate, models = that.visualSearch.searchQuery.where({category:"End Date"}); @@ -609,7 +624,14 @@ define(function(require) { var fullTrxLogListForTrxId = new VXTrxLogList(); fullTrxLogListForTrxId.getFullTrxLogListForTrxId(this.model.get('transactionId'),{ - cache : false + cache : false, + error : function(response , error){ + if (response && response.status === 419 ) { + XAUtils.defaultErrorHandler(error , response); + } else { + XAUtils.showErrorMsg(response.responseJSON.msgDesc); + } + } }).done(function(coll,mm){ XAUtils.blockUI('unblock'); fullTrxLogListForTrxId = new VXTrxLogList(coll.vXTrxLogs); @@ -726,9 +748,9 @@ define(function(require) { hasAction = ["EXPORT JSON", "EXPORT EXCEL", "EXPORT CSV", "IMPORT START", "IMPORT END"]; if($.inArray(action,hasAction)>=0){ if(action == "EXPORT JSON" || action == "EXPORT EXCEL" || action == "EXPORT CSV") - return html = 'Exported policies'; + return 'Exported policies'; else - return html = action; + return action; } else{ if(rawValue == XAEnums.ClassTypes.CLASS_TYPE_XA_ASSET.value || rawValue == XAEnums.ClassTypes.CLASS_TYPE_RANGER_SERVICE.value) html = 'Service '+action+'d '+''+name+''; @@ -835,19 +857,18 @@ define(function(require) { Backgrid.Row.prototype.initialize.apply(this, args); }, onClick: function (e) { - var self = this; - if($(e.target).hasClass('tagsColumn') || $(e.target).closest('td').hasClass("tagsColumn")){ - return; - } + var self = this ; + if($(e.target).hasClass('tagsColumn') || $(e.target).closest('td').hasClass("tagsColumn")){ + return; + } + if(this.model.get('repoType')){ + var repoType = this.model.get('repoType'); + } var policyId = this.model.get('policyId'); if(policyId == -1){ return; } - var serviceDef = that.serviceDefList.findWhere({'id':this.model.get('repoType')}); - if(_.isUndefined(serviceDef)){ - return ; - } - var eventTime = this.model.get('eventTime'); + var eventTime = this.model.get('eventTime'); var policy = new RangerPolicy({ id: policyId @@ -856,17 +877,19 @@ define(function(require) { var view = new RangerPolicyRO({ policy: policy, policyVersionList : policyVersionList, - serviceDef: serviceDef, - eventTime : eventTime + serviceDefList: that.serviceDefList, + eventTime : eventTime, + repoType : repoType }); var modal = new Backbone.BootstrapModal({ animate : true, content : view, title: localization.tt("h.policyDetails"), okText :localization.tt("lbl.ok"), - allowCancel : false, + allowCancel : true, escape : true }).open(); + modal.$el.find('.cancel').hide(); var policyVerEl = modal.$el.find('.modal-footer').prepend('
').find('.policyVer'); policyVerEl.append('Version '+ policy.get('version') +'').find('#preVer').click(function(e){ view.previousVer(e); @@ -945,7 +968,7 @@ define(function(require) { formatter: _.extend({}, Backgrid.CellFormatter.prototype, { fromRaw: function (rawValue, model) { return '
'+_.escape(rawValue)+'
\ -
'+_.escape(model.get('serviceType'))+'
';; +
'+_.escape(model.get('serviceType'))+'
'; } }) }, diff --git a/security-admin/src/main/webapp/scripts/views/reports/PlugableServiceDiffDetail.js b/security-admin/src/main/webapp/scripts/views/reports/PlugableServiceDiffDetail.js index f24fe8767f..914f271354 100644 --- a/security-admin/src/main/webapp/scripts/views/reports/PlugableServiceDiffDetail.js +++ b/security-admin/src/main/webapp/scripts/views/reports/PlugableServiceDiffDetail.js @@ -327,14 +327,20 @@ define(function(require){ if(itemType === 'Masked Policy Items') { // its for new created record for(var i = 0; i < newPolicyItems.length ; i++){ - if(newPolicyItems[i].DataMasklabel){ + if(newPolicyItems[i].DataMasklabel && newPolicyItems[i].DataMasklabel == "Custom"){ var maskingType = newPolicyItems[i].dataMaskInfo.dataMaskType; - newPolicyItems[i].dataMaskInfo.dataMaskType = newPolicyItems[i].DataMasklabel; + newPolicyItems[i].dataMaskInfo.dataMaskType = newPolicyItems[i].DataMasklabel +' : '+newPolicyItems[i].dataMaskInfo.valueExpr; + }else if(newPolicyItems[i].DataMasklabel){ + var maskingType = newPolicyItems[i].dataMaskInfo.dataMaskType; + newPolicyItems[i].dataMaskInfo.dataMaskType = newPolicyItems[i].DataMasklabel; } } for(var i = 0; i < oldPolicyItems.length ; i++){ - if(oldPolicyItems[i].DataMasklabel){ + if(oldPolicyItems[i].DataMasklabel && oldPolicyItems[i].DataMasklabel == "Custom"){ + var maskingType = oldPolicyItems[i].dataMaskInfo.dataMaskType; + oldPolicyItems[i].dataMaskInfo.dataMaskType = oldPolicyItems[i].DataMasklabel +' : '+oldPolicyItems[i].dataMaskInfo.valueExpr; + }else if(oldPolicyItems[i].DataMasklabel){ var maskingType = oldPolicyItems[i].dataMaskInfo.dataMaskType; oldPolicyItems[i].dataMaskInfo.dataMaskType = oldPolicyItems[i].DataMasklabel; } diff --git a/security-admin/src/main/webapp/scripts/views/reports/UserAccessLayout.js b/security-admin/src/main/webapp/scripts/views/reports/UserAccessLayout.js index 197bc84360..74bbd5f31b 100644 --- a/security-admin/src/main/webapp/scripts/views/reports/UserAccessLayout.js +++ b/security-admin/src/main/webapp/scripts/views/reports/UserAccessLayout.js @@ -141,10 +141,11 @@ define(function(require) {'use strict'; this.initializePlugins(); this.setupGroupAutoComplete(); this.renderComponentAndPolicyTypeSelect(); - //Show policies listing for each service and GET policies for each service + var policyType = this.ui.policyType.val(); +// Show policies listing for each service and GET policies for each service _.each(this.policyCollList, function(obj,i){ this.renderTable(obj.collName, obj.serviceDefName); - this.getResourceLists(obj.collName,obj.serviceDefName); + this.getResourceLists(obj.collName,obj.serviceDefName,policyType); },this); this.$el.find('[data-js="policyName"]').focus() var urlString = XAUtil.getBaseUrl(); @@ -161,11 +162,15 @@ define(function(require) {'use strict'; {text :'Search By' , info :localization.tt('msg.searchBy')}, {text :'Resource' , info :localization.tt('msg.resourceMsg')}] }, - getResourceLists: function(collName, serviceDefName){ + getResourceLists: function(collName, serviceDefName , policyType){ var that = this, coll = this[collName]; that.allowDownload = false; coll.queryParams.serviceType = serviceDefName; + if(policyType){ +// to set default value access type in policy type + coll.queryParams.policyType = policyType; + } coll.fetch({ cache : false, reset: true, @@ -178,9 +183,13 @@ define(function(require) {'use strict'; _.each(that[collName].models,function(model,ind){ if (XAUtil.isMaskingPolicy(model.get('policyType'))) { //'Collection' must be same as subgrid custom column name - model.attributes.allowCollection = model.get('dataMaskPolicyItems'); + model.attributes.maskCollection = model.get('dataMaskPolicyItems'); +// Add service type in masking condition + _.each(model.attributes.dataMaskPolicyItems , function(m){ + m.type = model.collection.queryParams.serviceType; + }) } else if (XAUtil.isRowFilterPolicy(model.get('policyType'))) { - model.attributes.allowCollection = model.get('rowFilterPolicyItems'); + model.attributes.rowlvlCollection = model.get('rowFilterPolicyItems'); } else { model.attributes.allowCollection = model.get('policyItems'); } @@ -215,16 +224,15 @@ define(function(require) {'use strict'; label: 'Groups', formatter: _.extend({}, Backgrid.CellFormatter.prototype, { fromRaw: function (rawValue,model, coll) { - var startSpanEle = '',endSpanEle = ''; var group_str = ''; if(_.isEmpty(model.get('groups'))){ return '
--
'; } else { _.each(model.get('groups'),function(group,index){ if(index < 4) { - group_str += '' + _.escape(group) + endSpanEle + " "; + group_str += '' + _.escape(group) + '' + " "; } else { - group_str += '