[3.1 backport] Introduced setting for disabling denormalized privilege data structures

CHANGELOG.md

@@ -6,6 +6,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 ## [Unreleased 3.1.x]
 ### Added
 
+* Introduced setting `plugins.security.privileges_evaluation.precomputed_privileges.enabled` ([#5483](https://github.com/opensearch-project/security/pull/5483))
 
 ### Changed
 

src/integrationTest/java/org/opensearch/security/privileges/ActionPrivilegesTest.java

@@ -960,6 +960,26 @@ public void aliasesOnDataStreamBackingIndices() throws Exception {
             );
             assertThat(resultForIndexNotCoveredByAlias, isForbidden());
         }
+
+        @Test
+        public void statefulDisabled() throws Exception {
+            SecurityDynamicConfiguration<RoleV7> roles = SecurityDynamicConfiguration.fromYaml(
+                "test_role:\n" + "  index_permissions:\n" + "  - index_patterns: ['test_*']\n" + "    allowed_actions: ['indices:*']",
+                CType.ROLES
+            );
+            Map<String, IndexAbstraction> metadata = indices("test_1", "test_2", "test_3")//
+                .build()
+                .getIndicesLookup();
+
+            ActionPrivileges subject = new ActionPrivileges(
+                roles,
+                FlattenedActionGroups.EMPTY,
+                Map::of,
+                Settings.builder().put(ActionPrivileges.PRECOMPUTED_PRIVILEGES_ENABLED.getKey(), false).build()
+            );
+            subject.updateStatefulIndexPrivileges(metadata, 1);
+            assertEquals(0, subject.getEstimatedStatefulIndexByteSize());
+        }
     }
 
     /**

src/integrationTest/java/org/opensearch/security/privileges/dlsfls/DocumentPrivilegesTest.java

@@ -51,6 +51,7 @@
 import org.opensearch.index.query.QueryBuilder;
 import org.opensearch.index.query.QueryBuilders;
 import org.opensearch.index.query.TermQueryBuilder;
+import org.opensearch.security.privileges.ActionPrivileges;
 import org.opensearch.security.privileges.PrivilegesConfigurationValidationException;
 import org.opensearch.security.privileges.PrivilegesEvaluationContext;
 import org.opensearch.security.privileges.PrivilegesEvaluationException;
@@ -537,7 +538,13 @@ private DocumentPrivileges createSubject(SecurityDynamicConfiguration<RoleV7> ro
                 roleConfig,
                 statefulness == Statefulness.STATEFUL ? INDEX_METADATA.getIndicesLookup() : Map.of(),
                 xContentRegistry,
-                Settings.builder().put("plugins.security.dfm_empty_overrides_all", this.dfmEmptyOverridesAll).build()
+                Settings.builder()
+                    .put("plugins.security.dfm_empty_overrides_all", this.dfmEmptyOverridesAll)
+                    .put(
+                        ActionPrivileges.PRECOMPUTED_PRIVILEGES_ENABLED.getKey(),
+                        statefulness == Statefulness.STATEFUL || statefulness == Statefulness.NON_STATEFUL
+                    )
+                    .build()
             );
         }
     }
@@ -852,7 +859,13 @@ private DocumentPrivileges createSubject(SecurityDynamicConfiguration<RoleV7> ro
                 roleConfig,
                 statefulness == Statefulness.STATEFUL ? INDEX_METADATA.getIndicesLookup() : Map.of(),
                 xContentRegistry,
-                Settings.builder().put("plugins.security.dfm_empty_overrides_all", this.dfmEmptyOverridesAll).build()
+                Settings.builder()
+                    .put("plugins.security.dfm_empty_overrides_all", this.dfmEmptyOverridesAll)
+                    .put(
+                        ActionPrivileges.PRECOMPUTED_PRIVILEGES_ENABLED.getKey(),
+                        statefulness == Statefulness.STATEFUL || statefulness == Statefulness.NON_STATEFUL
+                    )
+                    .build()
             );
         }
     }
@@ -1104,7 +1117,13 @@ private DocumentPrivileges createSubject(SecurityDynamicConfiguration<RoleV7> ro
                 roleConfig,
                 statefulness == Statefulness.STATEFUL ? INDEX_METADATA.getIndicesLookup() : Map.of(),
                 xContentRegistry,
-                Settings.builder().put("plugins.security.dfm_empty_overrides_all", this.dfmEmptyOverridesAll).build()
+                Settings.builder()
+                    .put("plugins.security.dfm_empty_overrides_all", this.dfmEmptyOverridesAll)
+                    .put(
+                        ActionPrivileges.PRECOMPUTED_PRIVILEGES_ENABLED.getKey(),
+                        statefulness == Statefulness.STATEFUL || statefulness == Statefulness.NON_STATEFUL
+                    )
+                    .build()
             );
         }
 
@@ -1231,7 +1250,8 @@ public String toString() {
      */
     static enum Statefulness {
         STATEFUL,
-        NON_STATEFUL
+        NON_STATEFUL,
+        DISABLED
     }
 
     /**

src/main/java/org/opensearch/security/OpenSearchSecurityPlugin.java

@@ -2131,6 +2131,7 @@ public List<Setting<?>> getSettings() {
 
             // Privileges evaluation
             settings.add(ActionPrivileges.PRECOMPUTED_PRIVILEGES_MAX_HEAP_SIZE);
+            settings.add(ActionPrivileges.PRECOMPUTED_PRIVILEGES_ENABLED);
 
             // Resource Sharing
             settings.add(

src/main/java/org/opensearch/security/privileges/ActionPrivileges.java

@@ -70,6 +70,18 @@ public class ActionPrivileges extends ClusterStateMetadataDependentPrivileges {
         Setting.Property.NodeScope
     );
 
+    /**
+     * This setting controls whether the precomputed/denormalized index privileges (in the inner class StatefulIndexPrivileges)
+     * will be created or not. This is on by default to provide the best action throughput. It can make sense to
+     * disable this when it is seen that the initialisation process takes so much time/resources that it negatively
+     * affects the cluster performance. This come at the price of a reduced action throughput.
+     */
+    public static Setting<Boolean> PRECOMPUTED_PRIVILEGES_ENABLED = Setting.boolSetting(
+        "plugins.security.privileges_evaluation.precomputed_privileges.enabled",
+        true,
+        Setting.Property.NodeScope
+    );
+
     private static final Logger log = LogManager.getLogger(ActionPrivileges.class);
 
     private final ClusterPrivileges cluster;
@@ -80,6 +92,7 @@ public class ActionPrivileges extends ClusterStateMetadataDependentPrivileges {
     private final ImmutableSet<String> wellKnownIndexActions;
     private final Supplier<Map<String, IndexAbstraction>> indexMetadataSupplier;
     private final ByteSizeValue statefulIndexMaxHeapSize;
+    private final boolean statefulIndexEnabled;
 
     private final AtomicReference<StatefulIndexPrivileges> statefulIndex = new AtomicReference<>();
 
@@ -101,6 +114,7 @@ public ActionPrivileges(
         this.wellKnownIndexActions = wellKnownIndexActions;
         this.indexMetadataSupplier = indexMetadataSupplier;
         this.statefulIndexMaxHeapSize = PRECOMPUTED_PRIVILEGES_MAX_HEAP_SIZE.get(settings);
+        this.statefulIndexEnabled = PRECOMPUTED_PRIVILEGES_ENABLED.get(settings);
     }
 
     public ActionPrivileges(
@@ -238,6 +252,10 @@ public PrivilegesEvaluatorResponse hasExplicitIndexPrivilege(
      * updateStatefulIndexPrivilegesAsync(). Package visible for testing.
      */
     void updateStatefulIndexPrivileges(Map<String, IndexAbstraction> indices, long metadataVersion) {
+        if (!this.statefulIndexEnabled) {
+            return;
+        }
+
         StatefulIndexPrivileges statefulIndex = this.statefulIndex.get();
 
         indices = StatefulIndexPrivileges.relevantOnly(indices);
@@ -963,6 +981,8 @@ static class StatefulIndexPrivileges {
             long metadataVersion,
             ByteSizeValue statefulIndexMaxHeapSize
         ) {
+            long startTime = System.currentTimeMillis();
+
             Map<
                 String,
                 CompactMapGroupBuilder.MapBuilder<String, DeduplicatingCompactSubSetBuilder.SubSetBuilder<String>>> actionToIndexToRoles =
@@ -1082,6 +1102,14 @@ static class StatefulIndexPrivileges {
             this.indices = ImmutableMap.copyOf(indices);
             this.metadataVersion = metadataVersion;
             this.wellKnownIndexActions = wellKnownIndexActions;
+
+            long duration = System.currentTimeMillis() - startTime;
+
+            if (duration > 30000) {
+                log.warn("Creation of StatefulIndexPrivileges took {} ms", duration);
+            } else {
+                log.debug("Creation of StatefulIndexPrivileges took {} ms", duration);
+            }
         }
 
         /**

src/main/java/org/opensearch/security/privileges/dlsfls/AbstractRuleBasedPrivileges.java

@@ -26,6 +26,7 @@
 
 import org.opensearch.cluster.metadata.IndexAbstraction;
 import org.opensearch.common.settings.Settings;
+import org.opensearch.security.privileges.ActionPrivileges;
 import org.opensearch.security.privileges.IndexPattern;
 import org.opensearch.security.privileges.PrivilegesConfigurationValidationException;
 import org.opensearch.security.privileges.PrivilegesEvaluationContext;
@@ -91,6 +92,11 @@ abstract class AbstractRuleBasedPrivileges<SingleRule, JoinedRule extends Abstra
      */
     private final boolean dfmEmptyOverridesAll;
 
+    /**
+     * Corresponds to the setting plugins.security.privileges_evaluation.precomputed_privileges.enabled
+     */
+    private final boolean statefulIndexEnabled;
+
     public AbstractRuleBasedPrivileges(
         SecurityDynamicConfiguration<RoleV7> roles,
         Map<String, IndexAbstraction> indexMetadata,
@@ -101,7 +107,8 @@ public AbstractRuleBasedPrivileges(
         this.roleToRuleFunction = roleToRuleFunction;
         this.staticRules = new StaticRules<>(roles, roleToRuleFunction);
         this.dfmEmptyOverridesAll = settings.getAsBoolean(ConfigConstants.SECURITY_DFM_EMPTY_OVERRIDES_ALL, false);
-        this.statefulRules = new StatefulRules<>(roles, indexMetadata, roleToRuleFunction);
+        this.statefulIndexEnabled = ActionPrivileges.PRECOMPUTED_PRIVILEGES_ENABLED.get(settings);
+        this.statefulRules = this.statefulIndexEnabled ? new StatefulRules<>(roles, indexMetadata, roleToRuleFunction) : null;
     }
 
     /**
@@ -524,6 +531,10 @@ protected abstract JoinedRule compile(PrivilegesEvaluationContext context, Colle
         throws PrivilegesEvaluationException;
 
     synchronized void updateIndices(Map<String, IndexAbstraction> indexMetadata) {
+        if (!this.statefulIndexEnabled) {
+            return;
+        }
+
         StatefulRules<SingleRule> statefulRules = this.statefulRules;
 
         if (statefulRules == null || !statefulRules.indexMetadata.keySet().equals(indexMetadata.keySet())) {