Introduce API Tokens with cluster_permissions and index_permissions directly associated with the token

CHANGELOG.md

@@ -14,6 +14,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Make security plugin aware of FIPS build param (-Pcrypto.standard=FIPS-140-3) ([#5952](https://github.com/opensearch-project/security/pull/5952))
 - Hardens input validation for resource sharing APIs ([#5831](https://github.com/opensearch-project/security/pull/5831)
 - Optimize getFieldFilter to only return a predicate when index has FLS restrictions for user ([#5777](https://github.com/opensearch-project/security/pull/5777))
+- Introduce API Tokens with `cluster_permissions` and `index_permissions` directly associated with the token ([#5443](https://github.com/opensearch-project/security/pull/5443))
 - Performance optimizations for building internal authorization data structures upon config updates ([#5988](https://github.com/opensearch-project/security/pull/5988))
 - Make encryption_key optional for obo token authenticator ([#6017](https://github.com/opensearch-project/security/pull/6017)
 - [Resource Sharing] Using custom action prefixes for sample resource plugin ([#6020](https://github.com/opensearch-project/security/pull/6020)

src/integrationTest/java/org/opensearch/test/framework/ApiTokenConfig.java

@@ -0,0 +1,32 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ *
+ */
+package org.opensearch.test.framework;
+
+import java.io.IOException;
+
+import org.opensearch.core.xcontent.ToXContentObject;
+import org.opensearch.core.xcontent.XContentBuilder;
+
+public class ApiTokenConfig implements ToXContentObject {
+    private Boolean enabled;
+
+    public ApiTokenConfig enabled(Boolean enabled) {
+        this.enabled = enabled;
+        return this;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder xContentBuilder, Params params) throws IOException {
+        xContentBuilder.startObject();
+        xContentBuilder.field("enabled", enabled);
+        xContentBuilder.endObject();
+        return xContentBuilder;
+    }
+}

src/integrationTest/java/org/opensearch/test/framework/TestSecurityConfig.java

@@ -151,6 +151,11 @@ public TestSecurityConfig onBehalfOf(OnBehalfOfConfig onBehalfOfConfig) {
         return this;
     }
 
+    public TestSecurityConfig apiToken(ApiTokenConfig apiTokenConfig) {
+        config.apiTokenConfig(apiTokenConfig);
+        return this;
+    }
+
     public TestSecurityConfig authc(AuthcDomain authcDomain) {
         config.authc(authcDomain);
         return this;
@@ -268,6 +273,7 @@ public static class Config implements ToXContentObject {
         private Boolean doNotFailOnForbidden;
         private XffConfig xffConfig;
         private OnBehalfOfConfig onBehalfOfConfig;
+        private ApiTokenConfig apiTokenConfig;
         private Map<String, AuthcDomain> authcDomainMap = new LinkedHashMap<>();
 
         private AuthFailureListeners authFailureListeners;
@@ -293,6 +299,11 @@ public Config onBehalfOfConfig(OnBehalfOfConfig onBehalfOfConfig) {
             return this;
         }
 
+        public Config apiTokenConfig(ApiTokenConfig apiTokenConfig) {
+            this.apiTokenConfig = apiTokenConfig;
+            return this;
+        }
+
         public Config authc(AuthcDomain authcDomain) {
             authcDomainMap.put(authcDomain.id, authcDomain);
             return this;
@@ -317,6 +328,10 @@ public XContentBuilder toXContent(XContentBuilder xContentBuilder, Params params
                 xContentBuilder.field("on_behalf_of", onBehalfOfConfig);
             }
 
+            if (apiTokenConfig != null) {
+                xContentBuilder.field("api_tokens", apiTokenConfig);
+            }
+
             if (anonymousAuth || (xffConfig != null)) {
                 xContentBuilder.startObject("http");
                 xContentBuilder.field("anonymous_auth_enabled", anonymousAuth);

src/integrationTest/java/org/opensearch/test/framework/cluster/LocalCluster.java

@@ -54,6 +54,7 @@
 import org.opensearch.security.action.configupdate.ConfigUpdateResponse;
 import org.opensearch.security.securityconf.impl.CType;
 import org.opensearch.security.support.ConfigConstants;
+import org.opensearch.test.framework.ApiTokenConfig;
 import org.opensearch.test.framework.AuditConfiguration;
 import org.opensearch.test.framework.AuthFailureListeners;
 import org.opensearch.test.framework.AuthzDomain;
@@ -664,6 +665,11 @@ public Builder onBehalfOf(OnBehalfOfConfig onBehalfOfConfig) {
             return this;
         }
 
+        public Builder apiToken(ApiTokenConfig apiTokenConfig) {
+            testSecurityConfig.apiToken(apiTokenConfig);
+            return this;
+        }
+
         public Builder loadConfigurationIntoIndex(boolean loadConfigurationIntoIndex) {
             this.loadConfigurationIntoIndex = loadConfigurationIntoIndex;
             return this;

src/main/java/org/opensearch/security/OpenSearchSecurityPlugin.java

@@ -92,6 +92,7 @@
 import org.opensearch.common.util.BigArrays;
 import org.opensearch.common.util.PageCacheRecycler;
 import org.opensearch.common.util.concurrent.ThreadContext;
+import org.opensearch.core.action.ActionListener;
 import org.opensearch.core.action.ActionResponse;
 import org.opensearch.core.common.io.stream.NamedWriteableRegistry;
 import org.opensearch.core.index.Index;
@@ -133,6 +134,10 @@
 import org.opensearch.search.internal.SearchContext;
 import org.opensearch.search.query.QuerySearchResult;
 import org.opensearch.secure_sm.AccessController;
+import org.opensearch.security.action.apitokens.ApiTokenAction;
+import org.opensearch.security.action.apitokens.ApiTokenRepository;
+import org.opensearch.security.action.apitokens.ApiTokenUpdateAction;
+import org.opensearch.security.action.apitokens.TransportApiTokenUpdateAction;
 import org.opensearch.security.action.configupdate.ConfigUpdateAction;
 import org.opensearch.security.action.configupdate.TransportConfigUpdateAction;
 import org.opensearch.security.action.onbehalf.CreateOnBehalfOfTokenAction;
@@ -279,6 +284,7 @@ public final class OpenSearchSecurityPlugin extends OpenSearchSecuritySSLPlugin
     private volatile UserService userService;
     private volatile RestLayerPrivilegesEvaluator restLayerEvaluator;
     private volatile ConfigurationRepository cr;
+    private volatile ApiTokenRepository apiTokenRepository;
     private volatile AdminDNs adminDns;
     private volatile ClusterService cs;
     private volatile AtomicReference<DiscoveryNode> localNode = new AtomicReference<>();
@@ -682,6 +688,22 @@ public List<RestHandler> getRestHandlers(
                     )
                 );
                 handlers.add(new CreateOnBehalfOfTokenAction(tokenManager));
+                handlers.add(
+                    new ApiTokenAction(
+                        Objects.requireNonNull(threadPool),
+                        cr,
+                        privilegesConfiguration,
+                        settings,
+                        adminDns,
+                        auditLog,
+                        configPath,
+                        principalExtractor,
+                        apiTokenRepository,
+                        cs,
+                        indexNameExpressionResolver,
+                        roleMapper
+                    )
+                );
                 handlers.addAll(
                     SecurityRestApiActions.getHandler(
                         settings,
@@ -741,6 +763,7 @@ public UnaryOperator<RestHandler> getRestHandlerWrapper(final ThreadContext thre
         List<ActionHandler<? extends ActionRequest, ? extends ActionResponse>> actions = new ArrayList<>(1);
         if (!disabled && !SSLConfig.isSslOnlyMode()) {
             actions.add(new ActionHandler<>(ConfigUpdateAction.INSTANCE, TransportConfigUpdateAction.class));
+            actions.add(new ActionHandler<>(ApiTokenUpdateAction.INSTANCE, TransportApiTokenUpdateAction.class));
             // external storage does not support reload and does not provide SSL certs info
             if (!ExternalSecurityKeyStore.hasExternalSslContext(settings)) {
                 actions.add(new ActionHandler<>(CertificatesActionType.INSTANCE, TransportCertificatesInfoNodesAction.class));
@@ -1211,6 +1234,7 @@ public Collection<Object> createComponents(
         );
         this.roleMapper = roleMapper;
         tokenManager = new SecurityTokenManager(cs, threadPool, userService, roleMapper);
+        apiTokenRepository = new ApiTokenRepository(localClient, clusterService);
 
         PrivilegesConfiguration privilegesConfiguration = new PrivilegesConfiguration(
             cr,
@@ -1224,7 +1248,8 @@ public Collection<Object> createComponents(
             settings,
             cih::getReasonForUnavailability,
             irr,
-            xContentRegistry
+            xContentRegistry,
+            apiTokenRepository
         );
         this.privilegesConfiguration = privilegesConfiguration;
 
@@ -1305,7 +1330,7 @@ public Collection<Object> createComponents(
             configPath,
             compatConfig
         );
-        dcf = new DynamicConfigFactory(cr, settings, configPath, localClient, threadPool, cih, passwordHasher);
+        dcf = new DynamicConfigFactory(cr, settings, configPath, localClient, threadPool, cih, passwordHasher, apiTokenRepository);
         dcf.registerDCFListener(backendRegistry);
         dcf.registerDCFListener(compatConfig);
         dcf.registerDCFListener(irr);
@@ -1356,6 +1381,7 @@ public Collection<Object> createComponents(
         components.add(dcf);
         components.add(userService);
         components.add(passwordHasher);
+        components.add(apiTokenRepository);
 
         components.add(sslSettingsManager);
         if (isSslCertReloadEnabled(settings) && sslCertificatesHotReloadEnabled(settings)) {
@@ -2335,6 +2361,14 @@ public void onNodeStarted(DiscoveryNode localNode) {
         this.localNode.set(localNode);
         if (!SSLConfig.isSslOnlyMode() && !client && !disabled && !useClusterStateToInitSecurityConfig(settings)) {
             cr.initOnNodeStart();
+            if (apiTokenRepository != null) {
+                apiTokenRepository.reloadApiTokensFromIndex(
+                    ActionListener.wrap(
+                        unused -> log.debug("API tokens loaded on node start"),
+                        e -> log.warn("Failed to load API tokens on node start", e)
+                    )
+                );
+            }
         }
 
         // resourceSharingIndexManagementRepository will be null when sec plugin is disabled or is in SSLOnly mode, hence it will not be
@@ -2412,8 +2446,13 @@ public Collection<SystemIndexDescriptor> getSystemIndexDescriptors(Settings sett
             ConfigConstants.SECURITY_CONFIG_INDEX_NAME,
             ConfigConstants.OPENDISTRO_SECURITY_DEFAULT_CONFIG_INDEX
         );
+        final SystemIndexDescriptor apiTokenSystemIndexDescriptor = new SystemIndexDescriptor(
+            ConfigConstants.OPENSEARCH_API_TOKENS_INDEX,
+            "Security API token index"
+        );
         final SystemIndexDescriptor securityIndexDescriptor = new SystemIndexDescriptor(indexPattern, "Security index");
         systemIndexDescriptors.add(securityIndexDescriptor);
+        systemIndexDescriptors.add(apiTokenSystemIndexDescriptor);
 
         for (String resourceIndex : resourcePluginInfo.getResourceIndices()) {
             final SystemIndexDescriptor resourceSharingIndexDescriptor = new SystemIndexDescriptor(