diff --git a/distribution/src/config/log4j2.properties b/distribution/src/config/log4j2.properties index 2aac6f58dc4e6..d774635b56f03 100644 --- a/distribution/src/config/log4j2.properties +++ b/distribution/src/config/log4j2.properties @@ -72,10 +72,21 @@ appender.deprecation_rolling.strategy.type = DefaultRolloverStrategy appender.deprecation_rolling.strategy.max = 4 ################################################# + logger.deprecation.name = org.elasticsearch.deprecation logger.deprecation.level = warn logger.deprecation.appenderRef.deprecation_rolling.ref = deprecation_rolling logger.deprecation.additivity = false +############ +appender.deprecation_indexer.type = DeprecationIndexer +appender.deprecation_indexer.name = deprecation_indexer +appender.deprecation_indexer.layout.type = ECSJsonLayout +appender.deprecation_indexer.layout.type_name = deprecation_indexer + +logger.deprecation_indexer.name = deprecation_indexer +logger.deprecation_indexer.level = warn +logger.deprecation_indexer.appenderRef.deprecation_indexer.ref = deprecation_indexer +logger.deprecation_indexer.additivity = false ######## Search slowlog JSON #################### appender.index_search_slowlog_rolling.type = RollingFile diff --git a/server/src/main/java/org/elasticsearch/common/logging/DeprecationAppender.java b/server/src/main/java/org/elasticsearch/common/logging/DeprecationAppender.java new file mode 100644 index 0000000000000..dc338ba11e76c --- /dev/null +++ b/server/src/main/java/org/elasticsearch/common/logging/DeprecationAppender.java @@ -0,0 +1,115 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.common.logging; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.core.Appender; +import org.apache.logging.log4j.core.Core; +import org.apache.logging.log4j.core.Filter; +import org.apache.logging.log4j.core.Layout; +import org.apache.logging.log4j.core.LogEvent; +import org.apache.logging.log4j.core.appender.AbstractAppender; +import org.apache.logging.log4j.core.appender.FileAppender; +import org.apache.logging.log4j.core.config.plugins.Plugin; +import org.apache.logging.log4j.core.config.plugins.PluginBuilderFactory; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.DocWriteRequest; +import org.elasticsearch.action.index.IndexAction; +import org.elasticsearch.action.index.IndexRequestBuilder; +import org.elasticsearch.action.index.IndexResponse; +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.xcontent.XContentType; + +import java.io.Serializable; +import java.time.Instant; +import java.time.LocalDate; +import java.time.format.DateTimeFormatter; +import java.util.HashMap; +import java.util.Map; + +import static org.elasticsearch.common.Strings.isNullOrEmpty; + +@Plugin(name = DeprecationAppender.NAME, category = Core.CATEGORY_NAME, elementType = Appender.ELEMENT_TYPE, printObject = true) +public class DeprecationAppender extends AbstractAppender{ + public static final String NAME = "DeprecationIndexer"; + private static final Logger logger = LogManager.getLogger(DeprecationAppender.class); + + private static final String TEMPLATE_NAME = ".deprecation_logs"; + +// private final ClusterService clusterService; +// private final NodeClient nodeClient; + + private boolean isTemplateCreated = false; + private NodeClient nodeClient; + + public DeprecationAppender(String name, Filter filter, Layout layout) { + super(name, filter, layout); + } + + @Override + public void start() { + this.setStarting(); + if (getFilter() != null) { + getFilter().start(); + } +// this.setStarted(); + } + + public void start(NodeClient client) { + this.nodeClient = client; + setStarted();//sets volatile variable + } + + public static class Builder> extends AbstractAppender.Builder + implements org.apache.logging.log4j.core.util.Builder { + + @Override + public DeprecationAppender build() { + return new DeprecationAppender(getName(),getFilter(),getOrCreateLayout()); + } + } + + @PluginBuilderFactory + public static > B newBuilder() { + return new DeprecationAppender.Builder().asBuilder(); + } + + @Override + public void append(LogEvent event) { + byte[] payload = getLayout().toByteArray(event); + final String indexName = TEMPLATE_NAME + "." + DateTimeFormatter.ISO_LOCAL_DATE.format(LocalDate.now()); + + new IndexRequestBuilder(nodeClient, IndexAction.INSTANCE).setIndex(indexName) + .setOpType(DocWriteRequest.OpType.CREATE) + .setSource(payload, XContentType.JSON) + .execute(new ActionListener<>() { + @Override + public void onResponse(IndexResponse indexResponse) { + // Nothing to do + } + + @Override + public void onFailure(Exception e) { + logger.error("Failed to index deprecation message", e); + } + }); + } +} diff --git a/server/src/main/java/org/elasticsearch/common/logging/DeprecationIndexerAppender.java b/server/src/main/java/org/elasticsearch/common/logging/DeprecationIndexerAppender.java new file mode 100644 index 0000000000000..634618cfbe573 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/common/logging/DeprecationIndexerAppender.java @@ -0,0 +1,39 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.common.logging; + +import org.apache.logging.log4j.core.Filter; +import org.apache.logging.log4j.core.Layout; +import org.apache.logging.log4j.core.LogEvent; +import org.apache.logging.log4j.core.appender.AbstractAppender; + +import java.io.Serializable; + +public class DeprecationIndexerAppender extends AbstractAppender { + + protected DeprecationIndexerAppender(String name, Filter filter, Layout layout) { + super(name, filter, layout); + } + + @Override + public void append(LogEvent event) { + + } +} diff --git a/server/src/main/java/org/elasticsearch/common/logging/DeprecationLayout.java b/server/src/main/java/org/elasticsearch/common/logging/DeprecationLayout.java new file mode 100644 index 0000000000000..d26e80f8b0d5f --- /dev/null +++ b/server/src/main/java/org/elasticsearch/common/logging/DeprecationLayout.java @@ -0,0 +1,4 @@ +package org.elasticsearch.common.logging; + +public class DeprecationLayout { +} diff --git a/server/src/main/java/org/elasticsearch/common/logging/DeprecationLogger.java b/server/src/main/java/org/elasticsearch/common/logging/DeprecationLogger.java index b7716a9b91a8d..4d87d6bf88c3c 100644 --- a/server/src/main/java/org/elasticsearch/common/logging/DeprecationLogger.java +++ b/server/src/main/java/org/elasticsearch/common/logging/DeprecationLogger.java @@ -37,6 +37,7 @@ * @see ThrottlingAndHeaderWarningLogger for throttling and header warnings implementation details */ public class DeprecationLogger { + private final Logger deprecationIndexer = LogManager.getLogger("deprecation_indexer"); private final ThrottlingAndHeaderWarningLogger deprecationLogger; /** @@ -47,6 +48,7 @@ public class DeprecationLogger { */ public DeprecationLogger(Logger parentLogger) { deprecationLogger = new ThrottlingAndHeaderWarningLogger(deprecatedLoggerName(parentLogger)); + } private static Logger deprecatedLoggerName(Logger parentLogger) { @@ -83,6 +85,7 @@ public DeprecationLoggerBuilder withDeprecation(String key, String msg, Object[] String opaqueId = HeaderWarning.getXOpaqueId(); ESLogMessage deprecationMessage = DeprecatedMessage.of(opaqueId, msg, params); deprecationLogger.throttleLogAndAddWarning(key, deprecationMessage); + deprecationIndexer.warn(deprecationMessage); return this; } diff --git a/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java b/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java index 3dacd94afa64e..f050674f0138b 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java +++ b/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java @@ -59,6 +59,7 @@ import org.elasticsearch.cluster.service.ClusterApplierService; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.cluster.service.MasterService; +import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.common.network.NetworkModule; import org.elasticsearch.common.network.NetworkService; @@ -196,6 +197,7 @@ public void apply(Settings value, Settings current, Settings previous) { ClusterRebalanceAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE_SETTING, ConcurrentRebalanceAllocationDecider.CLUSTER_ROUTING_ALLOCATION_CLUSTER_CONCURRENT_REBALANCE_SETTING, DanglingIndicesState.AUTO_IMPORT_DANGLING_INDICES_SETTING, +// DeprecationLogger.WRITE_DEPRECATION_LOGS_TO_INDEX, EnableAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ENABLE_SETTING, EnableAllocationDecider.CLUSTER_ROUTING_REBALANCE_ENABLE_SETTING, FilterAllocationDecider.CLUSTER_ROUTING_INCLUDE_GROUP_SETTING, diff --git a/server/src/main/java/org/elasticsearch/node/Node.java b/server/src/main/java/org/elasticsearch/node/Node.java index 9d02ecc259902..a8103c59ce81b 100644 --- a/server/src/main/java/org/elasticsearch/node/Node.java +++ b/server/src/main/java/org/elasticsearch/node/Node.java @@ -21,6 +21,8 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.core.LoggerContext; +import org.apache.logging.log4j.core.config.Configuration; import org.apache.lucene.util.Constants; import org.apache.lucene.util.SetOnce; import org.elasticsearch.Assertions; @@ -69,6 +71,7 @@ import org.elasticsearch.common.inject.ModulesBuilder; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.lease.Releasables; +import org.elasticsearch.common.logging.DeprecationAppender; import org.elasticsearch.common.logging.HeaderWarning; import org.elasticsearch.common.logging.NodeAndClusterIdStateListener; import org.elasticsearch.common.network.NetworkAddress; @@ -652,6 +655,18 @@ protected Node(final Environment initialEnvironment, actionModule.initRestHandlers(() -> clusterService.state().nodes()); logger.info("initialized"); + final LoggerContext ctx = (LoggerContext) LogManager.getContext(false); + final Configuration config = ctx.getConfiguration(); + DeprecationAppender deprecation_appender = config.getAppender("deprecation_indexer"); + deprecation_appender.start(client); + + resourcesToClose.add(() -> deprecation_appender.stop()); + +// Logger rootLogger = LogManager.getRootLogger(); +// final DeprecationAppender deprecationAppender = new DeprecationAppender(clusterService, client); +// DeprecationLogger.setIndexer(deprecationAppender); +// resourcesToClose.add(() -> DeprecationLogger.removeIndexer(deprecationAppender)); + success = true; } catch (IOException ex) { throw new ElasticsearchException("failed to bind service", ex); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/SystemPrivilege.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/SystemPrivilege.java index e06e7226de04c..e9c344418d2d4 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/SystemPrivilege.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/SystemPrivilege.java @@ -21,6 +21,8 @@ public final class SystemPrivilege extends Privilege { private static final Predicate ALLOWED_ACTIONS = Automatons.predicate( "internal:*", + "indices:*", // DIRTY GREAT HACK + "indices:monitor/*", // added for monitoring "cluster:monitor/*", // added for monitoring "cluster:admin/bootstrap/*", // for the bootstrap service