From ee4fba2628b06ce51224df920e892b9380c07638 Mon Sep 17 00:00:00 2001 From: Przemyslaw Gomulka Date: Wed, 27 May 2020 10:28:09 +0200 Subject: [PATCH 1/3] rory's impl --- .../common/logging/DeprecationIndexer.java | 164 ++++++++++++++++++ .../common/logging/DeprecationLogger.java | 130 ++++++++++++++ .../common/settings/ClusterSettings.java | 2 + .../java/org/elasticsearch/node/Node.java | 5 + .../authz/privilege/SystemPrivilege.java | 2 + 5 files changed, 303 insertions(+) create mode 100644 server/src/main/java/org/elasticsearch/common/logging/DeprecationIndexer.java diff --git a/server/src/main/java/org/elasticsearch/common/logging/DeprecationIndexer.java b/server/src/main/java/org/elasticsearch/common/logging/DeprecationIndexer.java new file mode 100644 index 0000000000000..50c0979a3fc8d --- /dev/null +++ b/server/src/main/java/org/elasticsearch/common/logging/DeprecationIndexer.java @@ -0,0 +1,164 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.common.logging; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.DocWriteRequest; +import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest; +import org.elasticsearch.action.index.IndexAction; +import org.elasticsearch.action.index.IndexRequestBuilder; +import org.elasticsearch.action.index.IndexResponse; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.cluster.ClusterChangedEvent; +import org.elasticsearch.cluster.ClusterStateListener; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentType; + +import java.io.IOException; +import java.time.Instant; +import java.time.LocalDate; +import java.time.format.DateTimeFormatter; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.elasticsearch.common.Strings.isNullOrEmpty; + +public class DeprecationIndexer implements ClusterStateListener { + private static final Logger logger = LogManager.getLogger(DeprecationIndexer.class); + + private static final String TEMPLATE_NAME = ".deprecation_logs"; + + private final ClusterService clusterService; + private final NodeClient nodeClient; + + private boolean isTemplateCreated = false; + + public DeprecationIndexer(ClusterService clusterService, NodeClient nodeClient) { + this.clusterService = clusterService; + this.nodeClient = nodeClient; + + clusterService.addListener(this); + } + + @Override + public void clusterChanged(ClusterChangedEvent event) { + ensureIndexTemplate(); + clusterService.removeListener(this); + } + + private void ensureIndexTemplate() { + PutIndexTemplateRequest putRequest = new PutIndexTemplateRequest(TEMPLATE_NAME); + putRequest.patterns(List.of(TEMPLATE_NAME + ".*")); + putRequest.create(true); + putRequest.settings(Map.of("number_of_shards", 1)); + + try { + XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); + builder.startObject() + .startObject("properties") + .field("@timestamp", Map.of("type", "date")) + .field("message", Map.of("type", "text")) + .field("keys", Map.of("type", "keyword")) + .field("x-opaque-id", Map.of("type", "keyword")) + .field("params", Map.of("type", "keyword")) + .endObject() + .endObject(); + putRequest.mapping(builder); + } catch (IOException e) { + throw new RuntimeException(e); + } + + nodeClient.admin().indices().putTemplate(putRequest, new ActionListener<>() { + @Override + public void onResponse(AcknowledgedResponse acknowledgedResponse) { + isTemplateCreated = true; + } + + @Override + public void onFailure(Exception e) { + if (e instanceof IllegalArgumentException && e.getMessage().contains("already exists")) { + // Template already exists, that's OK + isTemplateCreated = true; + } else { + logger.error("Failed to create index template [" + TEMPLATE_NAME + "]", e); + } + } + }); + } + + /** + * Records a deprecation message to the `.deprecations` index. + * + * @param key the key that was used to determine if this deprecation should have been be logged. + * This is potentially useful when aggregating the recorded messages. + * @param message the message to log + * @param xOpaqueId the associated "X-Opaque-ID" header value, if any + * @param params parameters to the message, if any + */ + public void indexDeprecationMessage(String key, String message, String xOpaqueId, Object[] params) { + if (isTemplateCreated == false) { + return; + } + + Map payload = new HashMap<>(); + + // ECS fields + payload.put("@timestamp", Instant.now().toString()); + payload.put("message", message); + if (isNullOrEmpty(key) == false) { + payload.put("tags", key); + } + + // Other fields + if (isNullOrEmpty(xOpaqueId) == false) { + // I considered putting this under labels.x-opaque-id, per ECS, + // but wondered if that was a stretch? Also it may have high + // cardinality, meaning that describing it as a label might + // be a stretch. + payload.put("x-opaque-id", xOpaqueId); + } + if (params != null && params.length > 0) { + payload.put("params", params); + } + + final String indexName = TEMPLATE_NAME + "." + DateTimeFormatter.ISO_LOCAL_DATE.format(LocalDate.now()); + + new IndexRequestBuilder(nodeClient, IndexAction.INSTANCE).setIndex(indexName) + .setOpType(DocWriteRequest.OpType.CREATE) + .setSource(payload) + .execute(new ActionListener<>() { + @Override + public void onResponse(IndexResponse indexResponse) { + // Nothing to do + } + + @Override + public void onFailure(Exception e) { + logger.error("Failed to index deprecation message", e); + } + }); + } +} diff --git a/server/src/main/java/org/elasticsearch/common/logging/DeprecationLogger.java b/server/src/main/java/org/elasticsearch/common/logging/DeprecationLogger.java index 080e2f03929c0..99c5b95db664f 100644 --- a/server/src/main/java/org/elasticsearch/common/logging/DeprecationLogger.java +++ b/server/src/main/java/org/elasticsearch/common/logging/DeprecationLogger.java @@ -23,7 +23,13 @@ import org.apache.logging.log4j.Logger; import org.elasticsearch.Build; import org.elasticsearch.Version; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse; +import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.client.node.NodeClient; import org.elasticsearch.common.SuppressLoggerChecks; +import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.tasks.Task; @@ -33,8 +39,10 @@ import java.security.PrivilegedAction; import java.util.BitSet; import java.util.Collections; +import java.util.HashMap; import java.util.Iterator; import java.util.LinkedHashMap; +import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Objects; @@ -47,6 +55,18 @@ * A logger that logs deprecation notices. */ public class DeprecationLogger { + private static final Logger classLogger = LogManager.getLogger(DeprecationLogger.class); + + /** + * When enabled, write deprecation messages to the `.deprecations` index. + */ + public static final Setting WRITE_DEPRECATION_LOGS_TO_INDEX = Setting.boolSetting( + "cluster.deprecation_logs.write_to_index", + true, + Setting.Property.NodeScope, + Setting.Property.Dynamic + ); + private static DeprecationIndexer deprecationIndexer; private final Logger logger; @@ -61,6 +81,16 @@ public class DeprecationLogger { */ private static final CopyOnWriteArraySet THREAD_CONTEXT = new CopyOnWriteArraySet<>(); + /** + * This is set once by the {@code Node} constructor, but it uses {@link CopyOnWriteArraySet} to ensure that tests can run in parallel. + *

+ * Integration tests will create separate nodes within the same classloader, thus leading to a shared, {@code static} state. + * In order for all tests to appropriately be handled, this must be able to remember all {@link NodeClient}s that it is + * given in a thread safe manner. + *

+ * For actual usage, multiple nodes do not share the same JVM and therefore this will only be set once in practice. + */ + private static final CopyOnWriteArraySet INDEXERS = new CopyOnWriteArraySet<>(); /** * Set the {@link ThreadContext} used to add deprecation headers to network responses. *

@@ -118,7 +148,39 @@ protected boolean removeEldestEntry(final Map.Entry eldest) { return size() > 128; } })); + /** + * Set the {@link NodeClient} used to index deprecation logs. + *

+ * This is expected to only be invoked by the {@code Node}'s constructor (therefore once outside of tests). + * + * @param indexer The node client owned by the {@code Node} + * @throws IllegalStateException if this {@code client} has already been set + */ + public static void setIndexer(DeprecationIndexer indexer) { + Objects.requireNonNull(indexer, "Cannot register a null DeprecationIndexer"); + + // add returning false means it _did_ have it already + if (INDEXERS.add(indexer) == false) { + throw new IllegalStateException("Double-setting DeprecationIndexer not allowed!"); + } + } + + /** + * Remove the {@link DeprecationIndexer} used to index deprecation logs. + *

+ * This is expected to only be invoked by the {@code Node}'s {@code close} method (therefore once outside of tests). + * + * @param indexer The indexer owned by the {@code Node} + * @throws IllegalStateException if this {@code indexer} is unknown (and presumably already unset before) + */ + public static void removeIndexer(DeprecationIndexer indexer) { + assert indexer != null; + // remove returning false means it did not have it already + if (INDEXERS.remove(indexer) == false) { + throw new IllegalStateException("Removing unknown DeprecationIndexer not allowed!"); + } + } /** * Adds a formatted warning message as a response header on the thread context, and logs a deprecation message if the associated key has * not recently been seen. @@ -259,9 +321,28 @@ public Void run() { return null; } }); + indexDeprecationMessage(key, message, params); } } + /** + * Records a deprecation message to the `.deprecations` index. + * + * @param key the key that was used to determine if this deprecation should have been be logged. + * This is potentially useful when aggregating the recorded messages. + * @param message the message to log + * @param params parameters to the message, if any + */ + private void indexDeprecationMessage(String key, String message, Object[] params) { + final Iterator iterator = INDEXERS.iterator(); + + if (iterator.hasNext() == false) { + return; + } + + iterator.next().indexDeprecationMessage(key, message, getXOpaqueId(THREAD_CONTEXT), params); + } + public String getXOpaqueId(Set threadContexts) { return threadContexts.stream() .filter(t -> t.getHeader(Task.X_OPAQUE_ID) != null) @@ -413,5 +494,54 @@ private static char hex(int b) { return ch; } } + private static void ensureIndexTemplate(NodeClient client, Runnable onCreate) { + client.admin().indices().prepareGetTemplates(".deprecation_logs").execute(new ActionListener<>() { + @Override + public void onResponse(GetIndexTemplatesResponse getIndexTemplatesResponse) { + if (getIndexTemplatesResponse.getIndexTemplates().isEmpty() == false) { + // Template already exists + onCreate.run(); + return; + } + + PutIndexTemplateRequest putRequest = new PutIndexTemplateRequest(".deprecation_logs"); + putRequest.patterns(List.of(".deprecation_logs.*")); + putRequest.create(false); + putRequest.settings(Map.of("number_of_shards", 1)); + + Map _source = Map.of("enabled", false); + + Map properties = new HashMap<>(); + properties.put("@timestamp", "date"); + properties.put("message", "text"); + properties.put("tags", "keyword"); + properties.put("x-opaque-id", "keyword"); + // Do we even want to index params? + properties.put("params", "keyword"); + + Map mappings = new HashMap<>(); + mappings.put("_source", _source); + mappings.put("properties", properties); + + putRequest.source(mappings); + + client.admin().indices().putTemplate(putRequest, new ActionListener<>() { + @Override + public void onResponse(AcknowledgedResponse acknowledgedResponse) { + onCreate.run(); + } + + @Override + public void onFailure(Exception e) { + classLogger.error("Failed to create index template [.deprecation_logs]", e); + } + }); + } + @Override + public void onFailure(Exception e) { + classLogger.error("Failed to ensure that the index template [.deprecation_logs] exists", e); + } + }); + } } diff --git a/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java b/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java index dd6e5a4b12a25..047ee1432efa8 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java +++ b/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java @@ -59,6 +59,7 @@ import org.elasticsearch.cluster.service.ClusterApplierService; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.cluster.service.MasterService; +import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.common.network.NetworkModule; import org.elasticsearch.common.network.NetworkService; @@ -191,6 +192,7 @@ public void apply(Settings value, Settings current, Settings previous) { ClusterRebalanceAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE_SETTING, ConcurrentRebalanceAllocationDecider.CLUSTER_ROUTING_ALLOCATION_CLUSTER_CONCURRENT_REBALANCE_SETTING, DanglingIndicesState.AUTO_IMPORT_DANGLING_INDICES_SETTING, + DeprecationLogger.WRITE_DEPRECATION_LOGS_TO_INDEX, EnableAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ENABLE_SETTING, EnableAllocationDecider.CLUSTER_ROUTING_REBALANCE_ENABLE_SETTING, FilterAllocationDecider.CLUSTER_ROUTING_INCLUDE_GROUP_SETTING, diff --git a/server/src/main/java/org/elasticsearch/node/Node.java b/server/src/main/java/org/elasticsearch/node/Node.java index dd33a2bb6b789..1ab182b7ac1a0 100644 --- a/server/src/main/java/org/elasticsearch/node/Node.java +++ b/server/src/main/java/org/elasticsearch/node/Node.java @@ -69,6 +69,7 @@ import org.elasticsearch.common.inject.ModulesBuilder; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.lease.Releasables; +import org.elasticsearch.common.logging.DeprecationIndexer; import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.logging.NodeAndClusterIdStateListener; import org.elasticsearch.common.network.NetworkAddress; @@ -635,6 +636,10 @@ protected Node(final Environment initialEnvironment, actionModule.initRestHandlers(() -> clusterService.state().nodes()); logger.info("initialized"); + final DeprecationIndexer deprecationIndexer = new DeprecationIndexer(clusterService, client); + DeprecationLogger.setIndexer(deprecationIndexer); + resourcesToClose.add(() -> DeprecationLogger.removeIndexer(deprecationIndexer)); + success = true; } catch (IOException ex) { throw new ElasticsearchException("failed to bind service", ex); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/SystemPrivilege.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/SystemPrivilege.java index e06e7226de04c..e9c344418d2d4 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/SystemPrivilege.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/SystemPrivilege.java @@ -21,6 +21,8 @@ public final class SystemPrivilege extends Privilege { private static final Predicate ALLOWED_ACTIONS = Automatons.predicate( "internal:*", + "indices:*", // DIRTY GREAT HACK + "indices:monitor/*", // added for monitoring "cluster:monitor/*", // added for monitoring "cluster:admin/bootstrap/*", // for the bootstrap service From 139b6b4d29d844aac54c9e099e6b20143232c28f Mon Sep 17 00:00:00 2001 From: Przemyslaw Gomulka Date: Thu, 4 Jun 2020 09:53:34 +0200 Subject: [PATCH 2/3] appender --- distribution/src/config/log4j2.properties | 5 + ...nIndexer.java => DeprecationAppender.java} | 125 +++++++++--------- .../logging/DeprecationIndexerAppender.java | 39 ++++++ .../common/logging/DeprecationLayout.java | 4 + .../common/logging/DeprecationLogger.java | 14 +- .../java/org/elasticsearch/node/Node.java | 19 ++- 6 files changed, 135 insertions(+), 71 deletions(-) rename server/src/main/java/org/elasticsearch/common/logging/{DeprecationIndexer.java => DeprecationAppender.java} (59%) create mode 100644 server/src/main/java/org/elasticsearch/common/logging/DeprecationIndexerAppender.java create mode 100644 server/src/main/java/org/elasticsearch/common/logging/DeprecationLayout.java diff --git a/distribution/src/config/log4j2.properties b/distribution/src/config/log4j2.properties index 2aac6f58dc4e6..800b68fb4332f 100644 --- a/distribution/src/config/log4j2.properties +++ b/distribution/src/config/log4j2.properties @@ -72,9 +72,14 @@ appender.deprecation_rolling.strategy.type = DefaultRolloverStrategy appender.deprecation_rolling.strategy.max = 4 ################################################# +appender.deprecation_indexer.type = DeprecationAppender +appender.deprecation_indexer.name = deprecation_indexer +appender.deprecation_indexer.layout = ECSJsonLayout + logger.deprecation.name = org.elasticsearch.deprecation logger.deprecation.level = warn logger.deprecation.appenderRef.deprecation_rolling.ref = deprecation_rolling +logger.deprecation.appenderRef.deprecation_indexer.ref = deprecation_indexer logger.deprecation.additivity = false ######## Search slowlog JSON #################### diff --git a/server/src/main/java/org/elasticsearch/common/logging/DeprecationIndexer.java b/server/src/main/java/org/elasticsearch/common/logging/DeprecationAppender.java similarity index 59% rename from server/src/main/java/org/elasticsearch/common/logging/DeprecationIndexer.java rename to server/src/main/java/org/elasticsearch/common/logging/DeprecationAppender.java index 50c0979a3fc8d..cad47da72abc0 100644 --- a/server/src/main/java/org/elasticsearch/common/logging/DeprecationIndexer.java +++ b/server/src/main/java/org/elasticsearch/common/logging/DeprecationAppender.java @@ -21,94 +21,78 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.core.Appender; +import org.apache.logging.log4j.core.Core; +import org.apache.logging.log4j.core.Filter; +import org.apache.logging.log4j.core.Layout; +import org.apache.logging.log4j.core.LogEvent; +import org.apache.logging.log4j.core.appender.AbstractAppender; +import org.apache.logging.log4j.core.config.plugins.Plugin; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.DocWriteRequest; -import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest; import org.elasticsearch.action.index.IndexAction; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.index.IndexResponse; -import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.client.node.NodeClient; -import org.elasticsearch.cluster.ClusterChangedEvent; -import org.elasticsearch.cluster.ClusterStateListener; import org.elasticsearch.cluster.service.ClusterService; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentType; -import java.io.IOException; +import java.io.Serializable; import java.time.Instant; import java.time.LocalDate; import java.time.format.DateTimeFormatter; import java.util.HashMap; -import java.util.List; import java.util.Map; import static org.elasticsearch.common.Strings.isNullOrEmpty; -public class DeprecationIndexer implements ClusterStateListener { - private static final Logger logger = LogManager.getLogger(DeprecationIndexer.class); +@Plugin(name = DeprecationAppender.NAME, category = Core.CATEGORY_NAME, elementType = Appender.ELEMENT_TYPE, printObject = true) +public class DeprecationAppender extends AbstractAppender{ + public static final String NAME = "deprecation_indexer"; + private static final Logger logger = LogManager.getLogger(DeprecationAppender.class); private static final String TEMPLATE_NAME = ".deprecation_logs"; - private final ClusterService clusterService; - private final NodeClient nodeClient; +// private final ClusterService clusterService; +// private final NodeClient nodeClient; private boolean isTemplateCreated = false; + private NodeClient nodeClient; - public DeprecationIndexer(ClusterService clusterService, NodeClient nodeClient) { - this.clusterService = clusterService; - this.nodeClient = nodeClient; - - clusterService.addListener(this); + public DeprecationAppender(String name, Filter filter, Layout layout) { + super(name, filter, layout); } @Override - public void clusterChanged(ClusterChangedEvent event) { - ensureIndexTemplate(); - clusterService.removeListener(this); + public void start() { + this.setStarting(); + if (getFilter() != null) { + getFilter().start(); + } +// this.setStarted(); } - private void ensureIndexTemplate() { - PutIndexTemplateRequest putRequest = new PutIndexTemplateRequest(TEMPLATE_NAME); - putRequest.patterns(List.of(TEMPLATE_NAME + ".*")); - putRequest.create(true); - putRequest.settings(Map.of("number_of_shards", 1)); - - try { - XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); - builder.startObject() - .startObject("properties") - .field("@timestamp", Map.of("type", "date")) - .field("message", Map.of("type", "text")) - .field("keys", Map.of("type", "keyword")) - .field("x-opaque-id", Map.of("type", "keyword")) - .field("params", Map.of("type", "keyword")) - .endObject() - .endObject(); - putRequest.mapping(builder); - } catch (IOException e) { - throw new RuntimeException(e); - } + public void start(NodeClient client) { + this.nodeClient = client; + setStarted();//sets volatile variable + } - nodeClient.admin().indices().putTemplate(putRequest, new ActionListener<>() { - @Override - public void onResponse(AcknowledgedResponse acknowledgedResponse) { - isTemplateCreated = true; - } - - @Override - public void onFailure(Exception e) { - if (e instanceof IllegalArgumentException && e.getMessage().contains("already exists")) { - // Template already exists, that's OK - isTemplateCreated = true; - } else { - logger.error("Failed to create index template [" + TEMPLATE_NAME + "]", e); - } - } - }); + public static class Builder> extends AbstractAppender.Builder + implements org.apache.logging.log4j.core.util.Builder { + + @Override + public DeprecationAppender build() { + return new DeprecationAppender(getName(),getFilter(),getLayout()); + } } +// public DeprecationAppender(/*ClusterService clusterService, NodeClient nodeClient*/) { +// this.clusterService = clusterService; +// this.nodeClient = nodeClient; +// +// } + + /** * Records a deprecation message to the `.deprecations` index. * @@ -119,9 +103,9 @@ public void onFailure(Exception e) { * @param params parameters to the message, if any */ public void indexDeprecationMessage(String key, String message, String xOpaqueId, Object[] params) { - if (isTemplateCreated == false) { - return; - } +// if (isTemplateCreated == false) { +// return; +// } Map payload = new HashMap<>(); @@ -161,4 +145,25 @@ public void onFailure(Exception e) { } }); } + + @Override + public void append(LogEvent event) { + String payload = event.getMessage().getFormattedMessage(); + final String indexName = TEMPLATE_NAME + "." + DateTimeFormatter.ISO_LOCAL_DATE.format(LocalDate.now()); + + new IndexRequestBuilder(nodeClient, IndexAction.INSTANCE).setIndex(indexName) + .setOpType(DocWriteRequest.OpType.CREATE) + .setSource(payload, XContentType.JSON) + .execute(new ActionListener<>() { + @Override + public void onResponse(IndexResponse indexResponse) { + // Nothing to do + } + + @Override + public void onFailure(Exception e) { + logger.error("Failed to index deprecation message", e); + } + }); + } } diff --git a/server/src/main/java/org/elasticsearch/common/logging/DeprecationIndexerAppender.java b/server/src/main/java/org/elasticsearch/common/logging/DeprecationIndexerAppender.java new file mode 100644 index 0000000000000..634618cfbe573 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/common/logging/DeprecationIndexerAppender.java @@ -0,0 +1,39 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.common.logging; + +import org.apache.logging.log4j.core.Filter; +import org.apache.logging.log4j.core.Layout; +import org.apache.logging.log4j.core.LogEvent; +import org.apache.logging.log4j.core.appender.AbstractAppender; + +import java.io.Serializable; + +public class DeprecationIndexerAppender extends AbstractAppender { + + protected DeprecationIndexerAppender(String name, Filter filter, Layout layout) { + super(name, filter, layout); + } + + @Override + public void append(LogEvent event) { + + } +} diff --git a/server/src/main/java/org/elasticsearch/common/logging/DeprecationLayout.java b/server/src/main/java/org/elasticsearch/common/logging/DeprecationLayout.java new file mode 100644 index 0000000000000..d26e80f8b0d5f --- /dev/null +++ b/server/src/main/java/org/elasticsearch/common/logging/DeprecationLayout.java @@ -0,0 +1,4 @@ +package org.elasticsearch.common.logging; + +public class DeprecationLayout { +} diff --git a/server/src/main/java/org/elasticsearch/common/logging/DeprecationLogger.java b/server/src/main/java/org/elasticsearch/common/logging/DeprecationLogger.java index 99c5b95db664f..ec94c422ddb77 100644 --- a/server/src/main/java/org/elasticsearch/common/logging/DeprecationLogger.java +++ b/server/src/main/java/org/elasticsearch/common/logging/DeprecationLogger.java @@ -66,7 +66,7 @@ public class DeprecationLogger { Setting.Property.NodeScope, Setting.Property.Dynamic ); - private static DeprecationIndexer deprecationIndexer; + private static DeprecationAppender deprecationAppender; private final Logger logger; @@ -90,7 +90,7 @@ public class DeprecationLogger { *

* For actual usage, multiple nodes do not share the same JVM and therefore this will only be set once in practice. */ - private static final CopyOnWriteArraySet INDEXERS = new CopyOnWriteArraySet<>(); + private static final CopyOnWriteArraySet INDEXERS = new CopyOnWriteArraySet<>(); /** * Set the {@link ThreadContext} used to add deprecation headers to network responses. *

@@ -156,7 +156,7 @@ protected boolean removeEldestEntry(final Map.Entry eldest) { * @param indexer The node client owned by the {@code Node} * @throws IllegalStateException if this {@code client} has already been set */ - public static void setIndexer(DeprecationIndexer indexer) { + public static void setIndexer(DeprecationAppender indexer) { Objects.requireNonNull(indexer, "Cannot register a null DeprecationIndexer"); // add returning false means it _did_ have it already @@ -166,14 +166,14 @@ public static void setIndexer(DeprecationIndexer indexer) { } /** - * Remove the {@link DeprecationIndexer} used to index deprecation logs. + * Remove the {@link DeprecationAppender} used to index deprecation logs. *

* This is expected to only be invoked by the {@code Node}'s {@code close} method (therefore once outside of tests). * * @param indexer The indexer owned by the {@code Node} * @throws IllegalStateException if this {@code indexer} is unknown (and presumably already unset before) */ - public static void removeIndexer(DeprecationIndexer indexer) { + public static void removeIndexer(DeprecationAppender indexer) { assert indexer != null; // remove returning false means it did not have it already @@ -321,7 +321,7 @@ public Void run() { return null; } }); - indexDeprecationMessage(key, message, params); +// indexDeprecationMessage(key, message, params); } } @@ -334,7 +334,7 @@ public Void run() { * @param params parameters to the message, if any */ private void indexDeprecationMessage(String key, String message, Object[] params) { - final Iterator iterator = INDEXERS.iterator(); + final Iterator iterator = INDEXERS.iterator(); if (iterator.hasNext() == false) { return; diff --git a/server/src/main/java/org/elasticsearch/node/Node.java b/server/src/main/java/org/elasticsearch/node/Node.java index 1ab182b7ac1a0..1858593870bc4 100644 --- a/server/src/main/java/org/elasticsearch/node/Node.java +++ b/server/src/main/java/org/elasticsearch/node/Node.java @@ -21,6 +21,9 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.core.Appender; +import org.apache.logging.log4j.core.LoggerContext; +import org.apache.logging.log4j.core.config.Configuration; import org.apache.lucene.util.Constants; import org.apache.lucene.util.SetOnce; import org.elasticsearch.Assertions; @@ -69,7 +72,7 @@ import org.elasticsearch.common.inject.ModulesBuilder; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.lease.Releasables; -import org.elasticsearch.common.logging.DeprecationIndexer; +import org.elasticsearch.common.logging.DeprecationAppender; import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.logging.NodeAndClusterIdStateListener; import org.elasticsearch.common.network.NetworkAddress; @@ -636,9 +639,17 @@ protected Node(final Environment initialEnvironment, actionModule.initRestHandlers(() -> clusterService.state().nodes()); logger.info("initialized"); - final DeprecationIndexer deprecationIndexer = new DeprecationIndexer(clusterService, client); - DeprecationLogger.setIndexer(deprecationIndexer); - resourcesToClose.add(() -> DeprecationLogger.removeIndexer(deprecationIndexer)); + final LoggerContext ctx = (LoggerContext) LogManager.getContext(false); + final Configuration config = ctx.getConfiguration(); + DeprecationAppender deprecation_appender = (DeprecationAppender)config.getAppender(DeprecationAppender.NAME); + deprecation_appender.start(client); + + resourcesToClose.add(() -> deprecation_appender.stop()); + +// Logger rootLogger = LogManager.getRootLogger(); +// final DeprecationAppender deprecationAppender = new DeprecationAppender(clusterService, client); +// DeprecationLogger.setIndexer(deprecationAppender); +// resourcesToClose.add(() -> DeprecationLogger.removeIndexer(deprecationAppender)); success = true; } catch (IOException ex) { From 674e82adf8416d0d6981381a073b61700d9cda2b Mon Sep 17 00:00:00 2001 From: Przemyslaw Gomulka Date: Thu, 4 Jun 2020 13:58:42 +0200 Subject: [PATCH 3/3] draft2 --- distribution/src/config/log4j2.properties | 14 ++-- .../common/logging/DeprecationAppender.java | 70 +++---------------- .../common/logging/DeprecationLogger.java | 3 + .../common/settings/ClusterSettings.java | 2 +- .../java/org/elasticsearch/node/Node.java | 5 +- 5 files changed, 23 insertions(+), 71 deletions(-) diff --git a/distribution/src/config/log4j2.properties b/distribution/src/config/log4j2.properties index 800b68fb4332f..d774635b56f03 100644 --- a/distribution/src/config/log4j2.properties +++ b/distribution/src/config/log4j2.properties @@ -72,15 +72,21 @@ appender.deprecation_rolling.strategy.type = DefaultRolloverStrategy appender.deprecation_rolling.strategy.max = 4 ################################################# -appender.deprecation_indexer.type = DeprecationAppender -appender.deprecation_indexer.name = deprecation_indexer -appender.deprecation_indexer.layout = ECSJsonLayout logger.deprecation.name = org.elasticsearch.deprecation logger.deprecation.level = warn logger.deprecation.appenderRef.deprecation_rolling.ref = deprecation_rolling -logger.deprecation.appenderRef.deprecation_indexer.ref = deprecation_indexer logger.deprecation.additivity = false +############ +appender.deprecation_indexer.type = DeprecationIndexer +appender.deprecation_indexer.name = deprecation_indexer +appender.deprecation_indexer.layout.type = ECSJsonLayout +appender.deprecation_indexer.layout.type_name = deprecation_indexer + +logger.deprecation_indexer.name = deprecation_indexer +logger.deprecation_indexer.level = warn +logger.deprecation_indexer.appenderRef.deprecation_indexer.ref = deprecation_indexer +logger.deprecation_indexer.additivity = false ######## Search slowlog JSON #################### appender.index_search_slowlog_rolling.type = RollingFile diff --git a/server/src/main/java/org/elasticsearch/common/logging/DeprecationAppender.java b/server/src/main/java/org/elasticsearch/common/logging/DeprecationAppender.java index cad47da72abc0..dc338ba11e76c 100644 --- a/server/src/main/java/org/elasticsearch/common/logging/DeprecationAppender.java +++ b/server/src/main/java/org/elasticsearch/common/logging/DeprecationAppender.java @@ -27,14 +27,15 @@ import org.apache.logging.log4j.core.Layout; import org.apache.logging.log4j.core.LogEvent; import org.apache.logging.log4j.core.appender.AbstractAppender; +import org.apache.logging.log4j.core.appender.FileAppender; import org.apache.logging.log4j.core.config.plugins.Plugin; +import org.apache.logging.log4j.core.config.plugins.PluginBuilderFactory; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.DocWriteRequest; import org.elasticsearch.action.index.IndexAction; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.index.IndexResponse; import org.elasticsearch.client.node.NodeClient; -import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.xcontent.XContentType; import java.io.Serializable; @@ -48,7 +49,7 @@ @Plugin(name = DeprecationAppender.NAME, category = Core.CATEGORY_NAME, elementType = Appender.ELEMENT_TYPE, printObject = true) public class DeprecationAppender extends AbstractAppender{ - public static final String NAME = "deprecation_indexer"; + public static final String NAME = "DeprecationIndexer"; private static final Logger logger = LogManager.getLogger(DeprecationAppender.class); private static final String TEMPLATE_NAME = ".deprecation_logs"; @@ -82,73 +83,18 @@ public static class Builder> extends Ab @Override public DeprecationAppender build() { - return new DeprecationAppender(getName(),getFilter(),getLayout()); + return new DeprecationAppender(getName(),getFilter(),getOrCreateLayout()); } } -// public DeprecationAppender(/*ClusterService clusterService, NodeClient nodeClient*/) { -// this.clusterService = clusterService; -// this.nodeClient = nodeClient; -// -// } - - - /** - * Records a deprecation message to the `.deprecations` index. - * - * @param key the key that was used to determine if this deprecation should have been be logged. - * This is potentially useful when aggregating the recorded messages. - * @param message the message to log - * @param xOpaqueId the associated "X-Opaque-ID" header value, if any - * @param params parameters to the message, if any - */ - public void indexDeprecationMessage(String key, String message, String xOpaqueId, Object[] params) { -// if (isTemplateCreated == false) { -// return; -// } - - Map payload = new HashMap<>(); - - // ECS fields - payload.put("@timestamp", Instant.now().toString()); - payload.put("message", message); - if (isNullOrEmpty(key) == false) { - payload.put("tags", key); - } - - // Other fields - if (isNullOrEmpty(xOpaqueId) == false) { - // I considered putting this under labels.x-opaque-id, per ECS, - // but wondered if that was a stretch? Also it may have high - // cardinality, meaning that describing it as a label might - // be a stretch. - payload.put("x-opaque-id", xOpaqueId); - } - if (params != null && params.length > 0) { - payload.put("params", params); - } - - final String indexName = TEMPLATE_NAME + "." + DateTimeFormatter.ISO_LOCAL_DATE.format(LocalDate.now()); - - new IndexRequestBuilder(nodeClient, IndexAction.INSTANCE).setIndex(indexName) - .setOpType(DocWriteRequest.OpType.CREATE) - .setSource(payload) - .execute(new ActionListener<>() { - @Override - public void onResponse(IndexResponse indexResponse) { - // Nothing to do - } - - @Override - public void onFailure(Exception e) { - logger.error("Failed to index deprecation message", e); - } - }); + @PluginBuilderFactory + public static > B newBuilder() { + return new DeprecationAppender.Builder().asBuilder(); } @Override public void append(LogEvent event) { - String payload = event.getMessage().getFormattedMessage(); + byte[] payload = getLayout().toByteArray(event); final String indexName = TEMPLATE_NAME + "." + DateTimeFormatter.ISO_LOCAL_DATE.format(LocalDate.now()); new IndexRequestBuilder(nodeClient, IndexAction.INSTANCE).setIndex(indexName) diff --git a/server/src/main/java/org/elasticsearch/common/logging/DeprecationLogger.java b/server/src/main/java/org/elasticsearch/common/logging/DeprecationLogger.java index b7716a9b91a8d..4d87d6bf88c3c 100644 --- a/server/src/main/java/org/elasticsearch/common/logging/DeprecationLogger.java +++ b/server/src/main/java/org/elasticsearch/common/logging/DeprecationLogger.java @@ -37,6 +37,7 @@ * @see ThrottlingAndHeaderWarningLogger for throttling and header warnings implementation details */ public class DeprecationLogger { + private final Logger deprecationIndexer = LogManager.getLogger("deprecation_indexer"); private final ThrottlingAndHeaderWarningLogger deprecationLogger; /** @@ -47,6 +48,7 @@ public class DeprecationLogger { */ public DeprecationLogger(Logger parentLogger) { deprecationLogger = new ThrottlingAndHeaderWarningLogger(deprecatedLoggerName(parentLogger)); + } private static Logger deprecatedLoggerName(Logger parentLogger) { @@ -83,6 +85,7 @@ public DeprecationLoggerBuilder withDeprecation(String key, String msg, Object[] String opaqueId = HeaderWarning.getXOpaqueId(); ESLogMessage deprecationMessage = DeprecatedMessage.of(opaqueId, msg, params); deprecationLogger.throttleLogAndAddWarning(key, deprecationMessage); + deprecationIndexer.warn(deprecationMessage); return this; } diff --git a/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java b/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java index f3b932266733e..f050674f0138b 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java +++ b/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java @@ -197,7 +197,7 @@ public void apply(Settings value, Settings current, Settings previous) { ClusterRebalanceAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE_SETTING, ConcurrentRebalanceAllocationDecider.CLUSTER_ROUTING_ALLOCATION_CLUSTER_CONCURRENT_REBALANCE_SETTING, DanglingIndicesState.AUTO_IMPORT_DANGLING_INDICES_SETTING, - DeprecationLogger.WRITE_DEPRECATION_LOGS_TO_INDEX, +// DeprecationLogger.WRITE_DEPRECATION_LOGS_TO_INDEX, EnableAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ENABLE_SETTING, EnableAllocationDecider.CLUSTER_ROUTING_REBALANCE_ENABLE_SETTING, FilterAllocationDecider.CLUSTER_ROUTING_INCLUDE_GROUP_SETTING, diff --git a/server/src/main/java/org/elasticsearch/node/Node.java b/server/src/main/java/org/elasticsearch/node/Node.java index 217cb1458b259..a8103c59ce81b 100644 --- a/server/src/main/java/org/elasticsearch/node/Node.java +++ b/server/src/main/java/org/elasticsearch/node/Node.java @@ -21,7 +21,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.apache.logging.log4j.core.Appender; import org.apache.logging.log4j.core.LoggerContext; import org.apache.logging.log4j.core.config.Configuration; import org.apache.lucene.util.Constants; @@ -72,9 +71,7 @@ import org.elasticsearch.common.inject.ModulesBuilder; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.lease.Releasables; -import org.elasticsearch.common.logging.HeaderWarning; import org.elasticsearch.common.logging.DeprecationAppender; -import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.logging.HeaderWarning; import org.elasticsearch.common.logging.NodeAndClusterIdStateListener; import org.elasticsearch.common.network.NetworkAddress; @@ -660,7 +657,7 @@ protected Node(final Environment initialEnvironment, final LoggerContext ctx = (LoggerContext) LogManager.getContext(false); final Configuration config = ctx.getConfiguration(); - DeprecationAppender deprecation_appender = (DeprecationAppender)config.getAppender(DeprecationAppender.NAME); + DeprecationAppender deprecation_appender = config.getAppender("deprecation_indexer"); deprecation_appender.start(client); resourcesToClose.add(() -> deprecation_appender.stop());