From 31451e0ce75c616131b3d25dbbab4aadd55fcad8 Mon Sep 17 00:00:00 2001 From: rishabhmaurya Date: Mon, 10 Apr 2023 19:14:22 +0200 Subject: [PATCH 1/8] Otel initial changes Signed-off-by: Stevan Buzejic --- server/build.gradle | 25 ++ .../OTelContextPreservingActionListener.java | 115 +++++++ .../util/concurrent/OpenSearchExecutors.java | 3 +- .../LocalStorePeerRecoverySourceHandler.java | 6 +- .../recovery/PeerRecoverySourceService.java | 25 +- .../recovery/RecoverySourceHandler.java | 52 ++- .../RecoverySourceHandlerFactory.java | 10 +- .../RemoteStorePeerRecoverySourceHandler.java | 6 +- .../SegmentFileTransferHandler.java | 33 +- .../main/java/org/opensearch/node/Node.java | 315 ++++++++---------- .../opensearch/otel/OtelEventListener.java | 16 + .../java/org/opensearch/otel/OtelService.java | 150 +++++++++ .../java/org/opensearch/plugins/Plugin.java | 5 + .../threadpool/ScalingExecutorBuilder.java | 4 +- .../org/opensearch/threadpool/ThreadPool.java | 6 +- .../org/opensearch/bootstrap/security.policy | 6 + ...alStorePeerRecoverySourceHandlerTests.java | 11 +- .../PeerRecoverySourceServiceTests.java | 5 +- 18 files changed, 578 insertions(+), 215 deletions(-) create mode 100644 server/src/main/java/org/opensearch/action/support/OTelContextPreservingActionListener.java create mode 100644 server/src/main/java/org/opensearch/otel/OtelEventListener.java create mode 100644 server/src/main/java/org/opensearch/otel/OtelService.java diff --git a/server/build.gradle b/server/build.gradle index 474d6eb0efead..39558903e405c 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -143,6 +143,31 @@ dependencies { api "org.apache.logging.log4j:log4j-jul:${versions.log4j}" api "org.apache.logging.log4j:log4j-core:${versions.log4j}", optional + api 'io.opentelemetry:opentelemetry-api:1.23.1' + api 'io.opentelemetry:opentelemetry-api-logs:1.23.1-alpha' + api 'io.opentelemetry:opentelemetry-api-events:1.23.1-alpha' + api 'io.opentelemetry:opentelemetry-sdk:1.23.1' + api 'io.opentelemetry:opentelemetry-sdk-metrics:1.23.1' + api 'io.opentelemetry:opentelemetry-sdk-common:1.23.1' + api 'io.opentelemetry:opentelemetry-sdk-trace:1.23.1' + api 'io.opentelemetry:opentelemetry-context:1.23.1' + api 'io.opentelemetry:opentelemetry-sdk-logs:1.23.1-alpha' + api 'io.opentelemetry:opentelemetry-semconv:1.23.1-alpha' + + // exporters + api 'io.opentelemetry:opentelemetry-exporter-common:1.23.1' + //implementation 'io.opentelemetry:opentelemetry-exporter-jaeger:1.23.1' + api 'io.opentelemetry:opentelemetry-exporter-otlp:1.23.1' + api 'io.opentelemetry:opentelemetry-exporter-otlp-common:1.23.1' + api("com.squareup.okhttp3:okhttp:4.10.0") + api 'org.jetbrains.kotlin:kotlin-stdlib:1.6.20' + api 'com.squareup.okio:okio-jvm:3.0.0' + + //implementation "com.squareup.okio:okio:2.8.0" + + //implementation 'io.opentelemetry:opentelemetry-exporter-logging:1.23.1' + //implementation 'io.opentelemetry:opentelemetry-exporter-jaeger:1.23.1' + // jna api "net.java.dev.jna:jna:${versions.jna}" diff --git a/server/src/main/java/org/opensearch/action/support/OTelContextPreservingActionListener.java b/server/src/main/java/org/opensearch/action/support/OTelContextPreservingActionListener.java new file mode 100644 index 0000000000000..d250143bc1ec0 --- /dev/null +++ b/server/src/main/java/org/opensearch/action/support/OTelContextPreservingActionListener.java @@ -0,0 +1,115 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/* + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.action.support; + +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.context.Context; +import io.opentelemetry.context.Scope; +import org.opensearch.action.ActionListener; +import org.opensearch.common.util.concurrent.ThreadContext; +import org.opensearch.otel.OtelService; + +import java.util.Objects; + +import static io.opentelemetry.api.common.AttributeKey.longKey; +import static io.opentelemetry.api.common.AttributeKey.stringKey; + +/** + * Restores the given {@link ThreadContext.StoredContext} + * once the listener is invoked + * + * @opensearch.internal + */ +public final class OTelContextPreservingActionListener implements ActionListener { + + private final ActionListener delegate; + public final Context beforeAttachContext; + public final Context afterAttachContext; + + private final String spanID; + + public final long startCPUUsage; + public final long startMemoryUsage; + public final long startThreadContentionTime; + + public OTelContextPreservingActionListener(Context beforeAttachContext, Context afterAttachContext, ActionListener delegate, String spanID) { + this.delegate = delegate; + this.beforeAttachContext = beforeAttachContext; + this.afterAttachContext = afterAttachContext; + this.spanID = spanID; + startCPUUsage = OtelService.getCPUUsage(Thread.currentThread().getId()); + startMemoryUsage = OtelService.getMemoryUsage(Thread.currentThread().getId()); + startThreadContentionTime = OtelService.getThreadContentionTime(Thread.currentThread().getId()); + } + + @Override + public void onResponse(R r) { + try (Scope ignored = Objects.requireNonNull(afterAttachContext).makeCurrent()) { + Span span = Span.current(); + if (spanID != null && Span.current() != Span.getInvalid() && + span.getSpanContext().getSpanId().equals(spanID)) { + span.setAttribute(AttributeKey.longKey("CPUUsage"), + OtelService.getCPUUsage(Thread.currentThread().getId())- startCPUUsage); + span.setAttribute(AttributeKey.longKey("MemoryUsage"), + OtelService.getMemoryUsage(Thread.currentThread().getId())- startMemoryUsage); + span.setAttribute(AttributeKey.longKey("ContentionTime"), + OtelService.getThreadContentionTime(Thread.currentThread().getId())- startThreadContentionTime); + span.setAttribute(stringKey("finish-thread-name"), Thread.currentThread().getName()); + span.setAttribute(longKey("finish-thread-id"), Thread.currentThread().getId()); + Span.current().end(); + } + } + + try (Scope ignored = Objects.requireNonNull(beforeAttachContext).makeCurrent()) { + delegate.onResponse(r); + } + } + + @Override + public void onFailure(Exception e) { + try (Scope ignored = Objects.requireNonNull(afterAttachContext).makeCurrent()) { + if (spanID != null && Span.current() != Span.getInvalid() && + Span.current().getSpanContext().getSpanId().equals(spanID)) { + Span.current().end(); + } + } + try (Scope ignored = Objects.requireNonNull(beforeAttachContext).makeCurrent()) { + delegate.onFailure(e); + } + } + + @Override + public String toString() { + return getClass().getName() + "/" + delegate.toString(); + } +} diff --git a/server/src/main/java/org/opensearch/common/util/concurrent/OpenSearchExecutors.java b/server/src/main/java/org/opensearch/common/util/concurrent/OpenSearchExecutors.java index ec1024bbe5f30..14fa7d65bb1fe 100644 --- a/server/src/main/java/org/opensearch/common/util/concurrent/OpenSearchExecutors.java +++ b/server/src/main/java/org/opensearch/common/util/concurrent/OpenSearchExecutors.java @@ -32,6 +32,7 @@ package org.opensearch.common.util.concurrent; +import io.opentelemetry.context.Context; import org.opensearch.ExceptionsHelper; import org.opensearch.common.SuppressForbidden; import org.opensearch.common.logging.DeprecationLogger; @@ -352,7 +353,7 @@ public void execute(Runnable command) { * @return an {@link ExecutorService} that executes submitted tasks on the current thread */ public static ExecutorService newDirectExecutorService() { - return DIRECT_EXECUTOR_SERVICE; + return Context.taskWrapping(DIRECT_EXECUTOR_SERVICE); } public static String threadName(Settings settings, String namePrefix) { diff --git a/server/src/main/java/org/opensearch/indices/recovery/LocalStorePeerRecoverySourceHandler.java b/server/src/main/java/org/opensearch/indices/recovery/LocalStorePeerRecoverySourceHandler.java index c8d3b2d09ad28..ac8313403dad8 100644 --- a/server/src/main/java/org/opensearch/indices/recovery/LocalStorePeerRecoverySourceHandler.java +++ b/server/src/main/java/org/opensearch/indices/recovery/LocalStorePeerRecoverySourceHandler.java @@ -28,6 +28,7 @@ import org.opensearch.index.shard.IndexShard; import org.opensearch.index.translog.Translog; import org.opensearch.indices.RunUnderPrimaryPermit; +import org.opensearch.otel.OtelService; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.Transports; @@ -50,9 +51,10 @@ public LocalStorePeerRecoverySourceHandler( StartRecoveryRequest request, int fileChunkSizeInBytes, int maxConcurrentFileChunks, - int maxConcurrentOperations + int maxConcurrentOperations, + OtelService otelService ) { - super(shard, recoveryTarget, threadPool, request, fileChunkSizeInBytes, maxConcurrentFileChunks, maxConcurrentOperations); + super(shard, recoveryTarget, threadPool, request, fileChunkSizeInBytes, maxConcurrentFileChunks, maxConcurrentOperations, otelService); } @Override diff --git a/server/src/main/java/org/opensearch/indices/recovery/PeerRecoverySourceService.java b/server/src/main/java/org/opensearch/indices/recovery/PeerRecoverySourceService.java index 8bea14a1a1c86..101ebe6d574be 100644 --- a/server/src/main/java/org/opensearch/indices/recovery/PeerRecoverySourceService.java +++ b/server/src/main/java/org/opensearch/indices/recovery/PeerRecoverySourceService.java @@ -32,6 +32,7 @@ package org.opensearch.indices.recovery; +import io.opentelemetry.api.trace.Span; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.ExceptionsHelper; @@ -55,7 +56,9 @@ import org.opensearch.index.shard.IndexShard; import org.opensearch.index.shard.ShardId; import org.opensearch.indices.IndicesService; +import org.opensearch.otel.OtelService; import org.opensearch.tasks.Task; +import org.opensearch.tasks.TaskResourceTrackingService; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportChannel; import org.opensearch.transport.TransportRequestHandler; @@ -69,6 +72,10 @@ import java.util.List; import java.util.Map; +import static io.opentelemetry.api.common.AttributeKey.doubleKey; +import static io.opentelemetry.api.common.AttributeKey.longKey; +import static io.opentelemetry.api.common.AttributeKey.stringKey; + /** * The source recovery accepts recovery requests from other peer shards and start the recovery process from this * source shard to the target shard. @@ -94,12 +101,15 @@ public static class Actions { private final RecoverySettings recoverySettings; final OngoingRecoveries ongoingRecoveries = new OngoingRecoveries(); + private final OtelService otelService; @Inject - public PeerRecoverySourceService(TransportService transportService, IndicesService indicesService, RecoverySettings recoverySettings) { + public PeerRecoverySourceService(TransportService transportService, IndicesService indicesService, RecoverySettings recoverySettings, + OtelService otelService) { this.transportService = transportService; this.indicesService = indicesService; this.recoverySettings = recoverySettings; + this.otelService = otelService; // When the target node wants to start a peer recovery it sends a START_RECOVERY request to the source // node. Upon receiving START_RECOVERY, the source node will initiate the peer recovery. transportService.registerRequestHandler( @@ -157,9 +167,16 @@ public void clusterChanged(ClusterChangedEvent event) { } private void recover(StartRecoveryRequest request, ActionListener listener) { + listener = OtelService.startSpan("recover", listener); + Span span = Span.current(); + span.setAttribute(stringKey("index-name"), request.shardId().getIndexName()); + span.setAttribute(longKey("shard-id"), request.shardId().id()); + span.setAttribute(stringKey("source-node"), request.sourceNode().getId()); + span.setAttribute(stringKey("target-node"), request.targetNode().getId()); + otelService.emitResources(span, "resource-usage-start"); + final IndexService indexService = indicesService.indexServiceSafe(request.shardId().getIndex()); final IndexShard shard = indexService.getShard(request.shardId().id()); - final ShardRouting routingEntry = shard.routingEntry(); if (routingEntry.primary() == false || routingEntry.active() == false) { @@ -183,7 +200,9 @@ private void recover(StartRecoveryRequest request, ActionListener ongoingRecoveries.remove(shard, handler))); + otelService.emitResources(span, "resource-usage-end"); } private void reestablish(ReestablishRecoveryRequest request, ActionListener listener) { @@ -378,7 +397,7 @@ private Tuple createRecovery recoverySettings, throttleTime -> shard.recoveryStats().addThrottleTime(throttleTime) ); - handler = RecoverySourceHandlerFactory.create(shard, recoveryTarget, request, recoverySettings); + handler = RecoverySourceHandlerFactory.create(shard, recoveryTarget, request, recoverySettings, otelService); return Tuple.tuple(handler, recoveryTarget); } } diff --git a/server/src/main/java/org/opensearch/indices/recovery/RecoverySourceHandler.java b/server/src/main/java/org/opensearch/indices/recovery/RecoverySourceHandler.java index 052b86c901093..93d02d41fe837 100644 --- a/server/src/main/java/org/opensearch/indices/recovery/RecoverySourceHandler.java +++ b/server/src/main/java/org/opensearch/indices/recovery/RecoverySourceHandler.java @@ -32,6 +32,7 @@ package org.opensearch.indices.recovery; +import io.opentelemetry.api.trace.Span; import org.apache.logging.log4j.Logger; import org.apache.lucene.index.CorruptIndexException; import org.apache.lucene.index.IndexCommit; @@ -71,6 +72,7 @@ import org.opensearch.index.translog.Translog; import org.opensearch.indices.RunUnderPrimaryPermit; import org.opensearch.indices.replication.SegmentFileTransferHandler; +import org.opensearch.otel.OtelService; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.Transports; @@ -81,6 +83,7 @@ import java.util.Comparator; import java.util.List; import java.util.Locale; +import java.util.Objects; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; @@ -89,6 +92,9 @@ import java.util.function.IntSupplier; import java.util.stream.StreamSupport; +import static io.opentelemetry.api.common.AttributeKey.longKey; +import static io.opentelemetry.api.common.AttributeKey.stringKey; + /** * RecoverySourceHandler handles the three phases of shard recovery, which is * everything relating to copying the segment files as well as sending translog @@ -119,6 +125,7 @@ public abstract class RecoverySourceHandler { protected final ListenableFuture future = new ListenableFuture<>(); public static final String PEER_RECOVERY_NAME = "peer-recovery"; private final SegmentFileTransferHandler transferHandler; + private final OtelService otelService; RecoverySourceHandler( IndexShard shard, @@ -127,7 +134,8 @@ public abstract class RecoverySourceHandler { StartRecoveryRequest request, int fileChunkSizeInBytes, int maxConcurrentFileChunks, - int maxConcurrentOperations + int maxConcurrentOperations, + OtelService otelService ) { this.logger = Loggers.getLogger(RecoverySourceHandler.class, request.shardId(), "recover to " + request.targetNode().getName()); this.transferHandler = new SegmentFileTransferHandler( @@ -138,7 +146,8 @@ public abstract class RecoverySourceHandler { threadPool, cancellableThreads, fileChunkSizeInBytes, - maxConcurrentFileChunks + maxConcurrentFileChunks, + otelService ); this.shard = shard; this.threadPool = threadPool; @@ -148,6 +157,7 @@ public abstract class RecoverySourceHandler { this.chunkSizeInBytes = fileChunkSizeInBytes; // if the target is on an old version, it won't be able to handle out-of-order file chunks. this.maxConcurrentOperations = maxConcurrentOperations; + this.otelService = otelService; } public StartRecoveryRequest getRequest() { @@ -191,6 +201,7 @@ public void recoverToTarget(ActionListener listener) { protected abstract void innerRecoveryToTarget(ActionListener listener, Consumer onFailure) throws IOException; + protected void finalizeStepAndCompleteFuture( long startingSeqNo, StepListener sendSnapshotStep, @@ -431,18 +442,19 @@ void phase1( phase1ExistingFileNames.size(), new ByteSizeValue(existingTotalSizeInBytes) ); - final StepListener sendFileInfoStep = new StepListener<>(); + StepListener sendFileInfoStep = new StepListener<>(); final StepListener sendFilesStep = new StepListener<>(); final StepListener createRetentionLeaseStep = new StepListener<>(); final StepListener cleanFilesStep = new StepListener<>(); cancellableThreads.checkForCancel(); + ActionListener wrapperSendFileInfoStep = OtelService.startSpan("receiveFileInfo", sendFileInfoStep); recoveryTarget.receiveFileInfo( phase1FileNames, phase1FileSizes, phase1ExistingFileNames, phase1ExistingFileSizes, translogOps.getAsInt(), - sendFileInfoStep + wrapperSendFileInfoStep ); sendFileInfoStep.whenComplete( @@ -514,6 +526,10 @@ void phase1( } void sendFiles(Store store, StoreFileMetadata[] files, IntSupplier translogOps, ActionListener listener) { + if (otelService != null) { + listener = OtelService.startSpan("sendFiles", listener); + otelService.emitResources(Span.current(), "start-resources"); + } final MultiChunkTransfer transfer = transferHandler.createTransfer( store, files, @@ -522,9 +538,12 @@ void sendFiles(Store store, StoreFileMetadata[] files, IntSupplier translogOps, ); resources.add(transfer); transfer.start(); + otelService.emitResources(Span.current(), "end-resources"); } void createRetentionLease(final long startingSeqNo, ActionListener listener) { + listener = OtelService.startSpan("createRetentionLease", listener); + ActionListener finalListener = listener; RunUnderPrimaryPermit.run(() -> { // Clone the peer recovery retention lease belonging to the source shard. We are retaining history between the the local // checkpoint of the safe commit we're creating and this lease's retained seqno with the retention lock, and by cloning an @@ -541,7 +560,7 @@ void createRetentionLease(final long startingSeqNo, ActionListener(logger, shard.getThreadPool(), ThreadPool.Names.GENERIC, cloneRetentionLeaseStep, false) ); logger.trace("cloned primary's retention lease as [{}]", clonedLease); - cloneRetentionLeaseStep.whenComplete(rr -> listener.onResponse(clonedLease), listener::onFailure); + cloneRetentionLeaseStep.whenComplete(rr -> finalListener.onResponse(clonedLease), finalListener::onFailure); } catch (RetentionLeaseNotFoundException e) { // it's possible that the primary has no retention lease yet if we are doing a rolling upgrade from a version before // 7.4, and in that case we just create a lease using the local checkpoint of the safe commit which we're using for @@ -554,7 +573,7 @@ void createRetentionLease(final long startingSeqNo, ActionListener(logger, shard.getThreadPool(), ThreadPool.Names.GENERIC, addRetentionLeaseStep, false) ); - addRetentionLeaseStep.whenComplete(rr -> listener.onResponse(newLease), listener::onFailure); + addRetentionLeaseStep.whenComplete(rr -> finalListener.onResponse(newLease), finalListener::onFailure); logger.trace("created retention lease with estimated checkpoint of [{}]", estimatedGlobalCheckpoint); } }, shardId + " establishing retention lease for [" + request.targetAllocationId() + "]", shard, cancellableThreads, logger); @@ -598,13 +617,15 @@ boolean canSkipPhase1(Store.MetadataSnapshot source, Store.MetadataSnapshot targ } void prepareTargetForTranslog(int totalTranslogOps, ActionListener listener) { + listener = OtelService.startSpan("prepareForTranslog", listener); StopWatch stopWatch = new StopWatch().start(); + ActionListener finalListener = listener; final ActionListener wrappedListener = ActionListener.wrap(nullVal -> { stopWatch.stop(); final TimeValue tookTime = stopWatch.totalTime(); logger.trace("recovery [phase1]: remote engine start took [{}]", tookTime); - listener.onResponse(tookTime); - }, e -> listener.onFailure(new RecoveryEngineException(shard.shardId(), 1, "prepare target for translog failed", e))); + finalListener.onResponse(tookTime); + }, e -> finalListener.onFailure(new RecoveryEngineException(shard.shardId(), 1, "prepare target for translog failed", e))); // Send a request preparing the new shard's translog to receive operations. This ensures the shard engine is started and disables // garbage collection (not the JVM's GC!) of tombstone deletes. logger.trace("recovery [phase1]: prepare remote engine for translog"); @@ -635,8 +656,9 @@ void phase2( final long maxSeqNoOfUpdatesOrDeletes, final RetentionLeases retentionLeases, final long mappingVersion, - final ActionListener listener + ActionListener listener ) throws IOException { + listener = OtelService.startSpan("phase2", listener); if (shard.state() == IndexShardState.CLOSED) { throw new IndexShardClosedException(request.shardId()); } @@ -653,6 +675,7 @@ void phase2( mappingVersion, sendListener ); + ActionListener finalListener = listener; sendListener.whenComplete(ignored -> { final long skippedOps = sender.skippedOps.get(); final int totalSentOps = sender.sentOps.get(); @@ -668,7 +691,7 @@ void phase2( stopWatch.stop(); final TimeValue tookTime = stopWatch.totalTime(); logger.trace("recovery [phase2]: took [{}]", tookTime); - listener.onResponse(new SendSnapshotResult(targetLocalCheckpoint, totalSentOps, tookTime)); + finalListener.onResponse(new SendSnapshotResult(targetLocalCheckpoint, totalSentOps, tookTime)); }, listener::onFailure); sender.start(); } @@ -786,6 +809,8 @@ public void close() throws IOException { } void finalizeRecovery(long targetLocalCheckpoint, long trimAboveSeqNo, ActionListener listener) throws IOException { + listener = OtelService.startSpan("finalizeRecovery", listener); + if (shard.state() == IndexShardState.CLOSED) { throw new IndexShardClosedException(request.shardId()); } @@ -809,6 +834,7 @@ void finalizeRecovery(long targetLocalCheckpoint, long trimAboveSeqNo, ActionLis final StepListener finalizeListener = new StepListener<>(); cancellableThreads.checkForCancel(); recoveryTarget.finalizeRecovery(globalCheckpoint, trimAboveSeqNo, finalizeListener); + ActionListener finalListener = listener; finalizeListener.whenComplete(r -> { RunUnderPrimaryPermit.run( () -> shard.updateGlobalCheckpointForShard(request.targetAllocationId(), globalCheckpoint), @@ -840,7 +866,7 @@ void finalizeRecovery(long targetLocalCheckpoint, long trimAboveSeqNo, ActionLis } stopWatch.stop(); logger.info("finalizing recovery took [{}]", stopWatch.totalTime()); - listener.onResponse(null); + finalListener.onResponse(null); }, listener::onFailure); } @@ -896,12 +922,14 @@ private void cleanFiles( // Once the files have been renamed, any other files that are not // related to this recovery (out of date segments, for example) // are deleted + ActionListener wrappedListener = OtelService.startSpan("receiveFileInfo", listener); + cancellableThreads.checkForCancel(); recoveryTarget.cleanFiles( translogOps.getAsInt(), globalCheckpoint, sourceMetadata, - ActionListener.delegateResponse(listener, (l, e) -> ActionListener.completeWith(l, () -> { + ActionListener.delegateResponse(wrappedListener, (l, e) -> ActionListener.completeWith(l, () -> { StoreFileMetadata[] mds = StreamSupport.stream(sourceMetadata.spliterator(), false).toArray(StoreFileMetadata[]::new); ArrayUtil.timSort(mds, Comparator.comparingLong(StoreFileMetadata::length)); // check small files first transferHandler.handleErrorOnSendFiles(store, e, mds); diff --git a/server/src/main/java/org/opensearch/indices/recovery/RecoverySourceHandlerFactory.java b/server/src/main/java/org/opensearch/indices/recovery/RecoverySourceHandlerFactory.java index ea13ca18bbfca..db9c7da5d429c 100644 --- a/server/src/main/java/org/opensearch/indices/recovery/RecoverySourceHandlerFactory.java +++ b/server/src/main/java/org/opensearch/indices/recovery/RecoverySourceHandlerFactory.java @@ -9,6 +9,7 @@ package org.opensearch.indices.recovery; import org.opensearch.index.shard.IndexShard; +import org.opensearch.otel.OtelService; /** * Factory that supplies {@link RecoverySourceHandler}. @@ -21,7 +22,8 @@ public static RecoverySourceHandler create( IndexShard shard, RecoveryTargetHandler recoveryTarget, StartRecoveryRequest request, - RecoverySettings recoverySettings + RecoverySettings recoverySettings, + OtelService otelService ) { boolean isReplicaRecoveryWithRemoteTranslog = request.isPrimaryRelocation() == false && shard.isRemoteTranslogEnabled(); if (isReplicaRecoveryWithRemoteTranslog) { @@ -32,7 +34,8 @@ public static RecoverySourceHandler create( request, Math.toIntExact(recoverySettings.getChunkSize().getBytes()), recoverySettings.getMaxConcurrentFileChunks(), - recoverySettings.getMaxConcurrentOperations() + recoverySettings.getMaxConcurrentOperations(), + otelService ); } else { return new LocalStorePeerRecoverySourceHandler( @@ -42,7 +45,8 @@ public static RecoverySourceHandler create( request, Math.toIntExact(recoverySettings.getChunkSize().getBytes()), recoverySettings.getMaxConcurrentFileChunks(), - recoverySettings.getMaxConcurrentOperations() + recoverySettings.getMaxConcurrentOperations(), + otelService ); } } diff --git a/server/src/main/java/org/opensearch/indices/recovery/RemoteStorePeerRecoverySourceHandler.java b/server/src/main/java/org/opensearch/indices/recovery/RemoteStorePeerRecoverySourceHandler.java index ff218ef71e397..30af90f620e19 100644 --- a/server/src/main/java/org/opensearch/indices/recovery/RemoteStorePeerRecoverySourceHandler.java +++ b/server/src/main/java/org/opensearch/indices/recovery/RemoteStorePeerRecoverySourceHandler.java @@ -20,6 +20,7 @@ import org.opensearch.indices.RunUnderPrimaryPermit; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.Transports; +import org.opensearch.otel.OtelService; import java.io.IOException; import java.util.function.Consumer; @@ -38,9 +39,10 @@ public RemoteStorePeerRecoverySourceHandler( StartRecoveryRequest request, int fileChunkSizeInBytes, int maxConcurrentFileChunks, - int maxConcurrentOperations + int maxConcurrentOperations, + OtelService otelService ) { - super(shard, recoveryTarget, threadPool, request, fileChunkSizeInBytes, maxConcurrentFileChunks, maxConcurrentOperations); + super(shard, recoveryTarget, threadPool, request, fileChunkSizeInBytes, maxConcurrentFileChunks, maxConcurrentOperations, otelService); } @Override diff --git a/server/src/main/java/org/opensearch/indices/replication/SegmentFileTransferHandler.java b/server/src/main/java/org/opensearch/indices/replication/SegmentFileTransferHandler.java index 5411b6b2c9068..9720d0cc446be 100644 --- a/server/src/main/java/org/opensearch/indices/replication/SegmentFileTransferHandler.java +++ b/server/src/main/java/org/opensearch/indices/replication/SegmentFileTransferHandler.java @@ -8,6 +8,7 @@ package org.opensearch.indices.replication; +import io.opentelemetry.api.trace.Span; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; import org.apache.lucene.index.CorruptIndexException; @@ -28,6 +29,8 @@ import org.opensearch.index.store.StoreFileMetadata; import org.opensearch.indices.recovery.FileChunkWriter; import org.opensearch.indices.recovery.MultiChunkTransfer; +import org.opensearch.otel.OtelEventListener; +import org.opensearch.otel.OtelService; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.RemoteTransportException; import org.opensearch.transport.Transports; @@ -56,6 +59,7 @@ public final class SegmentFileTransferHandler { private final int maxConcurrentFileChunks; private final DiscoveryNode targetNode; private final CancellableThreads cancellableThreads; + private final OtelService otelService; public SegmentFileTransferHandler( IndexShard shard, @@ -66,6 +70,21 @@ public SegmentFileTransferHandler( CancellableThreads cancellableThreads, int fileChunkSizeInBytes, int maxConcurrentFileChunks + ) { + this(shard, targetNode, chunkWriter, logger, threadPool, cancellableThreads, + fileChunkSizeInBytes, maxConcurrentFileChunks, null); + } + + public SegmentFileTransferHandler( + IndexShard shard, + DiscoveryNode targetNode, + FileChunkWriter chunkWriter, + Logger logger, + ThreadPool threadPool, + CancellableThreads cancellableThreads, + int fileChunkSizeInBytes, + int maxConcurrentFileChunks, + OtelService otelService ) { this.shard = shard; this.targetNode = targetNode; @@ -76,6 +95,7 @@ public SegmentFileTransferHandler( this.chunkSizeInBytes = fileChunkSizeInBytes; // if the target is on an old version, it won't be able to handle out-of-order file chunks. this.maxConcurrentFileChunks = maxConcurrentFileChunks; + this.otelService = otelService; } /** @@ -145,13 +165,24 @@ protected FileChunk nextChunkRequest(StoreFileMetadata md) throws IOException { @Override protected void executeChunkRequest(FileChunk request, ActionListener listener1) { cancellableThreads.checkForCancel(); + if (otelService != null) { + otelService.emitResources(Span.current(), "StartExecuteChunkRequest"); + } + + ActionListener listener2 = ActionListener.runBefore(listener1, request::close); + ActionListener listener3 = ActionListener.runBefore(listener2, () -> { + if (otelService != null) { + otelService.emitResources(Span.current(), "EndExecuteChunkRequest"); + } + }); + chunkWriter.writeFileChunk( request.md, request.position, request.content, request.lastChunk, translogOps.getAsInt(), - ActionListener.runBefore(listener1, request::close) + listener3 ); } diff --git a/server/src/main/java/org/opensearch/node/Node.java b/server/src/main/java/org/opensearch/node/Node.java index 3827041a60aa3..9e3e9ba9bdbc7 100644 --- a/server/src/main/java/org/opensearch/node/Node.java +++ b/server/src/main/java/org/opensearch/node/Node.java @@ -35,17 +35,15 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.lucene.util.Constants; -import org.opensearch.ExceptionsHelper; import org.opensearch.common.SetOnce; -import org.opensearch.common.settings.SettingsException; -import org.opensearch.common.unit.ByteSizeUnit; -import org.opensearch.common.unit.ByteSizeValue; -import org.opensearch.common.util.FeatureFlags; import org.opensearch.cluster.routing.allocation.AwarenessReplicaBalance; import org.opensearch.index.IndexModule; import org.opensearch.index.IndexingPressureService; +import org.opensearch.otel.OtelService; +import org.opensearch.tasks.TaskResourceTrackingService; +import org.opensearch.threadpool.RunnableTaskExecutionListener; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.index.store.remote.filecache.FileCache; -import org.opensearch.index.store.remote.filecache.FileCacheCleaner; import org.opensearch.index.store.remote.filecache.FileCacheFactory; import org.opensearch.indices.replication.SegmentReplicationSourceFactory; import org.opensearch.indices.replication.SegmentReplicationTargetService; @@ -54,22 +52,19 @@ import org.opensearch.extensions.NoopExtensionsManager; import org.opensearch.monitor.fs.FsInfo; import org.opensearch.monitor.fs.FsProbe; -import org.opensearch.plugins.SearchPipelinePlugin; import org.opensearch.search.backpressure.SearchBackpressureService; import org.opensearch.search.backpressure.settings.SearchBackpressureSettings; -import org.opensearch.search.pipeline.SearchPipelineService; import org.opensearch.tasks.TaskResourceTrackingService; -import org.opensearch.tasks.consumer.TopNSearchTasksLogger; import org.opensearch.threadpool.RunnableTaskExecutionListener; import org.opensearch.index.store.RemoteSegmentStoreDirectoryFactory; import org.opensearch.watcher.ResourceWatcherService; -import org.opensearch.core.Assertions; +import org.opensearch.Assertions; import org.opensearch.Build; import org.opensearch.OpenSearchException; import org.opensearch.OpenSearchTimeoutException; import org.opensearch.Version; +import org.opensearch.ExceptionsHelper; import org.opensearch.action.ActionModule; -import org.opensearch.action.ActionModule.DynamicActionRegistry; import org.opensearch.action.ActionType; import org.opensearch.action.admin.cluster.snapshots.status.TransportNodesSnapshotsStatus; import org.opensearch.action.search.SearchExecutionStatsCollector; @@ -131,8 +126,8 @@ import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.BigArrays; import org.opensearch.common.util.PageCacheRecycler; -import org.opensearch.core.xcontent.NamedXContentRegistry; -import org.opensearch.common.util.io.IOUtils; +import org.opensearch.common.xcontent.NamedXContentRegistry; +import org.opensearch.core.internal.io.IOUtils; import org.opensearch.discovery.Discovery; import org.opensearch.discovery.DiscoveryModule; import org.opensearch.env.Environment; @@ -145,7 +140,6 @@ import org.opensearch.gateway.MetaStateService; import org.opensearch.gateway.PersistedClusterStateService; import org.opensearch.http.HttpServerTransport; -import org.opensearch.identity.IdentityService; import org.opensearch.index.IndexSettings; import org.opensearch.index.analysis.AnalysisRegistry; import org.opensearch.index.engine.EngineFactory; @@ -178,7 +172,6 @@ import org.opensearch.plugins.ClusterPlugin; import org.opensearch.plugins.DiscoveryPlugin; import org.opensearch.plugins.EnginePlugin; -import org.opensearch.plugins.IdentityPlugin; import org.opensearch.plugins.IndexStorePlugin; import org.opensearch.plugins.IngestPlugin; import org.opensearch.plugins.MapperPlugin; @@ -225,7 +218,7 @@ import java.io.IOException; import java.net.InetAddress; import java.net.InetSocketAddress; -import java.nio.charset.StandardCharsets; +import java.nio.charset.Charset; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardCopyOption; @@ -249,8 +242,7 @@ import java.util.stream.Stream; import static java.util.stream.Collectors.toList; -import static org.opensearch.common.util.FeatureFlags.SEARCH_PIPELINE; -import static org.opensearch.env.NodeEnvironment.collectFileCacheDataPath; +import static org.opensearch.common.util.FeatureFlags.REPLICATION_TYPE; import static org.opensearch.index.ShardIndexingPressureSettings.SHARD_INDEXING_PRESSURE_ENABLED_ATTRIBUTE_KEY; /** @@ -327,12 +319,6 @@ public class Node implements Closeable { } }, Setting.Property.NodeScope); - public static final Setting NODE_SEARCH_CACHE_SIZE_SETTING = Setting.byteSizeSetting( - "node.search.cache.size", - ByteSizeValue.ZERO, - Property.NodeScope - ); - private static final String CLIENT_TYPE = "node"; /** @@ -369,9 +355,10 @@ public static class DiscoverySettings { private final Collection pluginLifecycleComponents; private final LocalNodeFactory localNodeFactory; private final NodeService nodeService; + private final OtelService otelService; + final NamedWriteableRegistry namedWriteableRegistry; private final AtomicReference runnableTaskListener; - private FileCache fileCache; public Node(Environment environment) { this(environment, Collections.emptyList(), true); @@ -436,18 +423,18 @@ protected Node( if (logger.isDebugEnabled()) { logger.debug( "using config [{}], data [{}], logs [{}], plugins [{}]", - initialEnvironment.configDir(), + initialEnvironment.configFile(), Arrays.toString(initialEnvironment.dataFiles()), - initialEnvironment.logsDir(), - initialEnvironment.pluginsDir() + initialEnvironment.logsFile(), + initialEnvironment.pluginsFile() ); } this.pluginsService = new PluginsService( tmpSettings, - initialEnvironment.configDir(), - initialEnvironment.modulesDir(), - initialEnvironment.pluginsDir(), + initialEnvironment.configFile(), + initialEnvironment.modulesFile(), + initialEnvironment.pluginsFile(), classpathPlugins ); @@ -456,17 +443,8 @@ protected Node( // Ensure to initialize Feature Flags via the settings from opensearch.yml FeatureFlags.initializeFeatureFlags(settings); - final List identityPlugins = new ArrayList<>(); - if (FeatureFlags.isEnabled(FeatureFlags.IDENTITY)) { - // If identity is enabled load plugins implementing the extension point - logger.info("Identity on so found plugins implementing: " + pluginsService.filterPlugins(IdentityPlugin.class).toString()); - identityPlugins.addAll(pluginsService.filterPlugins(IdentityPlugin.class)); - } - - final IdentityService identityService = new IdentityService(settings, identityPlugins); - if (FeatureFlags.isEnabled(FeatureFlags.EXTENSIONS)) { - this.extensionsManager = new ExtensionsManager(initialEnvironment.extensionDir()); + this.extensionsManager = new ExtensionsManager(tmpSettings, initialEnvironment.extensionDir()); } else { this.extensionsManager = new NoopExtensionsManager(); } @@ -484,7 +462,7 @@ protected Node( * Create the environment based on the finalized view of the settings. This is to ensure that components get the same setting * values, no matter they ask for them from. */ - this.environment = new Environment(settings, initialEnvironment.configDir(), Node.NODE_LOCAL_STORAGE_SETTING.get(settings)); + this.environment = new Environment(settings, initialEnvironment.configFile(), Node.NODE_LOCAL_STORAGE_SETTING.get(settings)); Environment.assertEquivalent(initialEnvironment, this.environment); nodeEnvironment = new NodeEnvironment(tmpSettings, environment); logger.info( @@ -566,7 +544,6 @@ protected Node( pluginsService.filterPlugins(IngestPlugin.class), client ); - final SetOnce repositoriesServiceReference = new SetOnce<>(); final ClusterInfoService clusterInfoService = newClusterInfoService(settings, clusterService, threadPool, client); final UsageService usageService = new UsageService(); @@ -576,6 +553,7 @@ protected Node( for (Module pluginModule : pluginsService.createGuiceModules()) { modules.add(pluginModule); } + final MonitorService monitorService = new MonitorService(settings, nodeEnvironment, threadPool); final FsHealthService fsHealthService = new FsHealthService( settings, clusterService.getClusterSettings(), @@ -611,11 +589,6 @@ protected Node( pluginCircuitBreakers, settingsModule.getClusterSettings() ); - // File cache will be initialized by the node once circuit breakers are in place. - initializeFileCache(settings, circuitBreakerService.getBreaker(CircuitBreaker.REQUEST)); - final FileCacheCleaner fileCacheCleaner = new FileCacheCleaner(nodeEnvironment, fileCache); - final MonitorService monitorService = new MonitorService(settings, nodeEnvironment, threadPool, fileCache); - pluginsService.filterPlugins(CircuitBreakerPlugin.class).forEach(plugin -> { CircuitBreaker breaker = circuitBreakerService.getBreaker(plugin.getCircuitBreaker(settings).getName()); plugin.setCircuitBreaker(breaker); @@ -657,13 +630,13 @@ protected Node( final Collection>> engineFactoryProviders = enginePlugins.stream() .map(plugin -> (Function>) plugin::getEngineFactory) .collect(Collectors.toList()); - + // TODO: for now this is a single cache, later, this should read node and index settings + final FileCache remoteStoreFileCache = createRemoteStoreFileCache(); final Map builtInDirectoryFactories = IndexModule.createBuiltInDirectoryFactories( repositoriesServiceReference::get, threadPool, - fileCache + remoteStoreFileCache ); - final Map directoryFactories = new HashMap<>(); pluginsService.filterPlugins(IndexStorePlugin.class) .stream() @@ -678,6 +651,7 @@ protected Node( directoryFactories.put(k, v); }); directoryFactories.putAll(builtInDirectoryFactories); + this.otelService = new OtelService(pluginsService); final Map recoveryStateFactories = pluginsService.filterPlugins( IndexStorePlugin.class @@ -700,36 +674,63 @@ protected Node( rerouteServiceReference.set(rerouteService); clusterService.setRerouteService(rerouteService); - final IndexStorePlugin.DirectoryFactory remoteDirectoryFactory = new RemoteSegmentStoreDirectoryFactory( + final IndexStorePlugin.RemoteDirectoryFactory remoteDirectoryFactory = new RemoteSegmentStoreDirectoryFactory( repositoriesServiceReference::get ); - final IndicesService indicesService = new IndicesService( - settings, - pluginsService, - extensionsManager, - nodeEnvironment, - xContentRegistry, - analysisModule.getAnalysisRegistry(), - clusterModule.getIndexNameExpressionResolver(), - indicesModule.getMapperRegistry(), - namedWriteableRegistry, - threadPool, - settingsModule.getIndexScopedSettings(), - circuitBreakerService, - bigArrays, - scriptService, - clusterService, - client, - metaStateService, - engineFactoryProviders, - Map.copyOf(directoryFactories), - searchModule.getValuesSourceRegistry(), - recoveryStateFactories, - remoteDirectoryFactory, - repositoriesServiceReference::get, - fileCacheCleaner - ); + final IndicesService indicesService; + if (FeatureFlags.isEnabled(FeatureFlags.EXTENSIONS)) { + indicesService = new IndicesService( + settings, + pluginsService, + extensionsManager, + nodeEnvironment, + xContentRegistry, + analysisModule.getAnalysisRegistry(), + clusterModule.getIndexNameExpressionResolver(), + indicesModule.getMapperRegistry(), + namedWriteableRegistry, + threadPool, + settingsModule.getIndexScopedSettings(), + circuitBreakerService, + bigArrays, + scriptService, + clusterService, + client, + metaStateService, + engineFactoryProviders, + Map.copyOf(directoryFactories), + searchModule.getValuesSourceRegistry(), + recoveryStateFactories, + remoteDirectoryFactory, + repositoriesServiceReference::get + ); + } else { + indicesService = new IndicesService( + settings, + pluginsService, + nodeEnvironment, + xContentRegistry, + analysisModule.getAnalysisRegistry(), + clusterModule.getIndexNameExpressionResolver(), + indicesModule.getMapperRegistry(), + namedWriteableRegistry, + threadPool, + settingsModule.getIndexScopedSettings(), + circuitBreakerService, + bigArrays, + scriptService, + clusterService, + client, + metaStateService, + engineFactoryProviders, + Map.copyOf(directoryFactories), + searchModule.getValuesSourceRegistry(), + recoveryStateFactories, + remoteDirectoryFactory, + repositoriesServiceReference::get + ); + } final AliasValidator aliasValidator = new AliasValidator(); @@ -794,8 +795,7 @@ protected Node( client, circuitBreakerService, usageService, - systemIndices, - identityService + systemIndices ); modules.add(actionModule); @@ -844,16 +844,16 @@ protected Node( settingsModule.getClusterSettings(), taskHeaders ); - TopNSearchTasksLogger taskConsumer = new TopNSearchTasksLogger(settings, settingsModule.getClusterSettings()); - transportService.getTaskManager().registerTaskResourceConsumer(taskConsumer); - this.extensionsManager.initializeServicesAndRestHandler( - actionModule, - settingsModule, - transportService, - clusterService, - environment.settings(), - client - ); + if (FeatureFlags.isEnabled(FeatureFlags.EXTENSIONS)) { + this.extensionsManager.initializeServicesAndRestHandler( + restController, + settingsModule, + transportService, + clusterService, + environment.settings(), + client + ); + } final GatewayMetaState gatewayMetaState = new GatewayMetaState(); final ResponseCollectorService responseCollectorService = new ResponseCollectorService(clusterService); final SearchTransportService searchTransportService = new SearchTransportService( @@ -949,23 +949,11 @@ protected Node( clusterService.getClusterSettings(), pluginsService.filterPlugins(DiscoveryPlugin.class), clusterModule.getAllocationService(), - environment.configDir(), + environment.configFile(), gatewayMetaState, rerouteService, fsHealthService ); - final SearchPipelineService searchPipelineService = new SearchPipelineService( - clusterService, - threadPool, - this.environment, - scriptService, - analysisModule.getAnalysisRegistry(), - xContentRegistry, - namedWriteableRegistry, - pluginsService.filterPlugins(SearchPipelinePlugin.class), - client, - FeatureFlags.isEnabled(SEARCH_PIPELINE) - ); this.nodeService = new NodeService( settings, threadPool, @@ -984,9 +972,7 @@ protected Node( searchTransportService, indexingPressureService, searchModule.getValuesSourceRegistry().getUsageService(), - searchBackpressureService, - searchPipelineService, - fileCache + searchBackpressureService ); final SearchService searchService = newSearchService( @@ -1044,7 +1030,6 @@ protected Node( b.bind(ScriptService.class).toInstance(scriptService); b.bind(AnalysisRegistry.class).toInstance(analysisModule.getAnalysisRegistry()); b.bind(IngestService.class).toInstance(ingestService); - b.bind(SearchPipelineService.class).toInstance(searchPipelineService); b.bind(IndexingPressureService.class).toInstance(indexingPressureService); b.bind(TaskResourceTrackingService.class).toInstance(taskResourceTrackingService); b.bind(SearchBackpressureService.class).toInstance(searchBackpressureService); @@ -1075,21 +1060,26 @@ protected Node( { processRecoverySettings(settingsModule.getClusterSettings(), recoverySettings); b.bind(PeerRecoverySourceService.class) - .toInstance(new PeerRecoverySourceService(transportService, indicesService, recoverySettings)); + .toInstance(new PeerRecoverySourceService(transportService, indicesService, recoverySettings, otelService)); b.bind(PeerRecoveryTargetService.class) .toInstance(new PeerRecoveryTargetService(threadPool, transportService, recoverySettings, clusterService)); - b.bind(SegmentReplicationTargetService.class) - .toInstance( - new SegmentReplicationTargetService( - threadPool, - recoverySettings, - transportService, - new SegmentReplicationSourceFactory(transportService, recoverySettings, clusterService), - indicesService - ) - ); - b.bind(SegmentReplicationSourceService.class) - .toInstance(new SegmentReplicationSourceService(indicesService, transportService, recoverySettings)); + if (FeatureFlags.isEnabled(REPLICATION_TYPE)) { + b.bind(SegmentReplicationTargetService.class) + .toInstance( + new SegmentReplicationTargetService( + threadPool, + recoverySettings, + transportService, + new SegmentReplicationSourceFactory(transportService, recoverySettings, clusterService), + indicesService + ) + ); + b.bind(SegmentReplicationSourceService.class) + .toInstance(new SegmentReplicationSourceService(indicesService, transportService, recoverySettings)); + } else { + b.bind(SegmentReplicationTargetService.class).toInstance(SegmentReplicationTargetService.NO_OP); + b.bind(SegmentReplicationSourceService.class).toInstance(SegmentReplicationSourceService.NO_OP); + } } b.bind(HttpServerTransport.class).toInstance(httpServerTransport); pluginComponents.stream().forEach(p -> b.bind((Class) p.getClass()).toInstance(p)); @@ -1105,7 +1095,6 @@ protected Node( b.bind(ShardLimitValidator.class).toInstance(shardLimitValidator); b.bind(FsHealthService.class).toInstance(fsHealthService); b.bind(SystemIndices.class).toInstance(systemIndices); - b.bind(IdentityService.class).toInstance(identityService); }); injector = modules.createInjector(); @@ -1126,15 +1115,8 @@ protected Node( resourcesToClose.addAll(pluginLifecycleComponents); resourcesToClose.add(injector.getInstance(PeerRecoverySourceService.class)); this.pluginLifecycleComponents = Collections.unmodifiableList(pluginLifecycleComponents); - DynamicActionRegistry dynamicActionRegistry = actionModule.getDynamicActionRegistry(); - dynamicActionRegistry.registerUnmodifiableActionMap(injector.getInstance(new Key>() { - })); - client.initialize( - dynamicActionRegistry, - () -> clusterService.localNode().getId(), - transportService.getRemoteClusterService(), - namedWriteableRegistry - ); + client.initialize(injector.getInstance(new Key>() { + }), () -> clusterService.localNode().getId(), transportService.getRemoteClusterService(), namedWriteableRegistry); this.namedWriteableRegistry = namedWriteableRegistry; logger.debug("initializing HTTP handlers ..."); @@ -1151,6 +1133,16 @@ protected Node( } } + private FileCache createRemoteStoreFileCache() { + // TODO: implement more custom logic to create named caches, using multiple node paths, more capacity computation options and + // capacity reservation logic + FsInfo.Path info = ExceptionsHelper.catchAsRuntimeException(() -> FsProbe.getFSInfo(nodeEnvironment.nodePaths()[0])); + long diskCapacity = info.getTotal().getBytes(); + // hard coded as 50% for now + long capacity = (long) (diskCapacity * 0.50); + return FileCacheFactory.createConcurrentLRUFileCache(capacity); + } + protected TransportService newTransportService( Settings settings, Transport transport, @@ -1241,7 +1233,9 @@ public Node start() throws NodeValidationException { assert transportService.getLocalNode().equals(localNodeFactory.getNode()) : "transportService has a different local node than the factory provided"; injector.getInstance(PeerRecoverySourceService.class).start(); - injector.getInstance(SegmentReplicationSourceService.class).start(); + if (FeatureFlags.isEnabled(REPLICATION_TYPE)) { + injector.getInstance(SegmentReplicationSourceService.class).start(); + } // Load (and maybe upgrade) the metadata stored on disk final GatewayMetaState gatewayMetaState = injector.getInstance(GatewayMetaState.class); @@ -1286,7 +1280,9 @@ public Node start() throws NodeValidationException { assert clusterService.localNode().equals(localNodeFactory.getNode()) : "clusterService has a different local node than the factory provided"; transportService.acceptIncomingRequests(); - extensionsManager.initialize(); + if (FeatureFlags.isEnabled(FeatureFlags.EXTENSIONS)) { + extensionsManager.initialize(); + } discovery.startInitialJoin(); final TimeValue initialStateTimeout = DiscoverySettings.INITIAL_STATE_TIMEOUT_SETTING.get(settings()); configureNodeAndClusterIdStateListener(clusterService); @@ -1419,7 +1415,9 @@ public synchronized void close() throws IOException { // close filter/fielddata caches after indices toClose.add(injector.getInstance(IndicesStore.class)); toClose.add(injector.getInstance(PeerRecoverySourceService.class)); - toClose.add(injector.getInstance(SegmentReplicationSourceService.class)); + if (FeatureFlags.isEnabled(REPLICATION_TYPE)) { + toClose.add(injector.getInstance(SegmentReplicationSourceService.class)); + } toClose.add(() -> stopWatch.stop().start("cluster")); toClose.add(injector.getInstance(ClusterService.class)); toClose.add(() -> stopWatch.stop().start("node_connections_service")); @@ -1523,8 +1521,8 @@ protected void validateNodeBeforeAcceptingRequests( /** Writes a file to the logs dir containing the ports for the given transport type */ private void writePortsFile(String type, BoundTransportAddress boundAddress) { - Path tmpPortsFile = environment.logsDir().resolve(type + ".ports.tmp"); - try (BufferedWriter writer = Files.newBufferedWriter(tmpPortsFile, StandardCharsets.UTF_8)) { + Path tmpPortsFile = environment.logsFile().resolve(type + ".ports.tmp"); + try (BufferedWriter writer = Files.newBufferedWriter(tmpPortsFile, Charset.forName("UTF-8"))) { for (TransportAddress address : boundAddress.boundAddresses()) { InetAddress inetAddress = InetAddress.getByName(address.getAddress()); writer.write(NetworkAddress.format(new InetSocketAddress(inetAddress, address.getPort())) + "\n"); @@ -1532,7 +1530,7 @@ private void writePortsFile(String type, BoundTransportAddress boundAddress) { } catch (IOException e) { throw new RuntimeException("Failed to write ports file", e); } - Path portsFile = environment.logsDir().resolve(type + ".ports"); + Path portsFile = environment.logsFile().resolve(type + ".ports"); try { Files.move(tmpPortsFile, portsFile, StandardCopyOption.ATOMIC_MOVE); } catch (IOException e) { @@ -1674,49 +1672,4 @@ DiscoveryNode getNode() { return localNode.get(); } } - - /** - * Initializes the search cache with a defined capacity. - * The capacity of the cache is based on user configuration for {@link Node#NODE_SEARCH_CACHE_SIZE_SETTING}. - * If the user doesn't configure the cache size, it fails if the node is a data + search node. - * Else it configures the size to 80% of available capacity for a dedicated search node, if not explicitly defined. - */ - private void initializeFileCache(Settings settings, CircuitBreaker circuitBreaker) throws IOException { - if (DiscoveryNode.isSearchNode(settings)) { - NodeEnvironment.NodePath fileCacheNodePath = nodeEnvironment.fileCacheNodePath(); - long capacity = NODE_SEARCH_CACHE_SIZE_SETTING.get(settings).getBytes(); - FsInfo.Path info = ExceptionsHelper.catchAsRuntimeException(() -> FsProbe.getFSInfo(fileCacheNodePath)); - long availableCapacity = info.getAvailable().getBytes(); - - // Initialize default values for cache if NODE_SEARCH_CACHE_SIZE_SETTING is not set. - if (capacity == 0) { - // If node is not a dedicated search node without configuration, prevent cache initialization - if (DiscoveryNode.getRolesFromSettings(settings).stream().anyMatch(role -> !DiscoveryNodeRole.SEARCH_ROLE.equals(role))) { - throw new SettingsException( - "Unable to initialize the " - + DiscoveryNodeRole.SEARCH_ROLE.roleName() - + "-" - + DiscoveryNodeRole.DATA_ROLE.roleName() - + " node: Missing value for configuration " - + NODE_SEARCH_CACHE_SIZE_SETTING.getKey() - ); - } else { - capacity = 80 * availableCapacity / 100; - } - } - capacity = Math.min(capacity, availableCapacity); - fileCacheNodePath.fileCacheReservedSize = new ByteSizeValue(capacity, ByteSizeUnit.BYTES); - this.fileCache = FileCacheFactory.createConcurrentLRUFileCache(capacity, circuitBreaker); - List fileCacheDataPaths = collectFileCacheDataPath(fileCacheNodePath); - this.fileCache.restoreFromDirectory(fileCacheDataPaths); - } - } - - /** - * Returns the {@link FileCache} instance for remote search node - * Note: Visible for testing - */ - public FileCache fileCache() { - return this.fileCache; - } } diff --git a/server/src/main/java/org/opensearch/otel/OtelEventListener.java b/server/src/main/java/org/opensearch/otel/OtelEventListener.java new file mode 100644 index 0000000000000..8751e06447beb --- /dev/null +++ b/server/src/main/java/org/opensearch/otel/OtelEventListener.java @@ -0,0 +1,16 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.otel; + +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.trace.Span; + +public interface OtelEventListener { + Attributes onEvent(Span span); +} diff --git a/server/src/main/java/org/opensearch/otel/OtelService.java b/server/src/main/java/org/opensearch/otel/OtelService.java new file mode 100644 index 0000000000000..4889b56fdd21b --- /dev/null +++ b/server/src/main/java/org/opensearch/otel/OtelService.java @@ -0,0 +1,150 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.otel; + +import com.sun.management.ThreadMXBean; +import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.Tracer; +import io.opentelemetry.api.trace.propagation.W3CTraceContextPropagator; +import io.opentelemetry.context.Context; +import io.opentelemetry.context.Scope; +import io.opentelemetry.context.propagation.ContextPropagators; +import io.opentelemetry.exporter.otlp.http.trace.OtlpHttpSpanExporter; +import io.opentelemetry.exporter.otlp.metrics.OtlpGrpcMetricExporter; +import io.opentelemetry.sdk.OpenTelemetrySdk; +import io.opentelemetry.sdk.metrics.SdkMeterProvider; +import io.opentelemetry.sdk.metrics.export.PeriodicMetricReader; +import io.opentelemetry.sdk.resources.Resource; +import io.opentelemetry.sdk.trace.SdkTracerProvider; +import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor; +import io.opentelemetry.semconv.resource.attributes.ResourceAttributes; +import org.opensearch.action.ActionListener; +import org.opensearch.action.support.OTelContextPreservingActionListener; +import org.opensearch.plugins.Plugin; +import org.opensearch.plugins.PluginsService; + +import java.lang.management.ManagementFactory; +import java.util.List; +import java.util.stream.Collectors; + +import static io.opentelemetry.api.common.AttributeKey.longKey; +import static io.opentelemetry.api.common.AttributeKey.stringKey; + + +public class OtelService { + public static Resource resource; + public static SdkTracerProvider sdkTracerProvider; + public static SdkMeterProvider sdkMeterProvider; + public static OpenTelemetry openTelemetry; + private static final ThreadMXBean threadMXBean = (ThreadMXBean) ManagementFactory.getThreadMXBean(); + public List otelEventListenerList; + static { + +// ManagedChannel jaegerChannel = ManagedChannelBuilder.forAddress("localhost", 3336) +// .usePlaintext() +// .build(); + +// JaegerGrpcSpanExporter jaegerExporter = JaegerGrpcSpanExporter.builder() +// .setEndpoint("http://localhost:3336") +// .setTimeout(30, TimeUnit.SECONDS) +// .build(); + +// SdkTracerProvider sdkTracerProvider = SdkTracerProvider.builder() +// .addSpanProcessor(SimpleSpanProcessor.create(jaegerExporter)) +// .build(); +// Meter recoverMeter = OtelService.sdkMeterProvider.meterBuilder("recover").build(); + +// DoubleGaugeBuilder cpuGauge = recoverMeter.gaugeBuilder("cpu"); +// recoverMeter +// .gaugeBuilder("cpu_usage") +// .setDescription("CPU Usage") +// .setUnit("ms") +// .buildWithCallback(measurement -> { +// measurement.record(, Attributes.of(stringKey("operation"), "recover-start")); +// }); + +// recoverMeter +// .gaugeBuilder("cpu_usage") +// .setDescription("CPU Usage") +// .setUnit("ms") +// .buildWithCallback(measurement -> { +// measurement.record(OtelService.getCPUUsage(threadId), Attributes.of(stringKey("operation"), "recover-start")); +// }); + + resource = Resource.getDefault() + .merge(Resource.create(Attributes.of(ResourceAttributes.SERVICE_NAME, "opensearch-tasks"))); + + sdkTracerProvider = SdkTracerProvider.builder() + //.addSpanProcessor(SimpleSpanProcessor.create(LoggingSpanExporter.create())) + .addSpanProcessor(SimpleSpanProcessor.create(OtlpHttpSpanExporter.builder().build())) + .setResource(resource) + .build(); + + sdkMeterProvider = SdkMeterProvider.builder() + .registerMetricReader(PeriodicMetricReader.builder(OtlpGrpcMetricExporter.builder().build()).build()) + .setResource(resource) + .build(); + + openTelemetry = OpenTelemetrySdk.builder() + .setTracerProvider(sdkTracerProvider) + .setMeterProvider(sdkMeterProvider) + .setPropagators(ContextPropagators.create(W3CTraceContextPropagator.getInstance())) + .buildAndRegisterGlobal(); + } + + public OtelService(PluginsService pluginsService) { + otelEventListenerList = pluginsService.filterPlugins(Plugin.class) + .stream() + .map(Plugin::getOtelEventListeners) + .flatMap(List::stream) + .collect(Collectors.toList()); + } + + public static long getCPUUsage(long threadId) { + return threadMXBean.getThreadCpuTime(threadId); + } + + public static long getMemoryUsage(long threadId) { + return threadMXBean.getThreadAllocatedBytes(threadId); + } + + public static long getThreadContentionTime(long threadId) { + return threadMXBean.getThreadInfo(threadId).getBlockedTime(); + } + + public static ActionListener startSpan(String spanName, ActionListener actionListener) { + Tracer tracer = OtelService.sdkTracerProvider.get("recover"); + Context beforeAttach = Context.current(); + Span span = tracer.spanBuilder(spanName).startSpan(); + span.setAttribute(stringKey("start-thread-name"), Thread.currentThread().getName()); + span.setAttribute(longKey("start-thread-id"), Thread.currentThread().getId()); + span.makeCurrent(); + return new OTelContextPreservingActionListener<>(beforeAttach, Context.current(), actionListener, + span.getSpanContext().getSpanId()); + } + + public void emitResources(Span span, String name) { + for (OtelEventListener eventListener : otelEventListenerList) { + eventListener.onEvent(span); + } + span.addEvent(name, + Attributes.of( + AttributeKey.longKey("ThreadID"), Thread.currentThread().getId(), + AttributeKey.stringKey("ThreadName"), Thread.currentThread().getName(), + AttributeKey.longKey("CPUUsage"), OtelService.getCPUUsage(Thread.currentThread().getId()), + AttributeKey.longKey("MemoryUsage"), OtelService.getMemoryUsage(Thread.currentThread().getId()), + AttributeKey.longKey("ContentionTime"), OtelService.getThreadContentionTime(Thread.currentThread().getId()) + ) + ); + } + +} diff --git a/server/src/main/java/org/opensearch/plugins/Plugin.java b/server/src/main/java/org/opensearch/plugins/Plugin.java index aff36a9884203..9b32bf71a9b67 100644 --- a/server/src/main/java/org/opensearch/plugins/Plugin.java +++ b/server/src/main/java/org/opensearch/plugins/Plugin.java @@ -32,6 +32,7 @@ package org.opensearch.plugins; +import org.opensearch.otel.OtelEventListener; import org.opensearch.watcher.ResourceWatcherService; import org.opensearch.bootstrap.BootstrapCheck; import org.opensearch.client.Client; @@ -206,6 +207,10 @@ public List> getSettingUpgraders() { return Collections.emptyList(); } + public List getOtelEventListeners() { + return Collections.emptyList(); + } + /** * Provides a function to modify index template meta data on startup. *

diff --git a/server/src/main/java/org/opensearch/threadpool/ScalingExecutorBuilder.java b/server/src/main/java/org/opensearch/threadpool/ScalingExecutorBuilder.java index 2690a3cc30238..18cef0c57be2e 100644 --- a/server/src/main/java/org/opensearch/threadpool/ScalingExecutorBuilder.java +++ b/server/src/main/java/org/opensearch/threadpool/ScalingExecutorBuilder.java @@ -111,7 +111,7 @@ ThreadPool.ExecutorHolder build(final ScalingExecutorSettings settings, final Th final ThreadFactory threadFactory = OpenSearchExecutors.daemonThreadFactory( OpenSearchExecutors.threadName(settings.nodeName, name()) ); - final ExecutorService executor = OpenSearchExecutors.newScaling( + final ExecutorService executor = io.opentelemetry.context.Context.taskWrapping(OpenSearchExecutors.newScaling( settings.nodeName + "/" + name(), core, max, @@ -119,7 +119,7 @@ ThreadPool.ExecutorHolder build(final ScalingExecutorSettings settings, final Th TimeUnit.MILLISECONDS, threadFactory, threadContext - ); + )); return new ThreadPool.ExecutorHolder(executor, info); } diff --git a/server/src/main/java/org/opensearch/threadpool/ThreadPool.java b/server/src/main/java/org/opensearch/threadpool/ThreadPool.java index 2c91d5aa33090..93bf28ed02af4 100644 --- a/server/src/main/java/org/opensearch/threadpool/ThreadPool.java +++ b/server/src/main/java/org/opensearch/threadpool/ThreadPool.java @@ -413,9 +413,11 @@ public ExecutorService executor(String name) { if (holder == null) { throw new IllegalArgumentException("no executor service found for [" + name + "]"); } - return holder.executor(); + return io.opentelemetry.context.Context.taskWrapping(holder.executor()); } + + /** * Schedules a one-shot command to run after a given delay. The command is run in the context of the calling thread. * @@ -693,7 +695,7 @@ static class ExecutorHolder { public final Info info; ExecutorHolder(ExecutorService executor, Info info) { - assert executor instanceof OpenSearchThreadPoolExecutor || executor == DIRECT_EXECUTOR; +// assert executor instanceof OpenSearchThreadPoolExecutor || executor == DIRECT_EXECUTOR; this.executor = executor; this.info = info; } diff --git a/server/src/main/resources/org/opensearch/bootstrap/security.policy b/server/src/main/resources/org/opensearch/bootstrap/security.policy index 3671782b9d12f..0b96ebfe49529 100644 --- a/server/src/main/resources/org/opensearch/bootstrap/security.policy +++ b/server/src/main/resources/org/opensearch/bootstrap/security.policy @@ -182,4 +182,10 @@ grant { permission java.io.FilePermission "/sys/fs/cgroup/memory", "read"; permission java.io.FilePermission "/sys/fs/cgroup/memory/-", "read"; + permission java.net.NetPermission "getProxySelector"; + permission java.util.PropertyPermission "*", "read,write"; + permission java.net.SocketPermission "127.0.0.1:4318", "connect,resolve"; + permission java.net.SocketPermission "127.0.0.1:4317", "connect,resolve"; + permission java.net.SocketPermission "[0:0:0:0:0:0:0:1]:4317", "connect,resolve"; + permission java.net.SocketPermission "[0:0:0:0:0:0:0:1]:4318", "connect,resolve"; }; diff --git a/server/src/test/java/org/opensearch/indices/recovery/LocalStorePeerRecoverySourceHandlerTests.java b/server/src/test/java/org/opensearch/indices/recovery/LocalStorePeerRecoverySourceHandlerTests.java index 39df2950d2552..9da32ced509ef 100644 --- a/server/src/test/java/org/opensearch/indices/recovery/LocalStorePeerRecoverySourceHandlerTests.java +++ b/server/src/test/java/org/opensearch/indices/recovery/LocalStorePeerRecoverySourceHandlerTests.java @@ -228,7 +228,7 @@ public void writeFileChunk( between(1, 5) ); PlainActionFuture sendFilesFuture = new PlainActionFuture<>(); - handler.sendFiles(store, metas.toArray(new StoreFileMetadata[0]), () -> 0, sendFilesFuture); + handler.sendFiles(store, metas.toArray(new StoreFileMetadata[0]), () -> 0, sendFilesFuture, null); sendFilesFuture.actionGet(); Store.MetadataSnapshot targetStoreMetadata = targetStore.getMetadata(); Store.RecoveryDiff recoveryDiff = targetStoreMetadata.recoveryDiff(metadata); @@ -570,7 +570,7 @@ public void writeFileChunk( store, metas.toArray(new StoreFileMetadata[0]), () -> 0, - new LatchedActionListener<>(ActionListener.wrap(r -> sendFilesError.set(null), e -> sendFilesError.set(e)), latch) + new LatchedActionListener<>(ActionListener.wrap(r -> sendFilesError.set(null), e -> sendFilesError.set(e)), latch), null ); latch.await(); assertThat(sendFilesError.get(), instanceOf(IOException.class)); @@ -640,7 +640,7 @@ public void writeFileChunk( between(1, 4) ); PlainActionFuture sendFilesFuture = new PlainActionFuture<>(); - handler.sendFiles(store, metas.toArray(new StoreFileMetadata[0]), () -> 0, sendFilesFuture); + handler.sendFiles(store, metas.toArray(new StoreFileMetadata[0]), () -> 0, sendFilesFuture, null); Exception ex = expectThrows(Exception.class, sendFilesFuture::actionGet); final IOException unwrappedCorruption = ExceptionsHelper.unwrapCorruption(ex); if (throwCorruptedIndexException) { @@ -808,7 +808,7 @@ public void writeFileChunk( List files = generateFiles(store, between(1, 10), () -> between(1, chunkSize * 20)); int totalChunks = files.stream().mapToInt(md -> ((int) md.length() + chunkSize - 1) / chunkSize).sum(); PlainActionFuture sendFilesFuture = new PlainActionFuture<>(); - handler.sendFiles(store, files.toArray(new StoreFileMetadata[0]), () -> 0, sendFilesFuture); + handler.sendFiles(store, files.toArray(new StoreFileMetadata[0]), () -> 0, sendFilesFuture, null); assertBusy(() -> { assertThat(sentChunks.get(), equalTo(Math.min(totalChunks, maxConcurrentChunks))); assertThat(unrepliedChunks, hasSize(sentChunks.get())); @@ -886,7 +886,8 @@ public void writeFileChunk( store, files.toArray(new StoreFileMetadata[0]), () -> 0, - new LatchedActionListener<>(ActionListener.wrap(r -> sendFilesError.set(null), e -> sendFilesError.set(e)), sendFilesLatch) + new LatchedActionListener<>(ActionListener.wrap(r -> sendFilesError.set(null), e -> sendFilesError.set(e)), sendFilesLatch), + null ); assertBusy(() -> assertThat(sentChunks.get(), equalTo(Math.min(totalChunks, maxConcurrentChunks)))); List failedChunks = randomSubsetOf(between(1, unrepliedChunks.size()), unrepliedChunks); diff --git a/server/src/test/java/org/opensearch/indices/recovery/PeerRecoverySourceServiceTests.java b/server/src/test/java/org/opensearch/indices/recovery/PeerRecoverySourceServiceTests.java index 4fbae4b0d53ca..c8bc5c86733b1 100644 --- a/server/src/test/java/org/opensearch/indices/recovery/PeerRecoverySourceServiceTests.java +++ b/server/src/test/java/org/opensearch/indices/recovery/PeerRecoverySourceServiceTests.java @@ -40,6 +40,8 @@ import org.opensearch.index.shard.IndexShardTestCase; import org.opensearch.index.store.Store; import org.opensearch.indices.IndicesService; +import org.opensearch.otel.OtelService; +import org.opensearch.plugins.PluginsService; import org.opensearch.test.NodeRoles; import org.opensearch.transport.TransportService; @@ -60,7 +62,8 @@ public void testDuplicateRecoveries() throws IOException { PeerRecoverySourceService peerRecoverySourceService = new PeerRecoverySourceService( mock(TransportService.class), indicesService, - new RecoverySettings(Settings.EMPTY, new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)) + new RecoverySettings(Settings.EMPTY, new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)), + new OtelService(mock(PluginsService.class)) ); StartRecoveryRequest startRecoveryRequest = new StartRecoveryRequest( primary.shardId(), From 89897d4f052fb2f9a5201d3807affa607236945c Mon Sep 17 00:00:00 2001 From: rishabhmaurya Date: Mon, 10 Apr 2023 19:15:52 +0200 Subject: [PATCH 2/8] otel debugging and testing infra related changes Signed-off-by: Stevan Buzejic --- .../org/opensearch/gradle/test/ClusterConfiguration.groovy | 2 +- .../java/org/opensearch/gradle/testclusters/RunTask.java | 2 +- .../org/opensearch/tools/launchers/SystemJvmOptions.java | 5 +++++ gradle.properties | 2 +- gradle/run.gradle | 2 -- .../java/org/opensearch/index/shard/IndexShardTestCase.java | 2 +- 6 files changed, 9 insertions(+), 6 deletions(-) diff --git a/buildSrc/src/main/groovy/org/opensearch/gradle/test/ClusterConfiguration.groovy b/buildSrc/src/main/groovy/org/opensearch/gradle/test/ClusterConfiguration.groovy index a5207933c3c72..2877d243bf838 100644 --- a/buildSrc/src/main/groovy/org/opensearch/gradle/test/ClusterConfiguration.groovy +++ b/buildSrc/src/main/groovy/org/opensearch/gradle/test/ClusterConfiguration.groovy @@ -42,7 +42,7 @@ class ClusterConfiguration { String distribution = 'archive' @Input - int numNodes = 1 + int numNodes = 2 @Input int numBwcNodes = 0 diff --git a/buildSrc/src/main/java/org/opensearch/gradle/testclusters/RunTask.java b/buildSrc/src/main/java/org/opensearch/gradle/testclusters/RunTask.java index c5035f3b082fe..e62827bda5fcc 100644 --- a/buildSrc/src/main/java/org/opensearch/gradle/testclusters/RunTask.java +++ b/buildSrc/src/main/java/org/opensearch/gradle/testclusters/RunTask.java @@ -64,7 +64,7 @@ public class RunTask extends DefaultTestClustersTask { private static final int DEFAULT_DEBUG_PORT = 5005; public static final String LOCALHOST_ADDRESS_PREFIX = "127.0.0.1:"; - private Boolean debug = false; + private Boolean debug = true; private Boolean debugServer = false; diff --git a/distribution/tools/launchers/src/main/java/org/opensearch/tools/launchers/SystemJvmOptions.java b/distribution/tools/launchers/src/main/java/org/opensearch/tools/launchers/SystemJvmOptions.java index aa3dfbe39ee96..9b469772ccc17 100644 --- a/distribution/tools/launchers/src/main/java/org/opensearch/tools/launchers/SystemJvmOptions.java +++ b/distribution/tools/launchers/src/main/java/org/opensearch/tools/launchers/SystemJvmOptions.java @@ -47,6 +47,11 @@ static List systemJvmOptions() { * networkaddress.cache.ttl; can be set to -1 to cache forever. */ "-Dopensearch.networkaddress.cache.ttl=60", + "-Djava.security.policy=/home/rishma/ws/pa/performance-analyzer/config/opensearch_security.policy", + "-Djdk.attach.allowAttachSelf=true", + "-Dclk.tck=100", + "--add-opens=jdk.attach/sun.tools.attach=ALL-UNNAMED", + /* * Cache ttl in seconds for negative DNS lookups noting that this overrides the JDK security property * networkaddress.cache.negative ttl; set to -1 to cache forever. diff --git a/gradle.properties b/gradle.properties index 73df0940ce181..02d578e1bc1e5 100644 --- a/gradle.properties +++ b/gradle.properties @@ -27,7 +27,7 @@ systemProp.org.gradle.dependency.duplicate.project.detection=false # Enforce the build to fail on deprecated gradle api usage systemProp.org.gradle.warning.mode=fail - +kotlin.stdlib.default.dependency=false # forcing to use TLS1.2 to avoid failure in vault # see https://github.com/hashicorp/vault/issues/8750#issuecomment-631236121 systemProp.jdk.tls.client.protocols=TLSv1.2 diff --git a/gradle/run.gradle b/gradle/run.gradle index 639479e97d28f..6a55d705cad9c 100644 --- a/gradle/run.gradle +++ b/gradle/run.gradle @@ -37,8 +37,6 @@ def numZones = findProperty('numZones') as Integer ?: 1 testClusters { runTask { testDistribution = 'archive' - if (numZones > 1) numberOfZones = numZones - if (numNodes > 1) numberOfNodes = numNodes } } diff --git a/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestCase.java b/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestCase.java index b785574ca52b2..a7dbedd0e72da 100644 --- a/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestCase.java +++ b/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestCase.java @@ -982,7 +982,7 @@ protected final void recoverUnstartedReplica( primary, new AsyncRecoveryTarget(recoveryTarget, threadPool.generic(), primary, replica, replicatePrimaryFunction), request, - recoverySettings + recoverySettings, null ); primary.updateShardState( primary.routingEntry(), From e608941525d3a7f2342c79a0ffbb5c7a0eb83124 Mon Sep 17 00:00:00 2001 From: Rishabh Maurya Date: Tue, 18 Apr 2023 01:52:27 +0200 Subject: [PATCH 3/8] Thread resource monitoring and fixes for context propagation Signed-off-by: Stevan Buzejic --- .../OTelContextPreservingActionListener.java | 115 ------- .../util/concurrent/OpenSearchExecutors.java | 4 +- .../OpenSearchThreadPoolExecutor.java | 2 +- .../index/seqno/RetentionLeaseSyncAction.java | 5 +- .../LocalStorePeerRecoverySourceHandler.java | 65 +++- .../recovery/PeerRecoverySourceService.java | 13 +- .../recovery/RecoverySourceHandler.java | 114 +++++-- .../recovery/RetryableTransportClient.java | 7 +- .../SegmentFileTransferHandler.java | 4 +- .../main/java/org/opensearch/node/Node.java | 12 +- .../java/org/opensearch/otel/OtelService.java | 306 ++++++++++++++---- .../threadpool/ScalingExecutorBuilder.java | 23 +- .../org/opensearch/threadpool/ThreadPool.java | 24 +- 13 files changed, 456 insertions(+), 238 deletions(-) delete mode 100644 server/src/main/java/org/opensearch/action/support/OTelContextPreservingActionListener.java diff --git a/server/src/main/java/org/opensearch/action/support/OTelContextPreservingActionListener.java b/server/src/main/java/org/opensearch/action/support/OTelContextPreservingActionListener.java deleted file mode 100644 index d250143bc1ec0..0000000000000 --- a/server/src/main/java/org/opensearch/action/support/OTelContextPreservingActionListener.java +++ /dev/null @@ -1,115 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -/* - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ - -package org.opensearch.action.support; - -import io.opentelemetry.api.common.AttributeKey; -import io.opentelemetry.api.trace.Span; -import io.opentelemetry.context.Context; -import io.opentelemetry.context.Scope; -import org.opensearch.action.ActionListener; -import org.opensearch.common.util.concurrent.ThreadContext; -import org.opensearch.otel.OtelService; - -import java.util.Objects; - -import static io.opentelemetry.api.common.AttributeKey.longKey; -import static io.opentelemetry.api.common.AttributeKey.stringKey; - -/** - * Restores the given {@link ThreadContext.StoredContext} - * once the listener is invoked - * - * @opensearch.internal - */ -public final class OTelContextPreservingActionListener implements ActionListener { - - private final ActionListener delegate; - public final Context beforeAttachContext; - public final Context afterAttachContext; - - private final String spanID; - - public final long startCPUUsage; - public final long startMemoryUsage; - public final long startThreadContentionTime; - - public OTelContextPreservingActionListener(Context beforeAttachContext, Context afterAttachContext, ActionListener delegate, String spanID) { - this.delegate = delegate; - this.beforeAttachContext = beforeAttachContext; - this.afterAttachContext = afterAttachContext; - this.spanID = spanID; - startCPUUsage = OtelService.getCPUUsage(Thread.currentThread().getId()); - startMemoryUsage = OtelService.getMemoryUsage(Thread.currentThread().getId()); - startThreadContentionTime = OtelService.getThreadContentionTime(Thread.currentThread().getId()); - } - - @Override - public void onResponse(R r) { - try (Scope ignored = Objects.requireNonNull(afterAttachContext).makeCurrent()) { - Span span = Span.current(); - if (spanID != null && Span.current() != Span.getInvalid() && - span.getSpanContext().getSpanId().equals(spanID)) { - span.setAttribute(AttributeKey.longKey("CPUUsage"), - OtelService.getCPUUsage(Thread.currentThread().getId())- startCPUUsage); - span.setAttribute(AttributeKey.longKey("MemoryUsage"), - OtelService.getMemoryUsage(Thread.currentThread().getId())- startMemoryUsage); - span.setAttribute(AttributeKey.longKey("ContentionTime"), - OtelService.getThreadContentionTime(Thread.currentThread().getId())- startThreadContentionTime); - span.setAttribute(stringKey("finish-thread-name"), Thread.currentThread().getName()); - span.setAttribute(longKey("finish-thread-id"), Thread.currentThread().getId()); - Span.current().end(); - } - } - - try (Scope ignored = Objects.requireNonNull(beforeAttachContext).makeCurrent()) { - delegate.onResponse(r); - } - } - - @Override - public void onFailure(Exception e) { - try (Scope ignored = Objects.requireNonNull(afterAttachContext).makeCurrent()) { - if (spanID != null && Span.current() != Span.getInvalid() && - Span.current().getSpanContext().getSpanId().equals(spanID)) { - Span.current().end(); - } - } - try (Scope ignored = Objects.requireNonNull(beforeAttachContext).makeCurrent()) { - delegate.onFailure(e); - } - } - - @Override - public String toString() { - return getClass().getName() + "/" + delegate.toString(); - } -} diff --git a/server/src/main/java/org/opensearch/common/util/concurrent/OpenSearchExecutors.java b/server/src/main/java/org/opensearch/common/util/concurrent/OpenSearchExecutors.java index 14fa7d65bb1fe..3f48faeef3a66 100644 --- a/server/src/main/java/org/opensearch/common/util/concurrent/OpenSearchExecutors.java +++ b/server/src/main/java/org/opensearch/common/util/concurrent/OpenSearchExecutors.java @@ -41,6 +41,7 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; import org.opensearch.node.Node; +import org.opensearch.otel.OtelService; import org.opensearch.threadpool.RunnableTaskExecutionListener; import org.opensearch.threadpool.TaskAwareRunnable; @@ -129,7 +130,6 @@ public static PrioritizedOpenSearchThreadPoolExecutor newSinglePrioritizing( ) { return new PrioritizedOpenSearchThreadPoolExecutor(name, 1, 1, 0L, TimeUnit.MILLISECONDS, threadFactory, contextHolder, timer); } - public static OpenSearchThreadPoolExecutor newScaling( String name, int min, @@ -353,7 +353,7 @@ public void execute(Runnable command) { * @return an {@link ExecutorService} that executes submitted tasks on the current thread */ public static ExecutorService newDirectExecutorService() { - return Context.taskWrapping(DIRECT_EXECUTOR_SERVICE); + return OtelService.taskWrapping(DIRECT_EXECUTOR_SERVICE); } public static String threadName(Settings settings, String namePrefix) { diff --git a/server/src/main/java/org/opensearch/common/util/concurrent/OpenSearchThreadPoolExecutor.java b/server/src/main/java/org/opensearch/common/util/concurrent/OpenSearchThreadPoolExecutor.java index d967b7423ca80..fd64559f28ea1 100644 --- a/server/src/main/java/org/opensearch/common/util/concurrent/OpenSearchThreadPoolExecutor.java +++ b/server/src/main/java/org/opensearch/common/util/concurrent/OpenSearchThreadPoolExecutor.java @@ -85,7 +85,7 @@ final String getName() { } @SuppressForbidden(reason = "properly rethrowing errors, see OpenSearchExecutors.rethrowErrors") - OpenSearchThreadPoolExecutor( + protected OpenSearchThreadPoolExecutor( String name, int corePoolSize, int maximumPoolSize, diff --git a/server/src/main/java/org/opensearch/index/seqno/RetentionLeaseSyncAction.java b/server/src/main/java/org/opensearch/index/seqno/RetentionLeaseSyncAction.java index afcf5c6766194..aa8bfd58815a1 100644 --- a/server/src/main/java/org/opensearch/index/seqno/RetentionLeaseSyncAction.java +++ b/server/src/main/java/org/opensearch/index/seqno/RetentionLeaseSyncAction.java @@ -32,6 +32,7 @@ package org.opensearch.index.seqno; +import io.opentelemetry.context.Context; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; @@ -60,6 +61,7 @@ import org.opensearch.index.shard.ShardId; import org.opensearch.indices.IndicesService; import org.opensearch.indices.SystemIndices; +import org.opensearch.otel.OtelService; import org.opensearch.tasks.Task; import org.opensearch.tasks.TaskId; import org.opensearch.threadpool.ThreadPool; @@ -129,8 +131,9 @@ final void sync( String primaryAllocationId, long primaryTerm, RetentionLeases retentionLeases, - ActionListener listener + ActionListener listener1 ) { + final ActionListener listener = new OtelService.SpanPreservingActionListener<>(listener1, Context.current()); final ThreadContext threadContext = threadPool.getThreadContext(); try (ThreadContext.StoredContext ignore = threadContext.stashContext()) { // we have to execute under the system context so that if security is enabled the sync is authorized diff --git a/server/src/main/java/org/opensearch/indices/recovery/LocalStorePeerRecoverySourceHandler.java b/server/src/main/java/org/opensearch/indices/recovery/LocalStorePeerRecoverySourceHandler.java index ac8313403dad8..93075acb1d43e 100644 --- a/server/src/main/java/org/opensearch/indices/recovery/LocalStorePeerRecoverySourceHandler.java +++ b/server/src/main/java/org/opensearch/indices/recovery/LocalStorePeerRecoverySourceHandler.java @@ -34,7 +34,9 @@ import java.io.Closeable; import java.io.IOException; +import java.util.function.BiFunction; import java.util.function.Consumer; +import java.util.function.IntSupplier; /** * This handler is used for node-to-node peer recovery when the recovery target is a replica/ or a relocating primary @@ -108,6 +110,7 @@ && isTargetSameHistory() logger.trace("performing sequence numbers based recovery. starting at [{}]", request.startingSeqNo()); startingSeqNo = request.startingSeqNo(); if (retentionLeaseRef.get() == null) { + createRetentionLease(startingSeqNo, ActionListener.map(sendFileStep, ignored -> SendFileResult.EMPTY)); } else { sendFileStep.onResponse(SendFileResult.EMPTY); @@ -146,15 +149,23 @@ && isTargetSameHistory() // If the target previously had a copy of this shard then a file-based recovery might move its global // checkpoint backwards. We must therefore remove any existing retention lease so that we can create a // new one later on in the recovery. - shard.removePeerRecoveryRetentionLease( - request.targetNode().getId(), + BiFunction, Void> removePeerRecoveryRetentionLeaseFun = + (args, actionListener) -> { + shard.removePeerRecoveryRetentionLease((String) args[0], + (ActionListener) actionListener); + return null; + }; + OtelService.callFunctionAndStartSpan( + "removePeerRecoveryRetentionLease", + removePeerRecoveryRetentionLeaseFun, new ThreadedActionListener<>( logger, shard.getThreadPool(), ThreadPool.Names.GENERIC, deleteRetentionLeaseStep, false - ) + ), + request.targetNode().getId() ); } catch (RetentionLeaseNotFoundException e) { logger.debug("no peer-recovery retention lease for " + request.targetAllocationId()); @@ -164,7 +175,21 @@ && isTargetSameHistory() deleteRetentionLeaseStep.whenComplete(ignored -> { assert Transports.assertNotTransportThread(this + "[phase1]"); - phase1(wrappedSafeCommit.get(), startingSeqNo, () -> estimateNumOps, sendFileStep, false); + BiFunction, Void> phase1Fun = + (args, actionListener) -> { + phase1((IndexCommit)args[0], (long) args[1], (IntSupplier) args[2], + (ActionListener) actionListener, (boolean) args[3]); + return null; + }; + OtelService.callFunctionAndStartSpan( + "phase1", + phase1Fun, + sendFileStep, + wrappedSafeCommit.get(), + startingSeqNo, + (IntSupplier)(() -> estimateNumOps), + false + ); }, onFailure); } catch (final Exception e) { @@ -176,7 +201,17 @@ && isTargetSameHistory() sendFileStep.whenComplete(r -> { assert Transports.assertNotTransportThread(this + "[prepareTargetForTranslog]"); // For a sequence based recovery, the target can keep its local translog - prepareTargetForTranslog(countNumberOfHistoryOperations(startingSeqNo), prepareEngineStep); + BiFunction, Void> prepareTargetForTranslogFun = + (args, actionListener) -> { + prepareTargetForTranslog((int)args[0], (ActionListener) actionListener); + return null; + }; + OtelService.callFunctionAndStartSpan( + "prepareTargetForTranslog", + prepareTargetForTranslogFun, + prepareEngineStep, + countNumberOfHistoryOperations(startingSeqNo) + ); }, onFailure); prepareEngineStep.whenComplete(prepareEngineTime -> { @@ -215,18 +250,30 @@ && isTargetSameHistory() final long maxSeqNoOfUpdatesOrDeletes = shard.getMaxSeqNoOfUpdatesOrDeletes(); final RetentionLeases retentionLeases = shard.getRetentionLeases(); final long mappingVersionOnPrimary = shard.indexSettings().getIndexMetadata().getMappingVersion(); - phase2( + BiFunction, Void> phase2Fun = + (args, actionListener) -> { + try { + phase2((long) args[0], (long) args[1], (Translog.Snapshot) args[2], (long)args[3], (long)args[4], + (RetentionLeases)args[5], (long)args[6], (ActionListener) actionListener); + } catch (IOException e) { + throw new RuntimeException(e); + } + return null; + }; + OtelService.callFunctionAndStartSpan( + "phase2", + phase2Fun, + sendSnapshotStep, startingSeqNo, endingSeqNo, phase2Snapshot, maxSeenAutoIdTimestamp, maxSeqNoOfUpdatesOrDeletes, retentionLeases, - mappingVersionOnPrimary, - sendSnapshotStep + mappingVersionOnPrimary ); - }, onFailure); + finalizeStepAndCompleteFuture(startingSeqNo, sendSnapshotStep, sendFileStep, prepareEngineStep, onFailure); } } diff --git a/server/src/main/java/org/opensearch/indices/recovery/PeerRecoverySourceService.java b/server/src/main/java/org/opensearch/indices/recovery/PeerRecoverySourceService.java index 101ebe6d574be..a57e2b9493a39 100644 --- a/server/src/main/java/org/opensearch/indices/recovery/PeerRecoverySourceService.java +++ b/server/src/main/java/org/opensearch/indices/recovery/PeerRecoverySourceService.java @@ -58,7 +58,6 @@ import org.opensearch.indices.IndicesService; import org.opensearch.otel.OtelService; import org.opensearch.tasks.Task; -import org.opensearch.tasks.TaskResourceTrackingService; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportChannel; import org.opensearch.transport.TransportRequestHandler; @@ -71,8 +70,8 @@ import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.function.BiFunction; -import static io.opentelemetry.api.common.AttributeKey.doubleKey; import static io.opentelemetry.api.common.AttributeKey.longKey; import static io.opentelemetry.api.common.AttributeKey.stringKey; @@ -167,13 +166,11 @@ public void clusterChanged(ClusterChangedEvent event) { } private void recover(StartRecoveryRequest request, ActionListener listener) { - listener = OtelService.startSpan("recover", listener); Span span = Span.current(); span.setAttribute(stringKey("index-name"), request.shardId().getIndexName()); span.setAttribute(longKey("shard-id"), request.shardId().id()); span.setAttribute(stringKey("source-node"), request.sourceNode().getId()); span.setAttribute(stringKey("target-node"), request.targetNode().getId()); - otelService.emitResources(span, "resource-usage-start"); final IndexService indexService = indicesService.indexServiceSafe(request.shardId().getIndex()); final IndexShard shard = indexService.getShard(request.shardId().id()); @@ -202,7 +199,6 @@ private void recover(StartRecoveryRequest request, ActionListener ongoingRecoveries.remove(shard, handler))); - otelService.emitResources(span, "resource-usage-end"); } private void reestablish(ReestablishRecoveryRequest request, ActionListener listener) { @@ -221,7 +217,12 @@ private void reestablish(ReestablishRecoveryRequest request, ActionListener { @Override public void messageReceived(final StartRecoveryRequest request, final TransportChannel channel, Task task) throws Exception { - recover(request, new ChannelActionListener<>(channel, Actions.START_RECOVERY, request)); + BiFunction, Void> recoverFunction = (args, actionListener) -> { + recover((StartRecoveryRequest) args[0], (ActionListener) actionListener); + return null; + }; + OtelService.callFunctionAndStartSpan("recover", recoverFunction, + new ChannelActionListener<>(channel, Actions.START_RECOVERY, request), request); } } diff --git a/server/src/main/java/org/opensearch/indices/recovery/RecoverySourceHandler.java b/server/src/main/java/org/opensearch/indices/recovery/RecoverySourceHandler.java index 93d02d41fe837..22b53604f6b24 100644 --- a/server/src/main/java/org/opensearch/indices/recovery/RecoverySourceHandler.java +++ b/server/src/main/java/org/opensearch/indices/recovery/RecoverySourceHandler.java @@ -33,6 +33,7 @@ package org.opensearch.indices.recovery; import io.opentelemetry.api.trace.Span; +import io.opentelemetry.context.Scope; import org.apache.logging.log4j.Logger; import org.apache.lucene.index.CorruptIndexException; import org.apache.lucene.index.IndexCommit; @@ -83,18 +84,15 @@ import java.util.Comparator; import java.util.List; import java.util.Locale; -import java.util.Objects; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; +import java.util.function.BiFunction; import java.util.function.Consumer; import java.util.function.IntSupplier; import java.util.stream.StreamSupport; -import static io.opentelemetry.api.common.AttributeKey.longKey; -import static io.opentelemetry.api.common.AttributeKey.stringKey; - /** * RecoverySourceHandler handles the three phases of shard recovery, which is * everything relating to copying the segment files as well as sending translog @@ -212,7 +210,26 @@ protected void finalizeStepAndCompleteFuture( final StepListener finalizeStep = new StepListener<>(); // Recovery target can trim all operations >= startingSeqNo as we have sent all these operations in the phase 2 final long trimAboveSeqNo = startingSeqNo - 1; - sendSnapshotStep.whenComplete(r -> finalizeRecovery(r.targetLocalCheckpoint, trimAboveSeqNo, finalizeStep), onFailure); + sendSnapshotStep.whenComplete(r -> { + BiFunction, Void> finalizeRecoveryFun = + (args, actionListener) -> { + try { + finalizeRecovery((long) args[0], (long) args[1], (ActionListener) actionListener); + } catch (IOException e) { + throw new RuntimeException(e); + } + return null; + }; + OtelService.callFunctionAndStartSpan( + "finalizeRecovery", + finalizeRecoveryFun, + finalizeStep, + r.targetLocalCheckpoint, + trimAboveSeqNo + ); + }, + onFailure + ); finalizeStep.whenComplete(r -> { final long phase1ThrottlingWaitTime = 0L; // TODO: return the actual throttle time @@ -447,26 +464,72 @@ void phase1( final StepListener createRetentionLeaseStep = new StepListener<>(); final StepListener cleanFilesStep = new StepListener<>(); cancellableThreads.checkForCancel(); - ActionListener wrapperSendFileInfoStep = OtelService.startSpan("receiveFileInfo", sendFileInfoStep); - recoveryTarget.receiveFileInfo( + BiFunction, Void> receiveFileInfoFunction = (args, actionListener) -> { + recoveryTarget.receiveFileInfo( + (List) args[0], + (List)args[1], + (List) args[2], + (List) args[3], + (int) args[4], + (ActionListener)actionListener + ); + return null; + }; + OtelService.callFunctionAndStartSpan( + "sendFileInfo", + receiveFileInfoFunction, + sendFileInfoStep, phase1FileNames, phase1FileSizes, phase1ExistingFileNames, phase1ExistingFileSizes, - translogOps.getAsInt(), - wrapperSendFileInfoStep + translogOps.getAsInt() ); sendFileInfoStep.whenComplete( - r -> sendFiles(store, phase1Files.toArray(new StoreFileMetadata[0]), translogOps, sendFilesStep), + r -> { + BiFunction, Void> sendFileFunction = (args, actionListener) -> { + sendFiles( + (Store) args[0], + (StoreFileMetadata[])args[1], + (IntSupplier) args[2], + (ActionListener) actionListener + ); + return null; + }; + OtelService.callFunctionAndStartSpan( + "sendFiles", + sendFileFunction, + sendFilesStep, + store, + phase1Files.toArray(new StoreFileMetadata[0]), + translogOps + ); + }, listener::onFailure ); + // When doing peer recovery of remote store enabled replica, retention leases are not required. if (skipCreateRetentionLeaseStep) { sendFilesStep.whenComplete(r -> createRetentionLeaseStep.onResponse(null), listener::onFailure); } else { - sendFilesStep.whenComplete(r -> createRetentionLease(startingSeqNo, createRetentionLeaseStep), listener::onFailure); + sendFilesStep.whenComplete(r -> { + BiFunction, Void> createRetentionLeaseFunction = (args, actionListener) -> { + createRetentionLease( + (long) args[0], + (ActionListener) actionListener + ); + return null; + }; + OtelService.callFunctionAndStartSpan( + "createRetentionLease", + createRetentionLeaseFunction, + createRetentionLeaseStep, + startingSeqNo + ); + }, + listener::onFailure); } createRetentionLeaseStep.whenComplete(retentionLease -> { @@ -526,10 +589,6 @@ void phase1( } void sendFiles(Store store, StoreFileMetadata[] files, IntSupplier translogOps, ActionListener listener) { - if (otelService != null) { - listener = OtelService.startSpan("sendFiles", listener); - otelService.emitResources(Span.current(), "start-resources"); - } final MultiChunkTransfer transfer = transferHandler.createTransfer( store, files, @@ -538,12 +597,9 @@ void sendFiles(Store store, StoreFileMetadata[] files, IntSupplier translogOps, ); resources.add(transfer); transfer.start(); - otelService.emitResources(Span.current(), "end-resources"); } void createRetentionLease(final long startingSeqNo, ActionListener listener) { - listener = OtelService.startSpan("createRetentionLease", listener); - ActionListener finalListener = listener; RunUnderPrimaryPermit.run(() -> { // Clone the peer recovery retention lease belonging to the source shard. We are retaining history between the the local // checkpoint of the safe commit we're creating and this lease's retained seqno with the retention lock, and by cloning an @@ -560,7 +616,7 @@ void createRetentionLease(final long startingSeqNo, ActionListener(logger, shard.getThreadPool(), ThreadPool.Names.GENERIC, cloneRetentionLeaseStep, false) ); logger.trace("cloned primary's retention lease as [{}]", clonedLease); - cloneRetentionLeaseStep.whenComplete(rr -> finalListener.onResponse(clonedLease), finalListener::onFailure); + cloneRetentionLeaseStep.whenComplete(rr -> listener.onResponse(clonedLease), listener::onFailure); } catch (RetentionLeaseNotFoundException e) { // it's possible that the primary has no retention lease yet if we are doing a rolling upgrade from a version before // 7.4, and in that case we just create a lease using the local checkpoint of the safe commit which we're using for @@ -573,7 +629,7 @@ void createRetentionLease(final long startingSeqNo, ActionListener(logger, shard.getThreadPool(), ThreadPool.Names.GENERIC, addRetentionLeaseStep, false) ); - addRetentionLeaseStep.whenComplete(rr -> finalListener.onResponse(newLease), finalListener::onFailure); + addRetentionLeaseStep.whenComplete(rr -> listener.onResponse(newLease), listener::onFailure); logger.trace("created retention lease with estimated checkpoint of [{}]", estimatedGlobalCheckpoint); } }, shardId + " establishing retention lease for [" + request.targetAllocationId() + "]", shard, cancellableThreads, logger); @@ -617,15 +673,13 @@ boolean canSkipPhase1(Store.MetadataSnapshot source, Store.MetadataSnapshot targ } void prepareTargetForTranslog(int totalTranslogOps, ActionListener listener) { - listener = OtelService.startSpan("prepareForTranslog", listener); StopWatch stopWatch = new StopWatch().start(); - ActionListener finalListener = listener; final ActionListener wrappedListener = ActionListener.wrap(nullVal -> { stopWatch.stop(); final TimeValue tookTime = stopWatch.totalTime(); logger.trace("recovery [phase1]: remote engine start took [{}]", tookTime); - finalListener.onResponse(tookTime); - }, e -> finalListener.onFailure(new RecoveryEngineException(shard.shardId(), 1, "prepare target for translog failed", e))); + listener.onResponse(tookTime); + }, e -> listener.onFailure(new RecoveryEngineException(shard.shardId(), 1, "prepare target for translog failed", e))); // Send a request preparing the new shard's translog to receive operations. This ensures the shard engine is started and disables // garbage collection (not the JVM's GC!) of tombstone deletes. logger.trace("recovery [phase1]: prepare remote engine for translog"); @@ -658,7 +712,6 @@ void phase2( final long mappingVersion, ActionListener listener ) throws IOException { - listener = OtelService.startSpan("phase2", listener); if (shard.state() == IndexShardState.CLOSED) { throw new IndexShardClosedException(request.shardId()); } @@ -675,7 +728,6 @@ void phase2( mappingVersion, sendListener ); - ActionListener finalListener = listener; sendListener.whenComplete(ignored -> { final long skippedOps = sender.skippedOps.get(); final int totalSentOps = sender.sentOps.get(); @@ -691,7 +743,7 @@ void phase2( stopWatch.stop(); final TimeValue tookTime = stopWatch.totalTime(); logger.trace("recovery [phase2]: took [{}]", tookTime); - finalListener.onResponse(new SendSnapshotResult(targetLocalCheckpoint, totalSentOps, tookTime)); + listener.onResponse(new SendSnapshotResult(targetLocalCheckpoint, totalSentOps, tookTime)); }, listener::onFailure); sender.start(); } @@ -809,8 +861,6 @@ public void close() throws IOException { } void finalizeRecovery(long targetLocalCheckpoint, long trimAboveSeqNo, ActionListener listener) throws IOException { - listener = OtelService.startSpan("finalizeRecovery", listener); - if (shard.state() == IndexShardState.CLOSED) { throw new IndexShardClosedException(request.shardId()); } @@ -834,7 +884,6 @@ void finalizeRecovery(long targetLocalCheckpoint, long trimAboveSeqNo, ActionLis final StepListener finalizeListener = new StepListener<>(); cancellableThreads.checkForCancel(); recoveryTarget.finalizeRecovery(globalCheckpoint, trimAboveSeqNo, finalizeListener); - ActionListener finalListener = listener; finalizeListener.whenComplete(r -> { RunUnderPrimaryPermit.run( () -> shard.updateGlobalCheckpointForShard(request.targetAllocationId(), globalCheckpoint), @@ -866,7 +915,7 @@ void finalizeRecovery(long targetLocalCheckpoint, long trimAboveSeqNo, ActionLis } stopWatch.stop(); logger.info("finalizing recovery took [{}]", stopWatch.totalTime()); - finalListener.onResponse(null); + listener.onResponse(null); }, listener::onFailure); } @@ -922,14 +971,13 @@ private void cleanFiles( // Once the files have been renamed, any other files that are not // related to this recovery (out of date segments, for example) // are deleted - ActionListener wrappedListener = OtelService.startSpan("receiveFileInfo", listener); cancellableThreads.checkForCancel(); recoveryTarget.cleanFiles( translogOps.getAsInt(), globalCheckpoint, sourceMetadata, - ActionListener.delegateResponse(wrappedListener, (l, e) -> ActionListener.completeWith(l, () -> { + ActionListener.delegateResponse(listener, (l, e) -> ActionListener.completeWith(l, () -> { StoreFileMetadata[] mds = StreamSupport.stream(sourceMetadata.spliterator(), false).toArray(StoreFileMetadata[]::new); ArrayUtil.timSort(mds, Comparator.comparingLong(StoreFileMetadata::length)); // check small files first transferHandler.handleErrorOnSendFiles(store, e, mds); diff --git a/server/src/main/java/org/opensearch/indices/recovery/RetryableTransportClient.java b/server/src/main/java/org/opensearch/indices/recovery/RetryableTransportClient.java index f7d75b356d6a3..2fd51b6669587 100644 --- a/server/src/main/java/org/opensearch/indices/recovery/RetryableTransportClient.java +++ b/server/src/main/java/org/opensearch/indices/recovery/RetryableTransportClient.java @@ -8,6 +8,7 @@ package org.opensearch.indices.recovery; +import io.opentelemetry.context.Context; import org.apache.logging.log4j.Logger; import org.opensearch.BaseExceptionsHelper; import org.opensearch.action.ActionListener; @@ -20,6 +21,8 @@ import org.opensearch.common.util.CancellableThreads; import org.opensearch.common.util.concurrent.ConcurrentCollections; import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; +import org.opensearch.common.util.concurrent.OpenSearchRejectedExecutionException; +import org.opensearch.otel.OtelService; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.ConnectTransportException; import org.opensearch.transport.RemoteTransportException; @@ -92,7 +95,9 @@ public void tryAction(ActionListener listener) { action, request, options, - new ActionListenerResponseHandler<>(listener, reader, ThreadPool.Names.GENERIC) + new ActionListenerResponseHandler<>( + new OtelService.SpanPreservingActionListener<>(listener, Context.current()), + reader, ThreadPool.Names.GENERIC) ); } diff --git a/server/src/main/java/org/opensearch/indices/replication/SegmentFileTransferHandler.java b/server/src/main/java/org/opensearch/indices/replication/SegmentFileTransferHandler.java index 9720d0cc446be..bbf9c34153c71 100644 --- a/server/src/main/java/org/opensearch/indices/replication/SegmentFileTransferHandler.java +++ b/server/src/main/java/org/opensearch/indices/replication/SegmentFileTransferHandler.java @@ -166,13 +166,13 @@ protected FileChunk nextChunkRequest(StoreFileMetadata md) throws IOException { protected void executeChunkRequest(FileChunk request, ActionListener listener1) { cancellableThreads.checkForCancel(); if (otelService != null) { - otelService.emitResources(Span.current(), "StartExecuteChunkRequest"); + // OtelService.emitResources(Span.current(), "StartExecuteChunkRequest", otelService.otelEventListenerList); } ActionListener listener2 = ActionListener.runBefore(listener1, request::close); ActionListener listener3 = ActionListener.runBefore(listener2, () -> { if (otelService != null) { - otelService.emitResources(Span.current(), "EndExecuteChunkRequest"); + // OtelService.emitResources(Span.current(), "EndExecuteChunkRequest", otelService.otelEventListenerList); } }); diff --git a/server/src/main/java/org/opensearch/node/Node.java b/server/src/main/java/org/opensearch/node/Node.java index 9e3e9ba9bdbc7..b561e33eea56e 100644 --- a/server/src/main/java/org/opensearch/node/Node.java +++ b/server/src/main/java/org/opensearch/node/Node.java @@ -39,6 +39,7 @@ import org.opensearch.cluster.routing.allocation.AwarenessReplicaBalance; import org.opensearch.index.IndexModule; import org.opensearch.index.IndexingPressureService; +import org.opensearch.otel.OtelEventListener; import org.opensearch.otel.OtelService; import org.opensearch.tasks.TaskResourceTrackingService; import org.opensearch.threadpool.RunnableTaskExecutionListener; @@ -481,7 +482,15 @@ protected Node( final List> executorBuilders = pluginsService.getExecutorBuilders(settings); runnableTaskListener = new AtomicReference<>(); - final ThreadPool threadPool = new ThreadPool(settings, runnableTaskListener, executorBuilders.toArray(new ExecutorBuilder[0])); + List otelEventListenerList = pluginsService.filterPlugins(Plugin.class) + .stream() + .map(Plugin::getOtelEventListeners) + .flatMap(List::stream) + .collect(Collectors.toList()); + this.otelService = new OtelService(otelEventListenerList); + + final ThreadPool threadPool = new ThreadPool(settings, runnableTaskListener, otelService, + executorBuilders.toArray(new ExecutorBuilder[0])); resourcesToClose.add(() -> ThreadPool.terminate(threadPool, 10, TimeUnit.SECONDS)); final ResourceWatcherService resourceWatcherService = new ResourceWatcherService(settings, threadPool); resourcesToClose.add(resourceWatcherService); @@ -651,7 +660,6 @@ protected Node( directoryFactories.put(k, v); }); directoryFactories.putAll(builtInDirectoryFactories); - this.otelService = new OtelService(pluginsService); final Map recoveryStateFactories = pluginsService.filterPlugins( IndexStorePlugin.class diff --git a/server/src/main/java/org/opensearch/otel/OtelService.java b/server/src/main/java/org/opensearch/otel/OtelService.java index 4889b56fdd21b..b2a94247b4723 100644 --- a/server/src/main/java/org/opensearch/otel/OtelService.java +++ b/server/src/main/java/org/opensearch/otel/OtelService.java @@ -13,6 +13,7 @@ import io.opentelemetry.api.common.AttributeKey; import io.opentelemetry.api.common.Attributes; import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.StatusCode; import io.opentelemetry.api.trace.Tracer; import io.opentelemetry.api.trace.propagation.W3CTraceContextPropagator; import io.opentelemetry.context.Context; @@ -28,13 +29,19 @@ import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor; import io.opentelemetry.semconv.resource.attributes.ResourceAttributes; import org.opensearch.action.ActionListener; -import org.opensearch.action.support.OTelContextPreservingActionListener; -import org.opensearch.plugins.Plugin; -import org.opensearch.plugins.PluginsService; import java.lang.management.ManagementFactory; +import java.util.ArrayList; +import java.util.Collection; import java.util.List; -import java.util.stream.Collectors; +import java.util.Objects; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.function.BiFunction; import static io.opentelemetry.api.common.AttributeKey.longKey; import static io.opentelemetry.api.common.AttributeKey.stringKey; @@ -48,38 +55,6 @@ public class OtelService { private static final ThreadMXBean threadMXBean = (ThreadMXBean) ManagementFactory.getThreadMXBean(); public List otelEventListenerList; static { - -// ManagedChannel jaegerChannel = ManagedChannelBuilder.forAddress("localhost", 3336) -// .usePlaintext() -// .build(); - -// JaegerGrpcSpanExporter jaegerExporter = JaegerGrpcSpanExporter.builder() -// .setEndpoint("http://localhost:3336") -// .setTimeout(30, TimeUnit.SECONDS) -// .build(); - -// SdkTracerProvider sdkTracerProvider = SdkTracerProvider.builder() -// .addSpanProcessor(SimpleSpanProcessor.create(jaegerExporter)) -// .build(); -// Meter recoverMeter = OtelService.sdkMeterProvider.meterBuilder("recover").build(); - -// DoubleGaugeBuilder cpuGauge = recoverMeter.gaugeBuilder("cpu"); -// recoverMeter -// .gaugeBuilder("cpu_usage") -// .setDescription("CPU Usage") -// .setUnit("ms") -// .buildWithCallback(measurement -> { -// measurement.record(, Attributes.of(stringKey("operation"), "recover-start")); -// }); - -// recoverMeter -// .gaugeBuilder("cpu_usage") -// .setDescription("CPU Usage") -// .setUnit("ms") -// .buildWithCallback(measurement -> { -// measurement.record(OtelService.getCPUUsage(threadId), Attributes.of(stringKey("operation"), "recover-start")); -// }); - resource = Resource.getDefault() .merge(Resource.create(Attributes.of(ResourceAttributes.SERVICE_NAME, "opensearch-tasks"))); @@ -101,12 +76,8 @@ public class OtelService { .buildAndRegisterGlobal(); } - public OtelService(PluginsService pluginsService) { - otelEventListenerList = pluginsService.filterPlugins(Plugin.class) - .stream() - .map(Plugin::getOtelEventListeners) - .flatMap(List::stream) - .collect(Collectors.toList()); + public OtelService(List otelEventListenerList) { + this.otelEventListenerList = otelEventListenerList; } public static long getCPUUsage(long threadId) { @@ -121,30 +92,245 @@ public static long getThreadContentionTime(long threadId) { return threadMXBean.getThreadInfo(threadId).getBlockedTime(); } - public static ActionListener startSpan(String spanName, ActionListener actionListener) { - Tracer tracer = OtelService.sdkTracerProvider.get("recover"); + public static void callFunctionAndStartSpan(String spanName, BiFunction, R> function, + ActionListener actionListener, Object... args) { Context beforeAttach = Context.current(); - Span span = tracer.spanBuilder(spanName).startSpan(); + Span span = startSpan(spanName); + try(Scope ignored = span.makeCurrent()) { + actionListener = new SpanPreservingActionListener<>(actionListener, beforeAttach, span.getSpanContext().getSpanId()); + emitResources(span, spanName + "-Start", null); + function.apply(args, actionListener); + } finally { + emitResources(span, spanName + "-End", null); + } + } + + private static Span startSpan(String spanName) { + Tracer tracer = OtelService.sdkTracerProvider.get("recover"); + Span span = tracer.spanBuilder(spanName).setParent(Context.current()).startSpan(); span.setAttribute(stringKey("start-thread-name"), Thread.currentThread().getName()); span.setAttribute(longKey("start-thread-id"), Thread.currentThread().getId()); - span.makeCurrent(); - return new OTelContextPreservingActionListener<>(beforeAttach, Context.current(), actionListener, - span.getSpanContext().getSpanId()); + return span; } - public void emitResources(Span span, String name) { - for (OtelEventListener eventListener : otelEventListenerList) { - eventListener.onEvent(span); + private static void emitResources(Span span, String name, List otelEventListenerList) { + emitResources(span, name, Thread.currentThread(), otelEventListenerList); + } + private static void emitResources(Span span, String name, Thread t, List otelEventListenerList) { + Context context = Context.current(); + if (context != Context.root()) { + try (Scope ignored = span.makeCurrent()) { + if (otelEventListenerList != null && !otelEventListenerList.isEmpty()) { + for (OtelEventListener eventListener : otelEventListenerList) { + eventListener.onEvent(span); + } + } + span.addEvent(name, + Attributes.of( + AttributeKey.longKey("ThreadID"), t.getId(), + AttributeKey.stringKey("ThreadName"), t.getName(), + AttributeKey.longKey("CPUUsage"), OtelService.getCPUUsage(t.getId()), + AttributeKey.longKey("MemoryUsage"), OtelService.getMemoryUsage(t.getId()), + AttributeKey.longKey("ContentionTime"), OtelService.getThreadContentionTime(t.getId()) + ) + ); + } } - span.addEvent(name, - Attributes.of( - AttributeKey.longKey("ThreadID"), Thread.currentThread().getId(), - AttributeKey.stringKey("ThreadName"), Thread.currentThread().getName(), - AttributeKey.longKey("CPUUsage"), OtelService.getCPUUsage(Thread.currentThread().getId()), - AttributeKey.longKey("MemoryUsage"), OtelService.getMemoryUsage(Thread.currentThread().getId()), - AttributeKey.longKey("ContentionTime"), OtelService.getThreadContentionTime(Thread.currentThread().getId()) - ) - ); } + public static ExecutorService taskWrapping(ExecutorService delegate) { + return new CurrentContextExecutorService(delegate); + } + + public static ExecutorService taskWrapping(ExecutorService delegate, List otelEventListeners) { + return new CurrentContextExecutorService(delegate, otelEventListeners); + } + private static Callable wrapTask(Callable callable, List otelEventListeners) { + return () -> { + try (Scope ignored = Context.current().makeCurrent()) { + emitResources(Span.current(), Span.current().getSpanContext().getSpanId() + "-" + + Thread.currentThread().getName() + "-Start", otelEventListeners); + return callable.call(); + } finally { + emitResources(Span.current(), Span.current().getSpanContext().getSpanId() + "-" + + Thread.currentThread().getName() + "-End", otelEventListeners); + } + }; + } + + static Runnable wrapTask(Runnable runnable, List otelEventListeners) { + return () -> { + try (Scope ignored = Context.current().makeCurrent()) { + emitResources(Span.current(), Span.current().getSpanContext().getSpanId() + "-" + + Thread.currentThread().getName() + "-Start", otelEventListeners); + runnable.run(); + emitResources(Span.current(), Span.current().getSpanContext().getSpanId() + "-" + + Thread.currentThread().getName() + "-End", otelEventListeners); + } + }; + } + + static final class CurrentContextExecutorService extends ForwardingExecutorService { + + private List otelEventListeners; + + CurrentContextExecutorService(ExecutorService delegate) { + this(delegate, null); + } + + CurrentContextExecutorService(ExecutorService delegate, List otelEventListeners) { + super(delegate); + this.otelEventListeners = otelEventListeners; + } + + public List getOtelEventListeners() { + return otelEventListeners; + } + @Override + public Future submit(Callable task) { + return delegate().submit(wrapTask(task, otelEventListeners)); + } + + @Override + public Future submit(Runnable task, T result) { + return delegate().submit(wrapTask(task, otelEventListeners), result); + } + + @Override + public Future submit(Runnable task) { + return delegate().submit(wrapTask(task, otelEventListeners)); + } + + @Override + public List> invokeAll(Collection> tasks) + throws InterruptedException { + return delegate().invokeAll(wrap(tasks, otelEventListeners)); + } + + @Override + public List> invokeAll( + Collection> tasks, long timeout, TimeUnit unit) + throws InterruptedException { + return delegate().invokeAll(wrap(tasks, otelEventListeners), timeout, unit); + } + + @Override + public T invokeAny(Collection> tasks) + throws InterruptedException, ExecutionException { + return delegate().invokeAny(wrap(tasks, otelEventListeners)); + } + + @Override + public T invokeAny(Collection> tasks, long timeout, TimeUnit unit) + throws InterruptedException, ExecutionException, TimeoutException { + return delegate().invokeAny(wrap(tasks, otelEventListeners), timeout, unit); + } + + @Override + public void execute(Runnable command) { + delegate().execute(wrapTask(command, otelEventListeners)); + } + } + + static abstract class ForwardingExecutorService implements ExecutorService { + + private final ExecutorService delegate; + + protected ForwardingExecutorService(ExecutorService delegate) { + this.delegate = delegate; + } + + ExecutorService delegate() { + return delegate; + } + + @Override + public final void shutdown() { + delegate.shutdown(); + } + + @Override + public final List shutdownNow() { + return delegate.shutdownNow(); + } + + @Override + public final boolean isShutdown() { + return delegate.isShutdown(); + } + + @Override + public final boolean isTerminated() { + return delegate.isTerminated(); + } + + @Override + public final boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedException { + return delegate.awaitTermination(timeout, unit); + } + + protected static Collection> wrap(Collection> tasks, + List otelEventListeners) { + List> wrapped = new ArrayList<>(); + for (Callable task : tasks) { + wrapped.add(wrapTask(task, otelEventListeners)); + } + return wrapped; + } + } + + public static final class SpanPreservingActionListener implements ActionListener { + private final ActionListener delegate; + private final Context beforeAttachContext; + private final Context afterAttachContext; + private final String spanID; + + public SpanPreservingActionListener(ActionListener delegate, Context beforeAttachContext, String spanID) { + this.delegate = delegate; + this.beforeAttachContext = beforeAttachContext; + this.afterAttachContext = Context.current(); + this.spanID = spanID; + } + + public SpanPreservingActionListener(ActionListener delegate, Context beforeAttachContext) { + this(delegate, beforeAttachContext, null); + } + + @Override + public void onResponse(R r) { + try (Scope ignored = Objects.requireNonNull(afterAttachContext).makeCurrent()) { + Span span = Span.current(); + closeCurrentScope(span); + } + try (Scope ignored = Objects.requireNonNull(beforeAttachContext).makeCurrent()) { + delegate.onResponse(r); + } + } + + @Override + public void onFailure(Exception e) { + try (Scope ignored = Objects.requireNonNull(afterAttachContext).makeCurrent()) { + Span span = Span.current(); + span.setStatus(StatusCode.ERROR); + closeCurrentScope(span); + } + try (Scope ignored = Objects.requireNonNull(beforeAttachContext).makeCurrent()) { + delegate.onFailure(e); + } + } + + private void closeCurrentScope(Span span) { + assert spanID == null || span.getSpanContext().getSpanId().equals(spanID); + span.setAttribute(stringKey("finish-thread-name"), Thread.currentThread().getName()); + span.setAttribute(longKey("finish-thread-id"), Thread.currentThread().getId()); + if (spanID != null) { + Span.current().end(); + } + } + + @Override + public String toString() { + return getClass().getName() + "/" + delegate.toString(); + } + } } diff --git a/server/src/main/java/org/opensearch/threadpool/ScalingExecutorBuilder.java b/server/src/main/java/org/opensearch/threadpool/ScalingExecutorBuilder.java index 18cef0c57be2e..9ce90a4fa28bb 100644 --- a/server/src/main/java/org/opensearch/threadpool/ScalingExecutorBuilder.java +++ b/server/src/main/java/org/opensearch/threadpool/ScalingExecutorBuilder.java @@ -38,6 +38,8 @@ import org.opensearch.common.util.concurrent.OpenSearchExecutors; import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.node.Node; +import org.opensearch.otel.OtelEventListener; +import org.opensearch.otel.OtelService; import java.util.Arrays; import java.util.List; @@ -57,6 +59,7 @@ public final class ScalingExecutorBuilder extends ExecutorBuilder maxSetting; private final Setting keepAliveSetting; + private final List otelEventListeners; /** * Construct a scaling executor builder; the settings will have the * key prefix "thread_pool." followed by the executor name. @@ -70,6 +73,15 @@ public final class ScalingExecutorBuilder extends ExecutorBuilder otelEventListeners) { + this(name, core, max, keepAlive, "thread_pool." + name, otelEventListeners); + } + + public ScalingExecutorBuilder(final String name, final int core, final int max, final TimeValue keepAlive, final String prefix) { + this(name, core, max, keepAlive, prefix, null); + } + /** * Construct a scaling executor builder; the settings will have the @@ -82,11 +94,13 @@ public ScalingExecutorBuilder(final String name, final int core, final int max, * threads will be kept alive * @param prefix the prefix for the settings keys */ - public ScalingExecutorBuilder(final String name, final int core, final int max, final TimeValue keepAlive, final String prefix) { + public ScalingExecutorBuilder(final String name, final int core, final int max, final TimeValue keepAlive, final String prefix, + List otelEventListeners) { super(name); this.coreSetting = Setting.intSetting(settingsKey(prefix, "core"), core, Setting.Property.NodeScope); this.maxSetting = Setting.intSetting(settingsKey(prefix, "max"), max, Setting.Property.NodeScope); this.keepAliveSetting = Setting.timeSetting(settingsKey(prefix, "keep_alive"), keepAlive, Setting.Property.NodeScope); + this.otelEventListeners = otelEventListeners; } @Override @@ -111,7 +125,7 @@ ThreadPool.ExecutorHolder build(final ScalingExecutorSettings settings, final Th final ThreadFactory threadFactory = OpenSearchExecutors.daemonThreadFactory( OpenSearchExecutors.threadName(settings.nodeName, name()) ); - final ExecutorService executor = io.opentelemetry.context.Context.taskWrapping(OpenSearchExecutors.newScaling( + ExecutorService executor = OpenSearchExecutors.newScaling( settings.nodeName + "/" + name(), core, max, @@ -119,7 +133,10 @@ ThreadPool.ExecutorHolder build(final ScalingExecutorSettings settings, final Th TimeUnit.MILLISECONDS, threadFactory, threadContext - )); + ); + if (otelEventListeners != null) { + executor = OtelService.taskWrapping(executor, otelEventListeners); + } return new ThreadPool.ExecutorHolder(executor, info); } diff --git a/server/src/main/java/org/opensearch/threadpool/ThreadPool.java b/server/src/main/java/org/opensearch/threadpool/ThreadPool.java index 93bf28ed02af4..ac809d463583f 100644 --- a/server/src/main/java/org/opensearch/threadpool/ThreadPool.java +++ b/server/src/main/java/org/opensearch/threadpool/ThreadPool.java @@ -48,12 +48,14 @@ import org.opensearch.common.util.concurrent.OpenSearchExecutors; import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; import org.opensearch.common.util.concurrent.OpenSearchThreadPoolExecutor; +import org.opensearch.common.util.concurrent.OpenSearchRejectedExecutionException; import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.common.util.concurrent.XRejectedExecutionHandler; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.node.Node; import org.opensearch.node.ReportingService; +import org.opensearch.otel.OtelService; import java.io.IOException; import java.util.ArrayList; @@ -202,6 +204,8 @@ public static ThreadPoolType fromType(String type) { private final ScheduledThreadPoolExecutor scheduler; + private final OtelService otelService; + public Collection builders() { return Collections.unmodifiableCollection(builders.values()); } @@ -222,14 +226,23 @@ public ThreadPool( final AtomicReference runnableTaskListener, final ExecutorBuilder... customBuilders ) { - assert Node.NODE_NAME_SETTING.exists(settings); + this(settings, runnableTaskListener, null, customBuilders); + } + public ThreadPool( + final Settings settings, + final AtomicReference runnableTaskListener, + OtelService otelService, + final ExecutorBuilder... customBuilders + ) { + assert Node.NODE_NAME_SETTING.exists(settings); + this.otelService = otelService; final Map builders = new HashMap<>(); final int allocatedProcessors = OpenSearchExecutors.allocatedProcessors(settings); final int halfProcMaxAt5 = halfAllocatedProcessorsMaxFive(allocatedProcessors); final int halfProcMaxAt10 = halfAllocatedProcessorsMaxTen(allocatedProcessors); final int genericThreadPoolMax = boundedBy(4 * allocatedProcessors, 128, 512); - builders.put(Names.GENERIC, new ScalingExecutorBuilder(Names.GENERIC, 4, genericThreadPoolMax, TimeValue.timeValueSeconds(30))); + builders.put(Names.GENERIC, new ScalingExecutorBuilder(Names.GENERIC, 4, genericThreadPoolMax, TimeValue.timeValueSeconds(30), otelService.otelEventListenerList)); builders.put(Names.WRITE, new FixedExecutorBuilder(settings, Names.WRITE, allocatedProcessors, 10000)); builders.put(Names.GET, new FixedExecutorBuilder(settings, Names.GET, allocatedProcessors, 1000)); builders.put(Names.ANALYZE, new FixedExecutorBuilder(settings, Names.ANALYZE, 1, 16)); @@ -413,7 +426,12 @@ public ExecutorService executor(String name) { if (holder == null) { throw new IllegalArgumentException("no executor service found for [" + name + "]"); } - return io.opentelemetry.context.Context.taskWrapping(holder.executor()); + if (name.equals(Names.GENERIC)) { + if (otelService != null) { + return OtelService.taskWrapping(holder.executor(), otelService.otelEventListenerList); + } + } + return holder.executor(); } From d588e86d455657306486b6b815685af04d4d5004 Mon Sep 17 00:00:00 2001 From: Rishabh Maurya Date: Tue, 18 Apr 2023 10:27:08 -0700 Subject: [PATCH 4/8] Lazy initialization of otel event listeners --- .../main/java/org/opensearch/node/Node.java | 2 +- .../java/org/opensearch/otel/OtelService.java | 20 ++++++++++++++++--- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/server/src/main/java/org/opensearch/node/Node.java b/server/src/main/java/org/opensearch/node/Node.java index b561e33eea56e..8fc31fd5c36a4 100644 --- a/server/src/main/java/org/opensearch/node/Node.java +++ b/server/src/main/java/org/opensearch/node/Node.java @@ -487,7 +487,7 @@ protected Node( .map(Plugin::getOtelEventListeners) .flatMap(List::stream) .collect(Collectors.toList()); - this.otelService = new OtelService(otelEventListenerList); + this.otelService = new OtelService(OtelService.OtelEventListeners.getInstance(otelEventListenerList)); final ThreadPool threadPool = new ThreadPool(settings, runnableTaskListener, otelService, executorBuilders.toArray(new ExecutorBuilder[0])); diff --git a/server/src/main/java/org/opensearch/otel/OtelService.java b/server/src/main/java/org/opensearch/otel/OtelService.java index b2a94247b4723..7d25b1aa87231 100644 --- a/server/src/main/java/org/opensearch/otel/OtelService.java +++ b/server/src/main/java/org/opensearch/otel/OtelService.java @@ -98,10 +98,10 @@ public static void callFunctionAndStartSpan(String spanName, BiFunction(actionListener, beforeAttach, span.getSpanContext().getSpanId()); - emitResources(span, spanName + "-Start", null); + emitResources(span, spanName + "-Start", OtelEventListeners.getInstance(null)); function.apply(args, actionListener); } finally { - emitResources(span, spanName + "-End", null); + emitResources(span, spanName + "-End", OtelEventListeners.getInstance(null)); } } @@ -138,8 +138,22 @@ private static void emitResources(Span span, String name, Thread t, List INSTANCE; + public static List getInstance(List otelEventListenerList) { + if (INSTANCE == null) { + synchronized (OtelEventListeners.class) { + if (INSTANCE == null) { + INSTANCE = otelEventListenerList; + } + } + } + return INSTANCE; + } + } + public static ExecutorService taskWrapping(ExecutorService delegate) { - return new CurrentContextExecutorService(delegate); + return new CurrentContextExecutorService(delegate, OtelEventListeners.getInstance(null)); } public static ExecutorService taskWrapping(ExecutorService delegate, List otelEventListeners) { From a0679257d5851476cd598a66c21e294d3b537c6a Mon Sep 17 00:00:00 2001 From: Rishabh Maurya Date: Thu, 20 Apr 2023 05:06:11 +0200 Subject: [PATCH 5/8] Code refactor of framework Signed-off-by: Stevan Buzejic --- .idea/runConfigurations/Debug_OpenSearch.xml | 11 - .idea/vcs.xml | 18 +- .../util/concurrent/OpenSearchExecutors.java | 5 +- .../index/seqno/RetentionLeaseSyncAction.java | 4 +- .../LocalStorePeerRecoverySourceHandler.java | 15 +- .../recovery/PeerRecoverySourceService.java | 11 +- .../recovery/RecoverySourceHandler.java | 20 +- .../RecoverySourceHandlerFactory.java | 10 +- .../RemoteStorePeerRecoverySourceHandler.java | 6 +- .../recovery/RetryableTransportClient.java | 4 +- .../SegmentFileTransferHandler.java | 35 +- .../main/java/org/opensearch/node/Node.java | 18 +- .../opensearch/otel/OtelEventListener.java | 16 - .../java/org/opensearch/otel/OtelService.java | 350 ------------------ .../java/org/opensearch/plugins/Plugin.java | 4 +- .../threadpool/ScalingExecutorBuilder.java | 22 +- .../org/opensearch/threadpool/ThreadPool.java | 23 +- .../opensearch/tracing/TaskEventListener.java | 50 +++ .../OTelContextPreservingActionListener.java | 85 +++++ .../OpenSearchConcurrentExecutorService.java | 119 ++++++ .../OpenSearchForwardingExecutorService.java | 51 +++ .../OpenTelemetryContextWrapper.java | 31 ++ .../opentelemetry/OpenTelemetryService.java | 182 +++++++++ .../PeerRecoverySourceServiceTests.java | 5 +- 24 files changed, 571 insertions(+), 524 deletions(-) delete mode 100644 .idea/runConfigurations/Debug_OpenSearch.xml delete mode 100644 server/src/main/java/org/opensearch/otel/OtelEventListener.java delete mode 100644 server/src/main/java/org/opensearch/otel/OtelService.java create mode 100644 server/src/main/java/org/opensearch/tracing/TaskEventListener.java create mode 100644 server/src/main/java/org/opensearch/tracing/opentelemetry/OTelContextPreservingActionListener.java create mode 100644 server/src/main/java/org/opensearch/tracing/opentelemetry/OpenSearchConcurrentExecutorService.java create mode 100644 server/src/main/java/org/opensearch/tracing/opentelemetry/OpenSearchForwardingExecutorService.java create mode 100644 server/src/main/java/org/opensearch/tracing/opentelemetry/OpenTelemetryContextWrapper.java create mode 100644 server/src/main/java/org/opensearch/tracing/opentelemetry/OpenTelemetryService.java diff --git a/.idea/runConfigurations/Debug_OpenSearch.xml b/.idea/runConfigurations/Debug_OpenSearch.xml deleted file mode 100644 index 0d8bf59823acf..0000000000000 --- a/.idea/runConfigurations/Debug_OpenSearch.xml +++ /dev/null @@ -1,11 +0,0 @@ - - - - diff --git a/.idea/vcs.xml b/.idea/vcs.xml index 48557884a8893..35eb1ddfbbc02 100644 --- a/.idea/vcs.xml +++ b/.idea/vcs.xml @@ -1,20 +1,6 @@ - - - - + - + \ No newline at end of file diff --git a/server/src/main/java/org/opensearch/common/util/concurrent/OpenSearchExecutors.java b/server/src/main/java/org/opensearch/common/util/concurrent/OpenSearchExecutors.java index 3f48faeef3a66..df0a3b030c345 100644 --- a/server/src/main/java/org/opensearch/common/util/concurrent/OpenSearchExecutors.java +++ b/server/src/main/java/org/opensearch/common/util/concurrent/OpenSearchExecutors.java @@ -32,7 +32,6 @@ package org.opensearch.common.util.concurrent; -import io.opentelemetry.context.Context; import org.opensearch.ExceptionsHelper; import org.opensearch.common.SuppressForbidden; import org.opensearch.common.logging.DeprecationLogger; @@ -41,9 +40,9 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; import org.opensearch.node.Node; -import org.opensearch.otel.OtelService; import org.opensearch.threadpool.RunnableTaskExecutionListener; import org.opensearch.threadpool.TaskAwareRunnable; +import org.opensearch.tracing.opentelemetry.OpenTelemetryContextWrapper; import java.util.List; import java.util.Optional; @@ -353,7 +352,7 @@ public void execute(Runnable command) { * @return an {@link ExecutorService} that executes submitted tasks on the current thread */ public static ExecutorService newDirectExecutorService() { - return OtelService.taskWrapping(DIRECT_EXECUTOR_SERVICE); + return OpenTelemetryContextWrapper.wrapTask(DIRECT_EXECUTOR_SERVICE); } public static String threadName(Settings settings, String namePrefix) { diff --git a/server/src/main/java/org/opensearch/index/seqno/RetentionLeaseSyncAction.java b/server/src/main/java/org/opensearch/index/seqno/RetentionLeaseSyncAction.java index aa8bfd58815a1..1775c633f5caf 100644 --- a/server/src/main/java/org/opensearch/index/seqno/RetentionLeaseSyncAction.java +++ b/server/src/main/java/org/opensearch/index/seqno/RetentionLeaseSyncAction.java @@ -61,10 +61,10 @@ import org.opensearch.index.shard.ShardId; import org.opensearch.indices.IndicesService; import org.opensearch.indices.SystemIndices; -import org.opensearch.otel.OtelService; import org.opensearch.tasks.Task; import org.opensearch.tasks.TaskId; import org.opensearch.threadpool.ThreadPool; +import org.opensearch.tracing.opentelemetry.OTelContextPreservingActionListener; import org.opensearch.transport.TransportException; import org.opensearch.transport.TransportResponseHandler; import org.opensearch.transport.TransportService; @@ -133,7 +133,7 @@ final void sync( RetentionLeases retentionLeases, ActionListener listener1 ) { - final ActionListener listener = new OtelService.SpanPreservingActionListener<>(listener1, Context.current()); + final ActionListener listener = new OTelContextPreservingActionListener<>(listener1, Context.current()); final ThreadContext threadContext = threadPool.getThreadContext(); try (ThreadContext.StoredContext ignore = threadContext.stashContext()) { // we have to execute under the system context so that if security is enabled the sync is authorized diff --git a/server/src/main/java/org/opensearch/indices/recovery/LocalStorePeerRecoverySourceHandler.java b/server/src/main/java/org/opensearch/indices/recovery/LocalStorePeerRecoverySourceHandler.java index 93075acb1d43e..2b0d5535904bf 100644 --- a/server/src/main/java/org/opensearch/indices/recovery/LocalStorePeerRecoverySourceHandler.java +++ b/server/src/main/java/org/opensearch/indices/recovery/LocalStorePeerRecoverySourceHandler.java @@ -28,7 +28,7 @@ import org.opensearch.index.shard.IndexShard; import org.opensearch.index.translog.Translog; import org.opensearch.indices.RunUnderPrimaryPermit; -import org.opensearch.otel.OtelService; +import org.opensearch.tracing.opentelemetry.OpenTelemetryService; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.Transports; @@ -53,10 +53,9 @@ public LocalStorePeerRecoverySourceHandler( StartRecoveryRequest request, int fileChunkSizeInBytes, int maxConcurrentFileChunks, - int maxConcurrentOperations, - OtelService otelService + int maxConcurrentOperations ) { - super(shard, recoveryTarget, threadPool, request, fileChunkSizeInBytes, maxConcurrentFileChunks, maxConcurrentOperations, otelService); + super(shard, recoveryTarget, threadPool, request, fileChunkSizeInBytes, maxConcurrentFileChunks, maxConcurrentOperations); } @Override @@ -155,7 +154,7 @@ && isTargetSameHistory() (ActionListener) actionListener); return null; }; - OtelService.callFunctionAndStartSpan( + OpenTelemetryService.callFunctionAndStartSpan( "removePeerRecoveryRetentionLease", removePeerRecoveryRetentionLeaseFun, new ThreadedActionListener<>( @@ -181,7 +180,7 @@ && isTargetSameHistory() (ActionListener) actionListener, (boolean) args[3]); return null; }; - OtelService.callFunctionAndStartSpan( + OpenTelemetryService.callFunctionAndStartSpan( "phase1", phase1Fun, sendFileStep, @@ -206,7 +205,7 @@ && isTargetSameHistory() prepareTargetForTranslog((int)args[0], (ActionListener) actionListener); return null; }; - OtelService.callFunctionAndStartSpan( + OpenTelemetryService.callFunctionAndStartSpan( "prepareTargetForTranslog", prepareTargetForTranslogFun, prepareEngineStep, @@ -260,7 +259,7 @@ && isTargetSameHistory() } return null; }; - OtelService.callFunctionAndStartSpan( + OpenTelemetryService.callFunctionAndStartSpan( "phase2", phase2Fun, sendSnapshotStep, diff --git a/server/src/main/java/org/opensearch/indices/recovery/PeerRecoverySourceService.java b/server/src/main/java/org/opensearch/indices/recovery/PeerRecoverySourceService.java index a57e2b9493a39..a40d1044975c1 100644 --- a/server/src/main/java/org/opensearch/indices/recovery/PeerRecoverySourceService.java +++ b/server/src/main/java/org/opensearch/indices/recovery/PeerRecoverySourceService.java @@ -56,9 +56,9 @@ import org.opensearch.index.shard.IndexShard; import org.opensearch.index.shard.ShardId; import org.opensearch.indices.IndicesService; -import org.opensearch.otel.OtelService; import org.opensearch.tasks.Task; import org.opensearch.threadpool.ThreadPool; +import org.opensearch.tracing.opentelemetry.OpenTelemetryService; import org.opensearch.transport.TransportChannel; import org.opensearch.transport.TransportRequestHandler; import org.opensearch.transport.TransportService; @@ -100,15 +100,12 @@ public static class Actions { private final RecoverySettings recoverySettings; final OngoingRecoveries ongoingRecoveries = new OngoingRecoveries(); - private final OtelService otelService; @Inject - public PeerRecoverySourceService(TransportService transportService, IndicesService indicesService, RecoverySettings recoverySettings, - OtelService otelService) { + public PeerRecoverySourceService(TransportService transportService, IndicesService indicesService, RecoverySettings recoverySettings) { this.transportService = transportService; this.indicesService = indicesService; this.recoverySettings = recoverySettings; - this.otelService = otelService; // When the target node wants to start a peer recovery it sends a START_RECOVERY request to the source // node. Upon receiving START_RECOVERY, the source node will initiate the peer recovery. transportService.registerRequestHandler( @@ -221,7 +218,7 @@ public void messageReceived(final StartRecoveryRequest request, final TransportC recover((StartRecoveryRequest) args[0], (ActionListener) actionListener); return null; }; - OtelService.callFunctionAndStartSpan("recover", recoverFunction, + OpenTelemetryService.callFunctionAndStartSpan("recover", recoverFunction, new ChannelActionListener<>(channel, Actions.START_RECOVERY, request), request); } } @@ -398,7 +395,7 @@ private Tuple createRecovery recoverySettings, throttleTime -> shard.recoveryStats().addThrottleTime(throttleTime) ); - handler = RecoverySourceHandlerFactory.create(shard, recoveryTarget, request, recoverySettings, otelService); + handler = RecoverySourceHandlerFactory.create(shard, recoveryTarget, request, recoverySettings); return Tuple.tuple(handler, recoveryTarget); } } diff --git a/server/src/main/java/org/opensearch/indices/recovery/RecoverySourceHandler.java b/server/src/main/java/org/opensearch/indices/recovery/RecoverySourceHandler.java index 22b53604f6b24..190389d2ec065 100644 --- a/server/src/main/java/org/opensearch/indices/recovery/RecoverySourceHandler.java +++ b/server/src/main/java/org/opensearch/indices/recovery/RecoverySourceHandler.java @@ -32,8 +32,6 @@ package org.opensearch.indices.recovery; -import io.opentelemetry.api.trace.Span; -import io.opentelemetry.context.Scope; import org.apache.logging.log4j.Logger; import org.apache.lucene.index.CorruptIndexException; import org.apache.lucene.index.IndexCommit; @@ -73,7 +71,7 @@ import org.opensearch.index.translog.Translog; import org.opensearch.indices.RunUnderPrimaryPermit; import org.opensearch.indices.replication.SegmentFileTransferHandler; -import org.opensearch.otel.OtelService; +import org.opensearch.tracing.opentelemetry.OpenTelemetryService; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.Transports; @@ -123,7 +121,6 @@ public abstract class RecoverySourceHandler { protected final ListenableFuture future = new ListenableFuture<>(); public static final String PEER_RECOVERY_NAME = "peer-recovery"; private final SegmentFileTransferHandler transferHandler; - private final OtelService otelService; RecoverySourceHandler( IndexShard shard, @@ -132,8 +129,7 @@ public abstract class RecoverySourceHandler { StartRecoveryRequest request, int fileChunkSizeInBytes, int maxConcurrentFileChunks, - int maxConcurrentOperations, - OtelService otelService + int maxConcurrentOperations ) { this.logger = Loggers.getLogger(RecoverySourceHandler.class, request.shardId(), "recover to " + request.targetNode().getName()); this.transferHandler = new SegmentFileTransferHandler( @@ -144,8 +140,7 @@ public abstract class RecoverySourceHandler { threadPool, cancellableThreads, fileChunkSizeInBytes, - maxConcurrentFileChunks, - otelService + maxConcurrentFileChunks ); this.shard = shard; this.threadPool = threadPool; @@ -155,7 +150,6 @@ public abstract class RecoverySourceHandler { this.chunkSizeInBytes = fileChunkSizeInBytes; // if the target is on an old version, it won't be able to handle out-of-order file chunks. this.maxConcurrentOperations = maxConcurrentOperations; - this.otelService = otelService; } public StartRecoveryRequest getRequest() { @@ -220,7 +214,7 @@ protected void finalizeStepAndCompleteFuture( } return null; }; - OtelService.callFunctionAndStartSpan( + OpenTelemetryService.callFunctionAndStartSpan( "finalizeRecovery", finalizeRecoveryFun, finalizeStep, @@ -475,7 +469,7 @@ void phase1( ); return null; }; - OtelService.callFunctionAndStartSpan( + OpenTelemetryService.callFunctionAndStartSpan( "sendFileInfo", receiveFileInfoFunction, sendFileInfoStep, @@ -497,7 +491,7 @@ void phase1( ); return null; }; - OtelService.callFunctionAndStartSpan( + OpenTelemetryService.callFunctionAndStartSpan( "sendFiles", sendFileFunction, sendFilesStep, @@ -522,7 +516,7 @@ void phase1( ); return null; }; - OtelService.callFunctionAndStartSpan( + OpenTelemetryService.callFunctionAndStartSpan( "createRetentionLease", createRetentionLeaseFunction, createRetentionLeaseStep, diff --git a/server/src/main/java/org/opensearch/indices/recovery/RecoverySourceHandlerFactory.java b/server/src/main/java/org/opensearch/indices/recovery/RecoverySourceHandlerFactory.java index db9c7da5d429c..ea13ca18bbfca 100644 --- a/server/src/main/java/org/opensearch/indices/recovery/RecoverySourceHandlerFactory.java +++ b/server/src/main/java/org/opensearch/indices/recovery/RecoverySourceHandlerFactory.java @@ -9,7 +9,6 @@ package org.opensearch.indices.recovery; import org.opensearch.index.shard.IndexShard; -import org.opensearch.otel.OtelService; /** * Factory that supplies {@link RecoverySourceHandler}. @@ -22,8 +21,7 @@ public static RecoverySourceHandler create( IndexShard shard, RecoveryTargetHandler recoveryTarget, StartRecoveryRequest request, - RecoverySettings recoverySettings, - OtelService otelService + RecoverySettings recoverySettings ) { boolean isReplicaRecoveryWithRemoteTranslog = request.isPrimaryRelocation() == false && shard.isRemoteTranslogEnabled(); if (isReplicaRecoveryWithRemoteTranslog) { @@ -34,8 +32,7 @@ public static RecoverySourceHandler create( request, Math.toIntExact(recoverySettings.getChunkSize().getBytes()), recoverySettings.getMaxConcurrentFileChunks(), - recoverySettings.getMaxConcurrentOperations(), - otelService + recoverySettings.getMaxConcurrentOperations() ); } else { return new LocalStorePeerRecoverySourceHandler( @@ -45,8 +42,7 @@ public static RecoverySourceHandler create( request, Math.toIntExact(recoverySettings.getChunkSize().getBytes()), recoverySettings.getMaxConcurrentFileChunks(), - recoverySettings.getMaxConcurrentOperations(), - otelService + recoverySettings.getMaxConcurrentOperations() ); } } diff --git a/server/src/main/java/org/opensearch/indices/recovery/RemoteStorePeerRecoverySourceHandler.java b/server/src/main/java/org/opensearch/indices/recovery/RemoteStorePeerRecoverySourceHandler.java index 30af90f620e19..ff218ef71e397 100644 --- a/server/src/main/java/org/opensearch/indices/recovery/RemoteStorePeerRecoverySourceHandler.java +++ b/server/src/main/java/org/opensearch/indices/recovery/RemoteStorePeerRecoverySourceHandler.java @@ -20,7 +20,6 @@ import org.opensearch.indices.RunUnderPrimaryPermit; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.Transports; -import org.opensearch.otel.OtelService; import java.io.IOException; import java.util.function.Consumer; @@ -39,10 +38,9 @@ public RemoteStorePeerRecoverySourceHandler( StartRecoveryRequest request, int fileChunkSizeInBytes, int maxConcurrentFileChunks, - int maxConcurrentOperations, - OtelService otelService + int maxConcurrentOperations ) { - super(shard, recoveryTarget, threadPool, request, fileChunkSizeInBytes, maxConcurrentFileChunks, maxConcurrentOperations, otelService); + super(shard, recoveryTarget, threadPool, request, fileChunkSizeInBytes, maxConcurrentFileChunks, maxConcurrentOperations); } @Override diff --git a/server/src/main/java/org/opensearch/indices/recovery/RetryableTransportClient.java b/server/src/main/java/org/opensearch/indices/recovery/RetryableTransportClient.java index 2fd51b6669587..163ead32ac617 100644 --- a/server/src/main/java/org/opensearch/indices/recovery/RetryableTransportClient.java +++ b/server/src/main/java/org/opensearch/indices/recovery/RetryableTransportClient.java @@ -22,8 +22,8 @@ import org.opensearch.common.util.concurrent.ConcurrentCollections; import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; import org.opensearch.common.util.concurrent.OpenSearchRejectedExecutionException; -import org.opensearch.otel.OtelService; import org.opensearch.threadpool.ThreadPool; +import org.opensearch.tracing.opentelemetry.OTelContextPreservingActionListener; import org.opensearch.transport.ConnectTransportException; import org.opensearch.transport.RemoteTransportException; import org.opensearch.transport.SendRequestTransportException; @@ -96,7 +96,7 @@ public void tryAction(ActionListener listener) { request, options, new ActionListenerResponseHandler<>( - new OtelService.SpanPreservingActionListener<>(listener, Context.current()), + new OTelContextPreservingActionListener<>(listener, Context.current()), reader, ThreadPool.Names.GENERIC) ); } diff --git a/server/src/main/java/org/opensearch/indices/replication/SegmentFileTransferHandler.java b/server/src/main/java/org/opensearch/indices/replication/SegmentFileTransferHandler.java index bbf9c34153c71..9e4e90765531a 100644 --- a/server/src/main/java/org/opensearch/indices/replication/SegmentFileTransferHandler.java +++ b/server/src/main/java/org/opensearch/indices/replication/SegmentFileTransferHandler.java @@ -8,7 +8,6 @@ package org.opensearch.indices.replication; -import io.opentelemetry.api.trace.Span; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; import org.apache.lucene.index.CorruptIndexException; @@ -29,8 +28,6 @@ import org.opensearch.index.store.StoreFileMetadata; import org.opensearch.indices.recovery.FileChunkWriter; import org.opensearch.indices.recovery.MultiChunkTransfer; -import org.opensearch.otel.OtelEventListener; -import org.opensearch.otel.OtelService; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.RemoteTransportException; import org.opensearch.transport.Transports; @@ -59,7 +56,6 @@ public final class SegmentFileTransferHandler { private final int maxConcurrentFileChunks; private final DiscoveryNode targetNode; private final CancellableThreads cancellableThreads; - private final OtelService otelService; public SegmentFileTransferHandler( IndexShard shard, @@ -70,21 +66,6 @@ public SegmentFileTransferHandler( CancellableThreads cancellableThreads, int fileChunkSizeInBytes, int maxConcurrentFileChunks - ) { - this(shard, targetNode, chunkWriter, logger, threadPool, cancellableThreads, - fileChunkSizeInBytes, maxConcurrentFileChunks, null); - } - - public SegmentFileTransferHandler( - IndexShard shard, - DiscoveryNode targetNode, - FileChunkWriter chunkWriter, - Logger logger, - ThreadPool threadPool, - CancellableThreads cancellableThreads, - int fileChunkSizeInBytes, - int maxConcurrentFileChunks, - OtelService otelService ) { this.shard = shard; this.targetNode = targetNode; @@ -95,7 +76,6 @@ public SegmentFileTransferHandler( this.chunkSizeInBytes = fileChunkSizeInBytes; // if the target is on an old version, it won't be able to handle out-of-order file chunks. this.maxConcurrentFileChunks = maxConcurrentFileChunks; - this.otelService = otelService; } /** @@ -163,18 +143,7 @@ protected FileChunk nextChunkRequest(StoreFileMetadata md) throws IOException { } @Override - protected void executeChunkRequest(FileChunk request, ActionListener listener1) { - cancellableThreads.checkForCancel(); - if (otelService != null) { - // OtelService.emitResources(Span.current(), "StartExecuteChunkRequest", otelService.otelEventListenerList); - } - - ActionListener listener2 = ActionListener.runBefore(listener1, request::close); - ActionListener listener3 = ActionListener.runBefore(listener2, () -> { - if (otelService != null) { - // OtelService.emitResources(Span.current(), "EndExecuteChunkRequest", otelService.otelEventListenerList); - } - }); + protected void executeChunkRequest(FileChunk request, ActionListener listener) { chunkWriter.writeFileChunk( request.md, @@ -182,7 +151,7 @@ protected void executeChunkRequest(FileChunk request, ActionListener liste request.content, request.lastChunk, translogOps.getAsInt(), - listener3 + ActionListener.runBefore(listener, request::close) ); } diff --git a/server/src/main/java/org/opensearch/node/Node.java b/server/src/main/java/org/opensearch/node/Node.java index 8fc31fd5c36a4..c87a78602a2d3 100644 --- a/server/src/main/java/org/opensearch/node/Node.java +++ b/server/src/main/java/org/opensearch/node/Node.java @@ -39,8 +39,8 @@ import org.opensearch.cluster.routing.allocation.AwarenessReplicaBalance; import org.opensearch.index.IndexModule; import org.opensearch.index.IndexingPressureService; -import org.opensearch.otel.OtelEventListener; -import org.opensearch.otel.OtelService; +import org.opensearch.tracing.TaskEventListener; +import org.opensearch.tracing.opentelemetry.OpenTelemetryService; import org.opensearch.tasks.TaskResourceTrackingService; import org.opensearch.threadpool.RunnableTaskExecutionListener; import org.opensearch.common.util.FeatureFlags; @@ -55,8 +55,6 @@ import org.opensearch.monitor.fs.FsProbe; import org.opensearch.search.backpressure.SearchBackpressureService; import org.opensearch.search.backpressure.settings.SearchBackpressureSettings; -import org.opensearch.tasks.TaskResourceTrackingService; -import org.opensearch.threadpool.RunnableTaskExecutionListener; import org.opensearch.index.store.RemoteSegmentStoreDirectoryFactory; import org.opensearch.watcher.ResourceWatcherService; import org.opensearch.Assertions; @@ -356,7 +354,6 @@ public static class DiscoverySettings { private final Collection pluginLifecycleComponents; private final LocalNodeFactory localNodeFactory; private final NodeService nodeService; - private final OtelService otelService; final NamedWriteableRegistry namedWriteableRegistry; private final AtomicReference runnableTaskListener; @@ -482,15 +479,14 @@ protected Node( final List> executorBuilders = pluginsService.getExecutorBuilders(settings); runnableTaskListener = new AtomicReference<>(); - List otelEventListenerList = pluginsService.filterPlugins(Plugin.class) + List taskEventListeners = pluginsService.filterPlugins(Plugin.class) .stream() - .map(Plugin::getOtelEventListeners) + .map(Plugin::getTaskEventListeners) .flatMap(List::stream) .collect(Collectors.toList()); - this.otelService = new OtelService(OtelService.OtelEventListeners.getInstance(otelEventListenerList)); + OpenTelemetryService.TaskEventListeners.getInstance(taskEventListeners); - final ThreadPool threadPool = new ThreadPool(settings, runnableTaskListener, otelService, - executorBuilders.toArray(new ExecutorBuilder[0])); + final ThreadPool threadPool = new ThreadPool(settings, runnableTaskListener, executorBuilders.toArray(new ExecutorBuilder[0])); resourcesToClose.add(() -> ThreadPool.terminate(threadPool, 10, TimeUnit.SECONDS)); final ResourceWatcherService resourceWatcherService = new ResourceWatcherService(settings, threadPool); resourcesToClose.add(resourceWatcherService); @@ -1068,7 +1064,7 @@ protected Node( { processRecoverySettings(settingsModule.getClusterSettings(), recoverySettings); b.bind(PeerRecoverySourceService.class) - .toInstance(new PeerRecoverySourceService(transportService, indicesService, recoverySettings, otelService)); + .toInstance(new PeerRecoverySourceService(transportService, indicesService, recoverySettings)); b.bind(PeerRecoveryTargetService.class) .toInstance(new PeerRecoveryTargetService(threadPool, transportService, recoverySettings, clusterService)); if (FeatureFlags.isEnabled(REPLICATION_TYPE)) { diff --git a/server/src/main/java/org/opensearch/otel/OtelEventListener.java b/server/src/main/java/org/opensearch/otel/OtelEventListener.java deleted file mode 100644 index 8751e06447beb..0000000000000 --- a/server/src/main/java/org/opensearch/otel/OtelEventListener.java +++ /dev/null @@ -1,16 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.otel; - -import io.opentelemetry.api.common.Attributes; -import io.opentelemetry.api.trace.Span; - -public interface OtelEventListener { - Attributes onEvent(Span span); -} diff --git a/server/src/main/java/org/opensearch/otel/OtelService.java b/server/src/main/java/org/opensearch/otel/OtelService.java deleted file mode 100644 index 7d25b1aa87231..0000000000000 --- a/server/src/main/java/org/opensearch/otel/OtelService.java +++ /dev/null @@ -1,350 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.otel; - -import com.sun.management.ThreadMXBean; -import io.opentelemetry.api.OpenTelemetry; -import io.opentelemetry.api.common.AttributeKey; -import io.opentelemetry.api.common.Attributes; -import io.opentelemetry.api.trace.Span; -import io.opentelemetry.api.trace.StatusCode; -import io.opentelemetry.api.trace.Tracer; -import io.opentelemetry.api.trace.propagation.W3CTraceContextPropagator; -import io.opentelemetry.context.Context; -import io.opentelemetry.context.Scope; -import io.opentelemetry.context.propagation.ContextPropagators; -import io.opentelemetry.exporter.otlp.http.trace.OtlpHttpSpanExporter; -import io.opentelemetry.exporter.otlp.metrics.OtlpGrpcMetricExporter; -import io.opentelemetry.sdk.OpenTelemetrySdk; -import io.opentelemetry.sdk.metrics.SdkMeterProvider; -import io.opentelemetry.sdk.metrics.export.PeriodicMetricReader; -import io.opentelemetry.sdk.resources.Resource; -import io.opentelemetry.sdk.trace.SdkTracerProvider; -import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor; -import io.opentelemetry.semconv.resource.attributes.ResourceAttributes; -import org.opensearch.action.ActionListener; - -import java.lang.management.ManagementFactory; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.Objects; -import java.util.concurrent.Callable; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Future; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; -import java.util.function.BiFunction; - -import static io.opentelemetry.api.common.AttributeKey.longKey; -import static io.opentelemetry.api.common.AttributeKey.stringKey; - - -public class OtelService { - public static Resource resource; - public static SdkTracerProvider sdkTracerProvider; - public static SdkMeterProvider sdkMeterProvider; - public static OpenTelemetry openTelemetry; - private static final ThreadMXBean threadMXBean = (ThreadMXBean) ManagementFactory.getThreadMXBean(); - public List otelEventListenerList; - static { - resource = Resource.getDefault() - .merge(Resource.create(Attributes.of(ResourceAttributes.SERVICE_NAME, "opensearch-tasks"))); - - sdkTracerProvider = SdkTracerProvider.builder() - //.addSpanProcessor(SimpleSpanProcessor.create(LoggingSpanExporter.create())) - .addSpanProcessor(SimpleSpanProcessor.create(OtlpHttpSpanExporter.builder().build())) - .setResource(resource) - .build(); - - sdkMeterProvider = SdkMeterProvider.builder() - .registerMetricReader(PeriodicMetricReader.builder(OtlpGrpcMetricExporter.builder().build()).build()) - .setResource(resource) - .build(); - - openTelemetry = OpenTelemetrySdk.builder() - .setTracerProvider(sdkTracerProvider) - .setMeterProvider(sdkMeterProvider) - .setPropagators(ContextPropagators.create(W3CTraceContextPropagator.getInstance())) - .buildAndRegisterGlobal(); - } - - public OtelService(List otelEventListenerList) { - this.otelEventListenerList = otelEventListenerList; - } - - public static long getCPUUsage(long threadId) { - return threadMXBean.getThreadCpuTime(threadId); - } - - public static long getMemoryUsage(long threadId) { - return threadMXBean.getThreadAllocatedBytes(threadId); - } - - public static long getThreadContentionTime(long threadId) { - return threadMXBean.getThreadInfo(threadId).getBlockedTime(); - } - - public static void callFunctionAndStartSpan(String spanName, BiFunction, R> function, - ActionListener actionListener, Object... args) { - Context beforeAttach = Context.current(); - Span span = startSpan(spanName); - try(Scope ignored = span.makeCurrent()) { - actionListener = new SpanPreservingActionListener<>(actionListener, beforeAttach, span.getSpanContext().getSpanId()); - emitResources(span, spanName + "-Start", OtelEventListeners.getInstance(null)); - function.apply(args, actionListener); - } finally { - emitResources(span, spanName + "-End", OtelEventListeners.getInstance(null)); - } - } - - private static Span startSpan(String spanName) { - Tracer tracer = OtelService.sdkTracerProvider.get("recover"); - Span span = tracer.spanBuilder(spanName).setParent(Context.current()).startSpan(); - span.setAttribute(stringKey("start-thread-name"), Thread.currentThread().getName()); - span.setAttribute(longKey("start-thread-id"), Thread.currentThread().getId()); - return span; - } - - private static void emitResources(Span span, String name, List otelEventListenerList) { - emitResources(span, name, Thread.currentThread(), otelEventListenerList); - } - private static void emitResources(Span span, String name, Thread t, List otelEventListenerList) { - Context context = Context.current(); - if (context != Context.root()) { - try (Scope ignored = span.makeCurrent()) { - if (otelEventListenerList != null && !otelEventListenerList.isEmpty()) { - for (OtelEventListener eventListener : otelEventListenerList) { - eventListener.onEvent(span); - } - } - span.addEvent(name, - Attributes.of( - AttributeKey.longKey("ThreadID"), t.getId(), - AttributeKey.stringKey("ThreadName"), t.getName(), - AttributeKey.longKey("CPUUsage"), OtelService.getCPUUsage(t.getId()), - AttributeKey.longKey("MemoryUsage"), OtelService.getMemoryUsage(t.getId()), - AttributeKey.longKey("ContentionTime"), OtelService.getThreadContentionTime(t.getId()) - ) - ); - } - } - } - - public static class OtelEventListeners { - static volatile List INSTANCE; - public static List getInstance(List otelEventListenerList) { - if (INSTANCE == null) { - synchronized (OtelEventListeners.class) { - if (INSTANCE == null) { - INSTANCE = otelEventListenerList; - } - } - } - return INSTANCE; - } - } - - public static ExecutorService taskWrapping(ExecutorService delegate) { - return new CurrentContextExecutorService(delegate, OtelEventListeners.getInstance(null)); - } - - public static ExecutorService taskWrapping(ExecutorService delegate, List otelEventListeners) { - return new CurrentContextExecutorService(delegate, otelEventListeners); - } - private static Callable wrapTask(Callable callable, List otelEventListeners) { - return () -> { - try (Scope ignored = Context.current().makeCurrent()) { - emitResources(Span.current(), Span.current().getSpanContext().getSpanId() + "-" + - Thread.currentThread().getName() + "-Start", otelEventListeners); - return callable.call(); - } finally { - emitResources(Span.current(), Span.current().getSpanContext().getSpanId() + "-" + - Thread.currentThread().getName() + "-End", otelEventListeners); - } - }; - } - - static Runnable wrapTask(Runnable runnable, List otelEventListeners) { - return () -> { - try (Scope ignored = Context.current().makeCurrent()) { - emitResources(Span.current(), Span.current().getSpanContext().getSpanId() + "-" + - Thread.currentThread().getName() + "-Start", otelEventListeners); - runnable.run(); - emitResources(Span.current(), Span.current().getSpanContext().getSpanId() + "-" + - Thread.currentThread().getName() + "-End", otelEventListeners); - } - }; - } - - static final class CurrentContextExecutorService extends ForwardingExecutorService { - - private List otelEventListeners; - - CurrentContextExecutorService(ExecutorService delegate) { - this(delegate, null); - } - - CurrentContextExecutorService(ExecutorService delegate, List otelEventListeners) { - super(delegate); - this.otelEventListeners = otelEventListeners; - } - - public List getOtelEventListeners() { - return otelEventListeners; - } - @Override - public Future submit(Callable task) { - return delegate().submit(wrapTask(task, otelEventListeners)); - } - - @Override - public Future submit(Runnable task, T result) { - return delegate().submit(wrapTask(task, otelEventListeners), result); - } - - @Override - public Future submit(Runnable task) { - return delegate().submit(wrapTask(task, otelEventListeners)); - } - - @Override - public List> invokeAll(Collection> tasks) - throws InterruptedException { - return delegate().invokeAll(wrap(tasks, otelEventListeners)); - } - - @Override - public List> invokeAll( - Collection> tasks, long timeout, TimeUnit unit) - throws InterruptedException { - return delegate().invokeAll(wrap(tasks, otelEventListeners), timeout, unit); - } - - @Override - public T invokeAny(Collection> tasks) - throws InterruptedException, ExecutionException { - return delegate().invokeAny(wrap(tasks, otelEventListeners)); - } - - @Override - public T invokeAny(Collection> tasks, long timeout, TimeUnit unit) - throws InterruptedException, ExecutionException, TimeoutException { - return delegate().invokeAny(wrap(tasks, otelEventListeners), timeout, unit); - } - - @Override - public void execute(Runnable command) { - delegate().execute(wrapTask(command, otelEventListeners)); - } - } - - static abstract class ForwardingExecutorService implements ExecutorService { - - private final ExecutorService delegate; - - protected ForwardingExecutorService(ExecutorService delegate) { - this.delegate = delegate; - } - - ExecutorService delegate() { - return delegate; - } - - @Override - public final void shutdown() { - delegate.shutdown(); - } - - @Override - public final List shutdownNow() { - return delegate.shutdownNow(); - } - - @Override - public final boolean isShutdown() { - return delegate.isShutdown(); - } - - @Override - public final boolean isTerminated() { - return delegate.isTerminated(); - } - - @Override - public final boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedException { - return delegate.awaitTermination(timeout, unit); - } - - protected static Collection> wrap(Collection> tasks, - List otelEventListeners) { - List> wrapped = new ArrayList<>(); - for (Callable task : tasks) { - wrapped.add(wrapTask(task, otelEventListeners)); - } - return wrapped; - } - } - - public static final class SpanPreservingActionListener implements ActionListener { - private final ActionListener delegate; - private final Context beforeAttachContext; - private final Context afterAttachContext; - private final String spanID; - - public SpanPreservingActionListener(ActionListener delegate, Context beforeAttachContext, String spanID) { - this.delegate = delegate; - this.beforeAttachContext = beforeAttachContext; - this.afterAttachContext = Context.current(); - this.spanID = spanID; - } - - public SpanPreservingActionListener(ActionListener delegate, Context beforeAttachContext) { - this(delegate, beforeAttachContext, null); - } - - @Override - public void onResponse(R r) { - try (Scope ignored = Objects.requireNonNull(afterAttachContext).makeCurrent()) { - Span span = Span.current(); - closeCurrentScope(span); - } - try (Scope ignored = Objects.requireNonNull(beforeAttachContext).makeCurrent()) { - delegate.onResponse(r); - } - } - - @Override - public void onFailure(Exception e) { - try (Scope ignored = Objects.requireNonNull(afterAttachContext).makeCurrent()) { - Span span = Span.current(); - span.setStatus(StatusCode.ERROR); - closeCurrentScope(span); - } - try (Scope ignored = Objects.requireNonNull(beforeAttachContext).makeCurrent()) { - delegate.onFailure(e); - } - } - - private void closeCurrentScope(Span span) { - assert spanID == null || span.getSpanContext().getSpanId().equals(spanID); - span.setAttribute(stringKey("finish-thread-name"), Thread.currentThread().getName()); - span.setAttribute(longKey("finish-thread-id"), Thread.currentThread().getId()); - if (spanID != null) { - Span.current().end(); - } - } - - @Override - public String toString() { - return getClass().getName() + "/" + delegate.toString(); - } - } -} diff --git a/server/src/main/java/org/opensearch/plugins/Plugin.java b/server/src/main/java/org/opensearch/plugins/Plugin.java index 9b32bf71a9b67..7b7d4b194c832 100644 --- a/server/src/main/java/org/opensearch/plugins/Plugin.java +++ b/server/src/main/java/org/opensearch/plugins/Plugin.java @@ -32,7 +32,7 @@ package org.opensearch.plugins; -import org.opensearch.otel.OtelEventListener; +import org.opensearch.tracing.TaskEventListener; import org.opensearch.watcher.ResourceWatcherService; import org.opensearch.bootstrap.BootstrapCheck; import org.opensearch.client.Client; @@ -207,7 +207,7 @@ public List> getSettingUpgraders() { return Collections.emptyList(); } - public List getOtelEventListeners() { + public List getTaskEventListeners() { return Collections.emptyList(); } diff --git a/server/src/main/java/org/opensearch/threadpool/ScalingExecutorBuilder.java b/server/src/main/java/org/opensearch/threadpool/ScalingExecutorBuilder.java index 9ce90a4fa28bb..a2e9cc0b3bbf5 100644 --- a/server/src/main/java/org/opensearch/threadpool/ScalingExecutorBuilder.java +++ b/server/src/main/java/org/opensearch/threadpool/ScalingExecutorBuilder.java @@ -38,8 +38,8 @@ import org.opensearch.common.util.concurrent.OpenSearchExecutors; import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.node.Node; -import org.opensearch.otel.OtelEventListener; -import org.opensearch.otel.OtelService; +import org.opensearch.tracing.opentelemetry.OpenTelemetryContextWrapper; +import org.opensearch.tracing.opentelemetry.OpenTelemetryService; import java.util.Arrays; import java.util.List; @@ -59,7 +59,6 @@ public final class ScalingExecutorBuilder extends ExecutorBuilder maxSetting; private final Setting keepAliveSetting; - private final List otelEventListeners; /** * Construct a scaling executor builder; the settings will have the * key prefix "thread_pool." followed by the executor name. @@ -73,15 +72,6 @@ public final class ScalingExecutorBuilder extends ExecutorBuilder otelEventListeners) { - this(name, core, max, keepAlive, "thread_pool." + name, otelEventListeners); - } - - public ScalingExecutorBuilder(final String name, final int core, final int max, final TimeValue keepAlive, final String prefix) { - this(name, core, max, keepAlive, prefix, null); - } - /** * Construct a scaling executor builder; the settings will have the @@ -94,13 +84,11 @@ public ScalingExecutorBuilder(final String name, final int core, final int max, * threads will be kept alive * @param prefix the prefix for the settings keys */ - public ScalingExecutorBuilder(final String name, final int core, final int max, final TimeValue keepAlive, final String prefix, - List otelEventListeners) { + public ScalingExecutorBuilder(final String name, final int core, final int max, final TimeValue keepAlive, final String prefix) { super(name); this.coreSetting = Setting.intSetting(settingsKey(prefix, "core"), core, Setting.Property.NodeScope); this.maxSetting = Setting.intSetting(settingsKey(prefix, "max"), max, Setting.Property.NodeScope); this.keepAliveSetting = Setting.timeSetting(settingsKey(prefix, "keep_alive"), keepAlive, Setting.Property.NodeScope); - this.otelEventListeners = otelEventListeners; } @Override @@ -134,8 +122,8 @@ ThreadPool.ExecutorHolder build(final ScalingExecutorSettings settings, final Th threadFactory, threadContext ); - if (otelEventListeners != null) { - executor = OtelService.taskWrapping(executor, otelEventListeners); + if (OpenTelemetryService.isThreadPoolAllowed(name())) { + executor = OpenTelemetryContextWrapper.wrapTask(executor); } return new ThreadPool.ExecutorHolder(executor, info); } diff --git a/server/src/main/java/org/opensearch/threadpool/ThreadPool.java b/server/src/main/java/org/opensearch/threadpool/ThreadPool.java index ac809d463583f..38ada9d63214d 100644 --- a/server/src/main/java/org/opensearch/threadpool/ThreadPool.java +++ b/server/src/main/java/org/opensearch/threadpool/ThreadPool.java @@ -55,7 +55,8 @@ import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.node.Node; import org.opensearch.node.ReportingService; -import org.opensearch.otel.OtelService; +import org.opensearch.tracing.opentelemetry.OpenTelemetryContextWrapper; +import org.opensearch.tracing.opentelemetry.OpenTelemetryService; import java.io.IOException; import java.util.ArrayList; @@ -204,8 +205,6 @@ public static ThreadPoolType fromType(String type) { private final ScheduledThreadPoolExecutor scheduler; - private final OtelService otelService; - public Collection builders() { return Collections.unmodifiableCollection(builders.values()); } @@ -225,24 +224,14 @@ public ThreadPool( final Settings settings, final AtomicReference runnableTaskListener, final ExecutorBuilder... customBuilders - ) { - this(settings, runnableTaskListener, null, customBuilders); - } - - public ThreadPool( - final Settings settings, - final AtomicReference runnableTaskListener, - OtelService otelService, - final ExecutorBuilder... customBuilders ) { assert Node.NODE_NAME_SETTING.exists(settings); - this.otelService = otelService; final Map builders = new HashMap<>(); final int allocatedProcessors = OpenSearchExecutors.allocatedProcessors(settings); final int halfProcMaxAt5 = halfAllocatedProcessorsMaxFive(allocatedProcessors); final int halfProcMaxAt10 = halfAllocatedProcessorsMaxTen(allocatedProcessors); final int genericThreadPoolMax = boundedBy(4 * allocatedProcessors, 128, 512); - builders.put(Names.GENERIC, new ScalingExecutorBuilder(Names.GENERIC, 4, genericThreadPoolMax, TimeValue.timeValueSeconds(30), otelService.otelEventListenerList)); + builders.put(Names.GENERIC, new ScalingExecutorBuilder(Names.GENERIC, 4, genericThreadPoolMax, TimeValue.timeValueSeconds(30))); builders.put(Names.WRITE, new FixedExecutorBuilder(settings, Names.WRITE, allocatedProcessors, 10000)); builders.put(Names.GET, new FixedExecutorBuilder(settings, Names.GET, allocatedProcessors, 1000)); builders.put(Names.ANALYZE, new FixedExecutorBuilder(settings, Names.ANALYZE, 1, 16)); @@ -426,10 +415,8 @@ public ExecutorService executor(String name) { if (holder == null) { throw new IllegalArgumentException("no executor service found for [" + name + "]"); } - if (name.equals(Names.GENERIC)) { - if (otelService != null) { - return OtelService.taskWrapping(holder.executor(), otelService.otelEventListenerList); - } + if (OpenTelemetryService.isThreadPoolAllowed(Names.GENERIC)) { + return OpenTelemetryContextWrapper.wrapTask(holder.executor()); } return holder.executor(); } diff --git a/server/src/main/java/org/opensearch/tracing/TaskEventListener.java b/server/src/main/java/org/opensearch/tracing/TaskEventListener.java new file mode 100644 index 0000000000000..e060a361247e5 --- /dev/null +++ b/server/src/main/java/org/opensearch/tracing/TaskEventListener.java @@ -0,0 +1,50 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.tracing; + +import io.opentelemetry.context.Context; + +import java.util.concurrent.ExecutorService; + +/** + * SPI for event listener when a thread picks a task to be executed associated with Context aware ExecutorService. + */ +public interface TaskEventListener { + + /** Invoked when a thread start working a task associated with Context aware ExecutorService {@link org.opensearch.tracing.opentelemetry.OpenTelemetryContextWrapper#wrapTask(ExecutorService)}. + * Context and Span{@link io.opentelemetry.api.trace.Span} information can be derived by calling Context{@link Context#current()} + * when OpenTelemetry implementation is chosen, as current thread will have the context propagated to it. Span events can be added as necessary. + * This can be used to start metering resource consumption by a thread. + * + * Note: current thread will have the Context set + * @param t Thread to start working on a task + */ + void onStart(String operationName, String eventName, Thread t); + + /** Invoked when a thread completes a task associated with Context aware ExecutorService{@link org.opensearch.tracing.opentelemetry.OpenTelemetryContextWrapper#wrapTask(ExecutorService)} + * for both success and failure scenarios. Context and Span{@link io.opentelemetry.api.trace.Span} information can + * be derived by calling Context{@link Context#current()} when OpenTelemetry implementation is chosen, as + * current thread will have the context propagated to it. Span events can be added as necessary. + * + * This can be used to stop metering resource consumption by a thread. + * + * @param t Thread which completed the task + */ + void onEnd(String operationName, String eventName, Thread t); + + /** + * This is used to check if the TaskEventListener should be called for provided operation and/or event. + * Contract here is, individual service provider should know which all operations are needs to be onboarded to + * this TaskEventListener. It doesn't make sense to call all available TaskEventListeners for all operations using + * current executor service. + * @param operationName name of the operation associated with a trace. + * @return + */ + boolean isApplicable(String operationName, String eventName); +} diff --git a/server/src/main/java/org/opensearch/tracing/opentelemetry/OTelContextPreservingActionListener.java b/server/src/main/java/org/opensearch/tracing/opentelemetry/OTelContextPreservingActionListener.java new file mode 100644 index 0000000000000..a97d1053e0c4f --- /dev/null +++ b/server/src/main/java/org/opensearch/tracing/opentelemetry/OTelContextPreservingActionListener.java @@ -0,0 +1,85 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.tracing.opentelemetry; + +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.StatusCode; +import io.opentelemetry.context.Context; +import io.opentelemetry.context.Scope; +import org.opensearch.action.ActionListener; + +import java.util.Objects; + +import static io.opentelemetry.api.common.AttributeKey.longKey; +import static io.opentelemetry.api.common.AttributeKey.stringKey; + +/** + * It does follow - + * 1. Pass the context to thread working on response/failure of delegated ActionListener. + * 2. Close the Span if one passed while its construction. + * 3. Set the scope back to previous context prior to starting the new Span. + * In case, no Span was started and needs to be closed + * {@link OTelContextPreservingActionListener#OTelContextPreservingActionListener(ActionListener, Context)} can be used + * with beforeAttachContext as {@link Context#current()}. + * @param Response object type + */ +public final class OTelContextPreservingActionListener implements ActionListener { + private final ActionListener delegate; + private final Context beforeAttachContext; + private final Context afterAttachContext; + private final String spanID; + + public OTelContextPreservingActionListener(ActionListener delegate, Context beforeAttachContext, String spanID) { + this.delegate = delegate; + this.beforeAttachContext = beforeAttachContext; + this.afterAttachContext = Context.current(); + this.spanID = spanID; + } + + public OTelContextPreservingActionListener(ActionListener delegate, Context beforeAttachContext) { + this(delegate, beforeAttachContext, null); + } + + @Override + public void onResponse(Response r) { + try (Scope ignored = Objects.requireNonNull(afterAttachContext).makeCurrent()) { + Span span = Span.current(); + closeCurrentScope(span); + } + try (Scope ignored = Objects.requireNonNull(beforeAttachContext).makeCurrent()) { + delegate.onResponse(r); + } + } + + @Override + public void onFailure(Exception e) { + try (Scope ignored = Objects.requireNonNull(afterAttachContext).makeCurrent()) { + Span span = Span.current(); + span.setStatus(StatusCode.ERROR); + closeCurrentScope(span); + } + try (Scope ignored = Objects.requireNonNull(beforeAttachContext).makeCurrent()) { + delegate.onFailure(e); + } + } + + private void closeCurrentScope(Span span) { + assert spanID == null || span.getSpanContext().getSpanId().equals(spanID); + span.setAttribute(stringKey("finish-thread-name"), Thread.currentThread().getName()); + span.setAttribute(longKey("finish-thread-id"), Thread.currentThread().getId()); + if (spanID != null) { + Span.current().end(); + } + } + + @Override + public String toString() { + return getClass().getName() + "/" + delegate.toString(); + } +} diff --git a/server/src/main/java/org/opensearch/tracing/opentelemetry/OpenSearchConcurrentExecutorService.java b/server/src/main/java/org/opensearch/tracing/opentelemetry/OpenSearchConcurrentExecutorService.java new file mode 100644 index 0000000000000..69381d9661d53 --- /dev/null +++ b/server/src/main/java/org/opensearch/tracing/opentelemetry/OpenSearchConcurrentExecutorService.java @@ -0,0 +1,119 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.tracing.opentelemetry; + +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.context.Context; +import io.opentelemetry.context.Scope; +import org.opensearch.tracing.TaskEventListener; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +final public class OpenSearchConcurrentExecutorService extends OpenSearchForwardingExecutorService { + private final List taskEventListeners; + + OpenSearchConcurrentExecutorService(ExecutorService delegate) { + this(delegate, OpenTelemetryService.TaskEventListeners.getInstance(null)); + } + + OpenSearchConcurrentExecutorService(ExecutorService delegate, List taskEventListeners) { + super(delegate); + this.taskEventListeners = taskEventListeners; + } + + @Override + public Future submit(Callable task) { + return delegate().submit(wrapTask(task, taskEventListeners)); + } + + @Override + public Future submit(Runnable task, T result) { + return delegate().submit(wrapTask(task, taskEventListeners), result); + } + + @Override + public Future submit(Runnable task) { + return delegate().submit(wrapTask(task, taskEventListeners)); + } + + @Override + public List> invokeAll(Collection> tasks) + throws InterruptedException { + return delegate().invokeAll(wrap(tasks, taskEventListeners)); + } + + @Override + public List> invokeAll( + Collection> tasks, long timeout, TimeUnit unit) + throws InterruptedException { + return delegate().invokeAll(wrap(tasks, taskEventListeners), timeout, unit); + } + + @Override + public T invokeAny(Collection> tasks) + throws InterruptedException, ExecutionException { + return delegate().invokeAny(wrap(tasks, taskEventListeners)); + } + + @Override + public T invokeAny(Collection> tasks, long timeout, TimeUnit unit) + throws InterruptedException, ExecutionException, TimeoutException { + return delegate().invokeAny(wrap(tasks, taskEventListeners), timeout, unit); + } + + @Override + public void execute(Runnable command) { + delegate().execute(wrapTask(command, taskEventListeners)); + } + + private static Collection> wrap(Collection> tasks, + List taskEventListeners) { + List> wrapped = new ArrayList<>(); + for (Callable task : tasks) { + wrapped.add(wrapTask(task, taskEventListeners)); + } + return wrapped; + } + + private static Callable wrapTask(Callable callable, List taskEventListeners) { + return () -> { + try (Scope ignored = Context.current().makeCurrent()) { + OpenTelemetryService.callTaskEventListeners(true, "", Span.current().getSpanContext().getSpanId() + "-" + + Thread.currentThread().getName() + "-Start", Thread.currentThread(), taskEventListeners); + return callable.call(); + } finally { + OpenTelemetryService.callTaskEventListeners(false, "", Span.current().getSpanContext().getSpanId() + "-" + + Thread.currentThread().getName() + "-End", Thread.currentThread(), taskEventListeners); + } + }; + } + + static Runnable wrapTask(Runnable runnable, List taskEventListeners) { + return () -> { + try (Scope ignored = Context.current().makeCurrent()) { + OpenTelemetryService.callTaskEventListeners(true, "", Span.current().getSpanContext().getSpanId() + "-" + + Thread.currentThread().getName() + "-Start", Thread.currentThread(), taskEventListeners); + runnable.run(); + } finally { + OpenTelemetryService.callTaskEventListeners(false, "", Span.current().getSpanContext().getSpanId() + "-" + + Thread.currentThread().getName() + "-End", Thread.currentThread(), taskEventListeners); + } + }; + } + + +} diff --git a/server/src/main/java/org/opensearch/tracing/opentelemetry/OpenSearchForwardingExecutorService.java b/server/src/main/java/org/opensearch/tracing/opentelemetry/OpenSearchForwardingExecutorService.java new file mode 100644 index 0000000000000..63435677f31c3 --- /dev/null +++ b/server/src/main/java/org/opensearch/tracing/opentelemetry/OpenSearchForwardingExecutorService.java @@ -0,0 +1,51 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.tracing.opentelemetry; + +import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.TimeUnit; + +abstract class OpenSearchForwardingExecutorService implements ExecutorService { + + private final ExecutorService delegate; + + protected OpenSearchForwardingExecutorService(ExecutorService delegate) { + this.delegate = delegate; + } + + ExecutorService delegate() { + return delegate; + } + + @Override + public final void shutdown() { + delegate.shutdown(); + } + + @Override + public final List shutdownNow() { + return delegate.shutdownNow(); + } + + @Override + public final boolean isShutdown() { + return delegate.isShutdown(); + } + + @Override + public final boolean isTerminated() { + return delegate.isTerminated(); + } + + @Override + public final boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedException { + return delegate.awaitTermination(timeout, unit); + } +} diff --git a/server/src/main/java/org/opensearch/tracing/opentelemetry/OpenTelemetryContextWrapper.java b/server/src/main/java/org/opensearch/tracing/opentelemetry/OpenTelemetryContextWrapper.java new file mode 100644 index 0000000000000..370fce1f7094a --- /dev/null +++ b/server/src/main/java/org/opensearch/tracing/opentelemetry/OpenTelemetryContextWrapper.java @@ -0,0 +1,31 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.tracing.opentelemetry; + +import java.util.concurrent.Executor; +import java.util.concurrent.ExecutorService; + +public class OpenTelemetryContextWrapper { + + /** + * Wraps the ExecutorService for context propagation across threads in the ThreadPool. + * @param executorService executor service to be wrapped + */ + public static ExecutorService wrapTask(ExecutorService executorService) { + return new OpenSearchConcurrentExecutorService(executorService); + } + + /** + * Passes the context to the provided delegate executor + * @param executor executor to be wrapped with. + */ + public static Executor wrapTask(Executor executor) { + return null; + } +} diff --git a/server/src/main/java/org/opensearch/tracing/opentelemetry/OpenTelemetryService.java b/server/src/main/java/org/opensearch/tracing/opentelemetry/OpenTelemetryService.java new file mode 100644 index 0000000000000..61f971aea7025 --- /dev/null +++ b/server/src/main/java/org/opensearch/tracing/opentelemetry/OpenTelemetryService.java @@ -0,0 +1,182 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.tracing.opentelemetry; + +import com.sun.management.ThreadMXBean; +import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.Tracer; +import io.opentelemetry.api.trace.propagation.W3CTraceContextPropagator; +import io.opentelemetry.context.Context; +import io.opentelemetry.context.Scope; +import io.opentelemetry.context.propagation.ContextPropagators; +import io.opentelemetry.exporter.otlp.http.trace.OtlpHttpSpanExporter; +import io.opentelemetry.exporter.otlp.metrics.OtlpGrpcMetricExporter; +import io.opentelemetry.sdk.OpenTelemetrySdk; +import io.opentelemetry.sdk.metrics.SdkMeterProvider; +import io.opentelemetry.sdk.metrics.export.PeriodicMetricReader; +import io.opentelemetry.sdk.resources.Resource; +import io.opentelemetry.sdk.trace.SdkTracerProvider; +import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor; +import io.opentelemetry.semconv.resource.attributes.ResourceAttributes; +import org.opensearch.action.ActionListener; +import org.opensearch.threadpool.ThreadPool; +import org.opensearch.tracing.TaskEventListener; + +import java.lang.management.ManagementFactory; +import java.util.List; +import java.util.function.BiFunction; + +import static io.opentelemetry.api.common.AttributeKey.longKey; +import static io.opentelemetry.api.common.AttributeKey.stringKey; + + +public class OpenTelemetryService { + public static Resource resource; + public static SdkTracerProvider sdkTracerProvider; + public static SdkMeterProvider sdkMeterProvider; + public static OpenTelemetry openTelemetry; + private static final List DEFAULT_TASK_EVENT_LISTENERS; + private static final List allowedThreadPools = List.of(ThreadPool.Names.GENERIC); + private static final ThreadMXBean threadMXBean = (ThreadMXBean) ManagementFactory.getThreadMXBean(); + + static { + resource = Resource.getDefault() + .merge(Resource.create(Attributes.of(ResourceAttributes.SERVICE_NAME, "opensearch-tasks"))); + + sdkTracerProvider = SdkTracerProvider.builder() + .addSpanProcessor(SimpleSpanProcessor.create(OtlpHttpSpanExporter.builder().build())) + .setResource(resource) + .build(); + + sdkMeterProvider = SdkMeterProvider.builder() + .registerMetricReader(PeriodicMetricReader.builder(OtlpGrpcMetricExporter.builder().build()).build()) + .setResource(resource) + .build(); + + openTelemetry = OpenTelemetrySdk.builder() + .setTracerProvider(sdkTracerProvider) + .setMeterProvider(sdkMeterProvider) + .setPropagators(ContextPropagators.create(W3CTraceContextPropagator.getInstance())) + .buildAndRegisterGlobal(); + + DEFAULT_TASK_EVENT_LISTENERS = List.of(new TaskEventListener() { + @Override + public void onStart(String operationName, String eventName, Thread t) { + Span span = Span.current(); + if (span != Span.getInvalid()) { + span.addEvent(eventName, + Attributes.of( + AttributeKey.longKey("ThreadID"), t.getId(), + AttributeKey.stringKey("ThreadName"), t.getName(), + AttributeKey.longKey("CPUUsage"), threadMXBean.getThreadCpuTime(t.getId()), + AttributeKey.longKey("MemoryUsage"), threadMXBean.getThreadAllocatedBytes(t.getId()), + AttributeKey.longKey("ContentionTime"), threadMXBean.getThreadInfo(t.getId()).getBlockedTime() + ) + ); + } + } + + @Override + public void onEnd(String operationName, String eventName, Thread t) { + Span span = Span.current(); + if (span != Span.getInvalid()) { + span.addEvent(eventName, + Attributes.of( + AttributeKey.longKey("ThreadID"), t.getId(), + AttributeKey.stringKey("ThreadName"), t.getName(), + AttributeKey.longKey("CPUUsage"), threadMXBean.getThreadCpuTime(t.getId()), + AttributeKey.longKey("MemoryUsage"), threadMXBean.getThreadAllocatedBytes(t.getId()), + AttributeKey.longKey("ContentionTime"), threadMXBean.getThreadInfo(t.getId()).getBlockedTime() + ) + ); + } + } + + @Override + public boolean isApplicable(String operationName, String eventName) { + return true; + } + }); + } + + public static boolean isThreadPoolAllowed(String threadPoolName) { + return allowedThreadPools.contains(threadPoolName); + } + + /** + * starts the span and invokes the function under the scope of new span, closes the scope when function is invoked. + * Wraps the ActionListener with {@link OTelContextPreservingActionListener} for context propagation and ends the span + * on response/failure of action listener. + */ + public static void callFunctionAndStartSpan(String spanName, BiFunction, R> function, + ActionListener actionListener, Object... args) { + Context beforeAttach = Context.current(); + Span span = startSpan(spanName); + try(Scope ignored = span.makeCurrent()) { + actionListener = new OTelContextPreservingActionListener<>(actionListener, beforeAttach, span.getSpanContext().getSpanId()); + callTaskEventListeners(true, "", spanName + "-Start", Thread.currentThread(), + TaskEventListeners.getInstance(null)); + function.apply(args, actionListener); + } finally { + callTaskEventListeners(false, "", spanName + "-End", Thread.currentThread(), + TaskEventListeners.getInstance(null)); + } + } + + /** + * TODO - to be replaced when OpenSearch tracing APIs are available + */ + private static Span startSpan(String spanName) { + Tracer tracer = OpenTelemetryService.sdkTracerProvider.get("recover"); + Span span = tracer.spanBuilder(spanName).setParent(Context.current()).startSpan(); + span.setAttribute(stringKey("start-thread-name"), Thread.currentThread().getName()); + span.setAttribute(longKey("start-thread-id"), Thread.currentThread().getId()); + return span; + } + + /** + * Lazy initialization of all TaskEventListener. + */ + public static class TaskEventListeners { + static volatile List INSTANCE; + public static List getInstance(List otelEventListenerList) { + if (INSTANCE == null) { + synchronized (TaskEventListener.class) { + if (INSTANCE == null) { + INSTANCE = otelEventListenerList; + INSTANCE.addAll(DEFAULT_TASK_EVENT_LISTENERS); + } + } + } + return INSTANCE; + } + } + + protected static void callTaskEventListeners(boolean startEvent, String operationName, String eventName, Thread t, + List taskEventListeners) { + if (Context.current() != Context.root()) { + try (Scope ignored = Span.current().makeCurrent()) { + if (taskEventListeners != null && !taskEventListeners.isEmpty()) { + for (TaskEventListener eventListener : taskEventListeners) { + if (eventListener.isApplicable(operationName, eventName)) { + if (startEvent) { + eventListener.onStart(operationName, eventName, t); + } else { + eventListener.onStart(operationName, eventName, t); + } + } + } + } + } + } + } +} diff --git a/server/src/test/java/org/opensearch/indices/recovery/PeerRecoverySourceServiceTests.java b/server/src/test/java/org/opensearch/indices/recovery/PeerRecoverySourceServiceTests.java index c8bc5c86733b1..4fbae4b0d53ca 100644 --- a/server/src/test/java/org/opensearch/indices/recovery/PeerRecoverySourceServiceTests.java +++ b/server/src/test/java/org/opensearch/indices/recovery/PeerRecoverySourceServiceTests.java @@ -40,8 +40,6 @@ import org.opensearch.index.shard.IndexShardTestCase; import org.opensearch.index.store.Store; import org.opensearch.indices.IndicesService; -import org.opensearch.otel.OtelService; -import org.opensearch.plugins.PluginsService; import org.opensearch.test.NodeRoles; import org.opensearch.transport.TransportService; @@ -62,8 +60,7 @@ public void testDuplicateRecoveries() throws IOException { PeerRecoverySourceService peerRecoverySourceService = new PeerRecoverySourceService( mock(TransportService.class), indicesService, - new RecoverySettings(Settings.EMPTY, new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)), - new OtelService(mock(PluginsService.class)) + new RecoverySettings(Settings.EMPTY, new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)) ); StartRecoveryRequest startRecoveryRequest = new StartRecoveryRequest( primary.shardId(), From 1995211caf9b1030ad6f4943e4ade53a8748b7fb Mon Sep 17 00:00:00 2001 From: Stevan Buzejic Date: Mon, 22 May 2023 16:51:50 +0200 Subject: [PATCH 6/8] Rishabh copy-paste Signed-off-by: Stevan Buzejic --- server/build.gradle | 3 +- .../recovery/RetryableTransportClient.java | 1 - .../main/java/org/opensearch/node/Node.java | 332 ++++++++++-------- .../listener/DiskStatsTaskEventListener.java | 190 ++++++++++ .../org/opensearch/threadpool/ThreadPool.java | 12 +- .../opentelemetry/OpenTelemetryService.java | 8 +- 6 files changed, 394 insertions(+), 152 deletions(-) create mode 100644 server/src/main/java/org/opensearch/performanceanalyzer/listener/DiskStatsTaskEventListener.java diff --git a/server/build.gradle b/server/build.gradle index 39558903e405c..83936ce003946 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -121,7 +121,6 @@ dependencies { api "org.apache.lucene:lucene-spatial-extras:${versions.lucene}" api "org.apache.lucene:lucene-spatial3d:${versions.lucene}" api "org.apache.lucene:lucene-suggest:${versions.lucene}" - // utilities api project(":libs:opensearch-cli") api 'com.carrotsearch:hppc:0.8.1' @@ -174,6 +173,8 @@ dependencies { // protobuf api "com.google.protobuf:protobuf-java:${versions.protobuf}" + implementation files('/home/stevan/git/opensearch/repo/performance-analyzer-commons/build/libs/opensearch-performance-analyzer-commons-3.0.0.0-SNAPSHOT.jar') + testImplementation(project(":test:framework")) { // tests use the locally compiled version of server exclude group: 'org.opensearch', module: 'server' diff --git a/server/src/main/java/org/opensearch/indices/recovery/RetryableTransportClient.java b/server/src/main/java/org/opensearch/indices/recovery/RetryableTransportClient.java index 163ead32ac617..70978362ff361 100644 --- a/server/src/main/java/org/opensearch/indices/recovery/RetryableTransportClient.java +++ b/server/src/main/java/org/opensearch/indices/recovery/RetryableTransportClient.java @@ -21,7 +21,6 @@ import org.opensearch.common.util.CancellableThreads; import org.opensearch.common.util.concurrent.ConcurrentCollections; import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; -import org.opensearch.common.util.concurrent.OpenSearchRejectedExecutionException; import org.opensearch.threadpool.ThreadPool; import org.opensearch.tracing.opentelemetry.OTelContextPreservingActionListener; import org.opensearch.transport.ConnectTransportException; diff --git a/server/src/main/java/org/opensearch/node/Node.java b/server/src/main/java/org/opensearch/node/Node.java index c87a78602a2d3..9cc0c1d59ce11 100644 --- a/server/src/main/java/org/opensearch/node/Node.java +++ b/server/src/main/java/org/opensearch/node/Node.java @@ -35,16 +35,17 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.lucene.util.Constants; +import org.opensearch.ExceptionsHelper; import org.opensearch.common.SetOnce; +import org.opensearch.common.settings.SettingsException; +import org.opensearch.common.unit.ByteSizeUnit; +import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.cluster.routing.allocation.AwarenessReplicaBalance; import org.opensearch.index.IndexModule; import org.opensearch.index.IndexingPressureService; -import org.opensearch.tracing.TaskEventListener; -import org.opensearch.tracing.opentelemetry.OpenTelemetryService; -import org.opensearch.tasks.TaskResourceTrackingService; -import org.opensearch.threadpool.RunnableTaskExecutionListener; -import org.opensearch.common.util.FeatureFlags; import org.opensearch.index.store.remote.filecache.FileCache; +import org.opensearch.index.store.remote.filecache.FileCacheCleaner; import org.opensearch.index.store.remote.filecache.FileCacheFactory; import org.opensearch.indices.replication.SegmentReplicationSourceFactory; import org.opensearch.indices.replication.SegmentReplicationTargetService; @@ -53,17 +54,24 @@ import org.opensearch.extensions.NoopExtensionsManager; import org.opensearch.monitor.fs.FsInfo; import org.opensearch.monitor.fs.FsProbe; +import org.opensearch.plugins.SearchPipelinePlugin; import org.opensearch.search.backpressure.SearchBackpressureService; import org.opensearch.search.backpressure.settings.SearchBackpressureSettings; +import org.opensearch.search.pipeline.SearchPipelineService; +import org.opensearch.tasks.TaskResourceTrackingService; +import org.opensearch.tasks.consumer.TopNSearchTasksLogger; +import org.opensearch.threadpool.RunnableTaskExecutionListener; import org.opensearch.index.store.RemoteSegmentStoreDirectoryFactory; +import org.opensearch.tracing.TaskEventListener; +import org.opensearch.tracing.opentelemetry.OpenTelemetryService; import org.opensearch.watcher.ResourceWatcherService; -import org.opensearch.Assertions; +import org.opensearch.core.Assertions; import org.opensearch.Build; import org.opensearch.OpenSearchException; import org.opensearch.OpenSearchTimeoutException; import org.opensearch.Version; -import org.opensearch.ExceptionsHelper; import org.opensearch.action.ActionModule; +import org.opensearch.action.ActionModule.DynamicActionRegistry; import org.opensearch.action.ActionType; import org.opensearch.action.admin.cluster.snapshots.status.TransportNodesSnapshotsStatus; import org.opensearch.action.search.SearchExecutionStatsCollector; @@ -125,8 +133,8 @@ import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.BigArrays; import org.opensearch.common.util.PageCacheRecycler; -import org.opensearch.common.xcontent.NamedXContentRegistry; -import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.discovery.Discovery; import org.opensearch.discovery.DiscoveryModule; import org.opensearch.env.Environment; @@ -139,6 +147,7 @@ import org.opensearch.gateway.MetaStateService; import org.opensearch.gateway.PersistedClusterStateService; import org.opensearch.http.HttpServerTransport; +import org.opensearch.identity.IdentityService; import org.opensearch.index.IndexSettings; import org.opensearch.index.analysis.AnalysisRegistry; import org.opensearch.index.engine.EngineFactory; @@ -171,6 +180,7 @@ import org.opensearch.plugins.ClusterPlugin; import org.opensearch.plugins.DiscoveryPlugin; import org.opensearch.plugins.EnginePlugin; +import org.opensearch.plugins.IdentityPlugin; import org.opensearch.plugins.IndexStorePlugin; import org.opensearch.plugins.IngestPlugin; import org.opensearch.plugins.MapperPlugin; @@ -217,7 +227,7 @@ import java.io.IOException; import java.net.InetAddress; import java.net.InetSocketAddress; -import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardCopyOption; @@ -241,7 +251,8 @@ import java.util.stream.Stream; import static java.util.stream.Collectors.toList; -import static org.opensearch.common.util.FeatureFlags.REPLICATION_TYPE; +import static org.opensearch.common.util.FeatureFlags.SEARCH_PIPELINE; +import static org.opensearch.env.NodeEnvironment.collectFileCacheDataPath; import static org.opensearch.index.ShardIndexingPressureSettings.SHARD_INDEXING_PRESSURE_ENABLED_ATTRIBUTE_KEY; /** @@ -278,12 +289,12 @@ public class Node implements Closeable { ); /** - * controls whether the node is allowed to persist things like metadata to disk - * Note that this does not control whether the node stores actual indices (see - * {@link #NODE_DATA_SETTING}). However, if this is false, {@link #NODE_DATA_SETTING} - * and {@link #NODE_MASTER_SETTING} must also be false. - * - */ + * controls whether the node is allowed to persist things like metadata to disk + * Note that this does not control whether the node stores actual indices (see + * {@link #NODE_DATA_SETTING}). However, if this is false, {@link #NODE_DATA_SETTING} + * and {@link #NODE_MASTER_SETTING} must also be false. + * + */ public static final Setting NODE_LOCAL_STORAGE_SETTING = Setting.boolSetting( "node.local_storage", true, @@ -318,6 +329,12 @@ public class Node implements Closeable { } }, Setting.Property.NodeScope); + public static final Setting NODE_SEARCH_CACHE_SIZE_SETTING = Setting.byteSizeSetting( + "node.search.cache.size", + ByteSizeValue.ZERO, + Property.NodeScope + ); + private static final String CLIENT_TYPE = "node"; /** @@ -354,9 +371,9 @@ public static class DiscoverySettings { private final Collection pluginLifecycleComponents; private final LocalNodeFactory localNodeFactory; private final NodeService nodeService; - final NamedWriteableRegistry namedWriteableRegistry; private final AtomicReference runnableTaskListener; + private FileCache fileCache; public Node(Environment environment) { this(environment, Collections.emptyList(), true); @@ -421,18 +438,18 @@ protected Node( if (logger.isDebugEnabled()) { logger.debug( "using config [{}], data [{}], logs [{}], plugins [{}]", - initialEnvironment.configFile(), + initialEnvironment.configDir(), Arrays.toString(initialEnvironment.dataFiles()), - initialEnvironment.logsFile(), - initialEnvironment.pluginsFile() + initialEnvironment.logsDir(), + initialEnvironment.pluginsDir() ); } this.pluginsService = new PluginsService( tmpSettings, - initialEnvironment.configFile(), - initialEnvironment.modulesFile(), - initialEnvironment.pluginsFile(), + initialEnvironment.configDir(), + initialEnvironment.modulesDir(), + initialEnvironment.pluginsDir(), classpathPlugins ); @@ -441,8 +458,17 @@ protected Node( // Ensure to initialize Feature Flags via the settings from opensearch.yml FeatureFlags.initializeFeatureFlags(settings); + final List identityPlugins = new ArrayList<>(); + if (FeatureFlags.isEnabled(FeatureFlags.IDENTITY)) { + // If identity is enabled load plugins implementing the extension point + logger.info("Identity on so found plugins implementing: " + pluginsService.filterPlugins(IdentityPlugin.class).toString()); + identityPlugins.addAll(pluginsService.filterPlugins(IdentityPlugin.class)); + } + + final IdentityService identityService = new IdentityService(settings, identityPlugins); + if (FeatureFlags.isEnabled(FeatureFlags.EXTENSIONS)) { - this.extensionsManager = new ExtensionsManager(tmpSettings, initialEnvironment.extensionDir()); + this.extensionsManager = new ExtensionsManager(initialEnvironment.extensionDir()); } else { this.extensionsManager = new NoopExtensionsManager(); } @@ -460,7 +486,7 @@ protected Node( * Create the environment based on the finalized view of the settings. This is to ensure that components get the same setting * values, no matter they ask for them from. */ - this.environment = new Environment(settings, initialEnvironment.configFile(), Node.NODE_LOCAL_STORAGE_SETTING.get(settings)); + this.environment = new Environment(settings, initialEnvironment.configDir(), Node.NODE_LOCAL_STORAGE_SETTING.get(settings)); Environment.assertEquivalent(initialEnvironment, this.environment); nodeEnvironment = new NodeEnvironment(tmpSettings, environment); logger.info( @@ -549,6 +575,7 @@ protected Node( pluginsService.filterPlugins(IngestPlugin.class), client ); + final SetOnce repositoriesServiceReference = new SetOnce<>(); final ClusterInfoService clusterInfoService = newClusterInfoService(settings, clusterService, threadPool, client); final UsageService usageService = new UsageService(); @@ -558,7 +585,6 @@ protected Node( for (Module pluginModule : pluginsService.createGuiceModules()) { modules.add(pluginModule); } - final MonitorService monitorService = new MonitorService(settings, nodeEnvironment, threadPool); final FsHealthService fsHealthService = new FsHealthService( settings, clusterService.getClusterSettings(), @@ -594,6 +620,11 @@ protected Node( pluginCircuitBreakers, settingsModule.getClusterSettings() ); + // File cache will be initialized by the node once circuit breakers are in place. + initializeFileCache(settings, circuitBreakerService.getBreaker(CircuitBreaker.REQUEST)); + final FileCacheCleaner fileCacheCleaner = new FileCacheCleaner(nodeEnvironment, fileCache); + final MonitorService monitorService = new MonitorService(settings, nodeEnvironment, threadPool, fileCache); + pluginsService.filterPlugins(CircuitBreakerPlugin.class).forEach(plugin -> { CircuitBreaker breaker = circuitBreakerService.getBreaker(plugin.getCircuitBreaker(settings).getName()); plugin.setCircuitBreaker(breaker); @@ -635,13 +666,13 @@ protected Node( final Collection>> engineFactoryProviders = enginePlugins.stream() .map(plugin -> (Function>) plugin::getEngineFactory) .collect(Collectors.toList()); - // TODO: for now this is a single cache, later, this should read node and index settings - final FileCache remoteStoreFileCache = createRemoteStoreFileCache(); + final Map builtInDirectoryFactories = IndexModule.createBuiltInDirectoryFactories( repositoriesServiceReference::get, threadPool, - remoteStoreFileCache + fileCache ); + final Map directoryFactories = new HashMap<>(); pluginsService.filterPlugins(IndexStorePlugin.class) .stream() @@ -658,8 +689,8 @@ protected Node( directoryFactories.putAll(builtInDirectoryFactories); final Map recoveryStateFactories = pluginsService.filterPlugins( - IndexStorePlugin.class - ) + IndexStorePlugin.class + ) .stream() .map(IndexStorePlugin::getRecoveryStateFactories) .flatMap(m -> m.entrySet().stream()) @@ -678,63 +709,36 @@ protected Node( rerouteServiceReference.set(rerouteService); clusterService.setRerouteService(rerouteService); - final IndexStorePlugin.RemoteDirectoryFactory remoteDirectoryFactory = new RemoteSegmentStoreDirectoryFactory( + final IndexStorePlugin.DirectoryFactory remoteDirectoryFactory = new RemoteSegmentStoreDirectoryFactory( repositoriesServiceReference::get ); - final IndicesService indicesService; - if (FeatureFlags.isEnabled(FeatureFlags.EXTENSIONS)) { - indicesService = new IndicesService( - settings, - pluginsService, - extensionsManager, - nodeEnvironment, - xContentRegistry, - analysisModule.getAnalysisRegistry(), - clusterModule.getIndexNameExpressionResolver(), - indicesModule.getMapperRegistry(), - namedWriteableRegistry, - threadPool, - settingsModule.getIndexScopedSettings(), - circuitBreakerService, - bigArrays, - scriptService, - clusterService, - client, - metaStateService, - engineFactoryProviders, - Map.copyOf(directoryFactories), - searchModule.getValuesSourceRegistry(), - recoveryStateFactories, - remoteDirectoryFactory, - repositoriesServiceReference::get - ); - } else { - indicesService = new IndicesService( - settings, - pluginsService, - nodeEnvironment, - xContentRegistry, - analysisModule.getAnalysisRegistry(), - clusterModule.getIndexNameExpressionResolver(), - indicesModule.getMapperRegistry(), - namedWriteableRegistry, - threadPool, - settingsModule.getIndexScopedSettings(), - circuitBreakerService, - bigArrays, - scriptService, - clusterService, - client, - metaStateService, - engineFactoryProviders, - Map.copyOf(directoryFactories), - searchModule.getValuesSourceRegistry(), - recoveryStateFactories, - remoteDirectoryFactory, - repositoriesServiceReference::get - ); - } + final IndicesService indicesService = new IndicesService( + settings, + pluginsService, + extensionsManager, + nodeEnvironment, + xContentRegistry, + analysisModule.getAnalysisRegistry(), + clusterModule.getIndexNameExpressionResolver(), + indicesModule.getMapperRegistry(), + namedWriteableRegistry, + threadPool, + settingsModule.getIndexScopedSettings(), + circuitBreakerService, + bigArrays, + scriptService, + clusterService, + client, + metaStateService, + engineFactoryProviders, + Map.copyOf(directoryFactories), + searchModule.getValuesSourceRegistry(), + recoveryStateFactories, + remoteDirectoryFactory, + repositoriesServiceReference::get, + fileCacheCleaner + ); final AliasValidator aliasValidator = new AliasValidator(); @@ -799,7 +803,8 @@ protected Node( client, circuitBreakerService, usageService, - systemIndices + systemIndices, + identityService ); modules.add(actionModule); @@ -848,16 +853,16 @@ protected Node( settingsModule.getClusterSettings(), taskHeaders ); - if (FeatureFlags.isEnabled(FeatureFlags.EXTENSIONS)) { - this.extensionsManager.initializeServicesAndRestHandler( - restController, - settingsModule, - transportService, - clusterService, - environment.settings(), - client - ); - } + TopNSearchTasksLogger taskConsumer = new TopNSearchTasksLogger(settings, settingsModule.getClusterSettings()); + transportService.getTaskManager().registerTaskResourceConsumer(taskConsumer); + this.extensionsManager.initializeServicesAndRestHandler( + actionModule, + settingsModule, + transportService, + clusterService, + environment.settings(), + client + ); final GatewayMetaState gatewayMetaState = new GatewayMetaState(); final ResponseCollectorService responseCollectorService = new ResponseCollectorService(clusterService); final SearchTransportService searchTransportService = new SearchTransportService( @@ -953,11 +958,23 @@ protected Node( clusterService.getClusterSettings(), pluginsService.filterPlugins(DiscoveryPlugin.class), clusterModule.getAllocationService(), - environment.configFile(), + environment.configDir(), gatewayMetaState, rerouteService, fsHealthService ); + final SearchPipelineService searchPipelineService = new SearchPipelineService( + clusterService, + threadPool, + this.environment, + scriptService, + analysisModule.getAnalysisRegistry(), + xContentRegistry, + namedWriteableRegistry, + pluginsService.filterPlugins(SearchPipelinePlugin.class), + client, + FeatureFlags.isEnabled(SEARCH_PIPELINE) + ); this.nodeService = new NodeService( settings, threadPool, @@ -976,7 +993,9 @@ protected Node( searchTransportService, indexingPressureService, searchModule.getValuesSourceRegistry().getUsageService(), - searchBackpressureService + searchBackpressureService, + searchPipelineService, + fileCache ); final SearchService searchService = newSearchService( @@ -1034,6 +1053,7 @@ protected Node( b.bind(ScriptService.class).toInstance(scriptService); b.bind(AnalysisRegistry.class).toInstance(analysisModule.getAnalysisRegistry()); b.bind(IngestService.class).toInstance(ingestService); + b.bind(SearchPipelineService.class).toInstance(searchPipelineService); b.bind(IndexingPressureService.class).toInstance(indexingPressureService); b.bind(TaskResourceTrackingService.class).toInstance(taskResourceTrackingService); b.bind(SearchBackpressureService.class).toInstance(searchBackpressureService); @@ -1067,23 +1087,18 @@ protected Node( .toInstance(new PeerRecoverySourceService(transportService, indicesService, recoverySettings)); b.bind(PeerRecoveryTargetService.class) .toInstance(new PeerRecoveryTargetService(threadPool, transportService, recoverySettings, clusterService)); - if (FeatureFlags.isEnabled(REPLICATION_TYPE)) { - b.bind(SegmentReplicationTargetService.class) - .toInstance( - new SegmentReplicationTargetService( - threadPool, - recoverySettings, - transportService, - new SegmentReplicationSourceFactory(transportService, recoverySettings, clusterService), - indicesService - ) - ); - b.bind(SegmentReplicationSourceService.class) - .toInstance(new SegmentReplicationSourceService(indicesService, transportService, recoverySettings)); - } else { - b.bind(SegmentReplicationTargetService.class).toInstance(SegmentReplicationTargetService.NO_OP); - b.bind(SegmentReplicationSourceService.class).toInstance(SegmentReplicationSourceService.NO_OP); - } + b.bind(SegmentReplicationTargetService.class) + .toInstance( + new SegmentReplicationTargetService( + threadPool, + recoverySettings, + transportService, + new SegmentReplicationSourceFactory(transportService, recoverySettings, clusterService), + indicesService + ) + ); + b.bind(SegmentReplicationSourceService.class) + .toInstance(new SegmentReplicationSourceService(indicesService, transportService, recoverySettings)); } b.bind(HttpServerTransport.class).toInstance(httpServerTransport); pluginComponents.stream().forEach(p -> b.bind((Class) p.getClass()).toInstance(p)); @@ -1099,6 +1114,7 @@ protected Node( b.bind(ShardLimitValidator.class).toInstance(shardLimitValidator); b.bind(FsHealthService.class).toInstance(fsHealthService); b.bind(SystemIndices.class).toInstance(systemIndices); + b.bind(IdentityService.class).toInstance(identityService); }); injector = modules.createInjector(); @@ -1119,8 +1135,15 @@ protected Node( resourcesToClose.addAll(pluginLifecycleComponents); resourcesToClose.add(injector.getInstance(PeerRecoverySourceService.class)); this.pluginLifecycleComponents = Collections.unmodifiableList(pluginLifecycleComponents); - client.initialize(injector.getInstance(new Key>() { - }), () -> clusterService.localNode().getId(), transportService.getRemoteClusterService(), namedWriteableRegistry); + DynamicActionRegistry dynamicActionRegistry = actionModule.getDynamicActionRegistry(); + dynamicActionRegistry.registerUnmodifiableActionMap(injector.getInstance(new Key>() { + })); + client.initialize( + dynamicActionRegistry, + () -> clusterService.localNode().getId(), + transportService.getRemoteClusterService(), + namedWriteableRegistry + ); this.namedWriteableRegistry = namedWriteableRegistry; logger.debug("initializing HTTP handlers ..."); @@ -1137,16 +1160,6 @@ protected Node( } } - private FileCache createRemoteStoreFileCache() { - // TODO: implement more custom logic to create named caches, using multiple node paths, more capacity computation options and - // capacity reservation logic - FsInfo.Path info = ExceptionsHelper.catchAsRuntimeException(() -> FsProbe.getFSInfo(nodeEnvironment.nodePaths()[0])); - long diskCapacity = info.getTotal().getBytes(); - // hard coded as 50% for now - long capacity = (long) (diskCapacity * 0.50); - return FileCacheFactory.createConcurrentLRUFileCache(capacity); - } - protected TransportService newTransportService( Settings settings, Transport transport, @@ -1237,9 +1250,7 @@ public Node start() throws NodeValidationException { assert transportService.getLocalNode().equals(localNodeFactory.getNode()) : "transportService has a different local node than the factory provided"; injector.getInstance(PeerRecoverySourceService.class).start(); - if (FeatureFlags.isEnabled(REPLICATION_TYPE)) { - injector.getInstance(SegmentReplicationSourceService.class).start(); - } + injector.getInstance(SegmentReplicationSourceService.class).start(); // Load (and maybe upgrade) the metadata stored on disk final GatewayMetaState gatewayMetaState = injector.getInstance(GatewayMetaState.class); @@ -1284,9 +1295,7 @@ public Node start() throws NodeValidationException { assert clusterService.localNode().equals(localNodeFactory.getNode()) : "clusterService has a different local node than the factory provided"; transportService.acceptIncomingRequests(); - if (FeatureFlags.isEnabled(FeatureFlags.EXTENSIONS)) { - extensionsManager.initialize(); - } + extensionsManager.initialize(); discovery.startInitialJoin(); final TimeValue initialStateTimeout = DiscoverySettings.INITIAL_STATE_TIMEOUT_SETTING.get(settings()); configureNodeAndClusterIdStateListener(clusterService); @@ -1419,9 +1428,7 @@ public synchronized void close() throws IOException { // close filter/fielddata caches after indices toClose.add(injector.getInstance(IndicesStore.class)); toClose.add(injector.getInstance(PeerRecoverySourceService.class)); - if (FeatureFlags.isEnabled(REPLICATION_TYPE)) { - toClose.add(injector.getInstance(SegmentReplicationSourceService.class)); - } + toClose.add(injector.getInstance(SegmentReplicationSourceService.class)); toClose.add(() -> stopWatch.stop().start("cluster")); toClose.add(injector.getInstance(ClusterService.class)); toClose.add(() -> stopWatch.stop().start("node_connections_service")); @@ -1525,8 +1532,8 @@ protected void validateNodeBeforeAcceptingRequests( /** Writes a file to the logs dir containing the ports for the given transport type */ private void writePortsFile(String type, BoundTransportAddress boundAddress) { - Path tmpPortsFile = environment.logsFile().resolve(type + ".ports.tmp"); - try (BufferedWriter writer = Files.newBufferedWriter(tmpPortsFile, Charset.forName("UTF-8"))) { + Path tmpPortsFile = environment.logsDir().resolve(type + ".ports.tmp"); + try (BufferedWriter writer = Files.newBufferedWriter(tmpPortsFile, StandardCharsets.UTF_8)) { for (TransportAddress address : boundAddress.boundAddresses()) { InetAddress inetAddress = InetAddress.getByName(address.getAddress()); writer.write(NetworkAddress.format(new InetSocketAddress(inetAddress, address.getPort())) + "\n"); @@ -1534,7 +1541,7 @@ private void writePortsFile(String type, BoundTransportAddress boundAddress) { } catch (IOException e) { throw new RuntimeException("Failed to write ports file", e); } - Path portsFile = environment.logsFile().resolve(type + ".ports"); + Path portsFile = environment.logsDir().resolve(type + ".ports"); try { Files.move(tmpPortsFile, portsFile, StandardCopyOption.ATOMIC_MOVE); } catch (IOException e) { @@ -1676,4 +1683,49 @@ DiscoveryNode getNode() { return localNode.get(); } } + + /** + * Initializes the search cache with a defined capacity. + * The capacity of the cache is based on user configuration for {@link Node#NODE_SEARCH_CACHE_SIZE_SETTING}. + * If the user doesn't configure the cache size, it fails if the node is a data + search node. + * Else it configures the size to 80% of available capacity for a dedicated search node, if not explicitly defined. + */ + private void initializeFileCache(Settings settings, CircuitBreaker circuitBreaker) throws IOException { + if (DiscoveryNode.isSearchNode(settings)) { + NodeEnvironment.NodePath fileCacheNodePath = nodeEnvironment.fileCacheNodePath(); + long capacity = NODE_SEARCH_CACHE_SIZE_SETTING.get(settings).getBytes(); + FsInfo.Path info = ExceptionsHelper.catchAsRuntimeException(() -> FsProbe.getFSInfo(fileCacheNodePath)); + long availableCapacity = info.getAvailable().getBytes(); + + // Initialize default values for cache if NODE_SEARCH_CACHE_SIZE_SETTING is not set. + if (capacity == 0) { + // If node is not a dedicated search node without configuration, prevent cache initialization + if (DiscoveryNode.getRolesFromSettings(settings).stream().anyMatch(role -> !DiscoveryNodeRole.SEARCH_ROLE.equals(role))) { + throw new SettingsException( + "Unable to initialize the " + + DiscoveryNodeRole.SEARCH_ROLE.roleName() + + "-" + + DiscoveryNodeRole.DATA_ROLE.roleName() + + " node: Missing value for configuration " + + NODE_SEARCH_CACHE_SIZE_SETTING.getKey() + ); + } else { + capacity = 80 * availableCapacity / 100; + } + } + capacity = Math.min(capacity, availableCapacity); + fileCacheNodePath.fileCacheReservedSize = new ByteSizeValue(capacity, ByteSizeUnit.BYTES); + this.fileCache = FileCacheFactory.createConcurrentLRUFileCache(capacity, circuitBreaker); + List fileCacheDataPaths = collectFileCacheDataPath(fileCacheNodePath); + this.fileCache.restoreFromDirectory(fileCacheDataPaths); + } + } + + /** + * Returns the {@link FileCache} instance for remote search node + * Note: Visible for testing + */ + public FileCache fileCache() { + return this.fileCache; + } } diff --git a/server/src/main/java/org/opensearch/performanceanalyzer/listener/DiskStatsTaskEventListener.java b/server/src/main/java/org/opensearch/performanceanalyzer/listener/DiskStatsTaskEventListener.java new file mode 100644 index 0000000000000..836e8d2e55ca6 --- /dev/null +++ b/server/src/main/java/org/opensearch/performanceanalyzer/listener/DiskStatsTaskEventListener.java @@ -0,0 +1,190 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.performanceanalyzer.listener; + +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.common.AttributesBuilder; +import io.opentelemetry.api.trace.Span; +import org.opensearch.performanceanalyzer.commons.OSMetricsGeneratorFactory; +import org.opensearch.performanceanalyzer.commons.collectors.OSMetricsCollector; +import org.opensearch.performanceanalyzer.commons.jvm.ThreadList; +import org.opensearch.performanceanalyzer.commons.metrics.AllMetrics; +import org.opensearch.performanceanalyzer.commons.metrics.ThreadIDUtil; +import org.opensearch.performanceanalyzer.commons.metrics_generator.CPUPagingActivityGenerator; +import org.opensearch.performanceanalyzer.commons.metrics_generator.SchedMetricsGenerator; +import org.opensearch.performanceanalyzer.commons.metrics_generator.linux.LinuxDiskIOMetricsGenerator; +import org.opensearch.performanceanalyzer.commons.os.ThreadDiskIO; +import org.opensearch.tracing.TaskEventListener; + +public class DiskStatsTaskEventListener implements TaskEventListener { + private final boolean threadContentionEnabled; + + public DiskStatsTaskEventListener() { + threadContentionEnabled = true; + } + + @Override + public void onStart(String operationName, String eventName, Thread t) { + emitOtherMetrics("Start-PA-" + t.getName() + eventName, t.getId(), false); + } + + @Override + public void onEnd(String operationName, String eventName, Thread t) { + emitOtherMetrics("End-PA-" + t.getName(), t.getId(), true); + } + + private synchronized void emitOtherMetrics(String eventName, long jTid, boolean emit) { + ThreadList.ThreadState threadState; + long nativeThreadID = ThreadIDUtil.INSTANCE.getNativeThreadId(jTid); + if (nativeThreadID == -1) { + // TODO - make it async + ThreadList.getNativeTidMap(threadContentionEnabled); + nativeThreadID = ThreadIDUtil.INSTANCE.getNativeThreadId(jTid); + + if (nativeThreadID == -1) { + return; + } + threadState = ThreadList.getThreadState(jTid); + } else { + threadState = ThreadList.getThreadState(jTid); + } + CPUPagingActivityGenerator threadCPUPagingActivityGenerator = + OSMetricsGeneratorFactory.getInstance().getPagingActivityGenerator(); + threadCPUPagingActivityGenerator.addSample(String.valueOf(nativeThreadID)); + + SchedMetricsGenerator schedMetricsGenerator = + OSMetricsGeneratorFactory.getInstance().getSchedMetricsGenerator(); + schedMetricsGenerator.addSample(String.valueOf(nativeThreadID)); + + ThreadDiskIO.addSample(String.valueOf(nativeThreadID)); + AttributesBuilder attrBuilder = Attributes.builder(); + + attrBuilder.put("eventName", eventName); + attrBuilder.put("nativeTid", nativeThreadID); + + if (emit) { + LinuxDiskIOMetricsGenerator diskIOMetricsGenerator = ThreadDiskIO.getIOUtilization(); + + if (diskIOMetricsGenerator.hasDiskIOMetrics(String.valueOf(nativeThreadID))) { + attrBuilder.put( + AttributeKey.doubleKey("ReadThroughputBps"), + diskIOMetricsGenerator.getAvgReadThroughputBps( + String.valueOf(nativeThreadID))); + attrBuilder.put( + AttributeKey.doubleKey("WriteThroughputBps"), + diskIOMetricsGenerator.getAvgWriteThroughputBps( + String.valueOf(nativeThreadID))); + attrBuilder.put( + AttributeKey.doubleKey("TotalThroughputBps"), + diskIOMetricsGenerator.getAvgTotalThroughputBps( + String.valueOf(nativeThreadID))); + attrBuilder.put( + AttributeKey.doubleKey("ReadSyscallRate"), + diskIOMetricsGenerator.getAvgReadSyscallRate( + String.valueOf(nativeThreadID))); + attrBuilder.put( + AttributeKey.doubleKey("WriteSyscallRate"), + diskIOMetricsGenerator.getAvgWriteSyscallRate( + String.valueOf(nativeThreadID))); + attrBuilder.put( + AttributeKey.doubleKey("TotalSyscallRate"), + diskIOMetricsGenerator.getAvgTotalSyscallRate( + String.valueOf(nativeThreadID))); + +// attrBuilder.put( +// AttributeKey.doubleKey("PageCacheReadThroughputBps"), +// diskIOMetricsGenerator.getAvgPageCacheReadThroughputBps( +// String.valueOf(nativeThreadID))); +// attrBuilder.put( +// AttributeKey.doubleKey("PageCacheWriteThroughputBps"), +// diskIOMetricsGenerator.getAvgPageCacheWriteThroughputBps( +// String.valueOf(nativeThreadID))); +// attrBuilder.put( +// AttributeKey.doubleKey("PageCacheTotalThroughputBps"), +// diskIOMetricsGenerator.getAvgPageCacheTotalThroughputBps( +// String.valueOf(nativeThreadID))); + } + + if (threadCPUPagingActivityGenerator.hasPagingActivity( + String.valueOf(nativeThreadID))) { + attrBuilder.put( + AttributeKey.doubleKey(AllMetrics.OSMetrics.CPU_UTILIZATION.toString()), + threadCPUPagingActivityGenerator.getCPUUtilization( + String.valueOf(nativeThreadID))); + + attrBuilder.put( + AttributeKey.doubleKey(AllMetrics.OSMetrics.PAGING_MAJ_FLT_RATE.toString()), + threadCPUPagingActivityGenerator.getMajorFault( + String.valueOf(nativeThreadID))); + attrBuilder.put( + AttributeKey.doubleKey(AllMetrics.OSMetrics.PAGING_MIN_FLT_RATE.toString()), + threadCPUPagingActivityGenerator.getMinorFault( + String.valueOf(nativeThreadID))); + attrBuilder.put( + AttributeKey.doubleKey(AllMetrics.OSMetrics.PAGING_RSS.toString()), + threadCPUPagingActivityGenerator.getResidentSetSize( + String.valueOf(nativeThreadID))); + } + if (schedMetricsGenerator.hasSchedMetrics(String.valueOf(nativeThreadID))) { + attrBuilder.put( + AttributeKey.doubleKey(AllMetrics.OSMetrics.SCHED_RUNTIME.toString()), + schedMetricsGenerator.getAvgRuntime(String.valueOf(nativeThreadID))); + attrBuilder.put( + AttributeKey.doubleKey(AllMetrics.OSMetrics.SCHED_WAITTIME.toString()), + schedMetricsGenerator.getAvgWaittime(String.valueOf(nativeThreadID))); + attrBuilder.put( + AttributeKey.doubleKey(AllMetrics.OSMetrics.SCHED_CTX_RATE.toString()), + schedMetricsGenerator.getContextSwitchRate(String.valueOf(nativeThreadID))); + } + if (threadState != null) { + attrBuilder.put( + AttributeKey.stringKey( + OSMetricsCollector.MetaDataFields.threadName.toString()), + threadState.threadName); + attrBuilder.put( + AttributeKey.doubleKey(AllMetrics.OSMetrics.HEAP_ALLOC_RATE.name()), + threadState.heapAllocRate); + + attrBuilder.put( + AttributeKey.longKey(AllMetrics.OSMetrics.THREAD_BLOCKED_EVENT.toString()), + threadState.blockedCount); + attrBuilder.put( + AttributeKey.longKey(AllMetrics.OSMetrics.THREAD_WAITED_TIME.toString()), + threadState.blockedTime); + attrBuilder.put( + AttributeKey.longKey(AllMetrics.OSMetrics.THREAD_WAITED_EVENT.toString()), + threadState.waitedCount); + attrBuilder.put( + AttributeKey.longKey(AllMetrics.OSMetrics.THREAD_WAITED_TIME.toString()), + threadState.waitedTime); + } + } + Attributes attr = attrBuilder.build(); + System.out.println(Span.current()); + attr.forEach( + (k, v) -> { + if (v instanceof Double) { + System.out.print(k + ":" + v + ", "); + } else if (v instanceof Long) { + System.out.print("Long=" + k + ":" + v + ", "); + + } else { + System.out.print("ND=" + k + ":" + v + ", "); + } + }); + System.out.println("\nDone"); + Span.current().addEvent(eventName, attrBuilder.build()); + } + + @Override + public boolean isApplicable(String s, String s1) { + return true; + } +} diff --git a/server/src/main/java/org/opensearch/threadpool/ThreadPool.java b/server/src/main/java/org/opensearch/threadpool/ThreadPool.java index 38ada9d63214d..90e0b015404bf 100644 --- a/server/src/main/java/org/opensearch/threadpool/ThreadPool.java +++ b/server/src/main/java/org/opensearch/threadpool/ThreadPool.java @@ -48,7 +48,6 @@ import org.opensearch.common.util.concurrent.OpenSearchExecutors; import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; import org.opensearch.common.util.concurrent.OpenSearchThreadPoolExecutor; -import org.opensearch.common.util.concurrent.OpenSearchRejectedExecutionException; import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.common.util.concurrent.XRejectedExecutionHandler; import org.opensearch.core.xcontent.ToXContentFragment; @@ -226,6 +225,7 @@ public ThreadPool( final ExecutorBuilder... customBuilders ) { assert Node.NODE_NAME_SETTING.exists(settings); + final Map builders = new HashMap<>(); final int allocatedProcessors = OpenSearchExecutors.allocatedProcessors(settings); final int halfProcMaxAt5 = halfAllocatedProcessorsMaxFive(allocatedProcessors); @@ -421,8 +421,6 @@ public ExecutorService executor(String name) { return holder.executor(); } - - /** * Schedules a one-shot command to run after a given delay. The command is run in the context of the calling thread. * @@ -700,7 +698,7 @@ static class ExecutorHolder { public final Info info; ExecutorHolder(ExecutorService executor, Info info) { -// assert executor instanceof OpenSearchThreadPoolExecutor || executor == DIRECT_EXECUTOR; + assert executor instanceof OpenSearchThreadPoolExecutor || executor == DIRECT_EXECUTOR; this.executor = executor; this.info = info; } @@ -902,9 +900,9 @@ public static boolean assertCurrentMethodIsNotCalledRecursively() { for (int i = 3; i < stackTraceElements.length; i++) { assert stackTraceElements[i].getClassName().equals(testingMethod.getClassName()) == false || stackTraceElements[i].getMethodName().equals(testingMethod.getMethodName()) == false : testingMethod.getClassName() - + "#" - + testingMethod.getMethodName() - + " is called recursively"; + + "#" + + testingMethod.getMethodName() + + " is called recursively"; } return true; } diff --git a/server/src/main/java/org/opensearch/tracing/opentelemetry/OpenTelemetryService.java b/server/src/main/java/org/opensearch/tracing/opentelemetry/OpenTelemetryService.java index 61f971aea7025..19a91a4415a19 100644 --- a/server/src/main/java/org/opensearch/tracing/opentelemetry/OpenTelemetryService.java +++ b/server/src/main/java/org/opensearch/tracing/opentelemetry/OpenTelemetryService.java @@ -28,6 +28,7 @@ import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor; import io.opentelemetry.semconv.resource.attributes.ResourceAttributes; import org.opensearch.action.ActionListener; +import org.opensearch.performanceanalyzer.listener.DiskStatsTaskEventListener; import org.opensearch.threadpool.ThreadPool; import org.opensearch.tracing.TaskEventListener; @@ -67,8 +68,7 @@ public class OpenTelemetryService { .setMeterProvider(sdkMeterProvider) .setPropagators(ContextPropagators.create(W3CTraceContextPropagator.getInstance())) .buildAndRegisterGlobal(); - - DEFAULT_TASK_EVENT_LISTENERS = List.of(new TaskEventListener() { + var defaultListener = new TaskEventListener() { @Override public void onStart(String operationName, String eventName, Thread t) { Span span = Span.current(); @@ -105,7 +105,9 @@ public void onEnd(String operationName, String eventName, Thread t) { public boolean isApplicable(String operationName, String eventName) { return true; } - }); + }; + + DEFAULT_TASK_EVENT_LISTENERS = List.of(defaultListener, new DiskStatsTaskEventListener()); } public static boolean isThreadPoolAllowed(String threadPoolName) { From bbb276c21372b3a625f1e4d1e76cdd3a6df4a83f Mon Sep 17 00:00:00 2001 From: Stevan Buzejic Date: Wed, 24 May 2023 14:44:31 +0200 Subject: [PATCH 7/8] Added DistStatsEventListener Signed-off-by: Stevan Buzejic --- build.gradle | 1 + gradle/run.gradle | 4 ++- server/build.gradle | 10 ++++++- .../listener/DiskStatsTaskEventListener.java | 29 ++++++++++--------- .../org/opensearch/threadpool/ThreadPool.java | 2 +- .../opentelemetry/OpenTelemetryService.java | 2 +- .../org/opensearch/bootstrap/security.policy | 18 ++++++++++++ 7 files changed, 48 insertions(+), 18 deletions(-) diff --git a/build.gradle b/build.gradle index 220f6f3daf5ac..b1281e7ca16b2 100644 --- a/build.gradle +++ b/build.gradle @@ -112,6 +112,7 @@ subprojects { } } repositories { + mavenLocal() maven { name = 'test' url = "${rootProject.buildDir}/local-test-repo" diff --git a/gradle/run.gradle b/gradle/run.gradle index 6a55d705cad9c..5b51f5801a795 100644 --- a/gradle/run.gradle +++ b/gradle/run.gradle @@ -31,12 +31,14 @@ import org.opensearch.gradle.testclusters.RunTask apply plugin: 'opensearch.testclusters' -def numNodes = findProperty('numNodes') as Integer ?: 1 +def numNodes = 2 //findProperty('numNodes') as Integer ?: 1 def numZones = findProperty('numZones') as Integer ?: 1 testClusters { runTask { testDistribution = 'archive' + if (numZones > 1) numberOfZones = numZones + if (numNodes > 1) numberOfNodes = numNodes } } diff --git a/server/build.gradle b/server/build.gradle index 83936ce003946..9964419a9d4bb 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -38,6 +38,8 @@ plugins { id('opensearch.internal-cluster-test') } +apply plugin: 'idea' + publishing { publications { nebula(MavenPublication) { @@ -173,7 +175,13 @@ dependencies { // protobuf api "com.google.protobuf:protobuf-java:${versions.protobuf}" - implementation files('/home/stevan/git/opensearch/repo/performance-analyzer-commons/build/libs/opensearch-performance-analyzer-commons-3.0.0.0-SNAPSHOT.jar') + // api files('/home/stevan/git/opensearch/repo/performance-analyzer-commons/build/libs/opensearch-performance-analyzer-commons-3.0.0.0-SNAPSHOT.jar') + api ("org.opensearch:performance-analyzer-commons:3.0.0.0-SNAPSHOT") { + exclude (group: 'com.fasterxml.jackson.core', module: 'jackson-core') + exclude (group: 'com.fasterxml.jackson.core', module: 'jackson-annotations') + exclude (group: 'com.fasterxml.jackson', module: 'jackson-bom') + exclude (group: 'com.fasterxml.jackson.core', module: 'jackson-databind') + } testImplementation(project(":test:framework")) { // tests use the locally compiled version of server diff --git a/server/src/main/java/org/opensearch/performanceanalyzer/listener/DiskStatsTaskEventListener.java b/server/src/main/java/org/opensearch/performanceanalyzer/listener/DiskStatsTaskEventListener.java index 836e8d2e55ca6..071b5d2732653 100644 --- a/server/src/main/java/org/opensearch/performanceanalyzer/listener/DiskStatsTaskEventListener.java +++ b/server/src/main/java/org/opensearch/performanceanalyzer/listener/DiskStatsTaskEventListener.java @@ -98,18 +98,18 @@ private synchronized void emitOtherMetrics(String eventName, long jTid, boolean diskIOMetricsGenerator.getAvgTotalSyscallRate( String.valueOf(nativeThreadID))); -// attrBuilder.put( -// AttributeKey.doubleKey("PageCacheReadThroughputBps"), -// diskIOMetricsGenerator.getAvgPageCacheReadThroughputBps( -// String.valueOf(nativeThreadID))); -// attrBuilder.put( -// AttributeKey.doubleKey("PageCacheWriteThroughputBps"), -// diskIOMetricsGenerator.getAvgPageCacheWriteThroughputBps( -// String.valueOf(nativeThreadID))); -// attrBuilder.put( -// AttributeKey.doubleKey("PageCacheTotalThroughputBps"), -// diskIOMetricsGenerator.getAvgPageCacheTotalThroughputBps( -// String.valueOf(nativeThreadID))); + attrBuilder.put( + AttributeKey.doubleKey("PageCacheReadThroughputBps"), + diskIOMetricsGenerator.getAvgPageCacheReadThroughputBps( + String.valueOf(nativeThreadID))); + attrBuilder.put( + AttributeKey.doubleKey("PageCacheWriteThroughputBps"), + diskIOMetricsGenerator.getAvgPageCacheWriteThroughputBps( + String.valueOf(nativeThreadID))); + attrBuilder.put( + AttributeKey.doubleKey("PageCacheTotalThroughputBps"), + diskIOMetricsGenerator.getAvgPageCacheTotalThroughputBps( + String.valueOf(nativeThreadID))); } if (threadCPUPagingActivityGenerator.hasPagingActivity( @@ -167,7 +167,8 @@ private synchronized void emitOtherMetrics(String eventName, long jTid, boolean } } Attributes attr = attrBuilder.build(); - System.out.println(Span.current()); + + /**System.out.println(Span.current()); attr.forEach( (k, v) -> { if (v instanceof Double) { @@ -179,7 +180,7 @@ private synchronized void emitOtherMetrics(String eventName, long jTid, boolean System.out.print("ND=" + k + ":" + v + ", "); } }); - System.out.println("\nDone"); + System.out.println("\nDone"); **/ Span.current().addEvent(eventName, attrBuilder.build()); } diff --git a/server/src/main/java/org/opensearch/threadpool/ThreadPool.java b/server/src/main/java/org/opensearch/threadpool/ThreadPool.java index 90e0b015404bf..46788912e5571 100644 --- a/server/src/main/java/org/opensearch/threadpool/ThreadPool.java +++ b/server/src/main/java/org/opensearch/threadpool/ThreadPool.java @@ -698,7 +698,7 @@ static class ExecutorHolder { public final Info info; ExecutorHolder(ExecutorService executor, Info info) { - assert executor instanceof OpenSearchThreadPoolExecutor || executor == DIRECT_EXECUTOR; +// assert executor instanceof OpenSearchThreadPoolExecutor || executor == DIRECT_EXECUTOR; this.executor = executor; this.info = info; } diff --git a/server/src/main/java/org/opensearch/tracing/opentelemetry/OpenTelemetryService.java b/server/src/main/java/org/opensearch/tracing/opentelemetry/OpenTelemetryService.java index 19a91a4415a19..da57fdb881a12 100644 --- a/server/src/main/java/org/opensearch/tracing/opentelemetry/OpenTelemetryService.java +++ b/server/src/main/java/org/opensearch/tracing/opentelemetry/OpenTelemetryService.java @@ -173,7 +173,7 @@ protected static void callTaskEventListeners(boolean startEvent, String operatio if (startEvent) { eventListener.onStart(operationName, eventName, t); } else { - eventListener.onStart(operationName, eventName, t); + eventListener.onEnd(operationName, eventName, t); } } } diff --git a/server/src/main/resources/org/opensearch/bootstrap/security.policy b/server/src/main/resources/org/opensearch/bootstrap/security.policy index 0b96ebfe49529..8c7b4dea17d4f 100644 --- a/server/src/main/resources/org/opensearch/bootstrap/security.policy +++ b/server/src/main/resources/org/opensearch/bootstrap/security.policy @@ -154,6 +154,7 @@ grant { // needed by jvminfo for monitoring the jvm permission java.lang.management.ManagementPermission "monitor"; + permission java.lang.management.ManagementPermission "control"; // needed by JDKESLoggerTests permission java.util.logging.LoggingPermission "control"; @@ -188,4 +189,21 @@ grant { permission java.net.SocketPermission "127.0.0.1:4317", "connect,resolve"; permission java.net.SocketPermission "[0:0:0:0:0:0:0:1]:4317", "connect,resolve"; permission java.net.SocketPermission "[0:0:0:0:0:0:0:1]:4318", "connect,resolve"; + + permission java.lang.reflect.ReflectPermission "suppressAccessChecks"; + permission java.lang.RuntimePermission "accessDeclaredMembers"; + //- Java 8 jdk.attach fucntionality needs write permissions for /proc/pid/cwd, which varies, + permission java.io.FilePermission "/-","read,readlink,write,delete,execute"; + //permission java.io.FilePermission "/dev/shm/-","read,readlink,write,delete,execute"; + //permission java.io.FilePermission "/proc/-","read,readlink,execute,write,delete"; + //permission java.io.FilePermission "/sys/block/-","read,readlink,execute,write,delete"; + permission java.io.FilePermission "build/tmp/junit_metrics", "read"; + permission com.sun.tools.attach.AttachPermission "attachVirtualMachine"; + permission com.sun.tools.attach.AttachPermission "createAttachProvider"; + permission java.lang.RuntimePermission "manageProcess"; + permission java.lang.RuntimePermission "loadLibrary.attach"; + permission java.lang.RuntimePermission "accessClassInPackage.jdk.internal.misc"; + permission java.lang.RuntimePermission "accessClassInPackage.sun.tools.attach"; + permission java.lang.RuntimePermission "createClassLoader"; + permission java.lang.RuntimePermission "defineClass"; }; From f67fddd94437079e032ada0dae943cf4f0121f22 Mon Sep 17 00:00:00 2001 From: Stevan Buzejic Date: Fri, 26 May 2023 10:54:33 +0200 Subject: [PATCH 8/8] Updated perf analyzer commons dependency Signed-off-by: Stevan Buzejic --- server/build.gradle | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/build.gradle b/server/build.gradle index 9964419a9d4bb..67d2ffb224390 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -175,8 +175,8 @@ dependencies { // protobuf api "com.google.protobuf:protobuf-java:${versions.protobuf}" - // api files('/home/stevan/git/opensearch/repo/performance-analyzer-commons/build/libs/opensearch-performance-analyzer-commons-3.0.0.0-SNAPSHOT.jar') - api ("org.opensearch:performance-analyzer-commons:3.0.0.0-SNAPSHOT") { + // api files('/home/stevan/.m2/repository/org/opensearch/performance-analyzer-commons/1.0.0-SNAPSHOT/performance-analyzer-commons-1.0.0-SNAPSHOT.jar') + api ("org.opensearch:performance-analyzer-commons:1.0.0-SNAPSHOT") { exclude (group: 'com.fasterxml.jackson.core', module: 'jackson-core') exclude (group: 'com.fasterxml.jackson.core', module: 'jackson-annotations') exclude (group: 'com.fasterxml.jackson', module: 'jackson-bom')