Main otel commit latest search

build.gradle

@@ -627,6 +627,12 @@ subprojects {
   }
 }
 
+
+testClusters.all {
+  numberOfNodes = 3
+}
+
+
 reporting {
   reports {
     testAggregateTestReport(AggregateTestReport) {

server/src/main/java/org/opensearch/common/settings/ClusterSettings.java

@@ -50,6 +50,7 @@
 import org.opensearch.tasks.TaskManager;
 import org.opensearch.tasks.TaskResourceTrackingService;
 import org.opensearch.tasks.consumer.TopNSearchTasksLogger;
+import org.opensearch.tracing.Tracer;
 import org.opensearch.watcher.ResourceWatcherService;
 import org.opensearch.action.admin.cluster.configuration.TransportAddVotingConfigExclusionsAction;
 import org.opensearch.action.admin.indices.close.TransportCloseIndexAction;
@@ -638,7 +639,10 @@ public void apply(Settings value, Settings current, Settings previous) {
                 SegmentReplicationPressureService.MAX_ALLOWED_STALE_SHARDS,
 
                 // Settings related to Searchable Snapshots
-                Node.NODE_SEARCH_CACHE_SIZE_SETTING
+                Node.NODE_SEARCH_CACHE_SIZE_SETTING,
+
+                // Settings related to Tracing framework
+                Tracer.TRACER_LEVEL_SETTING
             )
         )
     );

server/src/main/java/org/opensearch/common/util/concurrent/ThreadContext.java

@@ -44,8 +44,8 @@
 import org.opensearch.common.settings.Setting.Property;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.http.HttpTransportSettings;
-import org.opensearch.tracer.SpanHolder;
-import org.opensearch.tracer.TracerUtils;
+import org.opensearch.tracing.SpanHolder;
+import org.opensearch.tracing.TracerUtils;
 import org.opensearch.tasks.Task;
 
 import java.io.IOException;
@@ -60,7 +60,7 @@
 
 import static org.opensearch.http.HttpTransportSettings.SETTING_HTTP_MAX_WARNING_HEADER_COUNT;
 import static org.opensearch.http.HttpTransportSettings.SETTING_HTTP_MAX_WARNING_HEADER_SIZE;
-import static org.opensearch.tracer.DefaultTracer.T_CURRENT_SPAN_KEY;
+import static org.opensearch.tracing.DefaultTracer.CURRENT_SPAN;
 import static org.opensearch.tasks.TaskResourceTrackingService.TASK_ID;
 
 /**
@@ -142,8 +142,8 @@ public StoredContext stashContext() {
             threadContextStruct = threadContextStruct.putTransient(TASK_ID, context.transientHeaders.get(TASK_ID));
         }
 
-        if (context.transientHeaders.containsKey(T_CURRENT_SPAN_KEY)) {
-            threadContextStruct = threadContextStruct.putTransient(T_CURRENT_SPAN_KEY, new SpanHolder((SpanHolder) context.transientHeaders.get(T_CURRENT_SPAN_KEY)));
+        if (context.transientHeaders.containsKey(CURRENT_SPAN)) {
+            threadContextStruct = threadContextStruct.putTransient(CURRENT_SPAN, new SpanHolder((SpanHolder) context.transientHeaders.get(CURRENT_SPAN)));
         }
 
         threadLocal.set(threadContextStruct);
@@ -242,8 +242,8 @@ public StoredContext newStoredContext(boolean preserveResponseHeaders, Collectio
         // this is the context when this method returns
         final ThreadContextStruct newContext = threadLocal.get();
 
-        if (newContext.transientHeaders.containsKey(T_CURRENT_SPAN_KEY)) {
-            newContext.transientHeaders.put(T_CURRENT_SPAN_KEY, new SpanHolder((SpanHolder) newContext.transientHeaders.get(T_CURRENT_SPAN_KEY)));
+        if (newContext.transientHeaders.containsKey(CURRENT_SPAN)) {
+            newContext.transientHeaders.put(CURRENT_SPAN, new SpanHolder((SpanHolder) newContext.transientHeaders.get(CURRENT_SPAN)));
         }
 
         return () -> {
@@ -710,7 +710,7 @@ private ThreadContextStruct copyHeaders(Iterable<Map.Entry<String, String>> head
         }
 
         private void writeTo(StreamOutput out, Map<String, String> defaultHeaders) throws IOException {
-            TracerUtils.addTracerContextInHeader(this.requestHeaders, this.transientHeaders);
+            TracerUtils.addTracerContextToHeader(this.requestHeaders, this.transientHeaders);
 
             final Map<String, String> requestHeaders;
             if (defaultHeaders.isEmpty()) {

server/src/main/java/org/opensearch/http/DefaultRestChannel.java

@@ -43,6 +43,7 @@
 import org.opensearch.common.network.CloseableChannel;
 import org.opensearch.common.util.BigArrays;
 import org.opensearch.common.util.concurrent.ThreadContext;
+import org.opensearch.tracing.TracerFactory;
 import org.opensearch.rest.AbstractRestChannel;
 import org.opensearch.rest.RestChannel;
 import org.opensearch.rest.RestRequest;
@@ -164,6 +165,11 @@ public void sendResponse(RestResponse restResponse) {
             httpChannel.sendResponse(httpResponse, listener);
             success = true;
         } finally {
+            if (request.uri().startsWith("/_search")) {
+
+                System.out.println("ending base request:" + request.getRequestId());
+                TracerFactory.getInstance().endSpan();
+            }
             if (success == false) {
                 Releasables.close(toClose);
             }

server/src/main/java/org/opensearch/index/shard/SearchOperationListener.java

@@ -153,6 +153,9 @@ final class CompositeListener implements SearchOperationListener {
 
         CompositeListener(List<SearchOperationListener> listeners, Logger logger) {
             this.listeners = listeners;
+            if (listeners != null) {
+                listeners.add(new SearchOperationListenerTracerImpl());
+            }
             this.logger = logger;
         }
 

server/src/main/java/org/opensearch/index/shard/SearchOperationListenerTracerImpl.java

@@ -0,0 +1,41 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.shard;
+
+import org.opensearch.tracing.Tracer.Level;
+import org.opensearch.tracing.TracerFactory;
+import org.opensearch.search.internal.SearchContext;
+
+public class SearchOperationListenerTracerImpl implements SearchOperationListener {
+
+    @Override
+    public void onPreQueryPhase(SearchContext searchContext) {
+        SearchOperationListener.super.onPreQueryPhase(searchContext);
+         TracerFactory.getInstance().startSpan("onQueryPhase", null, Level.INFO);
+    }
+
+    @Override
+    public void onQueryPhase(SearchContext searchContext, long tookInNanos) {
+        SearchOperationListener.super.onQueryPhase(searchContext, tookInNanos);
+        TracerFactory.getInstance().endSpan();
+    }
+
+    @Override
+    public void onPreFetchPhase(SearchContext searchContext) {
+        SearchOperationListener.super.onPreFetchPhase(searchContext);
+        TracerFactory.getInstance().startSpan("onFetchPhase", null, Level.INFO);
+    }
+
+    @Override
+    public void onFetchPhase(SearchContext searchContext, long tookInNanos) {
+        SearchOperationListener.super.onFetchPhase(searchContext, tookInNanos);
+        TracerFactory.getInstance().endSpan();
+
+    }
+}

server/src/main/java/org/opensearch/node/Node.java

@@ -55,7 +55,7 @@
 import org.opensearch.monitor.fs.FsInfo;
 import org.opensearch.monitor.fs.FsProbe;
 import org.opensearch.plugins.SearchPipelinePlugin;
-import org.opensearch.tracer.TracerFactory;
+import org.opensearch.tracing.TracerFactory;
 import org.opensearch.search.backpressure.SearchBackpressureService;
 import org.opensearch.search.backpressure.settings.SearchBackpressureSettings;
 import org.opensearch.search.pipeline.SearchPipelineService;
@@ -1022,7 +1022,7 @@ protected Node(
                 searchModule.getIndexSearcherExecutor(threadPool)
             );
 
-            TracerFactory.initializeTracer(threadPool);
+            TracerFactory.initializeTracer(threadPool, clusterService);
 
             final List<PersistentTasksExecutor<?>> tasksExecutors = pluginsService.filterPlugins(PersistentTaskPlugin.class)
                 .stream()

server/src/main/java/org/opensearch/rest/RestController.java

@@ -51,6 +51,8 @@
 import org.opensearch.common.util.io.Streams;
 import org.opensearch.http.HttpServerTransport;
 import org.opensearch.indices.breaker.CircuitBreakerService;
+import org.opensearch.tracing.Tracer;
+import org.opensearch.tracing.TracerFactory;
 import org.opensearch.usage.UsageService;
 
 import java.io.ByteArrayOutputStream;
@@ -238,6 +240,15 @@ public void registerHandler(final RestHandler restHandler) {
     @Override
     public void dispatchRequest(RestRequest request, RestChannel channel, ThreadContext threadContext) {
         try {
+            if (request.uri().startsWith("/_search")) {
+                System.out.println("Request Id:" + request.getRequestId() + " uri:" + request.uri() );
+                Map<String, Object> map = new HashMap<>();
+                map.put("boolean", true);
+                map.put("long", 2l);
+                map.put("double", 2.0);
+                map.put("Request_id", String.valueOf(request.getRequestId()));
+                TracerFactory.getInstance().startSpan("Request_" + String.valueOf(request.getRequestId()), map, Tracer.Level.ROOT);
+            }
             tryAllHandlers(request, channel, threadContext);
         } catch (Exception e) {
             try {
@@ -571,6 +582,9 @@ public boolean detailedErrorsEnabled() {
         @Override
         public void sendResponse(RestResponse response) {
             close();
+            final RestRequest request = request();
+            System.out.println("Ending bbase request");
+            TracerFactory.getInstance().endSpan();
             delegate.sendResponse(response);
         }
 

server/src/main/java/org/opensearch/search/internal/ContextIndexSearcher.java

@@ -64,6 +64,8 @@
 import org.apache.lucene.util.SparseFixedBitSet;
 import org.opensearch.common.lease.Releasable;
 import org.opensearch.common.lucene.search.TopDocsAndMaxScore;
+import org.opensearch.tracing.Tracer.Level;
+import org.opensearch.tracing.TracerFactory;
 import org.opensearch.search.DocValueFormat;
 import org.opensearch.search.dfs.AggregatedDfs;
 import org.opensearch.search.profile.ContextualProfileBreakdown;
@@ -245,6 +247,7 @@ public void search(
         DocValueFormat[] formats,
         TotalHits totalHits
     ) throws IOException {
+        TracerFactory.getInstance().startSpan("IndexSearcher", null, Level.TERSE);
         final List<Collector> collectors = new ArrayList<>(leaves.size());
         for (LeafReaderContext ctx : leaves) {
             final Collector collector = manager.newCollector();
@@ -261,6 +264,8 @@ public void search(
             mergedTopDocs = new TopFieldDocs(totalHits, mergedTopDocs.scoreDocs, mergedTopDocs.fields);
         }
         result.topDocs(new TopDocsAndMaxScore(mergedTopDocs, Float.NaN), formats);
+
+        TracerFactory.getInstance().endSpan();
     }
 
     public void search(
@@ -284,6 +289,7 @@ public void search(
 
     @Override
     protected void search(List<LeafReaderContext> leaves, Weight weight, Collector collector) throws IOException {
+        TracerFactory.getInstance().startSpan("IndexSearcher", null, Level.TERSE);
         if (reverseLeafReaderContexts) {
             // reverse the segment search order if this flag is true.
             for (int i = leaves.size() - 1; i >= 0; i--) {
@@ -294,6 +300,7 @@ protected void search(List<LeafReaderContext> leaves, Weight weight, Collector c
                 searchLeaf(leaves.get(i), weight, collector);
             }
         }
+        TracerFactory.getInstance().endSpan();
     }
 
     /**
@@ -303,6 +310,7 @@ protected void search(List<LeafReaderContext> leaves, Weight weight, Collector c
      * the provided <code>ctx</code>.
      */
     private void searchLeaf(LeafReaderContext ctx, Weight weight, Collector collector) throws IOException {
+        TracerFactory.getInstance().startSpan("IndexSearcher-Leaf", null, Level.DEBUG);
         cancellable.checkCancelled();
         weight = wrapWeight(weight);
         // See please https://github.com/apache/lucene/pull/964
@@ -344,6 +352,7 @@ private void searchLeaf(LeafReaderContext ctx, Weight weight, Collector collecto
                 }
             }
         }
+        TracerFactory.getInstance().endSpan();
     }
 
     private Weight wrapWeight(Weight weight) {