From 399322213ebfc77c9115ad8cfaf74bf81788c9c8 Mon Sep 17 00:00:00 2001
From: suranjay <surajkumar.tu@gmail.com>
Date: Wed, 3 May 2023 13:09:16 +0530
Subject: [PATCH 1/3] Add tracer instrumentation in search code path

---
 .idea/runConfigurations/Debug_OpenSearch.xml  |  6 ++-
 build.gradle                                  |  6 +++
 .../opensearch/http/DefaultRestChannel.java   |  6 +++
 .../index/shard/SearchOperationListener.java  |  3 ++
 .../SearchOperationListenerTracerImpl.java    | 41 +++++++++++++++++++
 .../org/opensearch/rest/RestController.java   | 11 +++++
 .../search/internal/ContextIndexSearcher.java |  9 ++++
 7 files changed, 81 insertions(+), 1 deletion(-)
 create mode 100644 server/src/main/java/org/opensearch/index/shard/SearchOperationListenerTracerImpl.java

diff --git a/.idea/runConfigurations/Debug_OpenSearch.xml b/.idea/runConfigurations/Debug_OpenSearch.xml
index 0d8bf59823acf..c18046f873477 100644
--- a/.idea/runConfigurations/Debug_OpenSearch.xml
+++ b/.idea/runConfigurations/Debug_OpenSearch.xml
@@ -6,6 +6,10 @@
     <option name="HOST" value="localhost" />
     <option name="PORT" value="5005" />
     <option name="AUTO_RESTART" value="true" />
+    <RunnerSettings RunnerId="Debug">
+      <option name="DEBUG_PORT" value="5005" />
+      <option name="LOCAL" value="false" />
+    </RunnerSettings>
     <method v="2" />
   </configuration>
-</component>
+</component>
\ No newline at end of file
diff --git a/build.gradle b/build.gradle
index 220f6f3daf5ac..020e29494d1ec 100644
--- a/build.gradle
+++ b/build.gradle
@@ -627,6 +627,12 @@ subprojects {
   }
 }
 
+
+testClusters.all {
+  numberOfNodes = 3
+}
+
+
 reporting {
   reports {
     testAggregateTestReport(AggregateTestReport) {
diff --git a/server/src/main/java/org/opensearch/http/DefaultRestChannel.java b/server/src/main/java/org/opensearch/http/DefaultRestChannel.java
index 9df59cbb6f377..0a8e337ef48c7 100644
--- a/server/src/main/java/org/opensearch/http/DefaultRestChannel.java
+++ b/server/src/main/java/org/opensearch/http/DefaultRestChannel.java
@@ -43,6 +43,7 @@
 import org.opensearch.common.network.CloseableChannel;
 import org.opensearch.common.util.BigArrays;
 import org.opensearch.common.util.concurrent.ThreadContext;
+import org.opensearch.tracer.TracerFactory;
 import org.opensearch.rest.AbstractRestChannel;
 import org.opensearch.rest.RestChannel;
 import org.opensearch.rest.RestRequest;
@@ -164,6 +165,11 @@ public void sendResponse(RestResponse restResponse) {
             httpChannel.sendResponse(httpResponse, listener);
             success = true;
         } finally {
+            if (request.uri().startsWith("/_search")) {
+
+                System.out.println("ending base request:" + request.getRequestId());
+                TracerFactory.getInstance().endTrace();
+            }
             if (success == false) {
                 Releasables.close(toClose);
             }
diff --git a/server/src/main/java/org/opensearch/index/shard/SearchOperationListener.java b/server/src/main/java/org/opensearch/index/shard/SearchOperationListener.java
index 0a7c80f5e87d3..7da0d0d9bb227 100644
--- a/server/src/main/java/org/opensearch/index/shard/SearchOperationListener.java
+++ b/server/src/main/java/org/opensearch/index/shard/SearchOperationListener.java
@@ -153,6 +153,9 @@ final class CompositeListener implements SearchOperationListener {
 
         CompositeListener(List<SearchOperationListener> listeners, Logger logger) {
             this.listeners = listeners;
+            if (listeners != null) {
+                listeners.add(new SearchOperationListenerTracerImpl());
+            }
             this.logger = logger;
         }
 
diff --git a/server/src/main/java/org/opensearch/index/shard/SearchOperationListenerTracerImpl.java b/server/src/main/java/org/opensearch/index/shard/SearchOperationListenerTracerImpl.java
new file mode 100644
index 0000000000000..80e36d97f4b11
--- /dev/null
+++ b/server/src/main/java/org/opensearch/index/shard/SearchOperationListenerTracerImpl.java
@@ -0,0 +1,41 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.shard;
+
+import org.opensearch.tracer.Tracer.Level;
+import org.opensearch.tracer.TracerFactory;
+import org.opensearch.search.internal.SearchContext;
+
+public class SearchOperationListenerTracerImpl implements SearchOperationListener {
+
+    @Override
+    public void onPreQueryPhase(SearchContext searchContext) {
+        SearchOperationListener.super.onPreQueryPhase(searchContext);
+         TracerFactory.getInstance().startTrace("onQueryPhase", null, Level.MID);
+    }
+
+    @Override
+    public void onQueryPhase(SearchContext searchContext, long tookInNanos) {
+        SearchOperationListener.super.onQueryPhase(searchContext, tookInNanos);
+        TracerFactory.getInstance().endTrace();
+    }
+
+    @Override
+    public void onPreFetchPhase(SearchContext searchContext) {
+        SearchOperationListener.super.onPreFetchPhase(searchContext);
+        TracerFactory.getInstance().startTrace("onFetchPhase", null, Level.MID);
+    }
+
+    @Override
+    public void onFetchPhase(SearchContext searchContext, long tookInNanos) {
+        SearchOperationListener.super.onFetchPhase(searchContext, tookInNanos);
+        TracerFactory.getInstance().endTrace();
+
+    }
+}
diff --git a/server/src/main/java/org/opensearch/rest/RestController.java b/server/src/main/java/org/opensearch/rest/RestController.java
index 2554a1f376d4b..bdd33b2580c48 100644
--- a/server/src/main/java/org/opensearch/rest/RestController.java
+++ b/server/src/main/java/org/opensearch/rest/RestController.java
@@ -51,6 +51,8 @@
 import org.opensearch.common.util.io.Streams;
 import org.opensearch.http.HttpServerTransport;
 import org.opensearch.indices.breaker.CircuitBreakerService;
+import org.opensearch.tracer.Tracer;
+import org.opensearch.tracer.TracerFactory;
 import org.opensearch.usage.UsageService;
 
 import java.io.ByteArrayOutputStream;
@@ -238,6 +240,12 @@ public void registerHandler(final RestHandler restHandler) {
     @Override
     public void dispatchRequest(RestRequest request, RestChannel channel, ThreadContext threadContext) {
         try {
+            if (request.uri().startsWith("/_search")) {
+                System.out.println("Request Id:" + request.getRequestId() + " uri:" + request.uri() );
+                Map<String, String> map = new HashMap<>();
+                map.put("Request_id", String.valueOf(request.getRequestId()));
+                TracerFactory.getInstance().startTrace("Request_" + String.valueOf(request.getRequestId()), map, Tracer.Level.HIGH);
+            }
             tryAllHandlers(request, channel, threadContext);
         } catch (Exception e) {
             try {
@@ -571,6 +579,9 @@ public boolean detailedErrorsEnabled() {
         @Override
         public void sendResponse(RestResponse response) {
             close();
+            final RestRequest request = request();
+            System.out.println("Ending bbase request");
+            TracerFactory.getInstance().endTrace();
             delegate.sendResponse(response);
         }
 
diff --git a/server/src/main/java/org/opensearch/search/internal/ContextIndexSearcher.java b/server/src/main/java/org/opensearch/search/internal/ContextIndexSearcher.java
index 818da075fd7ec..6ad1e5902cc11 100644
--- a/server/src/main/java/org/opensearch/search/internal/ContextIndexSearcher.java
+++ b/server/src/main/java/org/opensearch/search/internal/ContextIndexSearcher.java
@@ -64,6 +64,8 @@
 import org.apache.lucene.util.SparseFixedBitSet;
 import org.opensearch.common.lease.Releasable;
 import org.opensearch.common.lucene.search.TopDocsAndMaxScore;
+import org.opensearch.tracer.Tracer.Level;
+import org.opensearch.tracer.TracerFactory;
 import org.opensearch.search.DocValueFormat;
 import org.opensearch.search.dfs.AggregatedDfs;
 import org.opensearch.search.profile.ContextualProfileBreakdown;
@@ -245,6 +247,7 @@ public void search(
         DocValueFormat[] formats,
         TotalHits totalHits
     ) throws IOException {
+        TracerFactory.getInstance().startTrace("IndexSearcher", null, Level.LOW);
         final List<Collector> collectors = new ArrayList<>(leaves.size());
         for (LeafReaderContext ctx : leaves) {
             final Collector collector = manager.newCollector();
@@ -261,6 +264,8 @@ public void search(
             mergedTopDocs = new TopFieldDocs(totalHits, mergedTopDocs.scoreDocs, mergedTopDocs.fields);
         }
         result.topDocs(new TopDocsAndMaxScore(mergedTopDocs, Float.NaN), formats);
+
+        TracerFactory.getInstance().endTrace();
     }
 
     public void search(
@@ -284,6 +289,7 @@ public void search(
 
     @Override
     protected void search(List<LeafReaderContext> leaves, Weight weight, Collector collector) throws IOException {
+        TracerFactory.getInstance().startTrace("IndexSearcher", null, Level.LOW);
         if (reverseLeafReaderContexts) {
             // reverse the segment search order if this flag is true.
             for (int i = leaves.size() - 1; i >= 0; i--) {
@@ -294,6 +300,7 @@ protected void search(List<LeafReaderContext> leaves, Weight weight, Collector c
                 searchLeaf(leaves.get(i), weight, collector);
             }
         }
+        TracerFactory.getInstance().endTrace();
     }
 
     /**
@@ -303,6 +310,7 @@ protected void search(List<LeafReaderContext> leaves, Weight weight, Collector c
      * the provided <code>ctx</code>.
      */
     private void searchLeaf(LeafReaderContext ctx, Weight weight, Collector collector) throws IOException {
+        TracerFactory.getInstance().startTrace("IndexSearcher-Leaf", null, Level.LOW);
         cancellable.checkCancelled();
         weight = wrapWeight(weight);
         // See please https://github.com/apache/lucene/pull/964
@@ -344,6 +352,7 @@ private void searchLeaf(LeafReaderContext ctx, Weight weight, Collector collecto
                 }
             }
         }
+        TracerFactory.getInstance().endTrace();
     }
 
     private Weight wrapWeight(Weight weight) {

From f417bf1019871cc52a453e19ec2055548c6a072e Mon Sep 17 00:00:00 2001
From: suranjay <surajkumar.tu@gmail.com>
Date: Sun, 7 May 2023 20:59:22 +0530
Subject: [PATCH 2/3] Refactor otel integration, add level logic with dynamic
 updates

---
 .../common/settings/ClusterSettings.java      |   6 +-
 .../common/util/concurrent/ThreadContext.java |  16 +-
 .../main/java/org/opensearch/node/Node.java   |   4 +-
 .../org/opensearch/tracer/DefaultTracer.java  | 195 ----------------
 .../org/opensearch/tracer/TracerFactory.java  |  32 ---
 .../org/opensearch/tracer/TracerUtils.java    |  52 -----
 .../org/opensearch/tracing/DefaultTracer.java | 213 ++++++++++++++++++
 .../java/org/opensearch/tracing/NoopSpan.java |  29 +++
 .../org/opensearch/tracing/NoopTracer.java    |  34 +++
 .../{tracer/Span.java => tracing/OSSpan.java} |  25 +-
 .../{tracer => tracing}/OTelMain.java         |  11 +-
 .../java/org/opensearch/tracing/Span.java     |  19 ++
 .../{tracer => tracing}/SpanHolder.java       |  10 +-
 .../{tracer => tracing}/Tracer.java           |  63 +++---
 .../org/opensearch/tracing/TracerFactory.java |  49 ++++
 .../org/opensearch/tracing/TracerUtils.java   |  75 ++++++
 16 files changed, 485 insertions(+), 348 deletions(-)
 delete mode 100644 server/src/main/java/org/opensearch/tracer/DefaultTracer.java
 delete mode 100644 server/src/main/java/org/opensearch/tracer/TracerFactory.java
 delete mode 100644 server/src/main/java/org/opensearch/tracer/TracerUtils.java
 create mode 100644 server/src/main/java/org/opensearch/tracing/DefaultTracer.java
 create mode 100644 server/src/main/java/org/opensearch/tracing/NoopSpan.java
 create mode 100644 server/src/main/java/org/opensearch/tracing/NoopTracer.java
 rename server/src/main/java/org/opensearch/{tracer/Span.java => tracing/OSSpan.java} (50%)
 rename server/src/main/java/org/opensearch/{tracer => tracing}/OTelMain.java (81%)
 create mode 100644 server/src/main/java/org/opensearch/tracing/Span.java
 rename server/src/main/java/org/opensearch/{tracer => tracing}/SpanHolder.java (75%)
 rename server/src/main/java/org/opensearch/{tracer => tracing}/Tracer.java (68%)
 create mode 100644 server/src/main/java/org/opensearch/tracing/TracerFactory.java
 create mode 100644 server/src/main/java/org/opensearch/tracing/TracerUtils.java

diff --git a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java
index 208a358d38395..a2b2c9c93ab15 100644
--- a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java
+++ b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java
@@ -50,6 +50,7 @@
 import org.opensearch.tasks.TaskManager;
 import org.opensearch.tasks.TaskResourceTrackingService;
 import org.opensearch.tasks.consumer.TopNSearchTasksLogger;
+import org.opensearch.tracing.Tracer;
 import org.opensearch.watcher.ResourceWatcherService;
 import org.opensearch.action.admin.cluster.configuration.TransportAddVotingConfigExclusionsAction;
 import org.opensearch.action.admin.indices.close.TransportCloseIndexAction;
@@ -638,7 +639,10 @@ public void apply(Settings value, Settings current, Settings previous) {
                 SegmentReplicationPressureService.MAX_ALLOWED_STALE_SHARDS,
 
                 // Settings related to Searchable Snapshots
-                Node.NODE_SEARCH_CACHE_SIZE_SETTING
+                Node.NODE_SEARCH_CACHE_SIZE_SETTING,
+
+                // Settings related to Tracing framework
+                Tracer.TRACER_LEVEL_SETTING
             )
         )
     );
diff --git a/server/src/main/java/org/opensearch/common/util/concurrent/ThreadContext.java b/server/src/main/java/org/opensearch/common/util/concurrent/ThreadContext.java
index db8e0eb920e79..15f8a6365c449 100644
--- a/server/src/main/java/org/opensearch/common/util/concurrent/ThreadContext.java
+++ b/server/src/main/java/org/opensearch/common/util/concurrent/ThreadContext.java
@@ -44,8 +44,8 @@
 import org.opensearch.common.settings.Setting.Property;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.http.HttpTransportSettings;
-import org.opensearch.tracer.SpanHolder;
-import org.opensearch.tracer.TracerUtils;
+import org.opensearch.tracing.SpanHolder;
+import org.opensearch.tracing.TracerUtils;
 import org.opensearch.tasks.Task;
 
 import java.io.IOException;
@@ -60,7 +60,7 @@
 
 import static org.opensearch.http.HttpTransportSettings.SETTING_HTTP_MAX_WARNING_HEADER_COUNT;
 import static org.opensearch.http.HttpTransportSettings.SETTING_HTTP_MAX_WARNING_HEADER_SIZE;
-import static org.opensearch.tracer.DefaultTracer.T_CURRENT_SPAN_KEY;
+import static org.opensearch.tracing.DefaultTracer.CURRENT_SPAN;
 import static org.opensearch.tasks.TaskResourceTrackingService.TASK_ID;
 
 /**
@@ -142,8 +142,8 @@ public StoredContext stashContext() {
             threadContextStruct = threadContextStruct.putTransient(TASK_ID, context.transientHeaders.get(TASK_ID));
         }
 
-        if (context.transientHeaders.containsKey(T_CURRENT_SPAN_KEY)) {
-            threadContextStruct = threadContextStruct.putTransient(T_CURRENT_SPAN_KEY, new SpanHolder((SpanHolder) context.transientHeaders.get(T_CURRENT_SPAN_KEY)));
+        if (context.transientHeaders.containsKey(CURRENT_SPAN)) {
+            threadContextStruct = threadContextStruct.putTransient(CURRENT_SPAN, new SpanHolder((SpanHolder) context.transientHeaders.get(CURRENT_SPAN)));
         }
 
         threadLocal.set(threadContextStruct);
@@ -242,8 +242,8 @@ public StoredContext newStoredContext(boolean preserveResponseHeaders, Collectio
         // this is the context when this method returns
         final ThreadContextStruct newContext = threadLocal.get();
 
-        if (newContext.transientHeaders.containsKey(T_CURRENT_SPAN_KEY)) {
-            newContext.transientHeaders.put(T_CURRENT_SPAN_KEY, new SpanHolder((SpanHolder) newContext.transientHeaders.get(T_CURRENT_SPAN_KEY)));
+        if (newContext.transientHeaders.containsKey(CURRENT_SPAN)) {
+            newContext.transientHeaders.put(CURRENT_SPAN, new SpanHolder((SpanHolder) newContext.transientHeaders.get(CURRENT_SPAN)));
         }
 
         return () -> {
@@ -710,7 +710,7 @@ private ThreadContextStruct copyHeaders(Iterable<Map.Entry<String, String>> head
         }
 
         private void writeTo(StreamOutput out, Map<String, String> defaultHeaders) throws IOException {
-            TracerUtils.addTracerContextInHeader(this.requestHeaders, this.transientHeaders);
+            TracerUtils.addTracerContextToHeader(this.requestHeaders, this.transientHeaders);
 
             final Map<String, String> requestHeaders;
             if (defaultHeaders.isEmpty()) {
diff --git a/server/src/main/java/org/opensearch/node/Node.java b/server/src/main/java/org/opensearch/node/Node.java
index b77bd0a1243fd..8ea0dc4c7ee9a 100644
--- a/server/src/main/java/org/opensearch/node/Node.java
+++ b/server/src/main/java/org/opensearch/node/Node.java
@@ -55,7 +55,7 @@
 import org.opensearch.monitor.fs.FsInfo;
 import org.opensearch.monitor.fs.FsProbe;
 import org.opensearch.plugins.SearchPipelinePlugin;
-import org.opensearch.tracer.TracerFactory;
+import org.opensearch.tracing.TracerFactory;
 import org.opensearch.search.backpressure.SearchBackpressureService;
 import org.opensearch.search.backpressure.settings.SearchBackpressureSettings;
 import org.opensearch.search.pipeline.SearchPipelineService;
@@ -1022,7 +1022,7 @@ protected Node(
                 searchModule.getIndexSearcherExecutor(threadPool)
             );
 
-            TracerFactory.initializeTracer(threadPool);
+            TracerFactory.initializeTracer(threadPool, clusterService);
 
             final List<PersistentTasksExecutor<?>> tasksExecutors = pluginsService.filterPlugins(PersistentTaskPlugin.class)
                 .stream()
diff --git a/server/src/main/java/org/opensearch/tracer/DefaultTracer.java b/server/src/main/java/org/opensearch/tracer/DefaultTracer.java
deleted file mode 100644
index c7cb070e09238..0000000000000
--- a/server/src/main/java/org/opensearch/tracer/DefaultTracer.java
+++ /dev/null
@@ -1,195 +0,0 @@
-package org.opensearch.tracer;
-
-
-import io.opentelemetry.context.Context;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-import org.opensearch.common.util.concurrent.ThreadContext;
-import org.opensearch.threadpool.ThreadPool;
-
-import java.util.Map;
-import java.util.Optional;
-
-/**
- * This class encapsulates the state needed to execute a search. It holds a reference to the
- * shards point in time snapshot (IndexReader / ContextIndexSearcher) and allows passing on
- * state from one query / fetch phase to another.
- *
- * @opensearch.internal
- */
-public class DefaultTracer implements Tracer {
-
-    public static final String T_CURRENT_SPAN_KEY = "T_CurrentSpan";
-
-    private static final Logger logger = LogManager.getLogger(DefaultTracer.class);
-    private static final String A_SPAN_ID_KEY = "SpanId";
-    private static final String A_PARENT_SPAN_ID_KEY = "ParentSpanId";
-
-    private final ThreadPool threadPool;
-    private final io.opentelemetry.api.trace.Tracer openTelemetryTracer;
-
-    public DefaultTracer(io.opentelemetry.api.trace.Tracer openTelemetryTracer, ThreadPool threadPool) {
-        this.openTelemetryTracer = openTelemetryTracer;
-        this.threadPool = threadPool;
-    }
-
-    @Override
-    public Span startTrace(String spanName, Map<String, String> attributes, Level level) {
-        return startTrace(spanName, attributes, null, level);
-    }
-
-    @Override
-    public Span startTrace(String spanName, Map<String, String> attributes, Span parentSpan, Level level) {
-        if (parentSpan == null) {
-            parentSpan = getCurrentSpan();
-        }
-        Level calculatedLevel = getLevel(parentSpan, level);
-        if (!isLevelEnabled(calculatedLevel)) {
-            return parentSpan;
-        }
-        io.opentelemetry.api.trace.Span span = createSpan(spanName, parentSpan);
-        Span osSpan = new Span(spanName, span, parentSpan, calculatedLevel);
-        logger.info("Starting span spanId:{} name:{}: traceId:{}",
-            span.getSpanContext().getSpanId(), osSpan.getSpanName(), span.getSpanContext().getTraceId());
-        setCurrentSpanInContext(osSpan);
-        setSpanAttributes(osSpan, attributes);
-        return osSpan;
-    }
-
-    @Override
-    public void endTrace() {
-        Span currentSpan = getCurrentSpan();
-        if (currentSpan != null) {
-            logger.info("Ending span spanId:{} name:{}: traceId:{}",
-                currentSpan.getSpanContext().getSpanId(), currentSpan.getSpanName(), currentSpan.getSpanContext().getTraceId());            currentSpan.getSpan().end();
-
-            Span parentSpan = currentSpan.getParentSpan();
-            if (parentSpan != null) {
-                setCurrentSpanInContext(parentSpan);
-            }
-        }
-    }
-
-    @Override
-    public void addAttribute(String key, String value) {
-        Span currentSpan = getCurrentSpan();
-        if (currentSpan != null) {
-            currentSpan.getSpan().setAttribute(key, value);
-        }
-    }
-
-    @Override
-    public void addAttribute(String key, boolean value) {
-        Span currentSpan = getCurrentSpan();
-        if (currentSpan != null) {
-            currentSpan.getSpan().setAttribute(key, value);
-        }
-    }
-
-    @Override
-    public void addAttribute(String key, long value) {
-        Span currentSpan = getCurrentSpan();
-        if (currentSpan != null) {
-            currentSpan.getSpan().setAttribute(key, value);
-        }
-    }
-
-    @Override
-    public void addAttribute(String key, double value) {
-        Span currentSpan = getCurrentSpan();
-        if (currentSpan != null) {
-            currentSpan.getSpan().setAttribute(key, value);
-        }
-    }
-
-    @Override
-    public void addEvent(String event) {
-        Span currentSpan = getCurrentSpan();
-        if (currentSpan != null) {
-            currentSpan.getSpan().addEvent(event);
-        }
-    }
-
-    private io.opentelemetry.api.trace.Span createSpan(String spanName, Span parentSpan) {
-        io.opentelemetry.api.trace.Span span;
-        if (parentSpan == null) {
-            span = openTelemetryTracer.spanBuilder(spanName).startSpan();
-        } else {
-            span = openTelemetryTracer.spanBuilder(spanName).setParent(Context.current().with(parentSpan.getSpan())).startSpan();
-        }
-        return span;
-    }
-
-    private boolean isLevelEnabled(Level calculatedLevel) {
-        return true;
-    }
-
-    private Level getLevel(Span parentSpan, Level level) { //TODO handle level
-        if (parentSpan != null && parentSpan.getLevel().isHigherThan(level)) {
-            return level;
-        }
-        return Level.UNKNOWN;
-    }
-
-    private void setCurrentSpanInContext(Span span) {
-        ThreadContext threadContext = threadPool.getThreadContext();
-        SpanHolder spanHolder = threadContext.getTransient(T_CURRENT_SPAN_KEY);
-        if (spanHolder == null) {
-            threadContext.putTransient(T_CURRENT_SPAN_KEY, new SpanHolder(span));
-        } else {
-            spanHolder.setSpan(span);
-        }
-    }
-
-    private Span getCurrentSpan() {
-        Optional<Span> optionalSpanFromContext = spanFromThreadContext();
-        return optionalSpanFromContext.orElse(spanFromHeader());
-    }
-
-    private void setSpanAttributes(Span span, Map<String, String> attributes) {
-        addTracerAttributes(span);
-        addUserAttributes(span, attributes);
-
-    }
-
-    public void addUserAttributes(Span span, Map<String, String> attributes) {
-        if (span != null && attributes != null) {
-            attributes.entrySet().stream().forEach(entry -> span.getSpan().setAttribute(entry.getKey(), entry.getValue()));
-        }
-    }
-
-    private void addTracerAttributes(Span span) { //TODO clean up this
-        if (span == null) {
-            return;
-        }
-        span.getSpan().setAttribute(A_SPAN_ID_KEY, span.getSpan().getSpanContext().getSpanId());
-        span.getSpan().setAttribute("TraceId", span.getSpan().getSpanContext().getTraceId());
-        span.getSpan().setAttribute("SpanKey", span.getSpanName());
-        span.getSpan().setAttribute("ThreadName", Thread.currentThread().getName());
-        if (span.getParentSpan() != null) {
-            span.getSpan().setAttribute(A_PARENT_SPAN_ID_KEY, span.getParentSpan().getSpan().getSpanContext().getSpanId());
-            span.getSpan().setAttribute("ParentSpanKey", span.getParentSpan().getSpanName());
-        }
-    }
-
-    private Span spanFromHeader() {
-        Context context = TracerUtils.extractTracerContextFromHeader(threadPool.getThreadContext().getHeaders());
-        if (context != null) {
-            io.opentelemetry.api.trace.Span span = io.opentelemetry.api.trace.Span.fromContext(context);
-            return new Span("RootSpan", span, null, Level.HIGH);
-        }
-        return null;
-    }
-
-
-    private Optional<Span> spanFromThreadContext() {
-        ThreadContext threadContext = threadPool.getThreadContext();
-        SpanHolder spanHolder = threadContext.getTransient(T_CURRENT_SPAN_KEY);
-        if (spanHolder == null) {
-            return Optional.empty();
-        }
-        return Optional.ofNullable(spanHolder.getSpan());
-    }
-
-
-}
diff --git a/server/src/main/java/org/opensearch/tracer/TracerFactory.java b/server/src/main/java/org/opensearch/tracer/TracerFactory.java
deleted file mode 100644
index d099ec42e97a0..0000000000000
--- a/server/src/main/java/org/opensearch/tracer/TracerFactory.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- *
- * The OpenSearch Contributors require contributions made to
- * this file be licensed under the Apache-2.0 license or a
- * compatible open source license.
- */
-
-package org.opensearch.tracer;
-
-import io.opentelemetry.api.OpenTelemetry;
-import org.opensearch.threadpool.ThreadPool;
-
-public class TracerFactory {
-
-    private static DefaultTracer defaultTracer;
-
-    public static synchronized void initializeTracer(ThreadPool threadPool) {
-        OpenTelemetry openTelemetry = OTelMain.openTelemetry;
-        io.opentelemetry.api.trace.Tracer openTelemetryTracer = openTelemetry.getTracer("opensearch-instrumentation", "1.0.0");
-        if (defaultTracer == null) {
-            defaultTracer = new DefaultTracer(openTelemetryTracer, threadPool);
-        } else {
-            throw new IllegalStateException("Double initialization not allowed!");
-        }
-    }
-
-    public static Tracer getInstance() {
-        return defaultTracer;
-    }
-
-}
diff --git a/server/src/main/java/org/opensearch/tracer/TracerUtils.java b/server/src/main/java/org/opensearch/tracer/TracerUtils.java
deleted file mode 100644
index 4fd0211431c06..0000000000000
--- a/server/src/main/java/org/opensearch/tracer/TracerUtils.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- *
- * The OpenSearch Contributors require contributions made to
- * this file be licensed under the Apache-2.0 license or a
- * compatible open source license.
- */
-
-package org.opensearch.tracer;
-
-import io.opentelemetry.context.Context;
-import io.opentelemetry.context.propagation.TextMapGetter;
-import io.opentelemetry.context.propagation.TextMapSetter;
-
-import java.util.Map;
-
-import static org.opensearch.tracer.DefaultTracer.T_CURRENT_SPAN_KEY;
-
-public class TracerUtils {
-
-    public static void addTracerContextInHeader(Map<String, String> requestHeaders, Map<String, Object> transientHeaders) {
-        TextMapSetter<Map> setter = (carrier, key, value) -> {
-            carrier.put(key, value);
-        };
-        if (transientHeaders != null && transientHeaders.containsKey(T_CURRENT_SPAN_KEY)) {
-            SpanHolder spanHolder = (SpanHolder) transientHeaders.get(T_CURRENT_SPAN_KEY);
-            Span currentSpan = spanHolder.getSpan();
-            OTelMain.openTelemetry.getPropagators().getTextMapPropagator().inject(
-                Context.current().with(io.opentelemetry.api.trace.Span.wrap(currentSpan.getSpanContext())), requestHeaders, setter
-            );
-        }
-    }
-
-    public static Context extractTracerContextFromHeader(Map<String, String> headers) {
-        TextMapGetter<Map<String, String>> getter = new TextMapGetter<>() {
-            @Override
-            public Iterable<String> keys(Map<String, String> headers) {
-                return headers.keySet();
-            }
-
-            @Override
-            public String get(Map<String, String> headers, String key) {
-                if (headers.containsKey(key)) {
-                    return headers.get(key);
-                }
-                return null;
-            }
-        };
-        return OTelMain.openTelemetry.getPropagators().getTextMapPropagator()
-            .extract(Context.current(), headers, getter);
-    }
-}
diff --git a/server/src/main/java/org/opensearch/tracing/DefaultTracer.java b/server/src/main/java/org/opensearch/tracing/DefaultTracer.java
new file mode 100644
index 0000000000000..899cda455e566
--- /dev/null
+++ b/server/src/main/java/org/opensearch/tracing/DefaultTracer.java
@@ -0,0 +1,213 @@
+package org.opensearch.tracing;
+
+
+import io.opentelemetry.context.Context;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.opensearch.cluster.service.ClusterService;
+import org.opensearch.common.util.concurrent.ThreadContext;
+import org.opensearch.threadpool.ThreadPool;
+
+import java.util.Map;
+import java.util.Optional;
+
+/**
+ * This class encapsulates the state needed to execute a search. It holds a reference to the
+ * shards point in time snapshot (IndexReader / ContextIndexSearcher) and allows passing on
+ * state from one query / fetch phase to another.
+ *
+ * @opensearch.internal
+ */
+public class DefaultTracer implements Tracer {
+
+    public static final String CURRENT_SPAN = "current_span";
+
+    private static final Logger logger = LogManager.getLogger(DefaultTracer.class);
+
+    private static final String TRACE_ID = "trace_id";
+    private static final String SPAN_ID = "span_id";
+    private static final String SPAN_NAME = "span_name";
+    private static final String PARENT_SPAN_ID = "parent_span_id";
+    private static final String THREAD_NAME = "thread_name";
+    private static final String PARENT_SPAN_NAME = "parent_span_name";
+    private static final String ROOT_SPAN = "RootSpan";
+
+    private final ThreadPool threadPool;
+    private final ClusterService clusterService;
+    private final io.opentelemetry.api.trace.Tracer openTelemetryTracer;
+
+    public DefaultTracer(io.opentelemetry.api.trace.Tracer openTelemetryTracer, ThreadPool threadPool, ClusterService clusterService) {
+        this.openTelemetryTracer = openTelemetryTracer;
+        this.threadPool = threadPool;
+        this.clusterService = clusterService;
+    }
+
+    @Override
+    public void startSpan(String spanName, Map<String, Object> attributes, Level level) {
+        startSpan(spanName, attributes, null, level);
+    }
+
+    @Override
+    public void startSpan(String spanName, Map<String, Object> attributes, Span parentSpan, Level level) {
+        if (parentSpan == null) {
+            parentSpan = getCurrentSpan();
+        }
+        Span span = createSpan(spanName, parentSpan, level);
+        setCurrentSpanInContext(span);
+        setSpanAttributes(span, attributes);
+    }
+
+    @Override
+    public void endSpan() {
+        Span currentSpan = getCurrentSpan();
+        if (currentSpan != null) {
+            endSpan(currentSpan);
+            setCurrentSpanInContext(currentSpan.getParentSpan());
+        }
+    }
+
+    @Override
+    public void addAttribute(String key, Object value) {
+        Span currentSpan = getCurrentSpan();
+        addSingleAttribute(key, value, currentSpan);
+    }
+
+    @Override
+    public void addEvent(String event) {
+        Span currentSpan = getCurrentSpan();
+        if (currentSpan instanceof OSSpan && ((OSSpan) currentSpan).getOtelSpan() != null) {
+            ((OSSpan) currentSpan).getOtelSpan().addEvent(event);
+        }
+    }
+
+    private Span createSpan(String spanName, Span parentSpan, Level level) {
+        return isLevelEnabled(level) ?
+                createOSSpan(spanName, parentSpan, level) :
+                createNoopSpan(spanName, parentSpan, level);
+    }
+
+    private Span createOSSpan(String spanName, Span parentSpan, Level level) {
+        OSSpan parentOSSpan = getLastOSSpanInChain(parentSpan);
+        io.opentelemetry.api.trace.Span otelSpan = createOtelSpan(spanName, parentOSSpan);
+        Span span = new OSSpan(spanName, otelSpan, parentOSSpan, level);
+        logger.info("Starting OtelSpan spanId:{} name:{}: traceId:{}", otelSpan.getSpanContext().getSpanId(),
+                span.getSpanName(), otelSpan.getSpanContext().getTraceId());
+        return span;
+    }
+
+    private NoopSpan createNoopSpan(String spanName, Span parentSpan, Level level) {
+        logger.info("Starting Noop span name:{}", spanName);
+        return new NoopSpan(spanName, parentSpan, level);
+    }
+
+    private OSSpan getLastOSSpanInChain(Span parentSpan) {
+        while (!(parentSpan instanceof OSSpan)) {
+            // TODO shall we have upper restriction on depth to break out from infinite loop in case of circular dependency
+            parentSpan = parentSpan.getParentSpan();
+        }
+        return (OSSpan) parentSpan;
+    }
+
+    private io.opentelemetry.api.trace.Span createOtelSpan(String spanName, OSSpan parentOSSpan) {
+        return parentOSSpan == null ?
+                openTelemetryTracer.spanBuilder(spanName).startSpan() :
+                openTelemetryTracer.spanBuilder(spanName)
+                        .setParent(Context.current().with(parentOSSpan.getOtelSpan())).startSpan();
+    }
+
+    private boolean isLevelEnabled(Level level) {
+        Level configuredLevel = clusterService.getClusterSettings().get(TRACER_LEVEL_SETTING);
+        return level.getOrder() >= configuredLevel.getOrder();
+    }
+
+    private void addSingleAttribute(String key, Object value, Span currentSpan) {
+        if (currentSpan instanceof OSSpan && ((OSSpan) currentSpan).getOtelSpan() != null) {
+            if (value instanceof String) {
+                ((OSSpan) currentSpan).getOtelSpan().setAttribute(key, (String) value);
+            } else if (value instanceof Long) {
+                ((OSSpan) currentSpan).getOtelSpan().setAttribute(key, (Long) value);
+            } else if (value instanceof Boolean) {
+                ((OSSpan) currentSpan).getOtelSpan().setAttribute(key, (Boolean) value);
+            } else if (value instanceof Double) {
+                ((OSSpan) currentSpan).getOtelSpan().setAttribute(key, (Double) value);
+            } else {
+                throw new IllegalArgumentException("Unsupported attribute value type found, " +
+                        "must be [String, long, boolean, double]");
+            }
+        }
+    }
+
+    private void setCurrentSpanInContext(Span span) {
+        if (span == null) {
+            return;
+        }
+        ThreadContext threadContext = threadPool.getThreadContext();
+        SpanHolder spanHolder = threadContext.getTransient(CURRENT_SPAN);
+        if (spanHolder == null) {
+            threadContext.putTransient(CURRENT_SPAN, new SpanHolder(span));
+        } else {
+            spanHolder.setSpan(span);
+        }
+    }
+
+    private Span getCurrentSpan() {
+        Optional<Span> optionalSpanFromContext = spanFromThreadContext();
+        return optionalSpanFromContext.orElse(spanFromHeader());
+    }
+
+    private void endSpan(Span span) {
+        if (span instanceof OSSpan) {
+            OSSpan osSpan = (OSSpan) span;
+            logger.info("Ending span spanId:{} name:{}: traceId:{}",
+                    osSpan.getSpanContext().getSpanId(),
+                    span.getSpanName(),
+                    osSpan.getSpanContext().getTraceId());
+            osSpan.getOtelSpan().end();
+        } else {
+            logger.info("Ending noop span name:{}", span.getSpanName());
+        }
+    }
+
+    private void setSpanAttributes(Span span, Map<String, Object> attributes) {
+        if (span instanceof NoopSpan) {
+            return;
+        }
+        addDefaultAttributes((OSSpan) span);
+        addUserAttributes((OSSpan) span, attributes);
+    }
+
+    public void addUserAttributes(OSSpan osSpan, Map<String, Object> attributes) {
+        if (attributes != null) {
+            attributes.forEach((key, value) -> addSingleAttribute(key, value, osSpan));
+        }
+    }
+
+    private void addDefaultAttributes(OSSpan osSpan) {
+        if (osSpan != null && osSpan.getOtelSpan() != null) {
+            osSpan.getOtelSpan().setAttribute(SPAN_ID, osSpan.getOtelSpan().getSpanContext().getSpanId());
+            osSpan.getOtelSpan().setAttribute(TRACE_ID, osSpan.getOtelSpan().getSpanContext().getTraceId());
+            osSpan.getOtelSpan().setAttribute(SPAN_NAME, osSpan.getSpanName());
+            osSpan.getOtelSpan().setAttribute(THREAD_NAME, Thread.currentThread().getName());
+            if (osSpan.getParentSpan() != null && osSpan.getParentSpan() instanceof OSSpan) {
+                osSpan.getOtelSpan().setAttribute(PARENT_SPAN_ID, ((OSSpan) osSpan.getParentSpan()).getOtelSpan().getSpanContext().getSpanId());
+                osSpan.getOtelSpan().setAttribute(PARENT_SPAN_NAME, osSpan.getParentSpan().getSpanName());
+            }
+        }
+    }
+
+    private Span spanFromHeader() {
+        Context context = TracerUtils.extractTracerContextFromHeader(threadPool.getThreadContext().getHeaders());
+        if (context != null) {
+            io.opentelemetry.api.trace.Span span = io.opentelemetry.api.trace.Span.fromContext(context);
+            return new OSSpan(ROOT_SPAN, span, null, Level.ROOT);
+        }
+        return null;
+    }
+
+    private Optional<Span> spanFromThreadContext() {
+        ThreadContext threadContext = threadPool.getThreadContext();
+        SpanHolder spanHolder = threadContext.getTransient(CURRENT_SPAN);
+        return (spanHolder == null) ? Optional.empty() : Optional.ofNullable(spanHolder.getSpan());
+    }
+
+}
diff --git a/server/src/main/java/org/opensearch/tracing/NoopSpan.java b/server/src/main/java/org/opensearch/tracing/NoopSpan.java
new file mode 100644
index 0000000000000..0552d947ee444
--- /dev/null
+++ b/server/src/main/java/org/opensearch/tracing/NoopSpan.java
@@ -0,0 +1,29 @@
+package org.opensearch.tracing;
+
+public class NoopSpan implements Span {
+    private final String spanName;
+    private final Span parentSpan;
+    private final Tracer.Level level;
+
+    public NoopSpan(String spanName, Span parentSpan, Tracer.Level level) {
+        this.spanName = spanName;
+        this.parentSpan = parentSpan;
+        this.level = level;
+    }
+
+    @Override
+    public Span getParentSpan() {
+        return parentSpan;
+    }
+
+    @Override
+    public Tracer.Level getLevel() {
+        return level;
+    }
+
+    @Override
+    public String getSpanName() {
+        return spanName;
+    }
+
+}
diff --git a/server/src/main/java/org/opensearch/tracing/NoopTracer.java b/server/src/main/java/org/opensearch/tracing/NoopTracer.java
new file mode 100644
index 0000000000000..900fd48ae2901
--- /dev/null
+++ b/server/src/main/java/org/opensearch/tracing/NoopTracer.java
@@ -0,0 +1,34 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.tracing;
+
+import java.util.Map;
+
+public class NoopTracer implements Tracer {
+
+    @Override
+    public void startSpan(String spanName, Map<String, Object> attributes, Span parentSpan, Level level) {
+
+    }
+
+    @Override
+    public void endSpan() {
+    }
+
+    @Override
+    public void addAttribute(String key, Object value) {
+
+    }
+
+
+    @Override
+    public void addEvent(String event) {
+
+    }
+}
diff --git a/server/src/main/java/org/opensearch/tracer/Span.java b/server/src/main/java/org/opensearch/tracing/OSSpan.java
similarity index 50%
rename from server/src/main/java/org/opensearch/tracer/Span.java
rename to server/src/main/java/org/opensearch/tracing/OSSpan.java
index 07f15afb03b37..d7737d0ce18f9 100644
--- a/server/src/main/java/org/opensearch/tracer/Span.java
+++ b/server/src/main/java/org/opensearch/tracing/OSSpan.java
@@ -1,38 +1,41 @@
-package org.opensearch.tracer;
+package org.opensearch.tracing;
 
 import io.opentelemetry.api.trace.SpanContext;
 
-public class Span { // TODO handle NPE in getters
+public class OSSpan implements Span {
     private final String spanName;
-    private final io.opentelemetry.api.trace.Span span;
+    private final io.opentelemetry.api.trace.Span otelSpan;
     private final Span parentSpan;
     private final Tracer.Level level;
 
-    public Span(String spanName, io.opentelemetry.api.trace.Span span, Span parentSpan, Tracer.Level level) {
+    public OSSpan(String spanName, io.opentelemetry.api.trace.Span span, OSSpan parentSpan, Tracer.Level level) {
         this.spanName = spanName;
-        this.span = span;
+        this.otelSpan = span;
         this.parentSpan = parentSpan;
         this.level = level;
     }
 
-    io.opentelemetry.api.trace.Span getSpan() {
-        return span;
-    }
-
+    @Override
     public Span getParentSpan() {
         return parentSpan;
     }
 
+    @Override
     public Tracer.Level getLevel() {
         return level;
     }
 
+    @Override
     public String getSpanName() {
         return spanName;
     }
 
-    public SpanContext getSpanContext() {
-        return span.getSpanContext();
+    io.opentelemetry.api.trace.Span getOtelSpan() {
+        return otelSpan;
+    }
+
+    SpanContext getSpanContext() {
+        return otelSpan.getSpanContext();
     }
 
 }
diff --git a/server/src/main/java/org/opensearch/tracer/OTelMain.java b/server/src/main/java/org/opensearch/tracing/OTelMain.java
similarity index 81%
rename from server/src/main/java/org/opensearch/tracer/OTelMain.java
rename to server/src/main/java/org/opensearch/tracing/OTelMain.java
index 69d30e4865609..d1798bbdbe9f2 100644
--- a/server/src/main/java/org/opensearch/tracer/OTelMain.java
+++ b/server/src/main/java/org/opensearch/tracing/OTelMain.java
@@ -6,7 +6,7 @@
  * compatible open source license.
  */
 
-package org.opensearch.tracer;
+package org.opensearch.tracing;
 
 import io.opentelemetry.api.OpenTelemetry;
 import io.opentelemetry.api.common.Attributes;
@@ -22,20 +22,19 @@
 import java.util.concurrent.TimeUnit;
 
 public class OTelMain {
-    public static final OpenTelemetry openTelemetry;
 
-    static {
-        Resource resource = Resource.getDefault()
+    static final class OTelHolder {
+        private static final Resource resource = Resource.getDefault()
             .merge(Resource.create(Attributes.of(ResourceAttributes.SERVICE_NAME, "OpenSearch")));
 
-        SdkTracerProvider sdkTracerProvider = SdkTracerProvider.builder()
+        private static final SdkTracerProvider sdkTracerProvider = SdkTracerProvider.builder()
             .addSpanProcessor(BatchSpanProcessor.builder(OtlpGrpcSpanExporter.builder()
                 .setTimeout(2, TimeUnit.SECONDS).build()).build())
             .setResource(resource)
             .build();
 
 
-        openTelemetry = OpenTelemetrySdk.builder()
+        static final OpenTelemetry OPEN_TELEMETRY = OpenTelemetrySdk.builder()
             .setTracerProvider(sdkTracerProvider)
             .setPropagators(ContextPropagators.create(W3CTraceContextPropagator.getInstance()))
             .buildAndRegisterGlobal();
diff --git a/server/src/main/java/org/opensearch/tracing/Span.java b/server/src/main/java/org/opensearch/tracing/Span.java
new file mode 100644
index 0000000000000..a7d261d8a8cab
--- /dev/null
+++ b/server/src/main/java/org/opensearch/tracing/Span.java
@@ -0,0 +1,19 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.tracing;
+
+public interface Span {
+
+    Span getParentSpan();
+
+    String getSpanName();
+
+    Tracer.Level getLevel();
+
+}
diff --git a/server/src/main/java/org/opensearch/tracer/SpanHolder.java b/server/src/main/java/org/opensearch/tracing/SpanHolder.java
similarity index 75%
rename from server/src/main/java/org/opensearch/tracer/SpanHolder.java
rename to server/src/main/java/org/opensearch/tracing/SpanHolder.java
index ad3bc5b92c9a0..ad1c133db9fdf 100644
--- a/server/src/main/java/org/opensearch/tracer/SpanHolder.java
+++ b/server/src/main/java/org/opensearch/tracing/SpanHolder.java
@@ -6,7 +6,7 @@
  * compatible open source license.
  */
 
-package org.opensearch.tracer;
+package org.opensearch.tracing;
 
 import java.util.concurrent.atomic.AtomicReference;
 
@@ -15,11 +15,13 @@ public class SpanHolder {
     private final AtomicReference<Span> span = new AtomicReference<>();
 
     public SpanHolder(SpanHolder spanHolder) {
-        this.span.getAndSet(spanHolder.span.get());
+        if (spanHolder != null) {
+            this.span.set(spanHolder.span.get());
+        }
     }
 
     public SpanHolder(Span span) {
-        this.span.getAndSet(span);
+        this.span.set(span);
     }
 
     public Span getSpan() {
@@ -27,6 +29,6 @@ public Span getSpan() {
     }
 
     public void setSpan(Span span) {
-        this.span.getAndSet(span);
+        this.span.set(span);
     }
 }
diff --git a/server/src/main/java/org/opensearch/tracer/Tracer.java b/server/src/main/java/org/opensearch/tracing/Tracer.java
similarity index 68%
rename from server/src/main/java/org/opensearch/tracer/Tracer.java
rename to server/src/main/java/org/opensearch/tracing/Tracer.java
index 9ebf620a89895..64495ef4b2418 100644
--- a/server/src/main/java/org/opensearch/tracer/Tracer.java
+++ b/server/src/main/java/org/opensearch/tracing/Tracer.java
@@ -6,8 +6,11 @@
  * compatible open source license.
  */
 
-package org.opensearch.tracer;
+package org.opensearch.tracing;
 
+import org.opensearch.common.settings.Setting;
+
+import java.util.Arrays;
 import java.util.Map;
 
 public interface Tracer {
@@ -17,8 +20,8 @@ public interface Tracer {
      * <p>
      * Returns a span reference which can be used for ending it.
      */
-    default Span startTrace(String spanName, Map<String, String> attributes, Level level) {
-        return startTrace(spanName, attributes, null, level);
+    default void startSpan(String spanName, Map<String, Object> attributes, Level level) {
+        startSpan(spanName, attributes, null, level);
     }
 
     /**
@@ -36,44 +39,20 @@ default Span startTrace(String spanName, Map<String, String> attributes, Level l
      * the configured level will be published. Level of a child span can't be higher than the parent span so that it shouldn't get into a situation where parent
      * span is filtered out based on the level and child still exists; it will lead to a parent child linking issue and the child will be orphaned.
      */
-    Span startTrace(String spanName, Map<String, String> attributes, Span parentSpan, Level level);
+    void startSpan(String spanName, Map<String, Object> attributes, Span parentSpan, Level level);
 
     /**
      * Ends the scope of the trace. It is mandatory to end each span explicitly.
      */
-    void endTrace();
+    void endSpan();
 
     /**
      * Adds string attribute to the span.
      *
      * @param key   attribute key
-     * @param value attribute String value
-     */
-    void addAttribute(String key, String value);
-
-    /**
-     * Adds boolean attribute to the span.
-     *
-     * @param key   attribute key
-     * @param value attribute boolean value
+     * @param value attribute value
      */
-    void addAttribute(String key, boolean value);
-
-    /**
-     * Adds long attribute to the span.
-     *
-     * @param key   attribute key
-     * @param value attribute long value
-     */
-    void addAttribute(String key, long value);
-
-    /**
-     * Adds double attribute to the span.
-     *
-     * @param key   attribute key
-     * @param value attribute double value
-     */
-    void addAttribute(String key, double value);
+    void addAttribute(String key, Object value);
 
     /**
      * Adds an event to the current span.
@@ -86,20 +65,30 @@ default Span startTrace(String spanName, Map<String, String> attributes, Level l
      * Level is an OpenSearch specific concept which
      */
     enum Level {
-        UNKNOWN(0), LOWEST(1), LOW(2), MID(3), HIGH(4);
+        TRACE(0), DEBUG(1), INFO(2), TERSE(3), ROOT(4), DISABLED(5);
         private final int order;
 
         Level(int order) {
             this.order = order;
         }
 
-        public boolean isHigherThan(Level level) {
-            if (level != null) {
-                return this.order >= level.order;
-            } else {
-                return false;
+        public int getOrder() {
+            return order;
+        }
+
+        public static Level fromString(String type) {
+            for (Level level : values()) {
+                if (level.name().equalsIgnoreCase(type)) {
+                    return level;
+                }
             }
+            throw new IllegalArgumentException("invalid value for tracing level [" + type + "], " +
+                    "must be in " + Arrays.asList(Level.values()));
         }
     }
 
+    Setting<Level> TRACER_LEVEL_SETTING = new Setting<>(
+        "tracer.level", Level.DISABLED.name(),
+        Level::fromString, Setting.Property.NodeScope, Setting.Property.Dynamic);
+
 }
diff --git a/server/src/main/java/org/opensearch/tracing/TracerFactory.java b/server/src/main/java/org/opensearch/tracing/TracerFactory.java
new file mode 100644
index 0000000000000..7bf8082cdf585
--- /dev/null
+++ b/server/src/main/java/org/opensearch/tracing/TracerFactory.java
@@ -0,0 +1,49 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.tracing;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.opensearch.cluster.service.ClusterService;
+import org.opensearch.threadpool.ThreadPool;
+
+import static org.opensearch.tracing.Tracer.TRACER_LEVEL_SETTING;
+
+public class TracerFactory {
+
+    private static final Logger logger = LogManager.getLogger(TracerFactory.class);
+
+    private static final Tracer noopTracer = new NoopTracer();
+
+    private static ThreadPool threadPool;
+
+    private static ClusterService clusterSvc;
+
+    public static synchronized void initializeTracer(ThreadPool thPool, ClusterService clusterService) {
+        threadPool = thPool;
+        clusterSvc = clusterService;
+    }
+
+    public static Tracer getInstance() {
+        return isTracingDisabled() ? noopTracer : DefaultTracerInstanceHolder.INSTANCE;
+    }
+
+    private static boolean isTracingDisabled() {
+        return clusterSvc != null &&
+                clusterSvc.getClusterSettings() != null &&
+                clusterSvc.getClusterSettings().get(TRACER_LEVEL_SETTING) == Tracer.Level.DISABLED;
+    }
+
+    private static class DefaultTracerInstanceHolder {
+        private static final io.opentelemetry.api.trace.Tracer openTelemetryTracer =
+                OTelMain.OTelHolder.OPEN_TELEMETRY.getTracer("os-tracer");
+        static final Tracer INSTANCE = new DefaultTracer(openTelemetryTracer, threadPool, clusterSvc);
+    }
+
+}
diff --git a/server/src/main/java/org/opensearch/tracing/TracerUtils.java b/server/src/main/java/org/opensearch/tracing/TracerUtils.java
new file mode 100644
index 0000000000000..ad276d4e45ee7
--- /dev/null
+++ b/server/src/main/java/org/opensearch/tracing/TracerUtils.java
@@ -0,0 +1,75 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.tracing;
+
+import io.opentelemetry.context.Context;
+import io.opentelemetry.context.propagation.TextMapGetter;
+import io.opentelemetry.context.propagation.TextMapSetter;
+
+import java.util.Map;
+
+import static org.opensearch.tracing.DefaultTracer.CURRENT_SPAN;
+
+public class TracerUtils {
+
+    public static void addTracerContextToHeader(Map<String, String> requestHeaders, Map<String, Object> transientHeaders) {
+        TextMapSetter<Map<String, String>> setter = textMapSetter();
+        if (transientHeaders != null && transientHeaders.containsKey(CURRENT_SPAN)) {
+            SpanHolder spanHolder = (SpanHolder) transientHeaders.get(CURRENT_SPAN);
+            Span currentSpan = spanHolder.getSpan();
+            OSSpan osSpan = getLastOSSpanInChain(currentSpan);
+            OTelMain.OTelHolder.OPEN_TELEMETRY.getPropagators().getTextMapPropagator()
+                    .inject(context(osSpan), requestHeaders, setter);
+        }
+    }
+
+    public static Context extractTracerContextFromHeader(Map<String, String> headers) {
+        TextMapGetter<Map<String, String>> getter = textMapGetter();
+        return OTelMain.OTelHolder.OPEN_TELEMETRY.getPropagators().getTextMapPropagator()
+                .extract(Context.current(), headers, getter);
+    }
+
+    private static Context context(OSSpan osSpan) {
+        return Context.current().with(io.opentelemetry.api.trace.Span.wrap(osSpan.getSpanContext()));
+    }
+
+    private static TextMapGetter<Map<String, String>> textMapGetter() {
+        TextMapGetter<Map<String, String>> getter = new TextMapGetter<>() {
+            @Override
+            public Iterable<String> keys(Map<String, String> headers) {
+                return headers.keySet();
+            }
+
+            @Override
+            public String get(Map<String, String> headers, String key) {
+                if (headers != null && headers.containsKey(key)) {
+                    return headers.get(key);
+                }
+                return null;
+            }
+        };
+        return getter;
+    }
+
+    private static TextMapSetter<Map<String, String>> textMapSetter() {
+        TextMapSetter<Map<String, String>> setter = (carrier, key, value) -> {
+            if (carrier != null) {
+                carrier.put(key, value);
+            }
+        };
+        return setter;
+    }
+
+    private static OSSpan getLastOSSpanInChain(Span span) {
+        while (!(span instanceof OSSpan)) {
+            span = span.getParentSpan();
+        }
+        return (OSSpan) span;
+    }
+}

From 3e93e36cedd35eaa6a0385dec4e7fa59420eb563 Mon Sep 17 00:00:00 2001
From: suranjay <surajkumar.tu@gmail.com>
Date: Sun, 7 May 2023 21:01:18 +0530
Subject: [PATCH 3/3] Refactor search instrumentation

---
 .../org/opensearch/http/DefaultRestChannel.java  |  4 ++--
 .../shard/SearchOperationListenerTracerImpl.java | 12 ++++++------
 .../java/org/opensearch/rest/RestController.java | 13 ++++++++-----
 .../search/internal/ContextIndexSearcher.java    | 16 ++++++++--------
 4 files changed, 24 insertions(+), 21 deletions(-)

diff --git a/server/src/main/java/org/opensearch/http/DefaultRestChannel.java b/server/src/main/java/org/opensearch/http/DefaultRestChannel.java
index 0a8e337ef48c7..0642f08236a9c 100644
--- a/server/src/main/java/org/opensearch/http/DefaultRestChannel.java
+++ b/server/src/main/java/org/opensearch/http/DefaultRestChannel.java
@@ -43,7 +43,7 @@
 import org.opensearch.common.network.CloseableChannel;
 import org.opensearch.common.util.BigArrays;
 import org.opensearch.common.util.concurrent.ThreadContext;
-import org.opensearch.tracer.TracerFactory;
+import org.opensearch.tracing.TracerFactory;
 import org.opensearch.rest.AbstractRestChannel;
 import org.opensearch.rest.RestChannel;
 import org.opensearch.rest.RestRequest;
@@ -168,7 +168,7 @@ public void sendResponse(RestResponse restResponse) {
             if (request.uri().startsWith("/_search")) {
 
                 System.out.println("ending base request:" + request.getRequestId());
-                TracerFactory.getInstance().endTrace();
+                TracerFactory.getInstance().endSpan();
             }
             if (success == false) {
                 Releasables.close(toClose);
diff --git a/server/src/main/java/org/opensearch/index/shard/SearchOperationListenerTracerImpl.java b/server/src/main/java/org/opensearch/index/shard/SearchOperationListenerTracerImpl.java
index 80e36d97f4b11..f0d5129e28a75 100644
--- a/server/src/main/java/org/opensearch/index/shard/SearchOperationListenerTracerImpl.java
+++ b/server/src/main/java/org/opensearch/index/shard/SearchOperationListenerTracerImpl.java
@@ -8,8 +8,8 @@
 
 package org.opensearch.index.shard;
 
-import org.opensearch.tracer.Tracer.Level;
-import org.opensearch.tracer.TracerFactory;
+import org.opensearch.tracing.Tracer.Level;
+import org.opensearch.tracing.TracerFactory;
 import org.opensearch.search.internal.SearchContext;
 
 public class SearchOperationListenerTracerImpl implements SearchOperationListener {
@@ -17,25 +17,25 @@ public class SearchOperationListenerTracerImpl implements SearchOperationListene
     @Override
     public void onPreQueryPhase(SearchContext searchContext) {
         SearchOperationListener.super.onPreQueryPhase(searchContext);
-         TracerFactory.getInstance().startTrace("onQueryPhase", null, Level.MID);
+         TracerFactory.getInstance().startSpan("onQueryPhase", null, Level.INFO);
     }
 
     @Override
     public void onQueryPhase(SearchContext searchContext, long tookInNanos) {
         SearchOperationListener.super.onQueryPhase(searchContext, tookInNanos);
-        TracerFactory.getInstance().endTrace();
+        TracerFactory.getInstance().endSpan();
     }
 
     @Override
     public void onPreFetchPhase(SearchContext searchContext) {
         SearchOperationListener.super.onPreFetchPhase(searchContext);
-        TracerFactory.getInstance().startTrace("onFetchPhase", null, Level.MID);
+        TracerFactory.getInstance().startSpan("onFetchPhase", null, Level.INFO);
     }
 
     @Override
     public void onFetchPhase(SearchContext searchContext, long tookInNanos) {
         SearchOperationListener.super.onFetchPhase(searchContext, tookInNanos);
-        TracerFactory.getInstance().endTrace();
+        TracerFactory.getInstance().endSpan();
 
     }
 }
diff --git a/server/src/main/java/org/opensearch/rest/RestController.java b/server/src/main/java/org/opensearch/rest/RestController.java
index bdd33b2580c48..2e7b1e22ceee0 100644
--- a/server/src/main/java/org/opensearch/rest/RestController.java
+++ b/server/src/main/java/org/opensearch/rest/RestController.java
@@ -51,8 +51,8 @@
 import org.opensearch.common.util.io.Streams;
 import org.opensearch.http.HttpServerTransport;
 import org.opensearch.indices.breaker.CircuitBreakerService;
-import org.opensearch.tracer.Tracer;
-import org.opensearch.tracer.TracerFactory;
+import org.opensearch.tracing.Tracer;
+import org.opensearch.tracing.TracerFactory;
 import org.opensearch.usage.UsageService;
 
 import java.io.ByteArrayOutputStream;
@@ -242,9 +242,12 @@ public void dispatchRequest(RestRequest request, RestChannel channel, ThreadCont
         try {
             if (request.uri().startsWith("/_search")) {
                 System.out.println("Request Id:" + request.getRequestId() + " uri:" + request.uri() );
-                Map<String, String> map = new HashMap<>();
+                Map<String, Object> map = new HashMap<>();
+                map.put("boolean", true);
+                map.put("long", 2l);
+                map.put("double", 2.0);
                 map.put("Request_id", String.valueOf(request.getRequestId()));
-                TracerFactory.getInstance().startTrace("Request_" + String.valueOf(request.getRequestId()), map, Tracer.Level.HIGH);
+                TracerFactory.getInstance().startSpan("Request_" + String.valueOf(request.getRequestId()), map, Tracer.Level.ROOT);
             }
             tryAllHandlers(request, channel, threadContext);
         } catch (Exception e) {
@@ -581,7 +584,7 @@ public void sendResponse(RestResponse response) {
             close();
             final RestRequest request = request();
             System.out.println("Ending bbase request");
-            TracerFactory.getInstance().endTrace();
+            TracerFactory.getInstance().endSpan();
             delegate.sendResponse(response);
         }
 
diff --git a/server/src/main/java/org/opensearch/search/internal/ContextIndexSearcher.java b/server/src/main/java/org/opensearch/search/internal/ContextIndexSearcher.java
index 6ad1e5902cc11..873b564be6009 100644
--- a/server/src/main/java/org/opensearch/search/internal/ContextIndexSearcher.java
+++ b/server/src/main/java/org/opensearch/search/internal/ContextIndexSearcher.java
@@ -64,8 +64,8 @@
 import org.apache.lucene.util.SparseFixedBitSet;
 import org.opensearch.common.lease.Releasable;
 import org.opensearch.common.lucene.search.TopDocsAndMaxScore;
-import org.opensearch.tracer.Tracer.Level;
-import org.opensearch.tracer.TracerFactory;
+import org.opensearch.tracing.Tracer.Level;
+import org.opensearch.tracing.TracerFactory;
 import org.opensearch.search.DocValueFormat;
 import org.opensearch.search.dfs.AggregatedDfs;
 import org.opensearch.search.profile.ContextualProfileBreakdown;
@@ -247,7 +247,7 @@ public void search(
         DocValueFormat[] formats,
         TotalHits totalHits
     ) throws IOException {
-        TracerFactory.getInstance().startTrace("IndexSearcher", null, Level.LOW);
+        TracerFactory.getInstance().startSpan("IndexSearcher", null, Level.TERSE);
         final List<Collector> collectors = new ArrayList<>(leaves.size());
         for (LeafReaderContext ctx : leaves) {
             final Collector collector = manager.newCollector();
@@ -265,7 +265,7 @@ public void search(
         }
         result.topDocs(new TopDocsAndMaxScore(mergedTopDocs, Float.NaN), formats);
 
-        TracerFactory.getInstance().endTrace();
+        TracerFactory.getInstance().endSpan();
     }
 
     public void search(
@@ -289,7 +289,7 @@ public void search(
 
     @Override
     protected void search(List<LeafReaderContext> leaves, Weight weight, Collector collector) throws IOException {
-        TracerFactory.getInstance().startTrace("IndexSearcher", null, Level.LOW);
+        TracerFactory.getInstance().startSpan("IndexSearcher", null, Level.TERSE);
         if (reverseLeafReaderContexts) {
             // reverse the segment search order if this flag is true.
             for (int i = leaves.size() - 1; i >= 0; i--) {
@@ -300,7 +300,7 @@ protected void search(List<LeafReaderContext> leaves, Weight weight, Collector c
                 searchLeaf(leaves.get(i), weight, collector);
             }
         }
-        TracerFactory.getInstance().endTrace();
+        TracerFactory.getInstance().endSpan();
     }
 
     /**
@@ -310,7 +310,7 @@ protected void search(List<LeafReaderContext> leaves, Weight weight, Collector c
      * the provided <code>ctx</code>.
      */
     private void searchLeaf(LeafReaderContext ctx, Weight weight, Collector collector) throws IOException {
-        TracerFactory.getInstance().startTrace("IndexSearcher-Leaf", null, Level.LOW);
+        TracerFactory.getInstance().startSpan("IndexSearcher-Leaf", null, Level.DEBUG);
         cancellable.checkCancelled();
         weight = wrapWeight(weight);
         // See please https://github.com/apache/lucene/pull/964
@@ -352,7 +352,7 @@ private void searchLeaf(LeafReaderContext ctx, Weight weight, Collector collecto
                 }
             }
         }
-        TracerFactory.getInstance().endTrace();
+        TracerFactory.getInstance().endSpan();
     }
 
     private Weight wrapWeight(Weight weight) {