From 0886f5966c4cb582911e54d077abaa3353c9df61 Mon Sep 17 00:00:00 2001 From: Anirudh Sharma Date: Sun, 25 Jan 2026 00:56:40 +0000 Subject: [PATCH 01/13] feat(dotnet): Add OpenTelemetry support for distributed tracing and metrics Add opt-in OpenTelemetry instrumentation to the .NET SDK, enabling distributed tracing and metrics collection for Copilot sessions. ## Features - ActivitySource "GitHub.Copilot.SDK" for distributed tracing - Meter "GitHub.Copilot.SDK" for metrics - Spans for session, turn, tool execution, subagent, hook, and inference events - Metrics for token usage, cost, tool executions, errors, and duration - Follows OpenTelemetry GenAI Semantic Conventions ## Enabling Telemetry Telemetry is disabled by default. Enable via: - AppContext switch: `GitHub.Copilot.EnableOpenTelemetry` - Environment variable: `GITHUB_COPILOT_ENABLE_OPEN_TELEMETRY=true` ## New Files - src/Telemetry/CopilotTelemetry.cs - Main telemetry class - src/Telemetry/OpenTelemetryConstants.cs - Span/attribute constants - src/Telemetry/SessionTelemetryTracker.cs - Event to span conversion - test/TelemetryTests.cs - Unit tests --- dotnet/README.md | 88 ++++ dotnet/src/GitHub.Copilot.SDK.csproj | 5 + dotnet/src/Session.cs | 7 + dotnet/src/Telemetry/CopilotTelemetry.cs | 234 +++++++++++ .../src/Telemetry/OpenTelemetryConstants.cs | 139 +++++++ .../src/Telemetry/SessionTelemetryTracker.cs | 384 ++++++++++++++++++ dotnet/test/TelemetryTests.cs | 336 +++++++++++++++ 7 files changed, 1193 insertions(+) create mode 100644 dotnet/src/Telemetry/CopilotTelemetry.cs create mode 100644 dotnet/src/Telemetry/OpenTelemetryConstants.cs create mode 100644 dotnet/src/Telemetry/SessionTelemetryTracker.cs create mode 100644 dotnet/test/TelemetryTests.cs diff --git a/dotnet/README.md b/dotnet/README.md index e176da40..724269bb 100644 --- a/dotnet/README.md +++ b/dotnet/README.md @@ -444,6 +444,94 @@ var session = await client.CreateSessionAsync(new SessionConfig }); ``` +## Observability + +The SDK supports OpenTelemetry-based distributed tracing and metrics. Telemetry is disabled by default. + +### Enabling Telemetry + +Enable telemetry using one of these methods: + +**Option 1: AppContext switch (recommended)** +```csharp +AppContext.SetSwitch("GitHub.Copilot.EnableOpenTelemetry", true); +``` + +**Option 2: Environment variable** +```bash +export GITHUB_COPILOT_ENABLE_OPEN_TELEMETRY=true +``` + +### Configuring OpenTelemetry + +Configure your `TracerProvider` and `MeterProvider` to listen to the SDK: + +```csharp +using OpenTelemetry; +using OpenTelemetry.Trace; +using OpenTelemetry.Metrics; + +// Enable telemetry +AppContext.SetSwitch("GitHub.Copilot.Experimental.EnableOpenTelemetry", true); + +// Configure OpenTelemetry +services.AddOpenTelemetry() + .WithTracing(tracing => tracing + .AddSource("GitHub.Copilot.SDK") // Traces + .AddConsoleExporter()) // Or your preferred exporter + .WithMetrics(metrics => metrics + .AddMeter("GitHub.Copilot.SDK") // Metrics + .AddConsoleExporter()); +``` + +### Spans + +The SDK emits the following spans following [GenAI Semantic Conventions](https://opentelemetry.io/docs/specs/semconv/gen-ai/): + +| Span Name | Description | Key Attributes | +|-----------|-------------|----------------| +| `copilot.session` | Root span for a session | `copilot.session.id`, `gen_ai.system`, `gen_ai.request.model` | +| `copilot.turn` | Assistant turn lifecycle | `copilot.turn.id` | +| `copilot.tool_execution` | Tool execution | `gen_ai.tool.name`, `gen_ai.tool.call_id`, `copilot.success` | +| `copilot.subagent` | Subagent execution | `copilot.subagent.name` | +| `copilot.hook` | Hook execution | `copilot.hook.type`, `copilot.hook.invocation_id` | +| `copilot.inference` | LLM inference call | `gen_ai.usage.input_tokens`, `gen_ai.usage.output_tokens`, `copilot.cost` | + +### Metrics + +The SDK emits the following metrics: + +| Metric Name | Type | Description | +|-------------|------|-------------| +| `copilot.tokens.input` | Counter | Number of input tokens used | +| `copilot.tokens.output` | Counter | Number of output tokens generated | +| `copilot.cost.total` | Counter | Total cost of operations | +| `copilot.tool_executions` | Counter | Number of tool executions | +| `copilot.errors` | Counter | Number of errors | +| `copilot.duration` | Histogram | Duration of operations in milliseconds | + +### Example Trace + +``` +copilot.session (root) +├── gen_ai.system = "github-copilot" +├── gen_ai.request.model = "gpt-4o" +├── copilot.session.id = "abc123" +│ +├── copilot.turn +│ ├── copilot.turn.id = "turn-1" +│ │ +│ ├── copilot.tool_execution +│ │ ├── gen_ai.tool.name = "file_edit" +│ │ ├── gen_ai.tool.call_id = "call-xyz" +│ │ └── copilot.success = true +│ │ +│ └── copilot.inference +│ ├── gen_ai.usage.input_tokens = 500 +│ ├── gen_ai.usage.output_tokens = 200 +│ └── copilot.cost = 0.003 +``` + ## Error Handling ```csharp diff --git a/dotnet/src/GitHub.Copilot.SDK.csproj b/dotnet/src/GitHub.Copilot.SDK.csproj index 10cfd98a..9e63588c 100644 --- a/dotnet/src/GitHub.Copilot.SDK.csproj +++ b/dotnet/src/GitHub.Copilot.SDK.csproj @@ -21,9 +21,14 @@ + + + + + diff --git a/dotnet/src/Session.cs b/dotnet/src/Session.cs index f1e47df8..63c93ec4 100644 --- a/dotnet/src/Session.cs +++ b/dotnet/src/Session.cs @@ -2,6 +2,7 @@ * Copyright (c) Microsoft Corporation. All rights reserved. *--------------------------------------------------------------------------------------------*/ +using GitHub.Copilot.SDK.Telemetry; using Microsoft.Extensions.AI; using StreamJsonRpc; using System.Text.Json; @@ -48,6 +49,7 @@ public partial class CopilotSession : IAsyncDisposable private readonly JsonRpc _rpc; private PermissionHandler? _permissionHandler; private readonly SemaphoreSlim _permissionHandlerLock = new(1, 1); + private readonly SessionTelemetryTracker _telemetryTracker; /// /// Gets the unique identifier for this session. @@ -78,6 +80,7 @@ internal CopilotSession(string sessionId, JsonRpc rpc, string? workspacePath = n SessionId = sessionId; _rpc = rpc; WorkspacePath = workspacePath; + _telemetryTracker = new SessionTelemetryTracker(sessionId); } /// @@ -237,6 +240,9 @@ public IDisposable On(SessionEventHandler handler) /// internal void DispatchEvent(SessionEvent sessionEvent) { + // Record telemetry for the event + _telemetryTracker.ProcessEvent(sessionEvent); + foreach (var handler in _eventHandlers.ToArray()) { // We allow handler exceptions to propagate so they are not lost @@ -421,6 +427,7 @@ await _rpc.InvokeWithCancellationAsync( _eventHandlers.Clear(); _toolHandlers.Clear(); + _telemetryTracker.Dispose(); await _permissionHandlerLock.WaitAsync(); try diff --git a/dotnet/src/Telemetry/CopilotTelemetry.cs b/dotnet/src/Telemetry/CopilotTelemetry.cs new file mode 100644 index 00000000..0189b6e9 --- /dev/null +++ b/dotnet/src/Telemetry/CopilotTelemetry.cs @@ -0,0 +1,234 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + *--------------------------------------------------------------------------------------------*/ + +using System.Diagnostics; +using System.Diagnostics.Metrics; + +namespace GitHub.Copilot.SDK.Telemetry; + +/// +/// Provides OpenTelemetry instrumentation for the GitHub Copilot SDK. +/// +/// +/// +/// Telemetry is disabled by default. Enable it using one of these methods: +/// +/// +/// Set the AppContext switch: AppContext.SetSwitch("GitHub.Copilot.EnableOpenTelemetry", true) +/// Set the environment variable: GITHUB_COPILOT_ENABLE_OPEN_TELEMETRY=true +/// +/// +/// Then configure your TracerProvider and MeterProvider to listen: +/// +/// +/// services.AddOpenTelemetry() +/// .WithTracing(tracing => tracing.AddSource("GitHub.Copilot.SDK")) +/// .WithMetrics(metrics => metrics.AddMeter("GitHub.Copilot.SDK")); +/// +/// +public static class CopilotTelemetry +{ + private static readonly Lazy s_isEnabled = new(DetermineIfEnabled); + + /// + /// Gets the ActivitySource for creating spans. + /// + internal static ActivitySource ActivitySource { get; } = new( + OpenTelemetryConstants.ActivitySourceName, + typeof(CopilotTelemetry).Assembly.GetName().Version?.ToString() ?? "1.0.0"); + + /// + /// Gets the Meter for recording metrics. + /// + internal static Meter Meter { get; } = new( + OpenTelemetryConstants.MeterName, + typeof(CopilotTelemetry).Assembly.GetName().Version?.ToString() ?? "1.0.0"); + + // Metrics instruments + internal static Counter TokensInputCounter { get; } = Meter.CreateCounter( + OpenTelemetryConstants.MetricTokensInput, + unit: "{token}", + description: "Number of input tokens used"); + + internal static Counter TokensOutputCounter { get; } = Meter.CreateCounter( + OpenTelemetryConstants.MetricTokensOutput, + unit: "{token}", + description: "Number of output tokens generated"); + + internal static Counter CostCounter { get; } = Meter.CreateCounter( + OpenTelemetryConstants.MetricCostTotal, + unit: "{dollar}", + description: "Total cost of operations"); + + internal static Counter ToolExecutionsCounter { get; } = Meter.CreateCounter( + OpenTelemetryConstants.MetricToolExecutions, + unit: "{execution}", + description: "Number of tool executions"); + + internal static Counter ErrorsCounter { get; } = Meter.CreateCounter( + OpenTelemetryConstants.MetricErrors, + unit: "{error}", + description: "Number of errors"); + + internal static Histogram DurationHistogram { get; } = Meter.CreateHistogram( + OpenTelemetryConstants.MetricDuration, + unit: "ms", + description: "Duration of operations in milliseconds"); + + /// + /// Gets a value indicating whether telemetry is enabled. + /// + public static bool IsEnabled => s_isEnabled.Value; + + private static bool DetermineIfEnabled() + { + // Check AppContext switch first + if (AppContext.TryGetSwitch(OpenTelemetryConstants.EnableTelemetrySwitch, out var isEnabled)) + { + return isEnabled; + } + + // Fall back to environment variable + var envValue = Environment.GetEnvironmentVariable(OpenTelemetryConstants.EnableTelemetryEnvVar); + return string.Equals(envValue, "true", StringComparison.OrdinalIgnoreCase) || + string.Equals(envValue, "1", StringComparison.Ordinal); + } + + /// + /// Starts an activity (span) if telemetry is enabled and there are listeners. + /// + internal static Activity? StartActivity(string name, ActivityKind kind = ActivityKind.Internal) + { + if (!IsEnabled) + { + return null; + } + + return ActivitySource.StartActivity(name, kind); + } + + /// + /// Sets common GenAI attributes on an activity. + /// + internal static void SetGenAiAttributes(Activity? activity, string? model = null) + { + if (activity is null) return; + + activity.SetTag(OpenTelemetryConstants.GenAiSystem, "github-copilot"); + + if (!string.IsNullOrEmpty(model)) + { + activity.SetTag(OpenTelemetryConstants.GenAiRequestModel, model); + } + } + + /// + /// Records token usage metrics. + /// + internal static void RecordTokenUsage( + long? inputTokens, + long? outputTokens, + double? cost, + string? model, + string? sessionId) + { + if (!IsEnabled) return; + + var tags = new TagList + { + { OpenTelemetryConstants.GenAiSystem, "github-copilot" } + }; + + if (!string.IsNullOrEmpty(model)) + { + tags.Add(OpenTelemetryConstants.GenAiRequestModel, model); + } + + if (!string.IsNullOrEmpty(sessionId)) + { + tags.Add(OpenTelemetryConstants.CopilotSessionId, sessionId); + } + + if (inputTokens.HasValue) + { + TokensInputCounter.Add(inputTokens.Value, tags); + } + + if (outputTokens.HasValue) + { + TokensOutputCounter.Add(outputTokens.Value, tags); + } + + if (cost.HasValue) + { + CostCounter.Add(cost.Value, tags); + } + } + + /// + /// Records a tool execution metric. + /// + internal static void RecordToolExecution(string toolName, bool success, string? sessionId) + { + if (!IsEnabled) return; + + var tags = new TagList + { + { OpenTelemetryConstants.GenAiToolName, toolName }, + { OpenTelemetryConstants.CopilotSuccess, success } + }; + + if (!string.IsNullOrEmpty(sessionId)) + { + tags.Add(OpenTelemetryConstants.CopilotSessionId, sessionId); + } + + ToolExecutionsCounter.Add(1, tags); + + if (!success) + { + ErrorsCounter.Add(1, tags); + } + } + + /// + /// Records an error metric. + /// + internal static void RecordError(string errorType, string? sessionId) + { + if (!IsEnabled) return; + + var tags = new TagList + { + { "error.type", errorType } + }; + + if (!string.IsNullOrEmpty(sessionId)) + { + tags.Add(OpenTelemetryConstants.CopilotSessionId, sessionId); + } + + ErrorsCounter.Add(1, tags); + } + + /// + /// Records duration metric. + /// + internal static void RecordDuration(double durationMs, string operationType, string? sessionId) + { + if (!IsEnabled) return; + + var tags = new TagList + { + { OpenTelemetryConstants.GenAiOperationName, operationType } + }; + + if (!string.IsNullOrEmpty(sessionId)) + { + tags.Add(OpenTelemetryConstants.CopilotSessionId, sessionId); + } + + DurationHistogram.Record(durationMs, tags); + } +} diff --git a/dotnet/src/Telemetry/OpenTelemetryConstants.cs b/dotnet/src/Telemetry/OpenTelemetryConstants.cs new file mode 100644 index 00000000..0f45bfb3 --- /dev/null +++ b/dotnet/src/Telemetry/OpenTelemetryConstants.cs @@ -0,0 +1,139 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + *--------------------------------------------------------------------------------------------*/ + +namespace GitHub.Copilot.SDK.Telemetry; + +/// +/// Constants for OpenTelemetry instrumentation following GenAI Semantic Conventions. +/// +/// +/// See: https://opentelemetry.io/docs/specs/semconv/gen-ai/ +/// +internal static class OpenTelemetryConstants +{ + /// + /// The AppContext switch to enable OpenTelemetry telemetry. + /// + public const string EnableTelemetrySwitch = "GitHub.Copilot.EnableOpenTelemetry"; + + /// + /// Environment variable to enable OpenTelemetry telemetry. + /// + public const string EnableTelemetryEnvVar = "GITHUB_COPILOT_ENABLE_OPEN_TELEMETRY"; + + /// + /// The ActivitySource name for GitHub Copilot SDK telemetry. + /// + public const string ActivitySourceName = "GitHub.Copilot.SDK"; + + /// + /// The Meter name for GitHub Copilot SDK metrics. + /// + public const string MeterName = "GitHub.Copilot.SDK"; + + // GenAI Semantic Convention attribute names + // See: https://opentelemetry.io/docs/specs/semconv/gen-ai/gen-ai-spans/ + + /// The name of the GenAI system (e.g., "github-copilot"). + public const string GenAiSystem = "gen_ai.system"; + + /// The operation being performed (e.g., "chat"). + public const string GenAiOperationName = "gen_ai.operation.name"; + + /// The model requested by the user. + public const string GenAiRequestModel = "gen_ai.request.model"; + + /// The model that generated the response. + public const string GenAiResponseModel = "gen_ai.response.model"; + + /// Number of input tokens used. + public const string GenAiUsageInputTokens = "gen_ai.usage.input_tokens"; + + /// Number of output tokens generated. + public const string GenAiUsageOutputTokens = "gen_ai.usage.output_tokens"; + + /// The name of the tool being called. + public const string GenAiToolName = "gen_ai.tool.name"; + + /// The unique identifier for the tool call. + public const string GenAiToolCallId = "gen_ai.tool.call_id"; + + // Copilot-specific attributes + + /// The session identifier. + public const string CopilotSessionId = "copilot.session.id"; + + /// The turn identifier within a session. + public const string CopilotTurnId = "copilot.turn.id"; + + /// The subagent name. + public const string CopilotSubagentName = "copilot.subagent.name"; + + /// The hook type. + public const string CopilotHookType = "copilot.hook.type"; + + /// The hook invocation identifier. + public const string CopilotHookInvocationId = "copilot.hook.invocation_id"; + + /// Whether the operation succeeded. + public const string CopilotSuccess = "copilot.success"; + + /// The error message if failed. + public const string CopilotErrorMessage = "copilot.error.message"; + + /// The cost of the operation. + public const string CopilotCost = "copilot.cost"; + + /// Duration of the operation in milliseconds. + public const string CopilotDurationMs = "copilot.duration_ms"; + + /// Cache read tokens. + public const string CopilotCacheReadTokens = "copilot.cache.read_tokens"; + + /// Cache write tokens. + public const string CopilotCacheWriteTokens = "copilot.cache.write_tokens"; + + // Span names + + /// Span name for a session. + public const string SpanNameSession = "copilot.session"; + + /// Span name for an assistant turn. + public const string SpanNameTurn = "copilot.turn"; + + /// Span name for tool execution. + public const string SpanNameToolExecution = "copilot.tool_execution"; + + /// Span name for subagent execution. + public const string SpanNameSubagent = "copilot.subagent"; + + /// Span name for hook execution. + public const string SpanNameHook = "copilot.hook"; + + /// Span name for inference/LLM call. + public const string SpanNameInference = "copilot.inference"; + + // Metric names + + /// Counter for total tokens used. + public const string MetricTokensTotal = "copilot.tokens.total"; + + /// Counter for input tokens. + public const string MetricTokensInput = "copilot.tokens.input"; + + /// Counter for output tokens. + public const string MetricTokensOutput = "copilot.tokens.output"; + + /// Counter for total cost. + public const string MetricCostTotal = "copilot.cost.total"; + + /// Histogram for operation duration. + public const string MetricDuration = "copilot.duration"; + + /// Counter for tool executions. + public const string MetricToolExecutions = "copilot.tool_executions"; + + /// Counter for session errors. + public const string MetricErrors = "copilot.errors"; +} diff --git a/dotnet/src/Telemetry/SessionTelemetryTracker.cs b/dotnet/src/Telemetry/SessionTelemetryTracker.cs new file mode 100644 index 00000000..3e7d4e8b --- /dev/null +++ b/dotnet/src/Telemetry/SessionTelemetryTracker.cs @@ -0,0 +1,384 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + *--------------------------------------------------------------------------------------------*/ + +using System.Collections.Concurrent; +using System.Diagnostics; + +namespace GitHub.Copilot.SDK.Telemetry; + +/// +/// Tracks active spans for a Copilot session based on session events. +/// +internal sealed class SessionTelemetryTracker : IDisposable +{ + private readonly string _sessionId; + private readonly ConcurrentDictionary _turnActivities = new(); + private readonly ConcurrentDictionary _toolActivities = new(); + private readonly ConcurrentDictionary _subagentActivities = new(); + private readonly ConcurrentDictionary _hookActivities = new(); + private Activity? _sessionActivity; + private string? _currentModel; + private bool _disposed; + + public SessionTelemetryTracker(string sessionId) + { + _sessionId = sessionId; + } + + /// + /// Processes a session event and creates/completes appropriate spans. + /// + public void ProcessEvent(SessionEvent sessionEvent) + { + if (!CopilotTelemetry.IsEnabled || _disposed) + { + return; + } + + switch (sessionEvent) + { + // Session lifecycle + case SessionStartEvent startEvent: + OnSessionStart(startEvent); + break; + case SessionIdleEvent: + case SessionErrorEvent errorEvent: + OnSessionEnd(sessionEvent as SessionErrorEvent); + break; + + // Model changes + case SessionModelChangeEvent modelChangeEvent: + _currentModel = modelChangeEvent.Data.NewModel; + _sessionActivity?.SetTag(OpenTelemetryConstants.GenAiResponseModel, _currentModel); + break; + + // Turn lifecycle + case AssistantTurnStartEvent turnStartEvent: + OnTurnStart(turnStartEvent); + break; + case AssistantTurnEndEvent turnEndEvent: + OnTurnEnd(turnEndEvent); + break; + + // Tool execution + case ToolExecutionStartEvent toolStartEvent: + OnToolExecutionStart(toolStartEvent); + break; + case ToolExecutionCompleteEvent toolCompleteEvent: + OnToolExecutionComplete(toolCompleteEvent); + break; + + // Subagent lifecycle + case SubagentStartedEvent subagentStartEvent: + OnSubagentStart(subagentStartEvent); + break; + case SubagentCompletedEvent subagentCompletedEvent: + OnSubagentComplete(subagentCompletedEvent, success: true); + break; + case SubagentFailedEvent subagentFailedEvent: + OnSubagentFailed(subagentFailedEvent); + break; + + // Hook lifecycle + case HookStartEvent hookStartEvent: + OnHookStart(hookStartEvent); + break; + case HookEndEvent hookEndEvent: + OnHookEnd(hookEndEvent); + break; + + // Usage/metrics + case AssistantUsageEvent usageEvent: + OnUsage(usageEvent); + break; + } + } + + private void OnSessionStart(SessionStartEvent startEvent) + { + _sessionActivity = CopilotTelemetry.StartActivity( + OpenTelemetryConstants.SpanNameSession, + ActivityKind.Server); + + if (_sessionActivity is null) return; + + _currentModel = startEvent.Data.SelectedModel; + + _sessionActivity.SetTag(OpenTelemetryConstants.CopilotSessionId, _sessionId); + CopilotTelemetry.SetGenAiAttributes(_sessionActivity, _currentModel); + _sessionActivity.SetTag(OpenTelemetryConstants.GenAiOperationName, "chat"); + + if (startEvent.Data.Context != null) + { + _sessionActivity.SetTag("copilot.context.cwd", startEvent.Data.Context.Cwd); + if (startEvent.Data.Context.Repository != null) + { + _sessionActivity.SetTag("copilot.context.repository", startEvent.Data.Context.Repository); + } + } + } + + private void OnSessionEnd(SessionErrorEvent? errorEvent) + { + if (_sessionActivity is null) return; + + if (errorEvent != null) + { + _sessionActivity.SetStatus(ActivityStatusCode.Error, errorEvent.Data.Message); + _sessionActivity.SetTag(OpenTelemetryConstants.CopilotErrorMessage, errorEvent.Data.Message); + _sessionActivity.SetTag("error.type", errorEvent.Data.ErrorType); + CopilotTelemetry.RecordError(errorEvent.Data.ErrorType, _sessionId); + } + else + { + _sessionActivity.SetStatus(ActivityStatusCode.Ok); + } + + _sessionActivity.Dispose(); + _sessionActivity = null; + } + + private void OnTurnStart(AssistantTurnStartEvent turnStartEvent) + { + var turnActivity = CopilotTelemetry.StartActivity( + OpenTelemetryConstants.SpanNameTurn, + ActivityKind.Internal); + + if (turnActivity is null) return; + + turnActivity.SetTag(OpenTelemetryConstants.CopilotSessionId, _sessionId); + turnActivity.SetTag(OpenTelemetryConstants.CopilotTurnId, turnStartEvent.Data.TurnId); + CopilotTelemetry.SetGenAiAttributes(turnActivity, _currentModel); + + _turnActivities[turnStartEvent.Data.TurnId] = turnActivity; + } + + private void OnTurnEnd(AssistantTurnEndEvent turnEndEvent) + { + if (_turnActivities.TryRemove(turnEndEvent.Data.TurnId, out var turnActivity)) + { + turnActivity.SetStatus(ActivityStatusCode.Ok); + turnActivity.Dispose(); + } + } + + private void OnToolExecutionStart(ToolExecutionStartEvent toolStartEvent) + { + var toolActivity = CopilotTelemetry.StartActivity( + OpenTelemetryConstants.SpanNameToolExecution, + ActivityKind.Internal); + + if (toolActivity is null) return; + + toolActivity.SetTag(OpenTelemetryConstants.CopilotSessionId, _sessionId); + toolActivity.SetTag(OpenTelemetryConstants.GenAiToolName, toolStartEvent.Data.ToolName); + toolActivity.SetTag(OpenTelemetryConstants.GenAiToolCallId, toolStartEvent.Data.ToolCallId); + + if (toolStartEvent.Data.ParentToolCallId != null) + { + toolActivity.SetTag("copilot.parent_tool_call_id", toolStartEvent.Data.ParentToolCallId); + } + + _toolActivities[toolStartEvent.Data.ToolCallId] = toolActivity; + } + + private void OnToolExecutionComplete(ToolExecutionCompleteEvent toolCompleteEvent) + { + if (_toolActivities.TryRemove(toolCompleteEvent.Data.ToolCallId, out var toolActivity)) + { + toolActivity.SetTag(OpenTelemetryConstants.CopilotSuccess, toolCompleteEvent.Data.Success); + + if (toolCompleteEvent.Data.Success) + { + toolActivity.SetStatus(ActivityStatusCode.Ok); + } + else + { + toolActivity.SetStatus(ActivityStatusCode.Error, toolCompleteEvent.Data.Error?.Message); + if (toolCompleteEvent.Data.Error != null) + { + toolActivity.SetTag(OpenTelemetryConstants.CopilotErrorMessage, toolCompleteEvent.Data.Error.Message); + } + } + + toolActivity.Dispose(); + + // Record metric - need to get tool name from activity tags + var toolName = toolActivity.GetTagItem(OpenTelemetryConstants.GenAiToolName)?.ToString() ?? "unknown"; + CopilotTelemetry.RecordToolExecution(toolName, toolCompleteEvent.Data.Success, _sessionId); + } + } + + private void OnSubagentStart(SubagentStartedEvent subagentStartEvent) + { + var subagentActivity = CopilotTelemetry.StartActivity( + OpenTelemetryConstants.SpanNameSubagent, + ActivityKind.Internal); + + if (subagentActivity is null) return; + + subagentActivity.SetTag(OpenTelemetryConstants.CopilotSessionId, _sessionId); + subagentActivity.SetTag(OpenTelemetryConstants.CopilotSubagentName, subagentStartEvent.Data.AgentName); + subagentActivity.SetTag("copilot.subagent.display_name", subagentStartEvent.Data.AgentDisplayName); + subagentActivity.SetTag(OpenTelemetryConstants.GenAiToolCallId, subagentStartEvent.Data.ToolCallId); + + _subagentActivities[subagentStartEvent.Data.ToolCallId] = subagentActivity; + } + + private void OnSubagentComplete(SubagentCompletedEvent subagentCompletedEvent, bool success) + { + if (_subagentActivities.TryRemove(subagentCompletedEvent.Data.ToolCallId, out var subagentActivity)) + { + subagentActivity.SetTag(OpenTelemetryConstants.CopilotSuccess, success); + subagentActivity.SetStatus(ActivityStatusCode.Ok); + subagentActivity.Dispose(); + } + } + + private void OnSubagentFailed(SubagentFailedEvent subagentFailedEvent) + { + if (_subagentActivities.TryRemove(subagentFailedEvent.Data.ToolCallId, out var subagentActivity)) + { + subagentActivity.SetTag(OpenTelemetryConstants.CopilotSuccess, false); + subagentActivity.SetStatus(ActivityStatusCode.Error, subagentFailedEvent.Data.Error); + subagentActivity.SetTag(OpenTelemetryConstants.CopilotErrorMessage, subagentFailedEvent.Data.Error); + subagentActivity.Dispose(); + + CopilotTelemetry.RecordError("subagent_failed", _sessionId); + } + } + + private void OnHookStart(HookStartEvent hookStartEvent) + { + var hookActivity = CopilotTelemetry.StartActivity( + OpenTelemetryConstants.SpanNameHook, + ActivityKind.Internal); + + if (hookActivity is null) return; + + hookActivity.SetTag(OpenTelemetryConstants.CopilotSessionId, _sessionId); + hookActivity.SetTag(OpenTelemetryConstants.CopilotHookType, hookStartEvent.Data.HookType); + hookActivity.SetTag(OpenTelemetryConstants.CopilotHookInvocationId, hookStartEvent.Data.HookInvocationId); + + _hookActivities[hookStartEvent.Data.HookInvocationId] = hookActivity; + } + + private void OnHookEnd(HookEndEvent hookEndEvent) + { + if (_hookActivities.TryRemove(hookEndEvent.Data.HookInvocationId, out var hookActivity)) + { + hookActivity.SetTag(OpenTelemetryConstants.CopilotSuccess, hookEndEvent.Data.Success); + + if (hookEndEvent.Data.Success) + { + hookActivity.SetStatus(ActivityStatusCode.Ok); + } + else + { + hookActivity.SetStatus(ActivityStatusCode.Error, hookEndEvent.Data.Error?.Message); + if (hookEndEvent.Data.Error != null) + { + hookActivity.SetTag(OpenTelemetryConstants.CopilotErrorMessage, hookEndEvent.Data.Error.Message); + } + } + + hookActivity.Dispose(); + } + } + + private void OnUsage(AssistantUsageEvent usageEvent) + { + var data = usageEvent.Data; + + // Create an inference span for the LLM call + using var inferenceActivity = CopilotTelemetry.StartActivity( + OpenTelemetryConstants.SpanNameInference, + ActivityKind.Client); + + if (inferenceActivity != null) + { + inferenceActivity.SetTag(OpenTelemetryConstants.CopilotSessionId, _sessionId); + inferenceActivity.SetTag(OpenTelemetryConstants.GenAiOperationName, "chat"); + + if (data.Model != null) + { + inferenceActivity.SetTag(OpenTelemetryConstants.GenAiResponseModel, data.Model); + } + + if (data.InputTokens.HasValue) + { + inferenceActivity.SetTag(OpenTelemetryConstants.GenAiUsageInputTokens, (long)data.InputTokens.Value); + } + + if (data.OutputTokens.HasValue) + { + inferenceActivity.SetTag(OpenTelemetryConstants.GenAiUsageOutputTokens, (long)data.OutputTokens.Value); + } + + if (data.Cost.HasValue) + { + inferenceActivity.SetTag(OpenTelemetryConstants.CopilotCost, data.Cost.Value); + } + + if (data.Duration.HasValue) + { + inferenceActivity.SetTag(OpenTelemetryConstants.CopilotDurationMs, data.Duration.Value); + CopilotTelemetry.RecordDuration(data.Duration.Value, "inference", _sessionId); + } + + if (data.CacheReadTokens.HasValue) + { + inferenceActivity.SetTag(OpenTelemetryConstants.CopilotCacheReadTokens, (long)data.CacheReadTokens.Value); + } + + if (data.CacheWriteTokens.HasValue) + { + inferenceActivity.SetTag(OpenTelemetryConstants.CopilotCacheWriteTokens, (long)data.CacheWriteTokens.Value); + } + + inferenceActivity.SetStatus(ActivityStatusCode.Ok); + } + + // Record metrics + CopilotTelemetry.RecordTokenUsage( + inputTokens: data.InputTokens.HasValue ? (long)data.InputTokens.Value : null, + outputTokens: data.OutputTokens.HasValue ? (long)data.OutputTokens.Value : null, + cost: data.Cost, + model: data.Model, + sessionId: _sessionId); + } + + public void Dispose() + { + if (_disposed) return; + _disposed = true; + + // Clean up any remaining activities + _sessionActivity?.Dispose(); + + foreach (var activity in _turnActivities.Values) + { + activity.Dispose(); + } + _turnActivities.Clear(); + + foreach (var activity in _toolActivities.Values) + { + activity.Dispose(); + } + _toolActivities.Clear(); + + foreach (var activity in _subagentActivities.Values) + { + activity.Dispose(); + } + _subagentActivities.Clear(); + + foreach (var activity in _hookActivities.Values) + { + activity.Dispose(); + } + _hookActivities.Clear(); + } +} diff --git a/dotnet/test/TelemetryTests.cs b/dotnet/test/TelemetryTests.cs new file mode 100644 index 00000000..a9188eb9 --- /dev/null +++ b/dotnet/test/TelemetryTests.cs @@ -0,0 +1,336 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + *--------------------------------------------------------------------------------------------*/ + +using GitHub.Copilot.SDK.Telemetry; +using System.Diagnostics; +using Xunit; + +namespace GitHub.Copilot.SDK.Test; + +public class TelemetryTests : IDisposable +{ + private readonly ActivityListener _listener; + private readonly List _recordedActivities = new(); + + public TelemetryTests() + { + // Enable telemetry for tests + AppContext.SetSwitch("GitHub.Copilot.EnableOpenTelemetry", true); + + // Set up an activity listener to capture spans + _listener = new ActivityListener + { + ShouldListenTo = source => source.Name == OpenTelemetryConstants.ActivitySourceName, + Sample = (ref ActivityCreationOptions options) => ActivitySamplingResult.AllData, + ActivityStarted = activity => { }, + ActivityStopped = activity => _recordedActivities.Add(activity) + }; + ActivitySource.AddActivityListener(_listener); + } + + public void Dispose() + { + _listener.Dispose(); + // Note: We can't easily reset the AppContext switch, but it won't affect other tests + // since IsEnabled is lazily evaluated once + } + + [Fact] + public void TelemetryIsEnabled_WhenAppContextSwitchSet() + { + Assert.True(CopilotTelemetry.IsEnabled); + } + + [Fact] + public void ActivitySourceName_IsCorrect() + { + Assert.Equal("GitHub.Copilot.SDK", CopilotTelemetry.ActivitySource.Name); + } + + [Fact] + public void MeterName_IsCorrect() + { + Assert.Equal("GitHub.Copilot.SDK", CopilotTelemetry.Meter.Name); + } + + [Fact] + public void SessionTelemetryTracker_ProcessesSessionStartEvent() + { + // Arrange + var tracker = new SessionTelemetryTracker("test-session-123"); + var sessionStartEvent = CreateSessionStartEvent("test-session-123", "gpt-4o"); + + // Act + tracker.ProcessEvent(sessionStartEvent); + + // Assert - check that an activity was started + // Note: The activity may not be stopped yet, so we check it exists + Assert.NotNull(CopilotTelemetry.ActivitySource); + } + + [Fact] + public void SessionTelemetryTracker_ProcessesToolExecutionEvents() + { + // Arrange + _recordedActivities.Clear(); + var tracker = new SessionTelemetryTracker("test-session-456"); + + var toolStartEvent = CreateToolExecutionStartEvent("call-123", "file_edit"); + var toolCompleteEvent = CreateToolExecutionCompleteEvent("call-123", success: true); + + // Act + tracker.ProcessEvent(toolStartEvent); + tracker.ProcessEvent(toolCompleteEvent); + + // Assert - tool execution span should be recorded + var toolActivity = _recordedActivities.FirstOrDefault(a => + a.OperationName == OpenTelemetryConstants.SpanNameToolExecution); + + Assert.NotNull(toolActivity); + Assert.Equal("file_edit", toolActivity!.GetTagItem(OpenTelemetryConstants.GenAiToolName)); + Assert.Equal("call-123", toolActivity.GetTagItem(OpenTelemetryConstants.GenAiToolCallId)); + Assert.Equal(true, toolActivity.GetTagItem(OpenTelemetryConstants.CopilotSuccess)); + Assert.Equal(ActivityStatusCode.Ok, toolActivity.Status); + } + + [Fact] + public void SessionTelemetryTracker_ProcessesFailedToolExecution() + { + // Arrange + _recordedActivities.Clear(); + var tracker = new SessionTelemetryTracker("test-session-789"); + + var toolStartEvent = CreateToolExecutionStartEvent("call-fail", "broken_tool"); + var toolCompleteEvent = CreateToolExecutionCompleteEvent("call-fail", success: false, errorMessage: "Something went wrong"); + + // Act + tracker.ProcessEvent(toolStartEvent); + tracker.ProcessEvent(toolCompleteEvent); + + // Assert + var toolActivity = _recordedActivities.FirstOrDefault(a => + a.OperationName == OpenTelemetryConstants.SpanNameToolExecution); + + Assert.NotNull(toolActivity); + Assert.Equal(false, toolActivity!.GetTagItem(OpenTelemetryConstants.CopilotSuccess)); + Assert.Equal(ActivityStatusCode.Error, toolActivity.Status); + Assert.Equal("Something went wrong", toolActivity.GetTagItem(OpenTelemetryConstants.CopilotErrorMessage)); + } + + [Fact] + public void SessionTelemetryTracker_ProcessesTurnEvents() + { + // Arrange + _recordedActivities.Clear(); + var tracker = new SessionTelemetryTracker("test-session-turn"); + + var turnStartEvent = CreateTurnStartEvent("turn-001"); + var turnEndEvent = CreateTurnEndEvent("turn-001"); + + // Act + tracker.ProcessEvent(turnStartEvent); + tracker.ProcessEvent(turnEndEvent); + + // Assert + var turnActivity = _recordedActivities.FirstOrDefault(a => + a.OperationName == OpenTelemetryConstants.SpanNameTurn); + + Assert.NotNull(turnActivity); + Assert.Equal("turn-001", turnActivity!.GetTagItem(OpenTelemetryConstants.CopilotTurnId)); + Assert.Equal(ActivityStatusCode.Ok, turnActivity.Status); + } + + [Fact] + public void SessionTelemetryTracker_ProcessesUsageEvent() + { + // Arrange + _recordedActivities.Clear(); + var tracker = new SessionTelemetryTracker("test-session-usage"); + + var usageEvent = CreateUsageEvent( + model: "gpt-4o", + inputTokens: 100, + outputTokens: 50, + cost: 0.005, + durationMs: 1500); + + // Act + tracker.ProcessEvent(usageEvent); + + // Assert - inference span should be recorded + var inferenceActivity = _recordedActivities.FirstOrDefault(a => + a.OperationName == OpenTelemetryConstants.SpanNameInference); + + Assert.NotNull(inferenceActivity); + Assert.Equal("gpt-4o", inferenceActivity!.GetTagItem(OpenTelemetryConstants.GenAiResponseModel)); + Assert.Equal(100L, inferenceActivity.GetTagItem(OpenTelemetryConstants.GenAiUsageInputTokens)); + Assert.Equal(50L, inferenceActivity.GetTagItem(OpenTelemetryConstants.GenAiUsageOutputTokens)); + Assert.Equal(0.005, inferenceActivity.GetTagItem(OpenTelemetryConstants.CopilotCost)); + } + + [Fact] + public void SessionTelemetryTracker_ProcessesSubagentEvents() + { + // Arrange + _recordedActivities.Clear(); + var tracker = new SessionTelemetryTracker("test-session-subagent"); + + var subagentStartEvent = CreateSubagentStartedEvent("call-sub-1", "code-reviewer", "Code Reviewer"); + var subagentCompleteEvent = CreateSubagentCompletedEvent("call-sub-1", "code-reviewer"); + + // Act + tracker.ProcessEvent(subagentStartEvent); + tracker.ProcessEvent(subagentCompleteEvent); + + // Assert + var subagentActivity = _recordedActivities.FirstOrDefault(a => + a.OperationName == OpenTelemetryConstants.SpanNameSubagent); + + Assert.NotNull(subagentActivity); + Assert.Equal("code-reviewer", subagentActivity!.GetTagItem(OpenTelemetryConstants.CopilotSubagentName)); + Assert.Equal(true, subagentActivity.GetTagItem(OpenTelemetryConstants.CopilotSuccess)); + } + + [Fact] + public void SessionTelemetryTracker_DisposeCleansUpActivities() + { + // Arrange + var tracker = new SessionTelemetryTracker("test-session-dispose"); + tracker.ProcessEvent(CreateSessionStartEvent("test-session-dispose", "gpt-4o")); + tracker.ProcessEvent(CreateTurnStartEvent("turn-orphan")); + + // Act + tracker.Dispose(); + + // Assert - should not throw, activities should be cleaned up + // Processing after dispose should be a no-op + tracker.ProcessEvent(CreateTurnEndEvent("turn-orphan")); + } + + #region Helper Methods for Creating Test Events + + private static SessionStartEvent CreateSessionStartEvent(string sessionId, string? model) + { + return new SessionStartEvent + { + Id = Guid.NewGuid(), + Timestamp = DateTimeOffset.UtcNow, + Data = new SessionStartData + { + SessionId = sessionId, + Version = 1.0, + Producer = "test", + CopilotVersion = "1.0.0", + StartTime = DateTimeOffset.UtcNow, + SelectedModel = model + } + }; + } + + private static ToolExecutionStartEvent CreateToolExecutionStartEvent(string toolCallId, string toolName) + { + return new ToolExecutionStartEvent + { + Id = Guid.NewGuid(), + Timestamp = DateTimeOffset.UtcNow, + Data = new ToolExecutionStartData + { + ToolCallId = toolCallId, + ToolName = toolName + } + }; + } + + private static ToolExecutionCompleteEvent CreateToolExecutionCompleteEvent( + string toolCallId, bool success, string? errorMessage = null) + { + return new ToolExecutionCompleteEvent + { + Id = Guid.NewGuid(), + Timestamp = DateTimeOffset.UtcNow, + Data = new ToolExecutionCompleteData + { + ToolCallId = toolCallId, + Success = success, + Error = errorMessage != null ? new ToolExecutionCompleteDataError { Message = errorMessage } : null + } + }; + } + + private static AssistantTurnStartEvent CreateTurnStartEvent(string turnId) + { + return new AssistantTurnStartEvent + { + Id = Guid.NewGuid(), + Timestamp = DateTimeOffset.UtcNow, + Data = new AssistantTurnStartData + { + TurnId = turnId + } + }; + } + + private static AssistantTurnEndEvent CreateTurnEndEvent(string turnId) + { + return new AssistantTurnEndEvent + { + Id = Guid.NewGuid(), + Timestamp = DateTimeOffset.UtcNow, + Data = new AssistantTurnEndData + { + TurnId = turnId + } + }; + } + + private static AssistantUsageEvent CreateUsageEvent( + string? model, double? inputTokens, double? outputTokens, double? cost, double? durationMs) + { + return new AssistantUsageEvent + { + Id = Guid.NewGuid(), + Timestamp = DateTimeOffset.UtcNow, + Data = new AssistantUsageData + { + Model = model, + InputTokens = inputTokens, + OutputTokens = outputTokens, + Cost = cost, + Duration = durationMs + } + }; + } + + private static SubagentStartedEvent CreateSubagentStartedEvent(string toolCallId, string agentName, string displayName) + { + return new SubagentStartedEvent + { + Id = Guid.NewGuid(), + Timestamp = DateTimeOffset.UtcNow, + Data = new SubagentStartedData + { + ToolCallId = toolCallId, + AgentName = agentName, + AgentDisplayName = displayName, + AgentDescription = "Test agent" + } + }; + } + + private static SubagentCompletedEvent CreateSubagentCompletedEvent(string toolCallId, string agentName) + { + return new SubagentCompletedEvent + { + Id = Guid.NewGuid(), + Timestamp = DateTimeOffset.UtcNow, + Data = new SubagentCompletedData + { + ToolCallId = toolCallId, + AgentName = agentName + } + }; + } + + #endregion +} From 4a03a891902dc7bd4e3cd242acf2952e553f205b Mon Sep 17 00:00:00 2001 From: Anirudh Sharma Date: Sun, 25 Jan 2026 01:13:27 +0000 Subject: [PATCH 02/13] Update dotnet/README.md Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- dotnet/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dotnet/README.md b/dotnet/README.md index 724269bb..dd131702 100644 --- a/dotnet/README.md +++ b/dotnet/README.md @@ -472,7 +472,7 @@ using OpenTelemetry.Trace; using OpenTelemetry.Metrics; // Enable telemetry -AppContext.SetSwitch("GitHub.Copilot.Experimental.EnableOpenTelemetry", true); +AppContext.SetSwitch("GitHub.Copilot.EnableOpenTelemetry", true); // Configure OpenTelemetry services.AddOpenTelemetry() From d3d9a8942cce3fadcbfe67a3f2868ac722021bad Mon Sep 17 00:00:00 2001 From: Anirudh Sharma Date: Sun, 25 Jan 2026 01:13:39 +0000 Subject: [PATCH 03/13] Update dotnet/src/Telemetry/SessionTelemetryTracker.cs Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- .../src/Telemetry/SessionTelemetryTracker.cs | 21 ++----------------- 1 file changed, 2 insertions(+), 19 deletions(-) diff --git a/dotnet/src/Telemetry/SessionTelemetryTracker.cs b/dotnet/src/Telemetry/SessionTelemetryTracker.cs index 3e7d4e8b..1437cd99 100644 --- a/dotnet/src/Telemetry/SessionTelemetryTracker.cs +++ b/dotnet/src/Telemetry/SessionTelemetryTracker.cs @@ -357,28 +357,11 @@ public void Dispose() // Clean up any remaining activities _sessionActivity?.Dispose(); - foreach (var activity in _turnActivities.Values) - { - activity.Dispose(); - } + // Clear dictionaries without disposing individual activities to avoid + // races with concurrent readers that may still be using them. _turnActivities.Clear(); - - foreach (var activity in _toolActivities.Values) - { - activity.Dispose(); - } _toolActivities.Clear(); - - foreach (var activity in _subagentActivities.Values) - { - activity.Dispose(); - } _subagentActivities.Clear(); - - foreach (var activity in _hookActivities.Values) - { - activity.Dispose(); - } _hookActivities.Clear(); } } From ff0da97247b0e33b531b63f5c4739e6acf79a017 Mon Sep 17 00:00:00 2001 From: Anirudh Sharma Date: Sun, 25 Jan 2026 01:14:08 +0000 Subject: [PATCH 04/13] Update dotnet/src/Telemetry/SessionTelemetryTracker.cs Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- dotnet/src/Telemetry/SessionTelemetryTracker.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dotnet/src/Telemetry/SessionTelemetryTracker.cs b/dotnet/src/Telemetry/SessionTelemetryTracker.cs index 1437cd99..9ea25fc6 100644 --- a/dotnet/src/Telemetry/SessionTelemetryTracker.cs +++ b/dotnet/src/Telemetry/SessionTelemetryTracker.cs @@ -43,7 +43,7 @@ public void ProcessEvent(SessionEvent sessionEvent) OnSessionStart(startEvent); break; case SessionIdleEvent: - case SessionErrorEvent errorEvent: + case SessionErrorEvent: OnSessionEnd(sessionEvent as SessionErrorEvent); break; From c34997342c6739fcc1ae234cd63d631b995b5fc3 Mon Sep 17 00:00:00 2001 From: Anirudh Sharma Date: Sun, 25 Jan 2026 01:33:38 +0000 Subject: [PATCH 05/13] Add tests for hooks and subagent failure events --- dotnet/test/TelemetryTests.cs | 119 ++++++++++++++++++++++++++++++++++ 1 file changed, 119 insertions(+) diff --git a/dotnet/test/TelemetryTests.cs b/dotnet/test/TelemetryTests.cs index a9188eb9..55bb9e22 100644 --- a/dotnet/test/TelemetryTests.cs +++ b/dotnet/test/TelemetryTests.cs @@ -192,6 +192,80 @@ public void SessionTelemetryTracker_ProcessesSubagentEvents() Assert.Equal(true, subagentActivity.GetTagItem(OpenTelemetryConstants.CopilotSuccess)); } + [Fact] + public void SessionTelemetryTracker_ProcessesHookEvents() + { + // Arrange + _recordedActivities.Clear(); + var tracker = new SessionTelemetryTracker("test-session-hook"); + + var hookStartEvent = CreateHookStartEvent("hook-inv-1", "pre-tool"); + var hookEndEvent = CreateHookEndEvent("hook-inv-1", "pre-tool", success: true); + + // Act + tracker.ProcessEvent(hookStartEvent); + tracker.ProcessEvent(hookEndEvent); + + // Assert + var hookActivity = _recordedActivities.FirstOrDefault(a => + a.OperationName == OpenTelemetryConstants.SpanNameHook); + + Assert.NotNull(hookActivity); + Assert.Equal("pre-tool", hookActivity!.GetTagItem(OpenTelemetryConstants.CopilotHookType)); + Assert.Equal("hook-inv-1", hookActivity.GetTagItem(OpenTelemetryConstants.CopilotHookInvocationId)); + Assert.Equal(true, hookActivity.GetTagItem(OpenTelemetryConstants.CopilotSuccess)); + Assert.Equal(ActivityStatusCode.Ok, hookActivity.Status); + } + + [Fact] + public void SessionTelemetryTracker_ProcessesFailedHookExecution() + { + // Arrange + _recordedActivities.Clear(); + var tracker = new SessionTelemetryTracker("test-session-hook-fail"); + + var hookStartEvent = CreateHookStartEvent("hook-inv-2", "post-tool"); + var hookEndEvent = CreateHookEndEvent("hook-inv-2", "post-tool", success: false, errorMessage: "Hook script failed"); + + // Act + tracker.ProcessEvent(hookStartEvent); + tracker.ProcessEvent(hookEndEvent); + + // Assert + var hookActivity = _recordedActivities.FirstOrDefault(a => + a.OperationName == OpenTelemetryConstants.SpanNameHook); + + Assert.NotNull(hookActivity); + Assert.Equal(false, hookActivity!.GetTagItem(OpenTelemetryConstants.CopilotSuccess)); + Assert.Equal(ActivityStatusCode.Error, hookActivity.Status); + Assert.Equal("Hook script failed", hookActivity.GetTagItem(OpenTelemetryConstants.CopilotErrorMessage)); + } + + [Fact] + public void SessionTelemetryTracker_ProcessesSubagentFailedEvent() + { + // Arrange + _recordedActivities.Clear(); + var tracker = new SessionTelemetryTracker("test-session-subagent-fail"); + + var subagentStartEvent = CreateSubagentStartedEvent("call-sub-fail", "code-reviewer", "Code Reviewer"); + var subagentFailedEvent = CreateSubagentFailedEvent("call-sub-fail", "code-reviewer", "Agent crashed unexpectedly"); + + // Act + tracker.ProcessEvent(subagentStartEvent); + tracker.ProcessEvent(subagentFailedEvent); + + // Assert + var subagentActivity = _recordedActivities.FirstOrDefault(a => + a.OperationName == OpenTelemetryConstants.SpanNameSubagent); + + Assert.NotNull(subagentActivity); + Assert.Equal("code-reviewer", subagentActivity!.GetTagItem(OpenTelemetryConstants.CopilotSubagentName)); + Assert.Equal(false, subagentActivity.GetTagItem(OpenTelemetryConstants.CopilotSuccess)); + Assert.Equal(ActivityStatusCode.Error, subagentActivity.Status); + Assert.Equal("Agent crashed unexpectedly", subagentActivity.GetTagItem(OpenTelemetryConstants.CopilotErrorMessage)); + } + [Fact] public void SessionTelemetryTracker_DisposeCleansUpActivities() { @@ -332,5 +406,50 @@ private static SubagentCompletedEvent CreateSubagentCompletedEvent(string toolCa }; } + private static SubagentFailedEvent CreateSubagentFailedEvent(string toolCallId, string agentName, string error) + { + return new SubagentFailedEvent + { + Id = Guid.NewGuid(), + Timestamp = DateTimeOffset.UtcNow, + Data = new SubagentFailedData + { + ToolCallId = toolCallId, + AgentName = agentName, + Error = error + } + }; + } + + private static HookStartEvent CreateHookStartEvent(string hookInvocationId, string hookType) + { + return new HookStartEvent + { + Id = Guid.NewGuid(), + Timestamp = DateTimeOffset.UtcNow, + Data = new HookStartData + { + HookInvocationId = hookInvocationId, + HookType = hookType + } + }; + } + + private static HookEndEvent CreateHookEndEvent(string hookInvocationId, string hookType, bool success, string? errorMessage = null) + { + return new HookEndEvent + { + Id = Guid.NewGuid(), + Timestamp = DateTimeOffset.UtcNow, + Data = new HookEndData + { + HookInvocationId = hookInvocationId, + HookType = hookType, + Success = success, + Error = errorMessage != null ? new HookEndDataError { Message = errorMessage } : null + } + }; + } + #endregion } From e8565772beccbfc53bcbde747f9ec812af7e29b4 Mon Sep 17 00:00:00 2001 From: Anirudh Sharma Date: Sun, 25 Jan 2026 02:08:37 +0000 Subject: [PATCH 06/13] Use lock for thread-safe disposal of activities - Add _disposeLock to synchronize ProcessEvent and Dispose - Properly dispose all orphaned activities during Dispose - Prevents race conditions and resource leaks --- .../src/Telemetry/SessionTelemetryTracker.cs | 52 ++++++++++++++----- 1 file changed, 39 insertions(+), 13 deletions(-) diff --git a/dotnet/src/Telemetry/SessionTelemetryTracker.cs b/dotnet/src/Telemetry/SessionTelemetryTracker.cs index 9ea25fc6..1c1aa3a9 100644 --- a/dotnet/src/Telemetry/SessionTelemetryTracker.cs +++ b/dotnet/src/Telemetry/SessionTelemetryTracker.cs @@ -17,6 +17,7 @@ internal sealed class SessionTelemetryTracker : IDisposable private readonly ConcurrentDictionary _toolActivities = new(); private readonly ConcurrentDictionary _subagentActivities = new(); private readonly ConcurrentDictionary _hookActivities = new(); + private readonly object _disposeLock = new(); private Activity? _sessionActivity; private string? _currentModel; private bool _disposed; @@ -31,11 +32,24 @@ public SessionTelemetryTracker(string sessionId) /// public void ProcessEvent(SessionEvent sessionEvent) { - if (!CopilotTelemetry.IsEnabled || _disposed) + if (!CopilotTelemetry.IsEnabled) { return; } + lock (_disposeLock) + { + if (_disposed) + { + return; + } + + ProcessEventCore(sessionEvent); + } + } + + private void ProcessEventCore(SessionEvent sessionEvent) + { switch (sessionEvent) { // Session lifecycle @@ -351,17 +365,29 @@ private void OnUsage(AssistantUsageEvent usageEvent) public void Dispose() { - if (_disposed) return; - _disposed = true; - - // Clean up any remaining activities - _sessionActivity?.Dispose(); - - // Clear dictionaries without disposing individual activities to avoid - // races with concurrent readers that may still be using them. - _turnActivities.Clear(); - _toolActivities.Clear(); - _subagentActivities.Clear(); - _hookActivities.Clear(); + lock (_disposeLock) + { + if (_disposed) return; + _disposed = true; + + // Clean up session activity + _sessionActivity?.Dispose(); + _sessionActivity = null; + + // Dispose all orphaned activities in each dictionary + DisposeActivities(_turnActivities); + DisposeActivities(_toolActivities); + DisposeActivities(_subagentActivities); + DisposeActivities(_hookActivities); + } + } + + private static void DisposeActivities(ConcurrentDictionary activities) + { + foreach (var kvp in activities) + { + kvp.Value.Dispose(); + } + activities.Clear(); } } From ba45a8a62bd0b47bdd3bd65a8e575036eeb89a83 Mon Sep 17 00:00:00 2001 From: Anirudh Sharma Date: Sun, 25 Jan 2026 02:10:30 +0000 Subject: [PATCH 07/13] Get tool name before disposing activity --- dotnet/src/Telemetry/SessionTelemetryTracker.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dotnet/src/Telemetry/SessionTelemetryTracker.cs b/dotnet/src/Telemetry/SessionTelemetryTracker.cs index 1c1aa3a9..2812b895 100644 --- a/dotnet/src/Telemetry/SessionTelemetryTracker.cs +++ b/dotnet/src/Telemetry/SessionTelemetryTracker.cs @@ -216,11 +216,11 @@ private void OnToolExecutionComplete(ToolExecutionCompleteEvent toolCompleteEven } } - toolActivity.Dispose(); - - // Record metric - need to get tool name from activity tags + // Record metric - get tool name before disposing var toolName = toolActivity.GetTagItem(OpenTelemetryConstants.GenAiToolName)?.ToString() ?? "unknown"; CopilotTelemetry.RecordToolExecution(toolName, toolCompleteEvent.Data.Success, _sessionId); + + toolActivity.Dispose(); } } From 779c7da740e8f897ca037d9bc18afdd68d522fc9 Mon Sep 17 00:00:00 2001 From: Anirudh Sharma Date: Sun, 25 Jan 2026 02:12:30 +0000 Subject: [PATCH 08/13] Record error metric for hook failures --- dotnet/src/Telemetry/SessionTelemetryTracker.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/dotnet/src/Telemetry/SessionTelemetryTracker.cs b/dotnet/src/Telemetry/SessionTelemetryTracker.cs index 2812b895..73559641 100644 --- a/dotnet/src/Telemetry/SessionTelemetryTracker.cs +++ b/dotnet/src/Telemetry/SessionTelemetryTracker.cs @@ -295,6 +295,7 @@ private void OnHookEnd(HookEndEvent hookEndEvent) { hookActivity.SetTag(OpenTelemetryConstants.CopilotErrorMessage, hookEndEvent.Data.Error.Message); } + CopilotTelemetry.RecordError("hook_failed", _sessionId); } hookActivity.Dispose(); From 54b6b36263d40ff517690feda7b442aa8aa7c176 Mon Sep 17 00:00:00 2001 From: Anirudh Sharma Date: Sun, 25 Jan 2026 02:14:24 +0000 Subject: [PATCH 09/13] Use static constructor for test initialization order --- dotnet/test/TelemetryTests.cs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/dotnet/test/TelemetryTests.cs b/dotnet/test/TelemetryTests.cs index 55bb9e22..503ea5d0 100644 --- a/dotnet/test/TelemetryTests.cs +++ b/dotnet/test/TelemetryTests.cs @@ -13,11 +13,15 @@ public class TelemetryTests : IDisposable private readonly ActivityListener _listener; private readonly List _recordedActivities = new(); - public TelemetryTests() + // Static constructor ensures the AppContext switch is set before any code + // accesses CopilotTelemetry.IsEnabled (which uses lazy evaluation) + static TelemetryTests() { - // Enable telemetry for tests AppContext.SetSwitch("GitHub.Copilot.EnableOpenTelemetry", true); + } + public TelemetryTests() + { // Set up an activity listener to capture spans _listener = new ActivityListener { @@ -32,8 +36,6 @@ public TelemetryTests() public void Dispose() { _listener.Dispose(); - // Note: We can't easily reset the AppContext switch, but it won't affect other tests - // since IsEnabled is lazily evaluated once } [Fact] From 64af6b9a91e66ebb37e59466cf369d38912ae581 Mon Sep 17 00:00:00 2001 From: Anirudh Sharma Date: Mon, 26 Jan 2026 17:14:19 +0000 Subject: [PATCH 10/13] Add session lifecycle hooks for observability --- dotnet/README.md | 88 ---- dotnet/src/Client.cs | 33 ++ dotnet/src/GitHub.Copilot.SDK.csproj | 1 - dotnet/src/Session.cs | 15 +- dotnet/src/Telemetry/CopilotTelemetry.cs | 234 --------- .../src/Telemetry/OpenTelemetryConstants.cs | 139 ------ .../src/Telemetry/SessionTelemetryTracker.cs | 394 --------------- dotnet/test/TelemetryTests.cs | 457 ------------------ 8 files changed, 41 insertions(+), 1320 deletions(-) delete mode 100644 dotnet/src/Telemetry/CopilotTelemetry.cs delete mode 100644 dotnet/src/Telemetry/OpenTelemetryConstants.cs delete mode 100644 dotnet/src/Telemetry/SessionTelemetryTracker.cs delete mode 100644 dotnet/test/TelemetryTests.cs diff --git a/dotnet/README.md b/dotnet/README.md index dd131702..e176da40 100644 --- a/dotnet/README.md +++ b/dotnet/README.md @@ -444,94 +444,6 @@ var session = await client.CreateSessionAsync(new SessionConfig }); ``` -## Observability - -The SDK supports OpenTelemetry-based distributed tracing and metrics. Telemetry is disabled by default. - -### Enabling Telemetry - -Enable telemetry using one of these methods: - -**Option 1: AppContext switch (recommended)** -```csharp -AppContext.SetSwitch("GitHub.Copilot.EnableOpenTelemetry", true); -``` - -**Option 2: Environment variable** -```bash -export GITHUB_COPILOT_ENABLE_OPEN_TELEMETRY=true -``` - -### Configuring OpenTelemetry - -Configure your `TracerProvider` and `MeterProvider` to listen to the SDK: - -```csharp -using OpenTelemetry; -using OpenTelemetry.Trace; -using OpenTelemetry.Metrics; - -// Enable telemetry -AppContext.SetSwitch("GitHub.Copilot.EnableOpenTelemetry", true); - -// Configure OpenTelemetry -services.AddOpenTelemetry() - .WithTracing(tracing => tracing - .AddSource("GitHub.Copilot.SDK") // Traces - .AddConsoleExporter()) // Or your preferred exporter - .WithMetrics(metrics => metrics - .AddMeter("GitHub.Copilot.SDK") // Metrics - .AddConsoleExporter()); -``` - -### Spans - -The SDK emits the following spans following [GenAI Semantic Conventions](https://opentelemetry.io/docs/specs/semconv/gen-ai/): - -| Span Name | Description | Key Attributes | -|-----------|-------------|----------------| -| `copilot.session` | Root span for a session | `copilot.session.id`, `gen_ai.system`, `gen_ai.request.model` | -| `copilot.turn` | Assistant turn lifecycle | `copilot.turn.id` | -| `copilot.tool_execution` | Tool execution | `gen_ai.tool.name`, `gen_ai.tool.call_id`, `copilot.success` | -| `copilot.subagent` | Subagent execution | `copilot.subagent.name` | -| `copilot.hook` | Hook execution | `copilot.hook.type`, `copilot.hook.invocation_id` | -| `copilot.inference` | LLM inference call | `gen_ai.usage.input_tokens`, `gen_ai.usage.output_tokens`, `copilot.cost` | - -### Metrics - -The SDK emits the following metrics: - -| Metric Name | Type | Description | -|-------------|------|-------------| -| `copilot.tokens.input` | Counter | Number of input tokens used | -| `copilot.tokens.output` | Counter | Number of output tokens generated | -| `copilot.cost.total` | Counter | Total cost of operations | -| `copilot.tool_executions` | Counter | Number of tool executions | -| `copilot.errors` | Counter | Number of errors | -| `copilot.duration` | Histogram | Duration of operations in milliseconds | - -### Example Trace - -``` -copilot.session (root) -├── gen_ai.system = "github-copilot" -├── gen_ai.request.model = "gpt-4o" -├── copilot.session.id = "abc123" -│ -├── copilot.turn -│ ├── copilot.turn.id = "turn-1" -│ │ -│ ├── copilot.tool_execution -│ │ ├── gen_ai.tool.name = "file_edit" -│ │ ├── gen_ai.tool.call_id = "call-xyz" -│ │ └── copilot.success = true -│ │ -│ └── copilot.inference -│ ├── gen_ai.usage.input_tokens = 500 -│ ├── gen_ai.usage.output_tokens = 200 -│ └── copilot.cost = 0.003 -``` - ## Error Handling ```csharp diff --git a/dotnet/src/Client.cs b/dotnet/src/Client.cs index ef7982cb..831bd9c4 100644 --- a/dotnet/src/Client.cs +++ b/dotnet/src/Client.cs @@ -59,6 +59,24 @@ public partial class CopilotClient : IDisposable, IAsyncDisposable private readonly int? _optionsPort; private readonly string? _optionsHost; + /// + /// Occurs when a new session is created. + /// + /// + /// Subscribe to this event to hook into session events globally. + /// The handler receives the newly created instance. + /// + public event Action? SessionCreated; + + /// + /// Occurs when a session is destroyed. + /// + /// + /// Subscribe to this event to perform cleanup when sessions end. + /// The handler receives the session ID of the destroyed session. + /// + public event Action? SessionDestroyed; + /// /// Creates a new instance of . /// @@ -362,6 +380,13 @@ public async Task CreateSessionAsync(SessionConfig? config = nul throw new InvalidOperationException($"Session {response.SessionId} already exists"); } + session.OnDisposed = (id) => + { + _sessions.TryRemove(id, out _); + SessionDestroyed?.Invoke(id); + }; + SessionCreated?.Invoke(session); + return session; } @@ -416,6 +441,14 @@ public async Task ResumeSessionAsync(string sessionId, ResumeSes // Replace any existing session entry to ensure new config (like permission handler) is used _sessions[response.SessionId] = session; + + session.OnDisposed = (id) => + { + _sessions.TryRemove(id, out _); + SessionDestroyed?.Invoke(id); + }; + SessionCreated?.Invoke(session); + return session; } diff --git a/dotnet/src/GitHub.Copilot.SDK.csproj b/dotnet/src/GitHub.Copilot.SDK.csproj index 9e63588c..6e1f2efd 100644 --- a/dotnet/src/GitHub.Copilot.SDK.csproj +++ b/dotnet/src/GitHub.Copilot.SDK.csproj @@ -28,7 +28,6 @@ - diff --git a/dotnet/src/Session.cs b/dotnet/src/Session.cs index 63c93ec4..dafd0923 100644 --- a/dotnet/src/Session.cs +++ b/dotnet/src/Session.cs @@ -2,7 +2,6 @@ * Copyright (c) Microsoft Corporation. All rights reserved. *--------------------------------------------------------------------------------------------*/ -using GitHub.Copilot.SDK.Telemetry; using Microsoft.Extensions.AI; using StreamJsonRpc; using System.Text.Json; @@ -49,7 +48,6 @@ public partial class CopilotSession : IAsyncDisposable private readonly JsonRpc _rpc; private PermissionHandler? _permissionHandler; private readonly SemaphoreSlim _permissionHandlerLock = new(1, 1); - private readonly SessionTelemetryTracker _telemetryTracker; /// /// Gets the unique identifier for this session. @@ -66,6 +64,12 @@ public partial class CopilotSession : IAsyncDisposable /// public string? WorkspacePath { get; } + /// + /// Internal callback invoked when the session is disposed. + /// Used by CopilotClient to fire the SessionDestroyed event. + /// + internal Action? OnDisposed { get; set; } + /// /// Initializes a new instance of the class. /// @@ -80,7 +84,6 @@ internal CopilotSession(string sessionId, JsonRpc rpc, string? workspacePath = n SessionId = sessionId; _rpc = rpc; WorkspacePath = workspacePath; - _telemetryTracker = new SessionTelemetryTracker(sessionId); } /// @@ -240,9 +243,6 @@ public IDisposable On(SessionEventHandler handler) /// internal void DispatchEvent(SessionEvent sessionEvent) { - // Record telemetry for the event - _telemetryTracker.ProcessEvent(sessionEvent); - foreach (var handler in _eventHandlers.ToArray()) { // We allow handler exceptions to propagate so they are not lost @@ -427,7 +427,6 @@ await _rpc.InvokeWithCancellationAsync( _eventHandlers.Clear(); _toolHandlers.Clear(); - _telemetryTracker.Dispose(); await _permissionHandlerLock.WaitAsync(); try @@ -438,6 +437,8 @@ await _rpc.InvokeWithCancellationAsync( { _permissionHandlerLock.Release(); } + + OnDisposed?.Invoke(SessionId); } private class OnDisposeCall(Action callback) : IDisposable diff --git a/dotnet/src/Telemetry/CopilotTelemetry.cs b/dotnet/src/Telemetry/CopilotTelemetry.cs deleted file mode 100644 index 0189b6e9..00000000 --- a/dotnet/src/Telemetry/CopilotTelemetry.cs +++ /dev/null @@ -1,234 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - *--------------------------------------------------------------------------------------------*/ - -using System.Diagnostics; -using System.Diagnostics.Metrics; - -namespace GitHub.Copilot.SDK.Telemetry; - -/// -/// Provides OpenTelemetry instrumentation for the GitHub Copilot SDK. -/// -/// -/// -/// Telemetry is disabled by default. Enable it using one of these methods: -/// -/// -/// Set the AppContext switch: AppContext.SetSwitch("GitHub.Copilot.EnableOpenTelemetry", true) -/// Set the environment variable: GITHUB_COPILOT_ENABLE_OPEN_TELEMETRY=true -/// -/// -/// Then configure your TracerProvider and MeterProvider to listen: -/// -/// -/// services.AddOpenTelemetry() -/// .WithTracing(tracing => tracing.AddSource("GitHub.Copilot.SDK")) -/// .WithMetrics(metrics => metrics.AddMeter("GitHub.Copilot.SDK")); -/// -/// -public static class CopilotTelemetry -{ - private static readonly Lazy s_isEnabled = new(DetermineIfEnabled); - - /// - /// Gets the ActivitySource for creating spans. - /// - internal static ActivitySource ActivitySource { get; } = new( - OpenTelemetryConstants.ActivitySourceName, - typeof(CopilotTelemetry).Assembly.GetName().Version?.ToString() ?? "1.0.0"); - - /// - /// Gets the Meter for recording metrics. - /// - internal static Meter Meter { get; } = new( - OpenTelemetryConstants.MeterName, - typeof(CopilotTelemetry).Assembly.GetName().Version?.ToString() ?? "1.0.0"); - - // Metrics instruments - internal static Counter TokensInputCounter { get; } = Meter.CreateCounter( - OpenTelemetryConstants.MetricTokensInput, - unit: "{token}", - description: "Number of input tokens used"); - - internal static Counter TokensOutputCounter { get; } = Meter.CreateCounter( - OpenTelemetryConstants.MetricTokensOutput, - unit: "{token}", - description: "Number of output tokens generated"); - - internal static Counter CostCounter { get; } = Meter.CreateCounter( - OpenTelemetryConstants.MetricCostTotal, - unit: "{dollar}", - description: "Total cost of operations"); - - internal static Counter ToolExecutionsCounter { get; } = Meter.CreateCounter( - OpenTelemetryConstants.MetricToolExecutions, - unit: "{execution}", - description: "Number of tool executions"); - - internal static Counter ErrorsCounter { get; } = Meter.CreateCounter( - OpenTelemetryConstants.MetricErrors, - unit: "{error}", - description: "Number of errors"); - - internal static Histogram DurationHistogram { get; } = Meter.CreateHistogram( - OpenTelemetryConstants.MetricDuration, - unit: "ms", - description: "Duration of operations in milliseconds"); - - /// - /// Gets a value indicating whether telemetry is enabled. - /// - public static bool IsEnabled => s_isEnabled.Value; - - private static bool DetermineIfEnabled() - { - // Check AppContext switch first - if (AppContext.TryGetSwitch(OpenTelemetryConstants.EnableTelemetrySwitch, out var isEnabled)) - { - return isEnabled; - } - - // Fall back to environment variable - var envValue = Environment.GetEnvironmentVariable(OpenTelemetryConstants.EnableTelemetryEnvVar); - return string.Equals(envValue, "true", StringComparison.OrdinalIgnoreCase) || - string.Equals(envValue, "1", StringComparison.Ordinal); - } - - /// - /// Starts an activity (span) if telemetry is enabled and there are listeners. - /// - internal static Activity? StartActivity(string name, ActivityKind kind = ActivityKind.Internal) - { - if (!IsEnabled) - { - return null; - } - - return ActivitySource.StartActivity(name, kind); - } - - /// - /// Sets common GenAI attributes on an activity. - /// - internal static void SetGenAiAttributes(Activity? activity, string? model = null) - { - if (activity is null) return; - - activity.SetTag(OpenTelemetryConstants.GenAiSystem, "github-copilot"); - - if (!string.IsNullOrEmpty(model)) - { - activity.SetTag(OpenTelemetryConstants.GenAiRequestModel, model); - } - } - - /// - /// Records token usage metrics. - /// - internal static void RecordTokenUsage( - long? inputTokens, - long? outputTokens, - double? cost, - string? model, - string? sessionId) - { - if (!IsEnabled) return; - - var tags = new TagList - { - { OpenTelemetryConstants.GenAiSystem, "github-copilot" } - }; - - if (!string.IsNullOrEmpty(model)) - { - tags.Add(OpenTelemetryConstants.GenAiRequestModel, model); - } - - if (!string.IsNullOrEmpty(sessionId)) - { - tags.Add(OpenTelemetryConstants.CopilotSessionId, sessionId); - } - - if (inputTokens.HasValue) - { - TokensInputCounter.Add(inputTokens.Value, tags); - } - - if (outputTokens.HasValue) - { - TokensOutputCounter.Add(outputTokens.Value, tags); - } - - if (cost.HasValue) - { - CostCounter.Add(cost.Value, tags); - } - } - - /// - /// Records a tool execution metric. - /// - internal static void RecordToolExecution(string toolName, bool success, string? sessionId) - { - if (!IsEnabled) return; - - var tags = new TagList - { - { OpenTelemetryConstants.GenAiToolName, toolName }, - { OpenTelemetryConstants.CopilotSuccess, success } - }; - - if (!string.IsNullOrEmpty(sessionId)) - { - tags.Add(OpenTelemetryConstants.CopilotSessionId, sessionId); - } - - ToolExecutionsCounter.Add(1, tags); - - if (!success) - { - ErrorsCounter.Add(1, tags); - } - } - - /// - /// Records an error metric. - /// - internal static void RecordError(string errorType, string? sessionId) - { - if (!IsEnabled) return; - - var tags = new TagList - { - { "error.type", errorType } - }; - - if (!string.IsNullOrEmpty(sessionId)) - { - tags.Add(OpenTelemetryConstants.CopilotSessionId, sessionId); - } - - ErrorsCounter.Add(1, tags); - } - - /// - /// Records duration metric. - /// - internal static void RecordDuration(double durationMs, string operationType, string? sessionId) - { - if (!IsEnabled) return; - - var tags = new TagList - { - { OpenTelemetryConstants.GenAiOperationName, operationType } - }; - - if (!string.IsNullOrEmpty(sessionId)) - { - tags.Add(OpenTelemetryConstants.CopilotSessionId, sessionId); - } - - DurationHistogram.Record(durationMs, tags); - } -} diff --git a/dotnet/src/Telemetry/OpenTelemetryConstants.cs b/dotnet/src/Telemetry/OpenTelemetryConstants.cs deleted file mode 100644 index 0f45bfb3..00000000 --- a/dotnet/src/Telemetry/OpenTelemetryConstants.cs +++ /dev/null @@ -1,139 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - *--------------------------------------------------------------------------------------------*/ - -namespace GitHub.Copilot.SDK.Telemetry; - -/// -/// Constants for OpenTelemetry instrumentation following GenAI Semantic Conventions. -/// -/// -/// See: https://opentelemetry.io/docs/specs/semconv/gen-ai/ -/// -internal static class OpenTelemetryConstants -{ - /// - /// The AppContext switch to enable OpenTelemetry telemetry. - /// - public const string EnableTelemetrySwitch = "GitHub.Copilot.EnableOpenTelemetry"; - - /// - /// Environment variable to enable OpenTelemetry telemetry. - /// - public const string EnableTelemetryEnvVar = "GITHUB_COPILOT_ENABLE_OPEN_TELEMETRY"; - - /// - /// The ActivitySource name for GitHub Copilot SDK telemetry. - /// - public const string ActivitySourceName = "GitHub.Copilot.SDK"; - - /// - /// The Meter name for GitHub Copilot SDK metrics. - /// - public const string MeterName = "GitHub.Copilot.SDK"; - - // GenAI Semantic Convention attribute names - // See: https://opentelemetry.io/docs/specs/semconv/gen-ai/gen-ai-spans/ - - /// The name of the GenAI system (e.g., "github-copilot"). - public const string GenAiSystem = "gen_ai.system"; - - /// The operation being performed (e.g., "chat"). - public const string GenAiOperationName = "gen_ai.operation.name"; - - /// The model requested by the user. - public const string GenAiRequestModel = "gen_ai.request.model"; - - /// The model that generated the response. - public const string GenAiResponseModel = "gen_ai.response.model"; - - /// Number of input tokens used. - public const string GenAiUsageInputTokens = "gen_ai.usage.input_tokens"; - - /// Number of output tokens generated. - public const string GenAiUsageOutputTokens = "gen_ai.usage.output_tokens"; - - /// The name of the tool being called. - public const string GenAiToolName = "gen_ai.tool.name"; - - /// The unique identifier for the tool call. - public const string GenAiToolCallId = "gen_ai.tool.call_id"; - - // Copilot-specific attributes - - /// The session identifier. - public const string CopilotSessionId = "copilot.session.id"; - - /// The turn identifier within a session. - public const string CopilotTurnId = "copilot.turn.id"; - - /// The subagent name. - public const string CopilotSubagentName = "copilot.subagent.name"; - - /// The hook type. - public const string CopilotHookType = "copilot.hook.type"; - - /// The hook invocation identifier. - public const string CopilotHookInvocationId = "copilot.hook.invocation_id"; - - /// Whether the operation succeeded. - public const string CopilotSuccess = "copilot.success"; - - /// The error message if failed. - public const string CopilotErrorMessage = "copilot.error.message"; - - /// The cost of the operation. - public const string CopilotCost = "copilot.cost"; - - /// Duration of the operation in milliseconds. - public const string CopilotDurationMs = "copilot.duration_ms"; - - /// Cache read tokens. - public const string CopilotCacheReadTokens = "copilot.cache.read_tokens"; - - /// Cache write tokens. - public const string CopilotCacheWriteTokens = "copilot.cache.write_tokens"; - - // Span names - - /// Span name for a session. - public const string SpanNameSession = "copilot.session"; - - /// Span name for an assistant turn. - public const string SpanNameTurn = "copilot.turn"; - - /// Span name for tool execution. - public const string SpanNameToolExecution = "copilot.tool_execution"; - - /// Span name for subagent execution. - public const string SpanNameSubagent = "copilot.subagent"; - - /// Span name for hook execution. - public const string SpanNameHook = "copilot.hook"; - - /// Span name for inference/LLM call. - public const string SpanNameInference = "copilot.inference"; - - // Metric names - - /// Counter for total tokens used. - public const string MetricTokensTotal = "copilot.tokens.total"; - - /// Counter for input tokens. - public const string MetricTokensInput = "copilot.tokens.input"; - - /// Counter for output tokens. - public const string MetricTokensOutput = "copilot.tokens.output"; - - /// Counter for total cost. - public const string MetricCostTotal = "copilot.cost.total"; - - /// Histogram for operation duration. - public const string MetricDuration = "copilot.duration"; - - /// Counter for tool executions. - public const string MetricToolExecutions = "copilot.tool_executions"; - - /// Counter for session errors. - public const string MetricErrors = "copilot.errors"; -} diff --git a/dotnet/src/Telemetry/SessionTelemetryTracker.cs b/dotnet/src/Telemetry/SessionTelemetryTracker.cs deleted file mode 100644 index 73559641..00000000 --- a/dotnet/src/Telemetry/SessionTelemetryTracker.cs +++ /dev/null @@ -1,394 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - *--------------------------------------------------------------------------------------------*/ - -using System.Collections.Concurrent; -using System.Diagnostics; - -namespace GitHub.Copilot.SDK.Telemetry; - -/// -/// Tracks active spans for a Copilot session based on session events. -/// -internal sealed class SessionTelemetryTracker : IDisposable -{ - private readonly string _sessionId; - private readonly ConcurrentDictionary _turnActivities = new(); - private readonly ConcurrentDictionary _toolActivities = new(); - private readonly ConcurrentDictionary _subagentActivities = new(); - private readonly ConcurrentDictionary _hookActivities = new(); - private readonly object _disposeLock = new(); - private Activity? _sessionActivity; - private string? _currentModel; - private bool _disposed; - - public SessionTelemetryTracker(string sessionId) - { - _sessionId = sessionId; - } - - /// - /// Processes a session event and creates/completes appropriate spans. - /// - public void ProcessEvent(SessionEvent sessionEvent) - { - if (!CopilotTelemetry.IsEnabled) - { - return; - } - - lock (_disposeLock) - { - if (_disposed) - { - return; - } - - ProcessEventCore(sessionEvent); - } - } - - private void ProcessEventCore(SessionEvent sessionEvent) - { - switch (sessionEvent) - { - // Session lifecycle - case SessionStartEvent startEvent: - OnSessionStart(startEvent); - break; - case SessionIdleEvent: - case SessionErrorEvent: - OnSessionEnd(sessionEvent as SessionErrorEvent); - break; - - // Model changes - case SessionModelChangeEvent modelChangeEvent: - _currentModel = modelChangeEvent.Data.NewModel; - _sessionActivity?.SetTag(OpenTelemetryConstants.GenAiResponseModel, _currentModel); - break; - - // Turn lifecycle - case AssistantTurnStartEvent turnStartEvent: - OnTurnStart(turnStartEvent); - break; - case AssistantTurnEndEvent turnEndEvent: - OnTurnEnd(turnEndEvent); - break; - - // Tool execution - case ToolExecutionStartEvent toolStartEvent: - OnToolExecutionStart(toolStartEvent); - break; - case ToolExecutionCompleteEvent toolCompleteEvent: - OnToolExecutionComplete(toolCompleteEvent); - break; - - // Subagent lifecycle - case SubagentStartedEvent subagentStartEvent: - OnSubagentStart(subagentStartEvent); - break; - case SubagentCompletedEvent subagentCompletedEvent: - OnSubagentComplete(subagentCompletedEvent, success: true); - break; - case SubagentFailedEvent subagentFailedEvent: - OnSubagentFailed(subagentFailedEvent); - break; - - // Hook lifecycle - case HookStartEvent hookStartEvent: - OnHookStart(hookStartEvent); - break; - case HookEndEvent hookEndEvent: - OnHookEnd(hookEndEvent); - break; - - // Usage/metrics - case AssistantUsageEvent usageEvent: - OnUsage(usageEvent); - break; - } - } - - private void OnSessionStart(SessionStartEvent startEvent) - { - _sessionActivity = CopilotTelemetry.StartActivity( - OpenTelemetryConstants.SpanNameSession, - ActivityKind.Server); - - if (_sessionActivity is null) return; - - _currentModel = startEvent.Data.SelectedModel; - - _sessionActivity.SetTag(OpenTelemetryConstants.CopilotSessionId, _sessionId); - CopilotTelemetry.SetGenAiAttributes(_sessionActivity, _currentModel); - _sessionActivity.SetTag(OpenTelemetryConstants.GenAiOperationName, "chat"); - - if (startEvent.Data.Context != null) - { - _sessionActivity.SetTag("copilot.context.cwd", startEvent.Data.Context.Cwd); - if (startEvent.Data.Context.Repository != null) - { - _sessionActivity.SetTag("copilot.context.repository", startEvent.Data.Context.Repository); - } - } - } - - private void OnSessionEnd(SessionErrorEvent? errorEvent) - { - if (_sessionActivity is null) return; - - if (errorEvent != null) - { - _sessionActivity.SetStatus(ActivityStatusCode.Error, errorEvent.Data.Message); - _sessionActivity.SetTag(OpenTelemetryConstants.CopilotErrorMessage, errorEvent.Data.Message); - _sessionActivity.SetTag("error.type", errorEvent.Data.ErrorType); - CopilotTelemetry.RecordError(errorEvent.Data.ErrorType, _sessionId); - } - else - { - _sessionActivity.SetStatus(ActivityStatusCode.Ok); - } - - _sessionActivity.Dispose(); - _sessionActivity = null; - } - - private void OnTurnStart(AssistantTurnStartEvent turnStartEvent) - { - var turnActivity = CopilotTelemetry.StartActivity( - OpenTelemetryConstants.SpanNameTurn, - ActivityKind.Internal); - - if (turnActivity is null) return; - - turnActivity.SetTag(OpenTelemetryConstants.CopilotSessionId, _sessionId); - turnActivity.SetTag(OpenTelemetryConstants.CopilotTurnId, turnStartEvent.Data.TurnId); - CopilotTelemetry.SetGenAiAttributes(turnActivity, _currentModel); - - _turnActivities[turnStartEvent.Data.TurnId] = turnActivity; - } - - private void OnTurnEnd(AssistantTurnEndEvent turnEndEvent) - { - if (_turnActivities.TryRemove(turnEndEvent.Data.TurnId, out var turnActivity)) - { - turnActivity.SetStatus(ActivityStatusCode.Ok); - turnActivity.Dispose(); - } - } - - private void OnToolExecutionStart(ToolExecutionStartEvent toolStartEvent) - { - var toolActivity = CopilotTelemetry.StartActivity( - OpenTelemetryConstants.SpanNameToolExecution, - ActivityKind.Internal); - - if (toolActivity is null) return; - - toolActivity.SetTag(OpenTelemetryConstants.CopilotSessionId, _sessionId); - toolActivity.SetTag(OpenTelemetryConstants.GenAiToolName, toolStartEvent.Data.ToolName); - toolActivity.SetTag(OpenTelemetryConstants.GenAiToolCallId, toolStartEvent.Data.ToolCallId); - - if (toolStartEvent.Data.ParentToolCallId != null) - { - toolActivity.SetTag("copilot.parent_tool_call_id", toolStartEvent.Data.ParentToolCallId); - } - - _toolActivities[toolStartEvent.Data.ToolCallId] = toolActivity; - } - - private void OnToolExecutionComplete(ToolExecutionCompleteEvent toolCompleteEvent) - { - if (_toolActivities.TryRemove(toolCompleteEvent.Data.ToolCallId, out var toolActivity)) - { - toolActivity.SetTag(OpenTelemetryConstants.CopilotSuccess, toolCompleteEvent.Data.Success); - - if (toolCompleteEvent.Data.Success) - { - toolActivity.SetStatus(ActivityStatusCode.Ok); - } - else - { - toolActivity.SetStatus(ActivityStatusCode.Error, toolCompleteEvent.Data.Error?.Message); - if (toolCompleteEvent.Data.Error != null) - { - toolActivity.SetTag(OpenTelemetryConstants.CopilotErrorMessage, toolCompleteEvent.Data.Error.Message); - } - } - - // Record metric - get tool name before disposing - var toolName = toolActivity.GetTagItem(OpenTelemetryConstants.GenAiToolName)?.ToString() ?? "unknown"; - CopilotTelemetry.RecordToolExecution(toolName, toolCompleteEvent.Data.Success, _sessionId); - - toolActivity.Dispose(); - } - } - - private void OnSubagentStart(SubagentStartedEvent subagentStartEvent) - { - var subagentActivity = CopilotTelemetry.StartActivity( - OpenTelemetryConstants.SpanNameSubagent, - ActivityKind.Internal); - - if (subagentActivity is null) return; - - subagentActivity.SetTag(OpenTelemetryConstants.CopilotSessionId, _sessionId); - subagentActivity.SetTag(OpenTelemetryConstants.CopilotSubagentName, subagentStartEvent.Data.AgentName); - subagentActivity.SetTag("copilot.subagent.display_name", subagentStartEvent.Data.AgentDisplayName); - subagentActivity.SetTag(OpenTelemetryConstants.GenAiToolCallId, subagentStartEvent.Data.ToolCallId); - - _subagentActivities[subagentStartEvent.Data.ToolCallId] = subagentActivity; - } - - private void OnSubagentComplete(SubagentCompletedEvent subagentCompletedEvent, bool success) - { - if (_subagentActivities.TryRemove(subagentCompletedEvent.Data.ToolCallId, out var subagentActivity)) - { - subagentActivity.SetTag(OpenTelemetryConstants.CopilotSuccess, success); - subagentActivity.SetStatus(ActivityStatusCode.Ok); - subagentActivity.Dispose(); - } - } - - private void OnSubagentFailed(SubagentFailedEvent subagentFailedEvent) - { - if (_subagentActivities.TryRemove(subagentFailedEvent.Data.ToolCallId, out var subagentActivity)) - { - subagentActivity.SetTag(OpenTelemetryConstants.CopilotSuccess, false); - subagentActivity.SetStatus(ActivityStatusCode.Error, subagentFailedEvent.Data.Error); - subagentActivity.SetTag(OpenTelemetryConstants.CopilotErrorMessage, subagentFailedEvent.Data.Error); - subagentActivity.Dispose(); - - CopilotTelemetry.RecordError("subagent_failed", _sessionId); - } - } - - private void OnHookStart(HookStartEvent hookStartEvent) - { - var hookActivity = CopilotTelemetry.StartActivity( - OpenTelemetryConstants.SpanNameHook, - ActivityKind.Internal); - - if (hookActivity is null) return; - - hookActivity.SetTag(OpenTelemetryConstants.CopilotSessionId, _sessionId); - hookActivity.SetTag(OpenTelemetryConstants.CopilotHookType, hookStartEvent.Data.HookType); - hookActivity.SetTag(OpenTelemetryConstants.CopilotHookInvocationId, hookStartEvent.Data.HookInvocationId); - - _hookActivities[hookStartEvent.Data.HookInvocationId] = hookActivity; - } - - private void OnHookEnd(HookEndEvent hookEndEvent) - { - if (_hookActivities.TryRemove(hookEndEvent.Data.HookInvocationId, out var hookActivity)) - { - hookActivity.SetTag(OpenTelemetryConstants.CopilotSuccess, hookEndEvent.Data.Success); - - if (hookEndEvent.Data.Success) - { - hookActivity.SetStatus(ActivityStatusCode.Ok); - } - else - { - hookActivity.SetStatus(ActivityStatusCode.Error, hookEndEvent.Data.Error?.Message); - if (hookEndEvent.Data.Error != null) - { - hookActivity.SetTag(OpenTelemetryConstants.CopilotErrorMessage, hookEndEvent.Data.Error.Message); - } - CopilotTelemetry.RecordError("hook_failed", _sessionId); - } - - hookActivity.Dispose(); - } - } - - private void OnUsage(AssistantUsageEvent usageEvent) - { - var data = usageEvent.Data; - - // Create an inference span for the LLM call - using var inferenceActivity = CopilotTelemetry.StartActivity( - OpenTelemetryConstants.SpanNameInference, - ActivityKind.Client); - - if (inferenceActivity != null) - { - inferenceActivity.SetTag(OpenTelemetryConstants.CopilotSessionId, _sessionId); - inferenceActivity.SetTag(OpenTelemetryConstants.GenAiOperationName, "chat"); - - if (data.Model != null) - { - inferenceActivity.SetTag(OpenTelemetryConstants.GenAiResponseModel, data.Model); - } - - if (data.InputTokens.HasValue) - { - inferenceActivity.SetTag(OpenTelemetryConstants.GenAiUsageInputTokens, (long)data.InputTokens.Value); - } - - if (data.OutputTokens.HasValue) - { - inferenceActivity.SetTag(OpenTelemetryConstants.GenAiUsageOutputTokens, (long)data.OutputTokens.Value); - } - - if (data.Cost.HasValue) - { - inferenceActivity.SetTag(OpenTelemetryConstants.CopilotCost, data.Cost.Value); - } - - if (data.Duration.HasValue) - { - inferenceActivity.SetTag(OpenTelemetryConstants.CopilotDurationMs, data.Duration.Value); - CopilotTelemetry.RecordDuration(data.Duration.Value, "inference", _sessionId); - } - - if (data.CacheReadTokens.HasValue) - { - inferenceActivity.SetTag(OpenTelemetryConstants.CopilotCacheReadTokens, (long)data.CacheReadTokens.Value); - } - - if (data.CacheWriteTokens.HasValue) - { - inferenceActivity.SetTag(OpenTelemetryConstants.CopilotCacheWriteTokens, (long)data.CacheWriteTokens.Value); - } - - inferenceActivity.SetStatus(ActivityStatusCode.Ok); - } - - // Record metrics - CopilotTelemetry.RecordTokenUsage( - inputTokens: data.InputTokens.HasValue ? (long)data.InputTokens.Value : null, - outputTokens: data.OutputTokens.HasValue ? (long)data.OutputTokens.Value : null, - cost: data.Cost, - model: data.Model, - sessionId: _sessionId); - } - - public void Dispose() - { - lock (_disposeLock) - { - if (_disposed) return; - _disposed = true; - - // Clean up session activity - _sessionActivity?.Dispose(); - _sessionActivity = null; - - // Dispose all orphaned activities in each dictionary - DisposeActivities(_turnActivities); - DisposeActivities(_toolActivities); - DisposeActivities(_subagentActivities); - DisposeActivities(_hookActivities); - } - } - - private static void DisposeActivities(ConcurrentDictionary activities) - { - foreach (var kvp in activities) - { - kvp.Value.Dispose(); - } - activities.Clear(); - } -} diff --git a/dotnet/test/TelemetryTests.cs b/dotnet/test/TelemetryTests.cs deleted file mode 100644 index 503ea5d0..00000000 --- a/dotnet/test/TelemetryTests.cs +++ /dev/null @@ -1,457 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - *--------------------------------------------------------------------------------------------*/ - -using GitHub.Copilot.SDK.Telemetry; -using System.Diagnostics; -using Xunit; - -namespace GitHub.Copilot.SDK.Test; - -public class TelemetryTests : IDisposable -{ - private readonly ActivityListener _listener; - private readonly List _recordedActivities = new(); - - // Static constructor ensures the AppContext switch is set before any code - // accesses CopilotTelemetry.IsEnabled (which uses lazy evaluation) - static TelemetryTests() - { - AppContext.SetSwitch("GitHub.Copilot.EnableOpenTelemetry", true); - } - - public TelemetryTests() - { - // Set up an activity listener to capture spans - _listener = new ActivityListener - { - ShouldListenTo = source => source.Name == OpenTelemetryConstants.ActivitySourceName, - Sample = (ref ActivityCreationOptions options) => ActivitySamplingResult.AllData, - ActivityStarted = activity => { }, - ActivityStopped = activity => _recordedActivities.Add(activity) - }; - ActivitySource.AddActivityListener(_listener); - } - - public void Dispose() - { - _listener.Dispose(); - } - - [Fact] - public void TelemetryIsEnabled_WhenAppContextSwitchSet() - { - Assert.True(CopilotTelemetry.IsEnabled); - } - - [Fact] - public void ActivitySourceName_IsCorrect() - { - Assert.Equal("GitHub.Copilot.SDK", CopilotTelemetry.ActivitySource.Name); - } - - [Fact] - public void MeterName_IsCorrect() - { - Assert.Equal("GitHub.Copilot.SDK", CopilotTelemetry.Meter.Name); - } - - [Fact] - public void SessionTelemetryTracker_ProcessesSessionStartEvent() - { - // Arrange - var tracker = new SessionTelemetryTracker("test-session-123"); - var sessionStartEvent = CreateSessionStartEvent("test-session-123", "gpt-4o"); - - // Act - tracker.ProcessEvent(sessionStartEvent); - - // Assert - check that an activity was started - // Note: The activity may not be stopped yet, so we check it exists - Assert.NotNull(CopilotTelemetry.ActivitySource); - } - - [Fact] - public void SessionTelemetryTracker_ProcessesToolExecutionEvents() - { - // Arrange - _recordedActivities.Clear(); - var tracker = new SessionTelemetryTracker("test-session-456"); - - var toolStartEvent = CreateToolExecutionStartEvent("call-123", "file_edit"); - var toolCompleteEvent = CreateToolExecutionCompleteEvent("call-123", success: true); - - // Act - tracker.ProcessEvent(toolStartEvent); - tracker.ProcessEvent(toolCompleteEvent); - - // Assert - tool execution span should be recorded - var toolActivity = _recordedActivities.FirstOrDefault(a => - a.OperationName == OpenTelemetryConstants.SpanNameToolExecution); - - Assert.NotNull(toolActivity); - Assert.Equal("file_edit", toolActivity!.GetTagItem(OpenTelemetryConstants.GenAiToolName)); - Assert.Equal("call-123", toolActivity.GetTagItem(OpenTelemetryConstants.GenAiToolCallId)); - Assert.Equal(true, toolActivity.GetTagItem(OpenTelemetryConstants.CopilotSuccess)); - Assert.Equal(ActivityStatusCode.Ok, toolActivity.Status); - } - - [Fact] - public void SessionTelemetryTracker_ProcessesFailedToolExecution() - { - // Arrange - _recordedActivities.Clear(); - var tracker = new SessionTelemetryTracker("test-session-789"); - - var toolStartEvent = CreateToolExecutionStartEvent("call-fail", "broken_tool"); - var toolCompleteEvent = CreateToolExecutionCompleteEvent("call-fail", success: false, errorMessage: "Something went wrong"); - - // Act - tracker.ProcessEvent(toolStartEvent); - tracker.ProcessEvent(toolCompleteEvent); - - // Assert - var toolActivity = _recordedActivities.FirstOrDefault(a => - a.OperationName == OpenTelemetryConstants.SpanNameToolExecution); - - Assert.NotNull(toolActivity); - Assert.Equal(false, toolActivity!.GetTagItem(OpenTelemetryConstants.CopilotSuccess)); - Assert.Equal(ActivityStatusCode.Error, toolActivity.Status); - Assert.Equal("Something went wrong", toolActivity.GetTagItem(OpenTelemetryConstants.CopilotErrorMessage)); - } - - [Fact] - public void SessionTelemetryTracker_ProcessesTurnEvents() - { - // Arrange - _recordedActivities.Clear(); - var tracker = new SessionTelemetryTracker("test-session-turn"); - - var turnStartEvent = CreateTurnStartEvent("turn-001"); - var turnEndEvent = CreateTurnEndEvent("turn-001"); - - // Act - tracker.ProcessEvent(turnStartEvent); - tracker.ProcessEvent(turnEndEvent); - - // Assert - var turnActivity = _recordedActivities.FirstOrDefault(a => - a.OperationName == OpenTelemetryConstants.SpanNameTurn); - - Assert.NotNull(turnActivity); - Assert.Equal("turn-001", turnActivity!.GetTagItem(OpenTelemetryConstants.CopilotTurnId)); - Assert.Equal(ActivityStatusCode.Ok, turnActivity.Status); - } - - [Fact] - public void SessionTelemetryTracker_ProcessesUsageEvent() - { - // Arrange - _recordedActivities.Clear(); - var tracker = new SessionTelemetryTracker("test-session-usage"); - - var usageEvent = CreateUsageEvent( - model: "gpt-4o", - inputTokens: 100, - outputTokens: 50, - cost: 0.005, - durationMs: 1500); - - // Act - tracker.ProcessEvent(usageEvent); - - // Assert - inference span should be recorded - var inferenceActivity = _recordedActivities.FirstOrDefault(a => - a.OperationName == OpenTelemetryConstants.SpanNameInference); - - Assert.NotNull(inferenceActivity); - Assert.Equal("gpt-4o", inferenceActivity!.GetTagItem(OpenTelemetryConstants.GenAiResponseModel)); - Assert.Equal(100L, inferenceActivity.GetTagItem(OpenTelemetryConstants.GenAiUsageInputTokens)); - Assert.Equal(50L, inferenceActivity.GetTagItem(OpenTelemetryConstants.GenAiUsageOutputTokens)); - Assert.Equal(0.005, inferenceActivity.GetTagItem(OpenTelemetryConstants.CopilotCost)); - } - - [Fact] - public void SessionTelemetryTracker_ProcessesSubagentEvents() - { - // Arrange - _recordedActivities.Clear(); - var tracker = new SessionTelemetryTracker("test-session-subagent"); - - var subagentStartEvent = CreateSubagentStartedEvent("call-sub-1", "code-reviewer", "Code Reviewer"); - var subagentCompleteEvent = CreateSubagentCompletedEvent("call-sub-1", "code-reviewer"); - - // Act - tracker.ProcessEvent(subagentStartEvent); - tracker.ProcessEvent(subagentCompleteEvent); - - // Assert - var subagentActivity = _recordedActivities.FirstOrDefault(a => - a.OperationName == OpenTelemetryConstants.SpanNameSubagent); - - Assert.NotNull(subagentActivity); - Assert.Equal("code-reviewer", subagentActivity!.GetTagItem(OpenTelemetryConstants.CopilotSubagentName)); - Assert.Equal(true, subagentActivity.GetTagItem(OpenTelemetryConstants.CopilotSuccess)); - } - - [Fact] - public void SessionTelemetryTracker_ProcessesHookEvents() - { - // Arrange - _recordedActivities.Clear(); - var tracker = new SessionTelemetryTracker("test-session-hook"); - - var hookStartEvent = CreateHookStartEvent("hook-inv-1", "pre-tool"); - var hookEndEvent = CreateHookEndEvent("hook-inv-1", "pre-tool", success: true); - - // Act - tracker.ProcessEvent(hookStartEvent); - tracker.ProcessEvent(hookEndEvent); - - // Assert - var hookActivity = _recordedActivities.FirstOrDefault(a => - a.OperationName == OpenTelemetryConstants.SpanNameHook); - - Assert.NotNull(hookActivity); - Assert.Equal("pre-tool", hookActivity!.GetTagItem(OpenTelemetryConstants.CopilotHookType)); - Assert.Equal("hook-inv-1", hookActivity.GetTagItem(OpenTelemetryConstants.CopilotHookInvocationId)); - Assert.Equal(true, hookActivity.GetTagItem(OpenTelemetryConstants.CopilotSuccess)); - Assert.Equal(ActivityStatusCode.Ok, hookActivity.Status); - } - - [Fact] - public void SessionTelemetryTracker_ProcessesFailedHookExecution() - { - // Arrange - _recordedActivities.Clear(); - var tracker = new SessionTelemetryTracker("test-session-hook-fail"); - - var hookStartEvent = CreateHookStartEvent("hook-inv-2", "post-tool"); - var hookEndEvent = CreateHookEndEvent("hook-inv-2", "post-tool", success: false, errorMessage: "Hook script failed"); - - // Act - tracker.ProcessEvent(hookStartEvent); - tracker.ProcessEvent(hookEndEvent); - - // Assert - var hookActivity = _recordedActivities.FirstOrDefault(a => - a.OperationName == OpenTelemetryConstants.SpanNameHook); - - Assert.NotNull(hookActivity); - Assert.Equal(false, hookActivity!.GetTagItem(OpenTelemetryConstants.CopilotSuccess)); - Assert.Equal(ActivityStatusCode.Error, hookActivity.Status); - Assert.Equal("Hook script failed", hookActivity.GetTagItem(OpenTelemetryConstants.CopilotErrorMessage)); - } - - [Fact] - public void SessionTelemetryTracker_ProcessesSubagentFailedEvent() - { - // Arrange - _recordedActivities.Clear(); - var tracker = new SessionTelemetryTracker("test-session-subagent-fail"); - - var subagentStartEvent = CreateSubagentStartedEvent("call-sub-fail", "code-reviewer", "Code Reviewer"); - var subagentFailedEvent = CreateSubagentFailedEvent("call-sub-fail", "code-reviewer", "Agent crashed unexpectedly"); - - // Act - tracker.ProcessEvent(subagentStartEvent); - tracker.ProcessEvent(subagentFailedEvent); - - // Assert - var subagentActivity = _recordedActivities.FirstOrDefault(a => - a.OperationName == OpenTelemetryConstants.SpanNameSubagent); - - Assert.NotNull(subagentActivity); - Assert.Equal("code-reviewer", subagentActivity!.GetTagItem(OpenTelemetryConstants.CopilotSubagentName)); - Assert.Equal(false, subagentActivity.GetTagItem(OpenTelemetryConstants.CopilotSuccess)); - Assert.Equal(ActivityStatusCode.Error, subagentActivity.Status); - Assert.Equal("Agent crashed unexpectedly", subagentActivity.GetTagItem(OpenTelemetryConstants.CopilotErrorMessage)); - } - - [Fact] - public void SessionTelemetryTracker_DisposeCleansUpActivities() - { - // Arrange - var tracker = new SessionTelemetryTracker("test-session-dispose"); - tracker.ProcessEvent(CreateSessionStartEvent("test-session-dispose", "gpt-4o")); - tracker.ProcessEvent(CreateTurnStartEvent("turn-orphan")); - - // Act - tracker.Dispose(); - - // Assert - should not throw, activities should be cleaned up - // Processing after dispose should be a no-op - tracker.ProcessEvent(CreateTurnEndEvent("turn-orphan")); - } - - #region Helper Methods for Creating Test Events - - private static SessionStartEvent CreateSessionStartEvent(string sessionId, string? model) - { - return new SessionStartEvent - { - Id = Guid.NewGuid(), - Timestamp = DateTimeOffset.UtcNow, - Data = new SessionStartData - { - SessionId = sessionId, - Version = 1.0, - Producer = "test", - CopilotVersion = "1.0.0", - StartTime = DateTimeOffset.UtcNow, - SelectedModel = model - } - }; - } - - private static ToolExecutionStartEvent CreateToolExecutionStartEvent(string toolCallId, string toolName) - { - return new ToolExecutionStartEvent - { - Id = Guid.NewGuid(), - Timestamp = DateTimeOffset.UtcNow, - Data = new ToolExecutionStartData - { - ToolCallId = toolCallId, - ToolName = toolName - } - }; - } - - private static ToolExecutionCompleteEvent CreateToolExecutionCompleteEvent( - string toolCallId, bool success, string? errorMessage = null) - { - return new ToolExecutionCompleteEvent - { - Id = Guid.NewGuid(), - Timestamp = DateTimeOffset.UtcNow, - Data = new ToolExecutionCompleteData - { - ToolCallId = toolCallId, - Success = success, - Error = errorMessage != null ? new ToolExecutionCompleteDataError { Message = errorMessage } : null - } - }; - } - - private static AssistantTurnStartEvent CreateTurnStartEvent(string turnId) - { - return new AssistantTurnStartEvent - { - Id = Guid.NewGuid(), - Timestamp = DateTimeOffset.UtcNow, - Data = new AssistantTurnStartData - { - TurnId = turnId - } - }; - } - - private static AssistantTurnEndEvent CreateTurnEndEvent(string turnId) - { - return new AssistantTurnEndEvent - { - Id = Guid.NewGuid(), - Timestamp = DateTimeOffset.UtcNow, - Data = new AssistantTurnEndData - { - TurnId = turnId - } - }; - } - - private static AssistantUsageEvent CreateUsageEvent( - string? model, double? inputTokens, double? outputTokens, double? cost, double? durationMs) - { - return new AssistantUsageEvent - { - Id = Guid.NewGuid(), - Timestamp = DateTimeOffset.UtcNow, - Data = new AssistantUsageData - { - Model = model, - InputTokens = inputTokens, - OutputTokens = outputTokens, - Cost = cost, - Duration = durationMs - } - }; - } - - private static SubagentStartedEvent CreateSubagentStartedEvent(string toolCallId, string agentName, string displayName) - { - return new SubagentStartedEvent - { - Id = Guid.NewGuid(), - Timestamp = DateTimeOffset.UtcNow, - Data = new SubagentStartedData - { - ToolCallId = toolCallId, - AgentName = agentName, - AgentDisplayName = displayName, - AgentDescription = "Test agent" - } - }; - } - - private static SubagentCompletedEvent CreateSubagentCompletedEvent(string toolCallId, string agentName) - { - return new SubagentCompletedEvent - { - Id = Guid.NewGuid(), - Timestamp = DateTimeOffset.UtcNow, - Data = new SubagentCompletedData - { - ToolCallId = toolCallId, - AgentName = agentName - } - }; - } - - private static SubagentFailedEvent CreateSubagentFailedEvent(string toolCallId, string agentName, string error) - { - return new SubagentFailedEvent - { - Id = Guid.NewGuid(), - Timestamp = DateTimeOffset.UtcNow, - Data = new SubagentFailedData - { - ToolCallId = toolCallId, - AgentName = agentName, - Error = error - } - }; - } - - private static HookStartEvent CreateHookStartEvent(string hookInvocationId, string hookType) - { - return new HookStartEvent - { - Id = Guid.NewGuid(), - Timestamp = DateTimeOffset.UtcNow, - Data = new HookStartData - { - HookInvocationId = hookInvocationId, - HookType = hookType - } - }; - } - - private static HookEndEvent CreateHookEndEvent(string hookInvocationId, string hookType, bool success, string? errorMessage = null) - { - return new HookEndEvent - { - Id = Guid.NewGuid(), - Timestamp = DateTimeOffset.UtcNow, - Data = new HookEndData - { - HookInvocationId = hookInvocationId, - HookType = hookType, - Success = success, - Error = errorMessage != null ? new HookEndDataError { Message = errorMessage } : null - } - }; - } - - #endregion -} From 3f249028be09493f32fd01c71e8b826366f94302 Mon Sep 17 00:00:00 2001 From: Anirudh Sharma Date: Mon, 26 Jan 2026 17:24:09 +0000 Subject: [PATCH 11/13] Remove unnecessary InternalsVisibleTo --- dotnet/src/GitHub.Copilot.SDK.csproj | 4 ---- 1 file changed, 4 deletions(-) diff --git a/dotnet/src/GitHub.Copilot.SDK.csproj b/dotnet/src/GitHub.Copilot.SDK.csproj index 6e1f2efd..10cfd98a 100644 --- a/dotnet/src/GitHub.Copilot.SDK.csproj +++ b/dotnet/src/GitHub.Copilot.SDK.csproj @@ -21,10 +21,6 @@ - - - - From d19847dd72197763769ebde8abb60a9402f202a8 Mon Sep 17 00:00:00 2001 From: Anirudh Sharma Date: Mon, 26 Jan 2026 17:37:57 +0000 Subject: [PATCH 12/13] Add tests for SessionCreated and SessionDestroyed events Tests verify: - SessionCreated fires when sessions are created - SessionDestroyed fires when sessions are disposed - Events fire correctly for multiple sessions - SessionCreated fires when sessions are resumed --- dotnet/test/ClientTests.cs | 86 +++++++++++++++++++++++++++++++++++++ dotnet/test/SessionTests.cs | 16 +++++++ 2 files changed, 102 insertions(+) diff --git a/dotnet/test/ClientTests.cs b/dotnet/test/ClientTests.cs index 23b0d9d9..862dd936 100644 --- a/dotnet/test/ClientTests.cs +++ b/dotnet/test/ClientTests.cs @@ -172,4 +172,90 @@ public async Task Should_List_Models_When_Authenticated() await client.ForceStopAsync(); } } + + [Fact] + public async Task Should_Fire_SessionCreated_When_Session_Is_Created() + { + using var client = new CopilotClient(new CopilotClientOptions { CliPath = _cliPath, UseStdio = true }); + + try + { + await client.StartAsync(); + + CopilotSession? createdSession = null; + client.SessionCreated += session => createdSession = session; + + var session = await client.CreateSessionAsync(); + + Assert.NotNull(createdSession); + Assert.Equal(session.SessionId, createdSession!.SessionId); + } + finally + { + await client.ForceStopAsync(); + } + } + + [Fact] + public async Task Should_Fire_SessionDestroyed_When_Session_Is_Disposed() + { + using var client = new CopilotClient(new CopilotClientOptions { CliPath = _cliPath, UseStdio = true }); + + try + { + await client.StartAsync(); + + string? destroyedSessionId = null; + client.SessionDestroyed += id => destroyedSessionId = id; + + var session = await client.CreateSessionAsync(); + var sessionId = session.SessionId; + + Assert.Null(destroyedSessionId); + + await session.DisposeAsync(); + + Assert.NotNull(destroyedSessionId); + Assert.Equal(sessionId, destroyedSessionId); + } + finally + { + await client.ForceStopAsync(); + } + } + + [Fact] + public async Task Should_Fire_Events_For_Multiple_Sessions() + { + using var client = new CopilotClient(new CopilotClientOptions { CliPath = _cliPath, UseStdio = true }); + + try + { + await client.StartAsync(); + + var createdIds = new List(); + var destroyedIds = new List(); + client.SessionCreated += session => createdIds.Add(session.SessionId); + client.SessionDestroyed += id => destroyedIds.Add(id); + + var session1 = await client.CreateSessionAsync(); + var session2 = await client.CreateSessionAsync(); + + Assert.Equal(2, createdIds.Count); + Assert.Contains(session1.SessionId, createdIds); + Assert.Contains(session2.SessionId, createdIds); + + await session1.DisposeAsync(); + Assert.Single(destroyedIds); + Assert.Equal(session1.SessionId, destroyedIds[0]); + + await session2.DisposeAsync(); + Assert.Equal(2, destroyedIds.Count); + Assert.Equal(session2.SessionId, destroyedIds[1]); + } + finally + { + await client.ForceStopAsync(); + } + } } diff --git a/dotnet/test/SessionTests.cs b/dotnet/test/SessionTests.cs index 845e604a..eec1c27e 100644 --- a/dotnet/test/SessionTests.cs +++ b/dotnet/test/SessionTests.cs @@ -196,6 +196,22 @@ public async Task Should_Throw_Error_When_Resuming_Non_Existent_Session() Client.ResumeSessionAsync("non-existent-session-id")); } + [Fact] + public async Task Should_Fire_SessionCreated_When_Session_Is_Resumed() + { + var session1 = await Client.CreateSessionAsync(); + var sessionId = session1.SessionId; + + CopilotSession? resumedSession = null; + Client.SessionCreated += s => resumedSession = s; + + var session2 = await Client.ResumeSessionAsync(sessionId); + + Assert.NotNull(resumedSession); + Assert.Equal(sessionId, resumedSession!.SessionId); + Assert.Same(session2, resumedSession); + } + [Fact] public async Task Should_Abort_A_Session() { From 63adfee491938bdb7c5929af66429097e5eab007 Mon Sep 17 00:00:00 2001 From: Anirudh Sharma Date: Mon, 26 Jan 2026 18:19:29 +0000 Subject: [PATCH 13/13] Fix orphaned OnDisposed callback when resuming sessions When ResumeSessionAsync replaces an existing session, clear the old session's OnDisposed callback to prevent it from firing SessionDestroyed if disposed later. Added test to verify old session disposal doesn't fire spurious events. --- dotnet/src/Client.cs | 7 +++++++ dotnet/test/SessionTests.cs | 25 +++++++++++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/dotnet/src/Client.cs b/dotnet/src/Client.cs index 831bd9c4..b080bcb5 100644 --- a/dotnet/src/Client.cs +++ b/dotnet/src/Client.cs @@ -439,6 +439,13 @@ public async Task ResumeSessionAsync(string sessionId, ResumeSes session.RegisterPermissionHandler(config.OnPermissionRequest); } + // Clear OnDisposed on the old session to prevent it from firing SessionDestroyed + // if it gets disposed after being replaced + if (_sessions.TryGetValue(response.SessionId, out var oldSession)) + { + oldSession.OnDisposed = null; + } + // Replace any existing session entry to ensure new config (like permission handler) is used _sessions[response.SessionId] = session; diff --git a/dotnet/test/SessionTests.cs b/dotnet/test/SessionTests.cs index eec1c27e..038bfc20 100644 --- a/dotnet/test/SessionTests.cs +++ b/dotnet/test/SessionTests.cs @@ -212,6 +212,31 @@ public async Task Should_Fire_SessionCreated_When_Session_Is_Resumed() Assert.Same(session2, resumedSession); } + [Fact] + public async Task Should_Not_Fire_SessionDestroyed_When_Old_Session_Is_Disposed_After_Resume() + { + var session1 = await Client.CreateSessionAsync(); + var sessionId = session1.SessionId; + + var destroyedIds = new List(); + Client.SessionDestroyed += id => destroyedIds.Add(id); + + // Resume creates a new session object for the same session ID + var session2 = await Client.ResumeSessionAsync(sessionId); + + // Disposing the old session object should NOT fire SessionDestroyed + // because session2 is now the active session for this ID + await session1.DisposeAsync(); + + Assert.Empty(destroyedIds); + + // Disposing the new session should fire SessionDestroyed + await session2.DisposeAsync(); + + Assert.Single(destroyedIds); + Assert.Equal(sessionId, destroyedIds[0]); + } + [Fact] public async Task Should_Abort_A_Session() {