diff --git a/CHANGELOG.md b/CHANGELOG.md index 2c40517406..47b101eecb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,7 @@ - Add `gen_ai.cost_calculation.result` metric to track AI cost calculation outcomes by integration and platform. ([#5560](https://github.com/getsentry/relay/pull/5560)) - Normalizes and validates trace metric names. ([#5589](https://github.com/getsentry/relay/pull/5589)) - Add manual category to cost calculation metric origin tag ([#5603](https://github.com/getsentry/relay/pull/5603)) +- Differentiate between reasons for missing cost calculation([#5611](https://github.com/getsentry/relay/pull/5611)) ## 26.1.0 diff --git a/relay-event-normalization/src/eap/ai.rs b/relay-event-normalization/src/eap/ai.rs index d5932befb6..b9ae812395 100644 --- a/relay-event-normalization/src/eap/ai.rs +++ b/relay-event-normalization/src/eap/ai.rs @@ -6,7 +6,7 @@ use relay_protocol::Annotated; use crate::ModelCosts; use crate::span::ai; -use crate::statsd::{map_origin_to_integration, platform_tag}; +use crate::statsd::{Counters, map_origin_to_integration, platform_tag}; /// Normalizes AI attributes. /// @@ -120,7 +120,18 @@ fn normalize_ai_costs(attributes: &mut Attributes, model_costs: Option<&ModelCos .and_then(|v| v.as_str()) .and_then(|model| model_costs?.cost_per_token(model)); - let Some(model_cost) = model_cost else { return }; + let integration = map_origin_to_integration(origin); + let platform_tag = platform_tag(platform); + + let Some(model_cost) = model_cost else { + relay_statsd::metric!( + counter(Counters::GenAiCostCalculationResult) += 1, + result = "calculation_no_model_cost_available", + integration = integration, + platform = platform_tag, + ); + return; + }; let get_tokens = |key| { attributes @@ -137,10 +148,7 @@ fn normalize_ai_costs(attributes: &mut Attributes, model_costs: Option<&ModelCos output_reasoning_tokens: get_tokens(GEN_AI_USAGE_OUTPUT_REASONING_TOKENS), }; - let integration = map_origin_to_integration(origin); - let platform = platform_tag(platform); - - let Some(costs) = ai::calculate_costs(model_cost, tokens, integration, platform) else { + let Some(costs) = ai::calculate_costs(model_cost, tokens, integration, platform_tag) else { return; }; diff --git a/relay-event-normalization/src/normalize/span/ai.rs b/relay-event-normalization/src/normalize/span/ai.rs index 36a3c9c578..d363486fe1 100644 --- a/relay-event-normalization/src/normalize/span/ai.rs +++ b/relay-event-normalization/src/normalize/span/ai.rs @@ -94,7 +94,7 @@ pub fn calculate_costs( if !tokens.has_usage() { relay_statsd::metric!( counter(Counters::GenAiCostCalculationResult) += 1, - result = "calculation_none", + result = "calculation_no_tokens", integration = integration, platform = platform, ); @@ -189,11 +189,20 @@ fn extract_ai_model_cost_data( origin: Option<&str>, platform: Option<&str>, ) { - let Some(model_cost) = model_cost else { return }; - - let used_tokens = UsedTokens::from_span_data(&*data); let integration = map_origin_to_integration(origin); let platform = platform_tag(platform); + + let Some(model_cost) = model_cost else { + relay_statsd::metric!( + counter(Counters::GenAiCostCalculationResult) += 1, + result = "calculation_no_model_cost_available", + integration = integration, + platform = platform, + ); + return; + }; + + let used_tokens = UsedTokens::from_span_data(&*data); let Some(costs) = calculate_costs(model_cost, used_tokens, integration, platform) else { return; }; @@ -288,6 +297,13 @@ fn extract_ai_data( origin, platform, ) + } else { + relay_statsd::metric!( + counter(Counters::GenAiCostCalculationResult) += 1, + result = "calculation_no_model_id_available", + integration = map_origin_to_integration(origin), + platform = platform_tag(platform), + ); } } diff --git a/relay-event-normalization/src/statsd.rs b/relay-event-normalization/src/statsd.rs index 088218f890..427f384b41 100644 --- a/relay-event-normalization/src/statsd.rs +++ b/relay-event-normalization/src/statsd.rs @@ -8,7 +8,9 @@ pub enum Counters { /// `calculation_negative`, /// `calculation_zero`, /// `calculation_positive`, - /// `calculation_none` + /// `calculation_no_tokens` + /// `calculation_no_model_cost_available` + /// `calculation_no_model_id_available` /// - `integration`: The integration used for the cost calculation. /// - `platform`: The platform used for the cost calculation. GenAiCostCalculationResult,