feat(ai-monitoring): Fetch model context size and rename task to fetch_ai_model_info (#112656)

Closes
https://linear.app/getsentry/issue/TET-2219/sentry-map-llm-context-size-to-relay-cost-calculation-config

Introduces a new `AIModelMetadata` schema that nests costs under a
`costs` field and adds an optional `contextSize`. Context size is
fetched from OpenRouter's `context_length` field and models.dev's
`limit.context` field, following the same precedence logic as costs
(OpenRouter takes priority).

## Architecture

Both tasks run independently on the same cron schedule (every 30 min):

| Task | Cache key | Format |
|------|-----------|--------|
| `fetch_ai_model_costs` (legacy, TODO remove) | `ai-model-costs:v2` |
Flat `AIModelCostV2` |
| `fetch_ai_model_metadata` (new) | `ai-model-metadata:v1` | Nested
`AIModelMetadata` |

They share helper functions (`_normalize_model_id`,
`_create_prefix_glob_model_name`, `safe_float_conversion`) but fetch and
cache independently. The old task + cache key will be removed once all
consumers have migrated.

`GlobalConfig` serves both fields side by side:
- `aiModelCosts` — legacy flat format (read by Relay today)
- `aiModelMetadata` — new nested format with `contextSize` (not yet
consumed by Relay)

## New schema (`ai-model-metadata:v1`)

```json
{
  "version": 1,
  "models": {
    "gpt-4": {
      "costs": {
        "inputPerToken": 0.0000003,
        "outputPerToken": 0.00000165,
        "outputReasoningPerToken": 0.0,
        "inputCachedPerToken": 0.0000015,
        "inputCacheWritePerToken": 0.00001875
      },
      "contextSize": 1000000
    },
    "claude-3-5-sonnet": {
      "costs": {
        "inputPerToken": 0.000003,
        "outputPerToken": 0.000015,
        "outputReasoningPerToken": 0.0,
        "inputCachedPerToken": 0.0000015,
        "inputCacheWritePerToken": 0.00000375
      }
    }
  }
}
```

`contextSize` is optional — only present when the source API provides
it.

## New types

| Type | Purpose |
|------|---------|
| `AIModelCost` | Token cost fields (nested under `costs`) |
| `AIModelMetadata` | Per-model entry: `costs` + optional `contextSize`
|
| `AIModelMetadataConfig` | Top-level config: `version` + `models` dict
|

Legacy types (`AIModelCostV2`, `AIModelCosts`) are kept unchanged for
the old task.

Co-Authored-By: Claude Sonnet 4 <noreply@anthropic.com>

---------

Co-authored-by: Claude Sonnet 4 <noreply@anthropic.com>
Co-authored-by: getsantry[bot] <66042841+getsantry[bot]@users.noreply.github.com>

Author: 3 people

pyproject.toml

@@ -89,7 +89,7 @@ dependencies = [
   "sentry-ophio>=1.1.3",
   "sentry-protos>=0.8.11",
   "sentry-redis-tools>=0.5.0",
-  "sentry-relay>=0.9.26",
+  "sentry-relay>=0.9.27",
   "sentry-sdk[http2]>=2.47.0",
   "sentry-usage-accountant>=0.0.10",
   # remove once there are no unmarked transitive dependencies on setuptools

src/sentry/conf/server.py

@@ -1205,10 +1205,15 @@ def SOCIAL_AUTH_DEFAULT_USERNAME() -> str:
         "task": "relocation:sentry.relocation.transfer.find_relocation_transfer_region",
         "schedule": crontab("*/5", "*", "*", "*", "*"),
     },
+    # TODO(constantinius): Remove fetch-ai-model-costs once all consumers have migrated to fetch-ai-model-metadata
     "fetch-ai-model-costs": {
         "task": "ai_agent_monitoring:sentry.tasks.ai_agent_monitoring.fetch_ai_model_costs",
         "schedule": crontab("*/30", "*", "*", "*", "*"),
     },
+    "fetch-ai-model-metadata": {
+        "task": "ai_agent_monitoring:sentry.tasks.ai_agent_monitoring.fetch_ai_model_metadata",
+        "schedule": crontab("*/30", "*", "*", "*", "*"),
+    },
     "llm-issue-detection": {
         "task": "issues:sentry.tasks.llm_issue_detection.run_llm_issue_detection",
         "schedule": crontab("0", "*", "*", "*", "*"),

src/sentry/relay/config/ai_model_costs.py

@@ -11,13 +11,20 @@
 type ModelId = str
 
 
-# Cache key for storing AI model costs
+# Legacy cache key for AI model costs (v2 flat format)
+# TODO(constantinius): Remove once all consumers have migrated to AI_MODEL_METADATA_CACHE_KEY
 AI_MODEL_COSTS_CACHE_KEY = "ai-model-costs:v2"
-# Cache timeout: 30 days (we re-fetch every 30 minutes, so this provides more than enough overlap)
 AI_MODEL_COSTS_CACHE_TTL = 30 * 24 * 60 * 60
 
+# Cache key for storing LLM model metadata (v1 nested format)
+AI_MODEL_METADATA_CACHE_KEY = "ai-model-metadata:v1"
+# Cache timeout: 30 days (we re-fetch every 30 minutes, so this provides more than enough overlap)
+AI_MODEL_METADATA_CACHE_TTL = 30 * 24 * 60 * 60
+
 
 class AIModelCostV2(TypedDict):
+    """Legacy flat format. TODO(constantinius): Remove once all consumers have migrated."""
+
     inputPerToken: float
     outputPerToken: float
     outputReasoningPerToken: float
@@ -26,18 +33,34 @@ class AIModelCostV2(TypedDict):
 
 
 class AIModelCosts(TypedDict):
+    """Legacy config type. TODO(constantinius): Remove once all consumers have migrated."""
+
     version: Required[int]
     models: Required[dict[ModelId, AIModelCostV2]]
 
 
+class AIModelCost(TypedDict):
+    inputPerToken: float
+    outputPerToken: float
+    outputReasoningPerToken: float
+    inputCachedPerToken: float
+    inputCacheWritePerToken: float
+
+
+class AIModelMetadata(TypedDict, total=False):
+    costs: Required[AIModelCost]
+    contextSize: int
+
+
+class AIModelMetadataConfig(TypedDict):
+    version: Required[int]
+    models: Required[dict[ModelId, AIModelMetadata]]
+
+
 def ai_model_costs_config() -> AIModelCosts | None:
     """
-    Get AI model costs configuration.
-    AI model costs are set in cache by a cron job,
-    if there are no costs, it should be investigated why.
-
-    Returns:
-        AIModelCosts object containing cost information for AI models
+    Legacy: Get AI model costs configuration.
+    TODO(constantinius): Remove once all consumers have migrated to ai_model_metadata_config.
     """
     if settings.SENTRY_AIR_GAP:
         return None
@@ -47,7 +70,29 @@ def ai_model_costs_config() -> AIModelCosts | None:
         return cached_costs
 
     if not settings.IS_DEV:
-        # in dev environment, we don't want to log this
         logger.warning("Empty model costs")
 
     return None
+
+
+def ai_model_metadata_config() -> AIModelMetadataConfig | None:
+    """
+    Get LLM model metadata configuration.
+    LLM model metadata is set in cache by a cron job,
+    if there is no metadata, it should be investigated why.
+
+    Returns:
+        AIModelMetadataConfig containing cost and context size information for LLM models
+    """
+    if settings.SENTRY_AIR_GAP:
+        return None
+
+    cached_metadata = cache.get(AI_MODEL_METADATA_CACHE_KEY)
+    if cached_metadata is not None:
+        return cached_metadata
+
+    if not settings.IS_DEV:
+        # in dev environment, we don't want to log this
+        logger.warning("Empty LLM model metadata")
+
+    return None

src/sentry/relay/globalconfig.py

@@ -1,7 +1,12 @@
 from typing import Any, TypedDict
 
 import sentry.options
-from sentry.relay.config.ai_model_costs import AIModelCosts, ai_model_costs_config
+from sentry.relay.config.ai_model_costs import (
+    AIModelCosts,
+    AIModelMetadataConfig,
+    ai_model_costs_config,
+    ai_model_metadata_config,
+)
 from sentry.relay.config.measurements import MeasurementsConfig, get_measurements_config
 from sentry.relay.config.metric_extraction import (
     MetricExtractionGroups,
@@ -39,7 +44,10 @@ class SpanOpDefaults(TypedDict):
 
 class GlobalConfig(TypedDict, total=False):
     measurements: MeasurementsConfig
-    aiModelCosts: AIModelCosts | None
+    aiModelCosts: (
+        AIModelCosts | None
+    )  # TODO(constantinius): Remove once all consumers use aiModelMetadata
+    aiModelMetadata: AIModelMetadataConfig | None
     metricExtraction: MetricExtractionGroups
     filters: GenericFiltersConfig | None
     spanOpDefaults: SpanOpDefaults
@@ -78,7 +86,8 @@ def get_global_config() -> GlobalConfig:
 
     global_config: GlobalConfig = {
         "measurements": get_measurements_config(),
-        "aiModelCosts": ai_model_costs_config(),
+        "aiModelCosts": ai_model_costs_config(),  # TODO(constantinius): Remove once all consumers use aiModelMetadata
+        "aiModelMetadata": ai_model_metadata_config(),
         "metricExtraction": global_metric_extraction_groups(),
         "spanOpDefaults": span_op_defaults(),
     }