feat(data-export): Exported attributes in data export should have public names

Exported attributes in full export of trace items should have public and user defined tags, no internal representation should be exposed

Author: manessaraj

src/sentry/data_export/endpoints/data_export_details.py

@@ -8,12 +8,11 @@
 from sentry.api.base import cell_silo_endpoint
 from sentry.api.bases.organization import OrganizationDataExportPermission, OrganizationEndpoint
 from sentry.api.serializers import serialize
+from sentry.data_export.models import ExportedData
 from sentry.models.organization import Organization
 from sentry.models.project import Project
 from sentry.utils import metrics
 
-from ..models import ExportedData
-
 
 @cell_silo_endpoint
 class DataExportDetailsEndpoint(OrganizationEndpoint):

src/sentry/data_export/processors/discover.py

@@ -4,6 +4,7 @@
 from sentry_relay.consts import SPAN_STATUS_CODE_TO_NAME
 
 from sentry.api.utils import get_date_range_from_params
+from sentry.data_export.base import ExportError
 from sentry.models.environment import Environment
 from sentry.models.group import Group
 from sentry.models.organization import Organization
@@ -13,8 +14,6 @@
 from sentry.snuba import discover
 from sentry.snuba.utils import get_dataset
 
-from ..base import ExportError
-
 logger = logging.getLogger(__name__)
 
 

src/sentry/data_export/processors/explore.py

@@ -21,7 +21,9 @@
     EAPResponse,
     FieldsACL,
     SearchResolverConfig,
+    SupportedTraceItemType,
 )
+from sentry.search.eap.utils import build_internal_to_public_export_map
 from sentry.search.events.types import SAMPLING_MODES, SnubaParams
 from sentry.snuba.ourlogs import OurLogs
 from sentry.snuba.referrer import Referrer
@@ -76,6 +78,11 @@ def __init__(
             if dataset == "spans"
             else TraceItemType.TRACE_ITEM_TYPE_LOG
         )
+        self._supported_trace_item_type = (
+            SupportedTraceItemType.LOGS
+            if self.trace_item_type == TraceItemType.TRACE_ITEM_TYPE_LOG
+            else SupportedTraceItemType.SPANS
+        )
 
         use_aggregate_conditions = explore_query.get("allowAggregateConditions", "1") == "1"
         disable_extrapolation = explore_query.get("disableAggregateExtrapolation", "0") == "1"
@@ -178,6 +185,7 @@ def __init__(
         super().__init__(organization, explore_query, output_mode=output_mode)
         self.page_token = page_token
         self._last_emitted_item_id_hex: str | None = last_emitted_item_id_hex
+        self._rename_mapping = build_internal_to_public_export_map(self._supported_trace_item_type)
 
     @property
     def last_emitted_item_id_hex(self) -> str | None:
@@ -215,18 +223,24 @@ def run_query(self, _offset: int, limit: int) -> list[dict[str, Any]]:
             token.ParseFromString(self.page_token)
             request.page_token.CopyFrom(token)
         http_resp = export_logs_rpc(request)
-        rows = list(iter_export_trace_items_rows(http_resp))
+        rows = list(
+            iter_export_trace_items_rows(
+                http_resp,
+                self._rename_mapping,
+                self._supported_trace_item_type,
+            )
+        )
 
         if self._last_emitted_item_id_hex is not None:
-            while rows and rows[0].get("item_id") == self._last_emitted_item_id_hex:
+            while rows and rows[0].get("id") == self._last_emitted_item_id_hex:
                 rows = rows[1:]
 
         self._sync_page_token_from_snuba_response(http_resp)
 
         if not rows:
             return []
 
-        last_id = rows[-1].get("item_id")
+        last_id = rows[-1].get("id")
         if isinstance(last_id, str):
             self._last_emitted_item_id_hex = last_id
         return rows

src/sentry/data_export/utils.py

@@ -10,12 +10,20 @@
 from sentry_protos.snuba.v1.request_common_pb2 import TraceItemType
 from sentry_protos.snuba.v1.trace_item_pb2 import AnyValue, TraceItem
 
+from sentry.data_export.base import ExportError
+from sentry.search.eap.types import SupportedTraceItemType
+from sentry.search.eap.utils import can_expose_attribute
 from sentry.search.events.constants import TIMEOUT_ERROR_MESSAGE
 from sentry.snuba import discover
 from sentry.utils import metrics, snuba
 from sentry.utils.sdk import capture_exception
 
-from .base import ExportError
+TRACE_ITEM_EXPORT_TOP_LEVEL_PUBLIC_KEYS: dict[str, str] = {
+    "organization_id": "organization.id",
+    "project_id": "project.id",
+    "trace_id": "trace",
+    "item_id": "id",
+}
 
 
 # Adapted into decorator from 'src/sentry/api/endpoints/organization_events.py'
@@ -94,7 +102,45 @@ def _ts_to_epoch(ts: Timestamp) -> float:
     return ts.seconds + ts.nanos / 1e9
 
 
-def trace_item_to_row(item: TraceItem) -> dict[str, Any]:
+def apply_public_names_to_trace_export_row(
+    row: dict[str, Any],
+    *,
+    rename_map: dict[str, str],
+    item_type: SupportedTraceItemType,
+) -> dict[str, Any]:
+    """
+    Rename known internal columns to EAP public aliases, keep extra/user attributes that are
+    exposable, and drop private or internal-only keys.
+    """
+    ordered_keys = [
+        *[k for k in TRACE_ITEM_EXPORT_TOP_LEVEL_PUBLIC_KEYS if k in row],
+        *[k for k in row if k not in TRACE_ITEM_EXPORT_TOP_LEVEL_PUBLIC_KEYS],
+    ]
+
+    out: dict[str, Any] = {}
+    for key in ordered_keys:
+        value = row[key]
+        if key in TRACE_ITEM_EXPORT_TOP_LEVEL_PUBLIC_KEYS:
+            new_key = TRACE_ITEM_EXPORT_TOP_LEVEL_PUBLIC_KEYS[key]
+        elif key in rename_map:
+            new_key = rename_map[key]
+        else:
+            if not can_expose_attribute(key, item_type, include_internal=False):
+                continue
+            new_key = key
+        if new_key not in out:
+            out[new_key] = value
+        elif out[new_key] is None and value is not None:
+            out[new_key] = value
+    return out
+
+
+def trace_item_to_row(
+    item: TraceItem,
+    *,
+    rename_mapping: dict[str, str],
+    item_type: SupportedTraceItemType,
+) -> dict[str, Any]:
     row: dict[str, Any] = {}
     for key, av in item.attributes.items():
         row[key] = None if av.WhichOneof("value") is None else anyvalue_to_python(av)
@@ -111,12 +157,15 @@ def trace_item_to_row(item: TraceItem) -> dict[str, Any]:
     row["server_sample_rate"] = item.server_sample_rate
     row["retention_days"] = item.retention_days
     row["downsampled_retention_days"] = item.downsampled_retention_days
-
-    return row
+    return apply_public_names_to_trace_export_row(
+        row, rename_map=rename_mapping, item_type=item_type
+    )
 
 
 def iter_export_trace_items_rows(
     resp: ExportTraceItemsResponse,
+    rename_mapping: dict[str, str],
+    item_type: SupportedTraceItemType,
 ) -> Iterator[dict[str, Any]]:
     for item in resp.trace_items:
-        yield trace_item_to_row(item)
+        yield trace_item_to_row(item, rename_mapping=rename_mapping, item_type=item_type)

src/sentry/search/eap/utils.py

@@ -207,3 +207,27 @@ def is_sentry_convention_replacement_attribute(
 def translate_to_sentry_conventions(public_alias: str, item_type: SupportedTraceItemType) -> str:
     mapping = SENTRY_CONVENTIONS_REPLACEMENT_MAPPINGS.get(item_type, {})
     return mapping.get(public_alias, public_alias)
+
+
+def build_internal_to_public_export_map(
+    item_type: SupportedTraceItemType,
+) -> dict[str, str]:
+    """
+    Map Snuba internal attribute names (e.g. ``sentry.item_id``) to a single **primary**
+    public alias per internal name.
+    """
+    definitions = PUBLIC_ALIAS_TO_INTERNAL_MAPPING.get(item_type, {})
+    by_internal: dict[str, str] = {}
+    for resolved in definitions.values():
+        if resolved.private or resolved.secondary_alias:
+            continue
+        if not can_expose_attribute(resolved.public_alias, item_type, include_internal=False):
+            continue
+        internal = resolved.internal_name
+        if internal not in by_internal:
+            by_internal[internal] = resolved.public_alias
+        else:
+            # Stable tie-break when multiple public aliases share one internal (e.g. project.id vs project_id).
+            by_internal[internal] = min(by_internal[internal], resolved.public_alias)
+
+    return by_internal

tests/sentry/data_export/test_tasks.py

@@ -855,7 +855,7 @@ def test_explore_logs_jsonl_format(self, emailer: MagicMock) -> None:
 
         lines = [ln for ln in content.split(b"\n") if ln]
         assert len(lines) == rows_exported
-        message_key = "log.body" if len(fields) else "sentry.body"
+        message_key = "log.body" if len(fields) else "message"
         bodies = {json.loads(ln.decode("utf-8"))[message_key] for ln in lines}
         assert bodies == {
             "jsonl log message",
@@ -930,11 +930,11 @@ def test_explore_logs_jsonl_full_dataset_rich_fields(self, emailer: MagicMock) -
                 "tags[code.line.number,number]": 148.0,
                 "logger.name": "sentry.access.api",
                 "origin": "auto.log.stdlib",
-                "sentry.body": "api.access.alpha",
+                "message": "api.access.alpha",
                 "rate_limited": "False",
-                "sentry.severity_text": "info",
+                "severity": "info",
                 "environment": "prod",
-                "sentry.severity_number": 9.0,
+                "severity_number": 9.0,
                 "payload_size": 981.0,
                 "tags[process.pid,number]": 6639.0,
                 "response": "200",
@@ -948,13 +948,13 @@ def test_explore_logs_jsonl_full_dataset_rich_fields(self, emailer: MagicMock) -
                 "tags[code.line.number,number]": 148.0,
                 "logger.name": "sentry.access.api",
                 "origin": "auto.log.stdlib",
-                "sentry.body": "api.access.beta",
+                "message": "api.access.beta",
                 "rate_limited": "False",
-                "sentry.severity_text": "info",
+                "severity": "info",
                 "environment": "prod",
                 "payload_size": 2048.0,
                 "tags[process.pid,number]": 6639.0,
-                "sentry.severity_number": 9.0,
+                "severity_number": 9.0,
                 "response": "201",
             },
         ]
@@ -967,15 +967,10 @@ def test_explore_logs_jsonl_full_dataset_rich_fields(self, emailer: MagicMock) -
                 "observed_timestamp",
                 "trace",
                 "id",
-                "item_id",
-                "sentry.timestamp_precise",
-                "sentry.observed_timestamp_nanos",
-                "organization_id",
-                "project_id",
-                "trace_id",
+                "organization.id",
+                "project.id",
                 "item_type",
                 "timestamp",
-                "sentry._internal.ingested_at",
                 "client_sample_rate",
                 "server_sample_rate",
                 "retention_days",