diff --git a/src/Elastic.CommonSchema.NLog/EcsLayout.cs b/src/Elastic.CommonSchema.NLog/EcsLayout.cs index c7583220..6fe3ced0 100644 --- a/src/Elastic.CommonSchema.NLog/EcsLayout.cs +++ b/src/Elastic.CommonSchema.NLog/EcsLayout.cs @@ -304,7 +304,9 @@ public NLogEcsDocument RenderEcsDocument(LogEventInfo logEvent) ecsEvent.Event = GetEvent(logEvent); ecsEvent.Process = GetProcess(logEvent); ecsEvent.Tags = GetTags(logEvent); +#pragma warning disable CS0618 // Obsolete Labels ecsEvent.Labels = GetLabels(logEvent); +#pragma warning restore CS0618 ecsEvent.Http = GetHttp(logEvent); ecsEvent.Url = GetUrl(logEvent); diff --git a/src/Elastic.CommonSchema/EcsDocument.Generated.cs b/src/Elastic.CommonSchema/EcsDocument.Generated.cs index fbdefb92..08951555 100644 --- a/src/Elastic.CommonSchema/EcsDocument.Generated.cs +++ b/src/Elastic.CommonSchema/EcsDocument.Generated.cs @@ -36,9 +36,10 @@ public partial class EcsDocument : BaseFieldSet , IAs, ICodeSignature, IElf, IEn /// Container for additional metadata against this event. /// /// When working with unknown fields use .
- /// This will try to assign valid ECS fields to their respective property + /// This will try to assign valid ECS fields to their respective property /// Failing that it will assign strings to and everything else to /// + [Obsolete("Use Attributes instead. Metadata values will be merged into Attributes during serialization.")] [JsonPropertyName("metadata"), DataMember(Name = "metadata")] [JsonConverter(typeof(MetadataDictionaryConverter))] public MetadataDictionary? Metadata { get; set; } diff --git a/src/Elastic.CommonSchema/EcsDocument.OTel.cs b/src/Elastic.CommonSchema/EcsDocument.OTel.cs new file mode 100644 index 00000000..b7df5ced --- /dev/null +++ b/src/Elastic.CommonSchema/EcsDocument.OTel.cs @@ -0,0 +1,41 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System.Collections.Generic; +using System.Runtime.Serialization; +using System.Text.Json.Serialization; + +namespace Elastic.CommonSchema; + +public partial class EcsDocument +{ + /// + /// Passthrough key-value store for OTel semantic convention attributes. + /// In Elasticsearch, attributes is a passthrough field type where the prefix can be omitted during queries. + /// In JSON source, data is nested under an attributes object. + /// + [JsonPropertyName("attributes"), DataMember(Name = "attributes")] + public MetadataDictionary? Attributes { get; set; } + + /// + /// Assigns an OTel attribute value. Always stores in . + /// If the OTel name maps to an ECS field (via ), + /// also sets the corresponding ECS property via . + /// + /// The OTel semantic convention attribute name + /// The value to assign + public void AssignOTelField(string otelName, object value) + { + // Always store in Attributes + Attributes ??= new MetadataDictionary(); + Attributes[otelName] = value; + + // If there's a mapping to an ECS field, also set it + if (OTelMappings.OTelToEcs.TryGetValue(otelName, out var ecsPath)) + AssignField(ecsPath, value); + else + // For match relations, the OTel name IS the ECS name — try direct assignment + AssignField(otelName, value); + } +} diff --git a/src/Elastic.CommonSchema/EcsDocument.cs b/src/Elastic.CommonSchema/EcsDocument.cs index a5c92774..55e515ce 100644 --- a/src/Elastic.CommonSchema/EcsDocument.cs +++ b/src/Elastic.CommonSchema/EcsDocument.cs @@ -97,6 +97,9 @@ public static TEcsDocument CreateNewWithDefaults( if (options?.IncludeActivityData is null or true) SetActivityData(doc); + // Process remaining OTel resource attributes through the mapping table + ApplyOTelResourceAttributes(doc, initialCache.OTelResourceAttributes); + return doc; } @@ -121,6 +124,48 @@ private static IDictionary ParseOTelResourceAttributes(string? r return keyValues; } + // Attributes already handled by specific code in GetService/GetHost + private static readonly HashSet HandledResourceAttributes = new() + { + "service.name", "service.version", "service.instance.id", + "deployment.environment", + "host.name", "host.type", "host.arch" + }; + + private static void ApplyOTelResourceAttributes(EcsDocument doc, IDictionary? resourceAttributes) + { + if (resourceAttributes == null || resourceAttributes.Count == 0) return; + + foreach (var kvp in resourceAttributes) + { + var attrName = kvp.Key; + var attrValue = kvp.Value; + + // Skip attributes already handled by specific code + if (HandledResourceAttributes.Contains(attrName)) continue; + + // Check if this OTel attribute name maps to an ECS field + if (OTelMappings.OTelToEcs.TryGetValue(attrName, out var ecsPath)) + { + // Equivalent mapping: set ECS property and store in Attributes + doc.AssignField(ecsPath, attrValue); + doc.Attributes ??= new MetadataDictionary(); + doc.Attributes[attrName] = attrValue; + } + else if (OTelMappings.AllBidirectionalEcsFields.Contains(attrName)) + { + // Match relation: OTel name IS the ECS name + doc.AssignField(attrName, attrValue); + } + else + { + // Unknown attribute: store in Attributes only + doc.Attributes ??= new MetadataDictionary(); + doc.Attributes[attrName] = attrValue; + } + } + } + /// /// Create an instance of that defaults to the assembly from diff --git a/src/Elastic.CommonSchema/FieldSets.Generated.cs b/src/Elastic.CommonSchema/FieldSets.Generated.cs index e7e46e1f..71d4e35e 100644 --- a/src/Elastic.CommonSchema/FieldSets.Generated.cs +++ b/src/Elastic.CommonSchema/FieldSets.Generated.cs @@ -173,6 +173,7 @@ public abstract class BaseFieldSet { /// Example: `docker` and `k8s` labels. /// {"application": "foo-bar", "env": "production"} /// + [Obsolete("Use EcsDocument.Attributes instead. Labels values will be merged into Attributes during serialization.")] [JsonPropertyName("labels"), DataMember(Name = "labels")] public Labels? Labels { get; set; } } @@ -605,6 +606,7 @@ public abstract class ContainerFieldSet { /// Image labels. /// /// + [Obsolete("Use EcsDocument.Attributes instead. Labels values will be merged into Attributes during serialization.")] [JsonPropertyName("labels"), DataMember(Name = "labels")] public ContainerLabels? Labels { get; set; } } diff --git a/src/Elastic.CommonSchema/IndexComponents.Generated.cs b/src/Elastic.CommonSchema/IndexComponents.Generated.cs index def12d37..5255fbfc 100644 --- a/src/Elastic.CommonSchema/IndexComponents.Generated.cs +++ b/src/Elastic.CommonSchema/IndexComponents.Generated.cs @@ -345,6 +345,10 @@ public static class IndexComponents ""@timestamp"": { ""type"": ""date"" }, + ""attributes"": { + ""type"": ""passthrough"", + ""priority"": 10 + }, ""labels"": { ""type"": ""object"" }, @@ -359,8 +363,7 @@ public static class IndexComponents } } } -} -" +}" }, { "ecs_9.3.0_client", diff --git a/src/Elastic.CommonSchema/IndexTemplates.Generated.cs b/src/Elastic.CommonSchema/IndexTemplates.Generated.cs index 669fa81f..5dfda513 100644 --- a/src/Elastic.CommonSchema/IndexTemplates.Generated.cs +++ b/src/Elastic.CommonSchema/IndexTemplates.Generated.cs @@ -110,7 +110,7 @@ public static string GetIndexTemplateForElasticsearchComposable(string indexPatt ""codec"": ""best_compression"", ""mapping"": { ""total_fields"": { - ""limit"": 3000 + ""limit"": 3212 } } } @@ -9562,7 +9562,7 @@ public static string GetIndexTemplateForElasticsearchLegacy(string indexPattern ""index"": { ""mapping"": { ""total_fields"": { - ""limit"": 10000 + ""limit"": 3212 } }, ""refresh_interval"": ""5s"" diff --git a/src/Elastic.CommonSchema/OTelMappings.Generated.cs b/src/Elastic.CommonSchema/OTelMappings.Generated.cs new file mode 100644 index 00000000..e3ba0e1d --- /dev/null +++ b/src/Elastic.CommonSchema/OTelMappings.Generated.cs @@ -0,0 +1,281 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +/* +IMPORTANT NOTE +============== +This file has been generated. +If you wish to submit a PR please modify the original csharp file and submit the PR with that change. Thanks! +*/ + +using System.Collections.Generic; +using System.Collections.ObjectModel; + +namespace Elastic.CommonSchema +{ + /// + /// Bidirectional mapping tables between ECS field paths and OTel semantic convention attribute names. + /// Only includes match and equivalent relations. + /// + public static class OTelMappings + { + /// + /// ECS field path to OTel attribute name(s) for fields where the names differ (equivalent relation). + /// For match relations the names are identical and not listed here. + /// + public static readonly IReadOnlyDictionary EcsToOTel = + new ReadOnlyDictionary(new Dictionary + { + { "cloud.service.name", new[] { "cloud.platform" } }, + { "container.image.hash.all", new[] { "container.image.repo_digests" } }, + { "container.image.tag", new[] { "container.image.tags" } }, + { "container.runtime", new[] { "container.runtime.name" } }, + { "error.message", new[] { "exception.message" } }, + { "error.stack_trace", new[] { "exception.stacktrace" } }, + { "faas.execution", new[] { "faas.invocation_id" } }, + { "faas.trigger.type", new[] { "faas.trigger" } }, + { "file.ctime", new[] { "file.changed" } }, + { "file.gid", new[] { "file.group.id" } }, + { "file.group", new[] { "file.group.name" } }, + { "file.mtime", new[] { "file.modified" } }, + { "file.owner", new[] { "file.owner.name" } }, + { "file.target_path", new[] { "file.symbolic_link.target_path" } }, + { "file.uid", new[] { "file.owner.id" } }, + { "gen_ai.system", new[] { "gen_ai.provider.name" } }, + { "geo.city_name", new[] { "geo.locality.name" } }, + { "geo.continent_code", new[] { "geo.continent.code" } }, + { "geo.country_iso_code", new[] { "geo.country.iso_code" } }, + { "geo.region_iso_code", new[] { "geo.region.iso_code" } }, + { "host.architecture", new[] { "host.arch" } }, + { "http.request.body.bytes", new[] { "http.request.body.size" } }, + { "http.request.bytes", new[] { "http.request.size" } }, + { "http.request.method", new[] { "http.request.method_original" } }, + { "http.response.body.bytes", new[] { "http.response.body.size" } }, + { "http.response.bytes", new[] { "http.response.size" } }, + { "network.protocol", new[] { "network.protocol.name" } }, + { "os.full", new[] { "os.description" } }, + { "process.args", new[] { "process.command_args" } }, + { "process.executable", new[] { "process.executable.path" } }, + { "service.environment", new[] { "deployment.environment.name" } }, + { "service.node.name", new[] { "service.instance.id" } }, + }); + + /// + /// OTel attribute name to ECS field path for fields where the names differ (equivalent relation). + /// For match relations the names are identical and not listed here. + /// + public static readonly IReadOnlyDictionary OTelToEcs = + new ReadOnlyDictionary(new Dictionary + { + { "cloud.platform", "cloud.service.name" }, + { "container.image.repo_digests", "container.image.hash.all" }, + { "container.image.tags", "container.image.tag" }, + { "container.runtime.name", "container.runtime" }, + { "deployment.environment.name", "service.environment" }, + { "exception.message", "error.message" }, + { "exception.stacktrace", "error.stack_trace" }, + { "faas.invocation_id", "faas.execution" }, + { "faas.trigger", "faas.trigger.type" }, + { "file.changed", "file.ctime" }, + { "file.group.id", "file.gid" }, + { "file.group.name", "file.group" }, + { "file.modified", "file.mtime" }, + { "file.owner.id", "file.uid" }, + { "file.owner.name", "file.owner" }, + { "file.symbolic_link.target_path", "file.target_path" }, + { "gen_ai.provider.name", "gen_ai.system" }, + { "geo.continent.code", "geo.continent_code" }, + { "geo.country.iso_code", "geo.country_iso_code" }, + { "geo.locality.name", "geo.city_name" }, + { "geo.region.iso_code", "geo.region_iso_code" }, + { "host.arch", "host.architecture" }, + { "http.request.body.size", "http.request.body.bytes" }, + { "http.request.method_original", "http.request.method" }, + { "http.request.size", "http.request.bytes" }, + { "http.response.body.size", "http.response.body.bytes" }, + { "http.response.size", "http.response.bytes" }, + { "network.protocol.name", "network.protocol" }, + { "os.description", "os.full" }, + { "process.command_args", "process.args" }, + { "process.executable.path", "process.executable" }, + { "service.instance.id", "service.node.name" }, + }); + + /// + /// All bidirectional ECS field paths (both match and equivalent). + /// These are ECS fields that have a corresponding OTel semantic convention. + /// + public static readonly IReadOnlyCollection AllBidirectionalEcsFields = new HashSet + { + "client.address", + "client.port", + "cloud.account.id", + "cloud.availability_zone", + "cloud.provider", + "cloud.region", + "cloud.service.name", + "container.id", + "container.image.hash.all", + "container.image.name", + "container.image.tag", + "container.name", + "container.runtime", + "destination.address", + "destination.port", + "device.id", + "device.manufacturer", + "device.model.identifier", + "device.model.name", + "dns.question.name", + "error.message", + "error.stack_trace", + "error.type", + "faas.coldstart", + "faas.execution", + "faas.name", + "faas.trigger.type", + "faas.version", + "file.accessed", + "file.attributes", + "file.created", + "file.ctime", + "file.directory", + "file.extension", + "file.fork_name", + "file.gid", + "file.group", + "file.inode", + "file.mode", + "file.mtime", + "file.name", + "file.owner", + "file.path", + "file.size", + "file.target_path", + "file.uid", + "gen_ai.agent.description", + "gen_ai.agent.id", + "gen_ai.agent.name", + "gen_ai.operation.name", + "gen_ai.output.type", + "gen_ai.request.choice.count", + "gen_ai.request.encoding_formats", + "gen_ai.request.frequency_penalty", + "gen_ai.request.max_tokens", + "gen_ai.request.model", + "gen_ai.request.presence_penalty", + "gen_ai.request.seed", + "gen_ai.request.stop_sequences", + "gen_ai.request.temperature", + "gen_ai.request.top_k", + "gen_ai.request.top_p", + "gen_ai.response.finish_reasons", + "gen_ai.response.id", + "gen_ai.response.model", + "gen_ai.system", + "gen_ai.token.type", + "gen_ai.tool.call.id", + "gen_ai.tool.name", + "gen_ai.tool.type", + "gen_ai.usage.input_tokens", + "gen_ai.usage.output_tokens", + "geo.city_name", + "geo.continent_code", + "geo.country_iso_code", + "geo.postal_code", + "geo.region_iso_code", + "host.architecture", + "host.id", + "host.ip", + "host.mac", + "host.name", + "host.type", + "http.request.body.bytes", + "http.request.bytes", + "http.request.method", + "http.response.body.bytes", + "http.response.bytes", + "http.response.status_code", + "log.file.path", + "network.protocol", + "network.transport", + "network.type", + "os.full", + "os.name", + "os.version", + "process.args", + "process.args_count", + "process.command_line", + "process.executable", + "process.group_leader.pid", + "process.interactive", + "process.pid", + "process.real_user.id", + "process.real_user.name", + "process.saved_user.id", + "process.saved_user.name", + "process.session_leader.pid", + "process.title", + "process.user.id", + "process.user.name", + "process.vpid", + "process.working_directory", + "server.address", + "server.port", + "service.environment", + "service.name", + "service.node.name", + "service.version", + "source.address", + "source.port", + "tls.cipher", + "tls.client.certificate", + "tls.client.certificate_chain", + "tls.client.hash.md5", + "tls.client.hash.sha1", + "tls.client.hash.sha256", + "tls.client.issuer", + "tls.client.ja3", + "tls.client.not_after", + "tls.client.not_before", + "tls.client.subject", + "tls.client.supported_ciphers", + "tls.curve", + "tls.established", + "tls.next_protocol", + "tls.resumed", + "tls.server.certificate", + "tls.server.certificate_chain", + "tls.server.hash.md5", + "tls.server.hash.sha1", + "tls.server.hash.sha256", + "tls.server.issuer", + "tls.server.ja3s", + "tls.server.not_after", + "tls.server.not_before", + "tls.server.subject", + "url.domain", + "url.extension", + "url.fragment", + "url.full", + "url.original", + "url.path", + "url.port", + "url.query", + "url.registered_domain", + "url.scheme", + "url.subdomain", + "url.top_level_domain", + "user_agent.name", + "user_agent.original", + "user_agent.version", + "user.email", + "user.full_name", + "user.hash", + "user.id", + "user.name", + "user.roles", + }; + } +} diff --git a/src/Elastic.CommonSchema/PropDispatch.Generated.cs b/src/Elastic.CommonSchema/PropDispatch.Generated.cs index 1bca65fe..3746756b 100644 --- a/src/Elastic.CommonSchema/PropDispatch.Generated.cs +++ b/src/Elastic.CommonSchema/PropDispatch.Generated.cs @@ -5,7 +5,7 @@ /* IMPORTANT NOTE ============== -This file has been generated. +This file has been generated. If you wish to submit a PR please modify the original csharp file and submit the PR with that change. Thanks! */ @@ -24,7 +24,7 @@ If you wish to submit a PR please modify the original csharp file and submit the namespace Elastic.CommonSchema { /// - public partial class EcsDocument : BaseFieldSet + public partial class EcsDocument : BaseFieldSet { /// /// Set ECS fields by name on . @@ -43,9 +43,9 @@ public partial class EcsDocument : BaseFieldSet public void AssignField(string path, object value) { var assigned = LogTemplateProperties.All.Contains(path) && TrySet(this, path, value); - if (!assigned && LogTemplateEntities.All.Contains(path)) + if (!assigned && LogTemplateEntities.All.Contains(path)) assigned = TrySetEntity(this, path, value); - if (!assigned) + if (!assigned) SetMetaOrLabel(this, path, value); } } @@ -688,9 +688,9 @@ bool TypeCheck(Dictionary templatedObject, string typeName) => } } - internal static bool TrySet(EcsDocument document, string path, object value) + internal static bool TrySet(EcsDocument document, string path, object value) { - switch (path) + switch (path) { case "@timestamp": case "Timestamp": @@ -3257,6 +3257,9 @@ internal static bool TrySet(EcsDocument document, string path, object value) case "X509VersionNumber": return TrySetX509(document, path, value); default: + // OTel equivalent name fallback: look up OTel name -> ECS path, then retry + if (OTelMappings.OTelToEcs.TryGetValue(path, out var ecsPath)) + return TrySet(document, ecsPath, value); return false; } } @@ -3304,7 +3307,7 @@ public static bool TrySetAgent(EcsDocument document, string path, object value) { var assign = TryAssignAgent(path); if (assign == null) return false; - + var entity = document.Agent ?? new Agent(); var assigned = assign(entity, value); if (assigned) document.Agent = entity; @@ -3327,7 +3330,7 @@ public static bool TrySetAs(IAs document, string path, object value) { var assign = TryAssignAs(path); if (assign == null) return false; - + var entity = document.As ?? new As(); var assigned = assign(entity, value); if (assigned) document.As = entity; @@ -3440,7 +3443,7 @@ public static bool TrySetClient(EcsDocument document, string path, object value) { var assign = TryAssignClient(path); if (assign == null) return false; - + var entity = document.Client ?? new Client(); var assigned = assign(entity, value); if (assigned) document.Client = entity; @@ -3473,6 +3476,7 @@ public static Func TryAssignCloud(string path) "CloudRegion" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Region = p), "cloud.service.name" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.ServiceName = p), "CloudServiceName" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.ServiceName = p), + "cloud.platform" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.ServiceName = p), "cloud.entity.display_name" => static (e, v) => TryAssignEntity("entity.display_name")(e.Entity ??= new Entity(),v), "CloudEntityDisplayName" => static (e, v) => TryAssignEntity("entity.display_name")(e.Entity ??= new Entity(),v), "cloud.entity.id" => static (e, v) => TryAssignEntity("entity.id")(e.Entity ??= new Entity(),v), @@ -3495,7 +3499,7 @@ public static bool TrySetCloud(EcsDocument document, string path, object value) { var assign = TryAssignCloud(path); if (assign == null) return false; - + var entity = document.Cloud ?? new Cloud(); var assigned = assign(entity, value); if (assigned) document.Cloud = entity; @@ -3536,7 +3540,7 @@ public static bool TrySetCodeSignature(ICodeSignature document, string path, obj { var assign = TryAssignCodeSignature(path); if (assign == null) return false; - + var entity = document.CodeSignature ?? new CodeSignature(); var assigned = assign(entity, value); if (assigned) document.CodeSignature = entity; @@ -3567,6 +3571,7 @@ public static Func TryAssignContainer(string path) "ContainerNetworkIngressBytes" => static (e, v) => TrySetLong(e, v, static (ee, p) => ee.NetworkIngressBytes = p), "container.runtime" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Runtime = p), "ContainerRuntime" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Runtime = p), + "container.runtime.name" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Runtime = p), "container.security_context.privileged" => static (e, v) => TrySetBool(e, v, static (ee, p) => ee.SecurityContextPrivileged = p), "ContainerSecurityContextPrivileged" => static (e, v) => TrySetBool(e, v, static (ee, p) => ee.SecurityContextPrivileged = p), _ => null @@ -3577,7 +3582,7 @@ public static bool TrySetContainer(EcsDocument document, string path, object val { var assign = TryAssignContainer(path); if (assign == null) return false; - + var entity = document.Container ?? new Container(); var assigned = assign(entity, value); if (assigned) document.Container = entity; @@ -3602,7 +3607,7 @@ public static bool TrySetDataStream(EcsDocument document, string path, object va { var assign = TryAssignDataStream(path); if (assign == null) return false; - + var entity = document.DataStream ?? new DataStream(); var assigned = assign(entity, value); if (assigned) document.DataStream = entity; @@ -3715,7 +3720,7 @@ public static bool TrySetDestination(EcsDocument document, string path, object v { var assign = TryAssignDestination(path); if (assign == null) return false; - + var entity = document.Destination ?? new Destination(); var assigned = assign(entity, value); if (assigned) document.Destination = entity; @@ -3754,7 +3759,7 @@ public static bool TrySetDevice(EcsDocument document, string path, object value) { var assign = TryAssignDevice(path); if (assign == null) return false; - + var entity = document.Device ?? new Device(); var assigned = assign(entity, value); if (assigned) document.Device = entity; @@ -3851,7 +3856,7 @@ public static bool TrySetDll(EcsDocument document, string path, object value) { var assign = TryAssignDll(path); if (assign == null) return false; - + var entity = document.Dll ?? new Dll(); var assigned = assign(entity, value); if (assigned) document.Dll = entity; @@ -3890,7 +3895,7 @@ public static bool TrySetDns(EcsDocument document, string path, object value) { var assign = TryAssignDns(path); if (assign == null) return false; - + var entity = document.Dns ?? new Dns(); var assigned = assign(entity, value); if (assigned) document.Dns = entity; @@ -3911,7 +3916,7 @@ public static bool TrySetEcs(EcsDocument document, string path, object value) { var assign = TryAssignEcs(path); if (assign == null) return false; - + var entity = document.Ecs ?? new Ecs(); var assigned = assign(entity, value); if (assigned) document.Ecs = entity; @@ -3972,7 +3977,7 @@ public static bool TrySetElf(IElf document, string path, object value) { var assign = TryAssignElf(path); if (assign == null) return false; - + var entity = document.Elf ?? new Elf(); var assigned = assign(entity, value); if (assigned) document.Elf = entity; @@ -4009,7 +4014,7 @@ public static bool TrySetEmail(EcsDocument document, string path, object value) { var assign = TryAssignEmail(path); if (assign == null) return false; - + var entity = document.Email ?? new Email(); var assigned = assign(entity, value); if (assigned) document.Email = entity; @@ -4042,7 +4047,7 @@ public static bool TrySetEntity(IEntity document, string path, object value) { var assign = TryAssignEntity(path); if (assign == null) return false; - + var entity = document.Entity ?? new Entity(); var assigned = assign(entity, value); if (assigned) document.Entity = entity; @@ -4059,8 +4064,10 @@ public static Func TryAssignError(string path) "ErrorId" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Id = p), "error.message" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Message = p), "ErrorMessage" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Message = p), + "exception.message" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Message = p), "error.stack_trace" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.StackTrace = p), "ErrorStackTrace" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.StackTrace = p), + "exception.stacktrace" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.StackTrace = p), "error.type" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Type = p), "ErrorType" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Type = p), _ => null @@ -4071,7 +4078,7 @@ public static bool TrySetError(EcsDocument document, string path, object value) { var assign = TryAssignError(path); if (assign == null) return false; - + var entity = document.Error ?? new Error(); var assigned = assign(entity, value); if (assigned) document.Error = entity; @@ -4138,7 +4145,7 @@ public static bool TrySetEvent(EcsDocument document, string path, object value) { var assign = TryAssignEvent(path); if (assign == null) return false; - + var entity = document.Event ?? new Event(); var assigned = assign(entity, value); if (assigned) document.Event = entity; @@ -4153,6 +4160,7 @@ public static Func TryAssignFaas(string path) "FaasColdstart" => static (e, v) => TrySetBool(e, v, static (ee, p) => ee.Coldstart = p), "faas.execution" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Execution = p), "FaasExecution" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Execution = p), + "faas.invocation_id" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Execution = p), "faas.id" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Id = p), "FaasId" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Id = p), "faas.name" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Name = p), @@ -4161,6 +4169,7 @@ public static Func TryAssignFaas(string path) "FaasTriggerRequestId" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.TriggerRequestId = p), "faas.trigger.type" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.TriggerType = p), "FaasTriggerType" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.TriggerType = p), + "faas.trigger" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.TriggerType = p), "faas.version" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Version = p), "FaasVersion" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Version = p), _ => null @@ -4171,7 +4180,7 @@ public static bool TrySetFaas(EcsDocument document, string path, object value) { var assign = TryAssignFaas(path); if (assign == null) return false; - + var entity = document.Faas ?? new Faas(); var assigned = assign(entity, value); if (assigned) document.Faas = entity; @@ -4188,6 +4197,7 @@ public static Func TryAssignFile(string path) "FileCreated" => static (e, v) => TrySetDateTimeOffset(e, v, static (ee, p) => ee.Created = p), "file.ctime" => static (e, v) => TrySetDateTimeOffset(e, v, static (ee, p) => ee.Ctime = p), "FileCtime" => static (e, v) => TrySetDateTimeOffset(e, v, static (ee, p) => ee.Ctime = p), + "file.changed" => static (e, v) => TrySetDateTimeOffset(e, v, static (ee, p) => ee.Ctime = p), "file.device" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Device = p), "FileDevice" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Device = p), "file.directory" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Directory = p), @@ -4200,8 +4210,10 @@ public static Func TryAssignFile(string path) "FileForkName" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.ForkName = p), "file.gid" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Gid = p), "FileGid" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Gid = p), + "file.group.id" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Gid = p), "file.group" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Group = p), "FileGroup" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Group = p), + "file.group.name" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Group = p), "file.inode" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Inode = p), "FileInode" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Inode = p), "file.mime_type" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.MimeType = p), @@ -4210,6 +4222,7 @@ public static Func TryAssignFile(string path) "FileMode" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Mode = p), "file.mtime" => static (e, v) => TrySetDateTimeOffset(e, v, static (ee, p) => ee.Mtime = p), "FileMtime" => static (e, v) => TrySetDateTimeOffset(e, v, static (ee, p) => ee.Mtime = p), + "file.modified" => static (e, v) => TrySetDateTimeOffset(e, v, static (ee, p) => ee.Mtime = p), "file.name" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Name = p), "FileName" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Name = p), "file.origin_referrer_url" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.OriginReferrerUrl = p), @@ -4218,16 +4231,19 @@ public static Func TryAssignFile(string path) "FileOriginUrl" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.OriginUrl = p), "file.owner" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Owner = p), "FileOwner" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Owner = p), + "file.owner.name" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Owner = p), "file.path" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Path = p), "FilePath" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Path = p), "file.size" => static (e, v) => TrySetLong(e, v, static (ee, p) => ee.Size = p), "FileSize" => static (e, v) => TrySetLong(e, v, static (ee, p) => ee.Size = p), "file.target_path" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.TargetPath = p), "FileTargetPath" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.TargetPath = p), + "file.symbolic_link.target_path" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.TargetPath = p), "file.type" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Type = p), "FileType" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Type = p), "file.uid" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Uid = p), "FileUid" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Uid = p), + "file.owner.id" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Uid = p), "file.hash.cdhash" => static (e, v) => TryAssignHash("hash.cdhash")(e.Hash ??= new Hash(),v), "FileHashCdhash" => static (e, v) => TryAssignHash("hash.cdhash")(e.Hash ??= new Hash(),v), "file.hash.md5" => static (e, v) => TryAssignHash("hash.md5")(e.Hash ??= new Hash(),v), @@ -4388,7 +4404,7 @@ public static bool TrySetFile(EcsDocument document, string path, object value) { var assign = TryAssignFile(path); if (assign == null) return false; - + var entity = document.File ?? new File(); var assigned = assign(entity, value); if (assigned) document.File = entity; @@ -4433,6 +4449,7 @@ public static Func TryAssignGenAi(string path) "GenAiResponseModel" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.ResponseModel = p), "gen_ai.system" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.System = p), "GenAiSystem" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.System = p), + "gen_ai.provider.name" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.System = p), "gen_ai.token.type" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.TokenType = p), "GenAiTokenType" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.TokenType = p), "gen_ai.tool.call.id" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.ToolCallId = p), @@ -4453,7 +4470,7 @@ public static bool TrySetGenAi(EcsDocument document, string path, object value) { var assign = TryAssignGenAi(path); if (assign == null) return false; - + var entity = document.GenAi ?? new GenAi(); var assigned = assign(entity, value); if (assigned) document.GenAi = entity; @@ -4466,12 +4483,15 @@ public static Func TryAssignGeo(string path) { "geo.city_name" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.CityName = p), "GeoCityName" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.CityName = p), + "geo.locality.name" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.CityName = p), "geo.continent_code" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.ContinentCode = p), "GeoContinentCode" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.ContinentCode = p), + "geo.continent.code" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.ContinentCode = p), "geo.continent_name" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.ContinentName = p), "GeoContinentName" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.ContinentName = p), "geo.country_iso_code" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.CountryIsoCode = p), "GeoCountryIsoCode" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.CountryIsoCode = p), + "geo.country.iso_code" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.CountryIsoCode = p), "geo.country_name" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.CountryName = p), "GeoCountryName" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.CountryName = p), "geo.name" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Name = p), @@ -4480,6 +4500,7 @@ public static Func TryAssignGeo(string path) "GeoPostalCode" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.PostalCode = p), "geo.region_iso_code" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.RegionIsoCode = p), "GeoRegionIsoCode" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.RegionIsoCode = p), + "geo.region.iso_code" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.RegionIsoCode = p), "geo.region_name" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.RegionName = p), "GeoRegionName" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.RegionName = p), "geo.timezone" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Timezone = p), @@ -4492,7 +4513,7 @@ public static bool TrySetGeo(IGeo document, string path, object value) { var assign = TryAssignGeo(path); if (assign == null) return false; - + var entity = document.Geo ?? new Geo(); var assigned = assign(entity, value); if (assigned) document.Geo = entity; @@ -4517,7 +4538,7 @@ public static bool TrySetGroup(IGroup document, string path, object value) { var assign = TryAssignGroup(path); if (assign == null) return false; - + var entity = document.Group ?? new Group(); var assigned = assign(entity, value); if (assigned) document.Group = entity; @@ -4552,7 +4573,7 @@ public static bool TrySetHash(IHash document, string path, object value) { var assign = TryAssignHash(path); if (assign == null) return false; - + var entity = document.Hash ?? new Hash(); var assigned = assign(entity, value); if (assigned) document.Hash = entity; @@ -4565,6 +4586,7 @@ public static Func TryAssignHost(string path) { "host.architecture" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Architecture = p), "HostArchitecture" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Architecture = p), + "host.arch" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Architecture = p), "host.boot.id" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.BootId = p), "HostBootId" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.BootId = p), "host.cpu.usage" => static (e, v) => TrySetFloat(e, v, static (ee, p) => ee.CpuUsage = p), @@ -4665,7 +4687,7 @@ public static bool TrySetHost(EcsDocument document, string path, object value) { var assign = TryAssignHost(path); if (assign == null) return false; - + var entity = document.Host ?? new Host(); var assigned = assign(entity, value); if (assigned) document.Host = entity; @@ -4678,24 +4700,29 @@ public static Func TryAssignHttp(string path) { "http.request.body.bytes" => static (e, v) => TrySetLong(e, v, static (ee, p) => ee.RequestBodyBytes = p), "HttpRequestBodyBytes" => static (e, v) => TrySetLong(e, v, static (ee, p) => ee.RequestBodyBytes = p), + "http.request.body.size" => static (e, v) => TrySetLong(e, v, static (ee, p) => ee.RequestBodyBytes = p), "http.request.body.content" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.RequestBodyContent = p), "HttpRequestBodyContent" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.RequestBodyContent = p), "http.request.bytes" => static (e, v) => TrySetLong(e, v, static (ee, p) => ee.RequestBytes = p), "HttpRequestBytes" => static (e, v) => TrySetLong(e, v, static (ee, p) => ee.RequestBytes = p), + "http.request.size" => static (e, v) => TrySetLong(e, v, static (ee, p) => ee.RequestBytes = p), "http.request.id" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.RequestId = p), "HttpRequestId" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.RequestId = p), "http.request.method" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.RequestMethod = p), "HttpRequestMethod" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.RequestMethod = p), + "http.request.method_original" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.RequestMethod = p), "http.request.mime_type" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.RequestMimeType = p), "HttpRequestMimeType" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.RequestMimeType = p), "http.request.referrer" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.RequestReferrer = p), "HttpRequestReferrer" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.RequestReferrer = p), "http.response.body.bytes" => static (e, v) => TrySetLong(e, v, static (ee, p) => ee.ResponseBodyBytes = p), "HttpResponseBodyBytes" => static (e, v) => TrySetLong(e, v, static (ee, p) => ee.ResponseBodyBytes = p), + "http.response.body.size" => static (e, v) => TrySetLong(e, v, static (ee, p) => ee.ResponseBodyBytes = p), "http.response.body.content" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.ResponseBodyContent = p), "HttpResponseBodyContent" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.ResponseBodyContent = p), "http.response.bytes" => static (e, v) => TrySetLong(e, v, static (ee, p) => ee.ResponseBytes = p), "HttpResponseBytes" => static (e, v) => TrySetLong(e, v, static (ee, p) => ee.ResponseBytes = p), + "http.response.size" => static (e, v) => TrySetLong(e, v, static (ee, p) => ee.ResponseBytes = p), "http.response.mime_type" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.ResponseMimeType = p), "HttpResponseMimeType" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.ResponseMimeType = p), "http.response.status_code" => static (e, v) => TrySetLong(e, v, static (ee, p) => ee.ResponseStatusCode = p), @@ -4710,7 +4737,7 @@ public static bool TrySetHttp(EcsDocument document, string path, object value) { var assign = TryAssignHttp(path); if (assign == null) return false; - + var entity = document.Http ?? new Http(); var assigned = assign(entity, value); if (assigned) document.Http = entity; @@ -4735,7 +4762,7 @@ public static bool TrySetInterface(EcsDocument document, string path, object val { var assign = TryAssignInterface(path); if (assign == null) return false; - + var entity = document.Interface ?? new Interface(); var assigned = assign(entity, value); if (assigned) document.Interface = entity; @@ -4766,7 +4793,7 @@ public static bool TrySetLog(EcsDocument document, string path, object value) { var assign = TryAssignLog(path); if (assign == null) return false; - + var entity = document.Log ?? new Log(); var assigned = assign(entity, value); if (assigned) document.Log = entity; @@ -4803,7 +4830,7 @@ public static bool TrySetMacho(IMacho document, string path, object value) { var assign = TryAssignMacho(path); if (assign == null) return false; - + var entity = document.Macho ?? new Macho(); var assigned = assign(entity, value); if (assigned) document.Macho = entity; @@ -4832,6 +4859,7 @@ public static Func TryAssignNetwork(string path) "NetworkPackets" => static (e, v) => TrySetLong(e, v, static (ee, p) => ee.Packets = p), "network.protocol" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Protocol = p), "NetworkProtocol" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Protocol = p), + "network.protocol.name" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Protocol = p), "network.transport" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Transport = p), "NetworkTransport" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Transport = p), "network.type" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Type = p), @@ -4848,7 +4876,7 @@ public static bool TrySetNetwork(EcsDocument document, string path, object value { var assign = TryAssignNetwork(path); if (assign == null) return false; - + var entity = document.Network ?? new Network(); var assigned = assign(entity, value); if (assigned) document.Network = entity; @@ -4917,7 +4945,7 @@ public static bool TrySetObserver(EcsDocument document, string path, object valu { var assign = TryAssignObserver(path); if (assign == null) return false; - + var entity = document.Observer ?? new Observer(); var assigned = assign(entity, value); if (assigned) document.Observer = entity; @@ -4974,7 +5002,7 @@ public static bool TrySetOrchestrator(EcsDocument document, string path, object { var assign = TryAssignOrchestrator(path); if (assign == null) return false; - + var entity = document.Orchestrator ?? new Orchestrator(); var assigned = assign(entity, value); if (assigned) document.Orchestrator = entity; @@ -4997,7 +5025,7 @@ public static bool TrySetOrganization(EcsDocument document, string path, object { var assign = TryAssignOrganization(path); if (assign == null) return false; - + var entity = document.Organization ?? new Organization(); var assigned = assign(entity, value); if (assigned) document.Organization = entity; @@ -5012,6 +5040,7 @@ public static Func TryAssignOs(string path) "OsFamily" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Family = p), "os.full" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Full = p), "OsFull" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Full = p), + "os.description" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Full = p), "os.kernel" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Kernel = p), "OsKernel" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Kernel = p), "os.name" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Name = p), @@ -5030,7 +5059,7 @@ public static bool TrySetOs(IOs document, string path, object value) { var assign = TryAssignOs(path); if (assign == null) return false; - + var entity = document.Os ?? new Os(); var assigned = assign(entity, value); if (assigned) document.Os = entity; @@ -5075,7 +5104,7 @@ public static bool TrySetPackage(EcsDocument document, string path, object value { var assign = TryAssignPackage(path); if (assign == null) return false; - + var entity = document.Package ?? new Package(); var assigned = assign(entity, value); if (assigned) document.Package = entity; @@ -5126,7 +5155,7 @@ public static bool TrySetPe(IPe document, string path, object value) { var assign = TryAssignPe(path); if (assign == null) return false; - + var entity = document.Pe ?? new Pe(); var assigned = assign(entity, value); if (assigned) document.Pe = entity; @@ -5147,6 +5176,7 @@ public static Func TryAssignProcess(string path) "ProcessEntityId" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.EntityId = p), "process.executable" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Executable = p), "ProcessExecutable" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Executable = p), + "process.executable.path" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Executable = p), "process.exit_code" => static (e, v) => TrySetLong(e, v, static (ee, p) => ee.ExitCode = p), "ProcessExitCode" => static (e, v) => TrySetLong(e, v, static (ee, p) => ee.ExitCode = p), "process.interactive" => static (e, v) => TrySetBool(e, v, static (ee, p) => ee.Interactive = p), @@ -5595,7 +5625,7 @@ public static bool TrySetProcess(EcsDocument document, string path, object value { var assign = TryAssignProcess(path); if (assign == null) return false; - + var entity = document.Process ?? new Process(); var assigned = assign(entity, value); if (assigned) document.Process = entity; @@ -5626,7 +5656,7 @@ public static bool TrySetRegistry(EcsDocument document, string path, object valu { var assign = TryAssignRegistry(path); if (assign == null) return false; - + var entity = document.Registry ?? new Registry(); var assigned = assign(entity, value); if (assigned) document.Registry = entity; @@ -5645,7 +5675,7 @@ public static bool TrySetRelated(EcsDocument document, string path, object value { var assign = TryAssignRelated(path); if (assign == null) return false; - + var entity = document.Related ?? new Related(); var assigned = assign(entity, value); if (assigned) document.Related = entity; @@ -5676,7 +5706,7 @@ public static bool TrySetRisk(IRisk document, string path, object value) { var assign = TryAssignRisk(path); if (assign == null) return false; - + var entity = document.Risk ?? new Risk(); var assigned = assign(entity, value); if (assigned) document.Risk = entity; @@ -5713,7 +5743,7 @@ public static bool TrySetRule(EcsDocument document, string path, object value) { var assign = TryAssignRule(path); if (assign == null) return false; - + var entity = document.Rule ?? new Rule(); var assigned = assign(entity, value); if (assigned) document.Rule = entity; @@ -5826,7 +5856,7 @@ public static bool TrySetServer(EcsDocument document, string path, object value) { var assign = TryAssignServer(path); if (assign == null) return false; - + var entity = document.Server ?? new Server(); var assigned = assign(entity, value); if (assigned) document.Server = entity; @@ -5841,6 +5871,7 @@ public static Func TryAssignService(string path) "ServiceAddress" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Address = p), "service.environment" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Environment = p), "ServiceEnvironment" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Environment = p), + "deployment.environment.name" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Environment = p), "service.ephemeral_id" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.EphemeralId = p), "ServiceEphemeralId" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.EphemeralId = p), "service.id" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Id = p), @@ -5849,6 +5880,7 @@ public static Func TryAssignService(string path) "ServiceName" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Name = p), "service.node.name" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.NodeName = p), "ServiceNodeName" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.NodeName = p), + "service.instance.id" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.NodeName = p), "service.node.role" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.NodeRole = p), "ServiceNodeRole" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.NodeRole = p), "service.state" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.State = p), @@ -5879,7 +5911,7 @@ public static bool TrySetService(EcsDocument document, string path, object value { var assign = TryAssignService(path); if (assign == null) return false; - + var entity = document.Service ?? new Service(); var assigned = assign(entity, value); if (assigned) document.Service = entity; @@ -5992,7 +6024,7 @@ public static bool TrySetSource(EcsDocument document, string path, object value) { var assign = TryAssignSource(path); if (assign == null) return false; - + var entity = document.Source ?? new Source(); var assigned = assign(entity, value); if (assigned) document.Source = entity; @@ -6353,7 +6385,7 @@ public static bool TrySetThreat(EcsDocument document, string path, object value) { var assign = TryAssignThreat(path); if (assign == null) return false; - + var entity = document.Threat ?? new Threat(); var assigned = assign(entity, value); if (assigned) document.Threat = entity; @@ -6446,7 +6478,7 @@ public static bool TrySetTls(EcsDocument document, string path, object value) { var assign = TryAssignTls(path); if (assign == null) return false; - + var entity = document.Tls ?? new Tls(); var assigned = assign(entity, value); if (assigned) document.Tls = entity; @@ -6493,7 +6525,7 @@ public static bool TrySetUrl(EcsDocument document, string path, object value) { var assign = TryAssignUrl(path); if (assign == null) return false; - + var entity = document.Url ?? new Url(); var assigned = assign(entity, value); if (assigned) document.Url = entity; @@ -6556,7 +6588,7 @@ public static bool TrySetUser(IUser document, string path, object value) { var assign = TryAssignUser(path); if (assign == null) return false; - + var entity = document.User ?? new User(); var assigned = assign(entity, value); if (assigned) document.User = entity; @@ -6597,7 +6629,7 @@ public static bool TrySetUserAgent(EcsDocument document, string path, object val { var assign = TryAssignUserAgent(path); if (assign == null) return false; - + var entity = document.UserAgent ?? new UserAgent(); var assigned = assign(entity, value); if (assigned) document.UserAgent = entity; @@ -6620,7 +6652,7 @@ public static bool TrySetVlan(IVlan document, string path, object value) { var assign = TryAssignVlan(path); if (assign == null) return false; - + var entity = document.Vlan ?? new Vlan(); var assigned = assign(entity, value); if (assigned) document.Vlan = entity; @@ -6671,7 +6703,7 @@ public static bool TrySetVolume(EcsDocument document, string path, object value) { var assign = TryAssignVolume(path); if (assign == null) return false; - + var entity = document.Volume ?? new Volume(); var assigned = assign(entity, value); if (assigned) document.Volume = entity; @@ -6714,7 +6746,7 @@ public static bool TrySetVulnerability(EcsDocument document, string path, object { var assign = TryAssignVulnerability(path); if (assign == null) return false; - + var entity = document.Vulnerability ?? new Vulnerability(); var assigned = assign(entity, value); if (assigned) document.Vulnerability = entity; @@ -6755,7 +6787,7 @@ public static bool TrySetX509(IX509 document, string path, object value) { var assign = TryAssignX509(path); if (assign == null) return false; - + var entity = document.X509 ?? new X509(); var assigned = assign(entity, value); if (assigned) document.X509 = entity; diff --git a/src/Elastic.CommonSchema/PropDispatch.cs b/src/Elastic.CommonSchema/PropDispatch.cs index 68806606..6aa98d72 100644 --- a/src/Elastic.CommonSchema/PropDispatch.cs +++ b/src/Elastic.CommonSchema/PropDispatch.cs @@ -30,21 +30,8 @@ internal static partial class PropDispatch { public static void SetMetaOrLabel(EcsDocument document, string path, object value) { - switch (value) - { - case string s: - document.Labels ??= new Labels(); - document.Labels[path] = s; - break; - case bool b: - document.Labels ??= new Labels(); - document.Labels[path] = b.ToString(CultureInfo.InvariantCulture).ToLowerInvariant(); - break; - default: - document.Metadata ??= new MetadataDictionary(); - document.Metadata[path] = value; - break; - } + document.Attributes ??= new MetadataDictionary(); + document.Attributes[path] = value; } private static bool TrySetLong(T target, object value, Action set) diff --git a/src/Elastic.CommonSchema/SemConv.Generated.cs b/src/Elastic.CommonSchema/SemConv.Generated.cs new file mode 100644 index 00000000..8663c5ec --- /dev/null +++ b/src/Elastic.CommonSchema/SemConv.Generated.cs @@ -0,0 +1,861 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +/* +IMPORTANT NOTE +============== +This file has been generated. +If you wish to submit a PR please modify the original csharp file and submit the PR with that change. Thanks! +*/ + +namespace Elastic.CommonSchema +{ + /// + /// OpenTelemetry semantic convention attribute name constants. + /// Use these with and . + /// + public static class SemConv + { + /// + /// OTel client.address (match: same as ECS) + /// + public const string ClientAddress = "client.address"; + + /// + /// OTel client.port (match: same as ECS) + /// + public const string ClientPort = "client.port"; + + /// + /// OTel cloud.account.id (match: same as ECS) + /// + public const string CloudAccountId = "cloud.account.id"; + + /// + /// OTel cloud.availability_zone (match: same as ECS) + /// + public const string CloudAvailabilityZone = "cloud.availability_zone"; + + /// + /// OTel cloud.platform (equivalent: ECS cloud.service.name) + /// + public const string CloudPlatform = "cloud.platform"; + + /// + /// OTel cloud.provider (match: same as ECS) + /// + public const string CloudProvider = "cloud.provider"; + + /// + /// OTel cloud.region (match: same as ECS) + /// + public const string CloudRegion = "cloud.region"; + + /// + /// OTel container.id (match: same as ECS) + /// + public const string ContainerId = "container.id"; + + /// + /// OTel container.image.name (match: same as ECS) + /// + public const string ContainerImageName = "container.image.name"; + + /// + /// OTel container.image.repo_digests (equivalent: ECS container.image.hash.all) + /// + public const string ContainerImageRepoDigests = "container.image.repo_digests"; + + /// + /// OTel container.image.tags (equivalent: ECS container.image.tag) + /// + public const string ContainerImageTags = "container.image.tags"; + + /// + /// OTel container.name (match: same as ECS) + /// + public const string ContainerName = "container.name"; + + /// + /// OTel container.runtime.name (equivalent: ECS container.runtime) + /// + public const string ContainerRuntimeName = "container.runtime.name"; + + /// + /// OTel deployment.environment.name (equivalent: ECS service.environment) + /// + public const string DeploymentEnvironmentName = "deployment.environment.name"; + + /// + /// OTel destination.address (match: same as ECS) + /// + public const string DestinationAddress = "destination.address"; + + /// + /// OTel destination.port (match: same as ECS) + /// + public const string DestinationPort = "destination.port"; + + /// + /// OTel device.id (match: same as ECS) + /// + public const string DeviceId = "device.id"; + + /// + /// OTel device.manufacturer (match: same as ECS) + /// + public const string DeviceManufacturer = "device.manufacturer"; + + /// + /// OTel device.model.identifier (match: same as ECS) + /// + public const string DeviceModelIdentifier = "device.model.identifier"; + + /// + /// OTel device.model.name (match: same as ECS) + /// + public const string DeviceModelName = "device.model.name"; + + /// + /// OTel dns.question.name (match: same as ECS) + /// + public const string DnsQuestionName = "dns.question.name"; + + /// + /// OTel error.type (match: same as ECS) + /// + public const string ErrorType = "error.type"; + + /// + /// OTel exception.message (equivalent: ECS error.message) + /// + public const string ExceptionMessage = "exception.message"; + + /// + /// OTel exception.stacktrace (equivalent: ECS error.stack_trace) + /// + public const string ExceptionStacktrace = "exception.stacktrace"; + + /// + /// OTel faas.coldstart (match: same as ECS) + /// + public const string FaasColdstart = "faas.coldstart"; + + /// + /// OTel faas.invocation_id (equivalent: ECS faas.execution) + /// + public const string FaasInvocationId = "faas.invocation_id"; + + /// + /// OTel faas.name (match: same as ECS) + /// + public const string FaasName = "faas.name"; + + /// + /// OTel faas.trigger (equivalent: ECS faas.trigger.type) + /// + public const string FaasTrigger = "faas.trigger"; + + /// + /// OTel faas.version (match: same as ECS) + /// + public const string FaasVersion = "faas.version"; + + /// + /// OTel file.accessed (match: same as ECS) + /// + public const string FileAccessed = "file.accessed"; + + /// + /// OTel file.attributes (match: same as ECS) + /// + public const string FileAttributes = "file.attributes"; + + /// + /// OTel file.changed (equivalent: ECS file.ctime) + /// + public const string FileChanged = "file.changed"; + + /// + /// OTel file.created (match: same as ECS) + /// + public const string FileCreated = "file.created"; + + /// + /// OTel file.directory (match: same as ECS) + /// + public const string FileDirectory = "file.directory"; + + /// + /// OTel file.extension (match: same as ECS) + /// + public const string FileExtension = "file.extension"; + + /// + /// OTel file.fork_name (match: same as ECS) + /// + public const string FileForkName = "file.fork_name"; + + /// + /// OTel file.group.id (equivalent: ECS file.gid) + /// + public const string FileGroupId = "file.group.id"; + + /// + /// OTel file.group.name (equivalent: ECS file.group) + /// + public const string FileGroupName = "file.group.name"; + + /// + /// OTel file.inode (match: same as ECS) + /// + public const string FileInode = "file.inode"; + + /// + /// OTel file.mode (match: same as ECS) + /// + public const string FileMode = "file.mode"; + + /// + /// OTel file.modified (equivalent: ECS file.mtime) + /// + public const string FileModified = "file.modified"; + + /// + /// OTel file.name (match: same as ECS) + /// + public const string FileName = "file.name"; + + /// + /// OTel file.owner.id (equivalent: ECS file.uid) + /// + public const string FileOwnerId = "file.owner.id"; + + /// + /// OTel file.owner.name (equivalent: ECS file.owner) + /// + public const string FileOwnerName = "file.owner.name"; + + /// + /// OTel file.path (match: same as ECS) + /// + public const string FilePath = "file.path"; + + /// + /// OTel file.size (match: same as ECS) + /// + public const string FileSize = "file.size"; + + /// + /// OTel file.symbolic_link.target_path (equivalent: ECS file.target_path) + /// + public const string FileSymbolicLinkTargetPath = "file.symbolic_link.target_path"; + + /// + /// OTel gen_ai.agent.description (match: same as ECS) + /// + public const string GenAiAgentDescription = "gen_ai.agent.description"; + + /// + /// OTel gen_ai.agent.id (match: same as ECS) + /// + public const string GenAiAgentId = "gen_ai.agent.id"; + + /// + /// OTel gen_ai.agent.name (match: same as ECS) + /// + public const string GenAiAgentName = "gen_ai.agent.name"; + + /// + /// OTel gen_ai.operation.name (match: same as ECS) + /// + public const string GenAiOperationName = "gen_ai.operation.name"; + + /// + /// OTel gen_ai.output.type (match: same as ECS) + /// + public const string GenAiOutputType = "gen_ai.output.type"; + + /// + /// OTel gen_ai.provider.name (equivalent: ECS gen_ai.system) + /// + public const string GenAiProviderName = "gen_ai.provider.name"; + + /// + /// OTel gen_ai.request.choice.count (match: same as ECS) + /// + public const string GenAiRequestChoiceCount = "gen_ai.request.choice.count"; + + /// + /// OTel gen_ai.request.encoding_formats (match: same as ECS) + /// + public const string GenAiRequestEncodingFormats = "gen_ai.request.encoding_formats"; + + /// + /// OTel gen_ai.request.frequency_penalty (match: same as ECS) + /// + public const string GenAiRequestFrequencyPenalty = "gen_ai.request.frequency_penalty"; + + /// + /// OTel gen_ai.request.max_tokens (match: same as ECS) + /// + public const string GenAiRequestMaxTokens = "gen_ai.request.max_tokens"; + + /// + /// OTel gen_ai.request.model (match: same as ECS) + /// + public const string GenAiRequestModel = "gen_ai.request.model"; + + /// + /// OTel gen_ai.request.presence_penalty (match: same as ECS) + /// + public const string GenAiRequestPresencePenalty = "gen_ai.request.presence_penalty"; + + /// + /// OTel gen_ai.request.seed (match: same as ECS) + /// + public const string GenAiRequestSeed = "gen_ai.request.seed"; + + /// + /// OTel gen_ai.request.stop_sequences (match: same as ECS) + /// + public const string GenAiRequestStopSequences = "gen_ai.request.stop_sequences"; + + /// + /// OTel gen_ai.request.temperature (match: same as ECS) + /// + public const string GenAiRequestTemperature = "gen_ai.request.temperature"; + + /// + /// OTel gen_ai.request.top_k (match: same as ECS) + /// + public const string GenAiRequestTopK = "gen_ai.request.top_k"; + + /// + /// OTel gen_ai.request.top_p (match: same as ECS) + /// + public const string GenAiRequestTopP = "gen_ai.request.top_p"; + + /// + /// OTel gen_ai.response.finish_reasons (match: same as ECS) + /// + public const string GenAiResponseFinishReasons = "gen_ai.response.finish_reasons"; + + /// + /// OTel gen_ai.response.id (match: same as ECS) + /// + public const string GenAiResponseId = "gen_ai.response.id"; + + /// + /// OTel gen_ai.response.model (match: same as ECS) + /// + public const string GenAiResponseModel = "gen_ai.response.model"; + + /// + /// OTel gen_ai.token.type (match: same as ECS) + /// + public const string GenAiTokenType = "gen_ai.token.type"; + + /// + /// OTel gen_ai.tool.call.id (match: same as ECS) + /// + public const string GenAiToolCallId = "gen_ai.tool.call.id"; + + /// + /// OTel gen_ai.tool.name (match: same as ECS) + /// + public const string GenAiToolName = "gen_ai.tool.name"; + + /// + /// OTel gen_ai.tool.type (match: same as ECS) + /// + public const string GenAiToolType = "gen_ai.tool.type"; + + /// + /// OTel gen_ai.usage.input_tokens (match: same as ECS) + /// + public const string GenAiUsageInputTokens = "gen_ai.usage.input_tokens"; + + /// + /// OTel gen_ai.usage.output_tokens (match: same as ECS) + /// + public const string GenAiUsageOutputTokens = "gen_ai.usage.output_tokens"; + + /// + /// OTel geo.continent.code (equivalent: ECS geo.continent_code) + /// + public const string GeoContinentCode = "geo.continent.code"; + + /// + /// OTel geo.country.iso_code (equivalent: ECS geo.country_iso_code) + /// + public const string GeoCountryIsoCode = "geo.country.iso_code"; + + /// + /// OTel geo.locality.name (equivalent: ECS geo.city_name) + /// + public const string GeoLocalityName = "geo.locality.name"; + + /// + /// OTel geo.postal_code (match: same as ECS) + /// + public const string GeoPostalCode = "geo.postal_code"; + + /// + /// OTel geo.region.iso_code (equivalent: ECS geo.region_iso_code) + /// + public const string GeoRegionIsoCode = "geo.region.iso_code"; + + /// + /// OTel host.arch (equivalent: ECS host.architecture) + /// + public const string HostArch = "host.arch"; + + /// + /// OTel host.id (match: same as ECS) + /// + public const string HostId = "host.id"; + + /// + /// OTel host.ip (match: same as ECS) + /// + public const string HostIp = "host.ip"; + + /// + /// OTel host.mac (match: same as ECS) + /// + public const string HostMac = "host.mac"; + + /// + /// OTel host.name (match: same as ECS) + /// + public const string HostName = "host.name"; + + /// + /// OTel host.type (match: same as ECS) + /// + public const string HostType = "host.type"; + + /// + /// OTel http.request.body.size (equivalent: ECS http.request.body.bytes) + /// + public const string HttpRequestBodySize = "http.request.body.size"; + + /// + /// OTel http.request.method_original (equivalent: ECS http.request.method) + /// + public const string HttpRequestMethodOriginal = "http.request.method_original"; + + /// + /// OTel http.request.size (equivalent: ECS http.request.bytes) + /// + public const string HttpRequestSize = "http.request.size"; + + /// + /// OTel http.response.body.size (equivalent: ECS http.response.body.bytes) + /// + public const string HttpResponseBodySize = "http.response.body.size"; + + /// + /// OTel http.response.size (equivalent: ECS http.response.bytes) + /// + public const string HttpResponseSize = "http.response.size"; + + /// + /// OTel http.response.status_code (match: same as ECS) + /// + public const string HttpResponseStatusCode = "http.response.status_code"; + + /// + /// OTel log.file.path (match: same as ECS) + /// + public const string LogFilePath = "log.file.path"; + + /// + /// OTel network.protocol.name (equivalent: ECS network.protocol) + /// + public const string NetworkProtocolName = "network.protocol.name"; + + /// + /// OTel network.transport (match: same as ECS) + /// + public const string NetworkTransport = "network.transport"; + + /// + /// OTel network.type (match: same as ECS) + /// + public const string NetworkType = "network.type"; + + /// + /// OTel os.description (equivalent: ECS os.full) + /// + public const string OsDescription = "os.description"; + + /// + /// OTel os.name (match: same as ECS) + /// + public const string OsName = "os.name"; + + /// + /// OTel os.version (match: same as ECS) + /// + public const string OsVersion = "os.version"; + + /// + /// OTel process.args_count (match: same as ECS) + /// + public const string ProcessArgsCount = "process.args_count"; + + /// + /// OTel process.command_args (equivalent: ECS process.args) + /// + public const string ProcessCommandArgs = "process.command_args"; + + /// + /// OTel process.command_line (match: same as ECS) + /// + public const string ProcessCommandLine = "process.command_line"; + + /// + /// OTel process.executable.path (equivalent: ECS process.executable) + /// + public const string ProcessExecutablePath = "process.executable.path"; + + /// + /// OTel process.group_leader.pid (match: same as ECS) + /// + public const string ProcessGroupLeaderPid = "process.group_leader.pid"; + + /// + /// OTel process.interactive (match: same as ECS) + /// + public const string ProcessInteractive = "process.interactive"; + + /// + /// OTel process.pid (match: same as ECS) + /// + public const string ProcessPid = "process.pid"; + + /// + /// OTel process.real_user.id (match: same as ECS) + /// + public const string ProcessRealUserId = "process.real_user.id"; + + /// + /// OTel process.real_user.name (match: same as ECS) + /// + public const string ProcessRealUserName = "process.real_user.name"; + + /// + /// OTel process.saved_user.id (match: same as ECS) + /// + public const string ProcessSavedUserId = "process.saved_user.id"; + + /// + /// OTel process.saved_user.name (match: same as ECS) + /// + public const string ProcessSavedUserName = "process.saved_user.name"; + + /// + /// OTel process.session_leader.pid (match: same as ECS) + /// + public const string ProcessSessionLeaderPid = "process.session_leader.pid"; + + /// + /// OTel process.title (match: same as ECS) + /// + public const string ProcessTitle = "process.title"; + + /// + /// OTel process.user.id (match: same as ECS) + /// + public const string ProcessUserId = "process.user.id"; + + /// + /// OTel process.user.name (match: same as ECS) + /// + public const string ProcessUserName = "process.user.name"; + + /// + /// OTel process.vpid (match: same as ECS) + /// + public const string ProcessVpid = "process.vpid"; + + /// + /// OTel process.working_directory (match: same as ECS) + /// + public const string ProcessWorkingDirectory = "process.working_directory"; + + /// + /// OTel server.address (match: same as ECS) + /// + public const string ServerAddress = "server.address"; + + /// + /// OTel server.port (match: same as ECS) + /// + public const string ServerPort = "server.port"; + + /// + /// OTel service.instance.id (equivalent: ECS service.node.name) + /// + public const string ServiceInstanceId = "service.instance.id"; + + /// + /// OTel service.name (match: same as ECS) + /// + public const string ServiceName = "service.name"; + + /// + /// OTel service.version (match: same as ECS) + /// + public const string ServiceVersion = "service.version"; + + /// + /// OTel source.address (match: same as ECS) + /// + public const string SourceAddress = "source.address"; + + /// + /// OTel source.port (match: same as ECS) + /// + public const string SourcePort = "source.port"; + + /// + /// OTel tls.cipher (match: same as ECS) + /// + public const string TlsCipher = "tls.cipher"; + + /// + /// OTel tls.client.certificate (match: same as ECS) + /// + public const string TlsClientCertificate = "tls.client.certificate"; + + /// + /// OTel tls.client.certificate_chain (match: same as ECS) + /// + public const string TlsClientCertificateChain = "tls.client.certificate_chain"; + + /// + /// OTel tls.client.hash.md5 (match: same as ECS) + /// + public const string TlsClientHashMd5 = "tls.client.hash.md5"; + + /// + /// OTel tls.client.hash.sha1 (match: same as ECS) + /// + public const string TlsClientHashSha1 = "tls.client.hash.sha1"; + + /// + /// OTel tls.client.hash.sha256 (match: same as ECS) + /// + public const string TlsClientHashSha256 = "tls.client.hash.sha256"; + + /// + /// OTel tls.client.issuer (match: same as ECS) + /// + public const string TlsClientIssuer = "tls.client.issuer"; + + /// + /// OTel tls.client.ja3 (match: same as ECS) + /// + public const string TlsClientJa3 = "tls.client.ja3"; + + /// + /// OTel tls.client.not_after (match: same as ECS) + /// + public const string TlsClientNotAfter = "tls.client.not_after"; + + /// + /// OTel tls.client.not_before (match: same as ECS) + /// + public const string TlsClientNotBefore = "tls.client.not_before"; + + /// + /// OTel tls.client.subject (match: same as ECS) + /// + public const string TlsClientSubject = "tls.client.subject"; + + /// + /// OTel tls.client.supported_ciphers (match: same as ECS) + /// + public const string TlsClientSupportedCiphers = "tls.client.supported_ciphers"; + + /// + /// OTel tls.curve (match: same as ECS) + /// + public const string TlsCurve = "tls.curve"; + + /// + /// OTel tls.established (match: same as ECS) + /// + public const string TlsEstablished = "tls.established"; + + /// + /// OTel tls.next_protocol (match: same as ECS) + /// + public const string TlsNextProtocol = "tls.next_protocol"; + + /// + /// OTel tls.resumed (match: same as ECS) + /// + public const string TlsResumed = "tls.resumed"; + + /// + /// OTel tls.server.certificate (match: same as ECS) + /// + public const string TlsServerCertificate = "tls.server.certificate"; + + /// + /// OTel tls.server.certificate_chain (match: same as ECS) + /// + public const string TlsServerCertificateChain = "tls.server.certificate_chain"; + + /// + /// OTel tls.server.hash.md5 (match: same as ECS) + /// + public const string TlsServerHashMd5 = "tls.server.hash.md5"; + + /// + /// OTel tls.server.hash.sha1 (match: same as ECS) + /// + public const string TlsServerHashSha1 = "tls.server.hash.sha1"; + + /// + /// OTel tls.server.hash.sha256 (match: same as ECS) + /// + public const string TlsServerHashSha256 = "tls.server.hash.sha256"; + + /// + /// OTel tls.server.issuer (match: same as ECS) + /// + public const string TlsServerIssuer = "tls.server.issuer"; + + /// + /// OTel tls.server.ja3s (match: same as ECS) + /// + public const string TlsServerJa3s = "tls.server.ja3s"; + + /// + /// OTel tls.server.not_after (match: same as ECS) + /// + public const string TlsServerNotAfter = "tls.server.not_after"; + + /// + /// OTel tls.server.not_before (match: same as ECS) + /// + public const string TlsServerNotBefore = "tls.server.not_before"; + + /// + /// OTel tls.server.subject (match: same as ECS) + /// + public const string TlsServerSubject = "tls.server.subject"; + + /// + /// OTel url.domain (match: same as ECS) + /// + public const string UrlDomain = "url.domain"; + + /// + /// OTel url.extension (match: same as ECS) + /// + public const string UrlExtension = "url.extension"; + + /// + /// OTel url.fragment (match: same as ECS) + /// + public const string UrlFragment = "url.fragment"; + + /// + /// OTel url.full (match: same as ECS) + /// + public const string UrlFull = "url.full"; + + /// + /// OTel url.original (match: same as ECS) + /// + public const string UrlOriginal = "url.original"; + + /// + /// OTel url.path (match: same as ECS) + /// + public const string UrlPath = "url.path"; + + /// + /// OTel url.port (match: same as ECS) + /// + public const string UrlPort = "url.port"; + + /// + /// OTel url.query (match: same as ECS) + /// + public const string UrlQuery = "url.query"; + + /// + /// OTel url.registered_domain (match: same as ECS) + /// + public const string UrlRegisteredDomain = "url.registered_domain"; + + /// + /// OTel url.scheme (match: same as ECS) + /// + public const string UrlScheme = "url.scheme"; + + /// + /// OTel url.subdomain (match: same as ECS) + /// + public const string UrlSubdomain = "url.subdomain"; + + /// + /// OTel url.top_level_domain (match: same as ECS) + /// + public const string UrlTopLevelDomain = "url.top_level_domain"; + + /// + /// OTel user_agent.name (match: same as ECS) + /// + public const string UserAgentName = "user_agent.name"; + + /// + /// OTel user_agent.original (match: same as ECS) + /// + public const string UserAgentOriginal = "user_agent.original"; + + /// + /// OTel user_agent.version (match: same as ECS) + /// + public const string UserAgentVersion = "user_agent.version"; + + /// + /// OTel user.email (match: same as ECS) + /// + public const string UserEmail = "user.email"; + + /// + /// OTel user.full_name (match: same as ECS) + /// + public const string UserFullName = "user.full_name"; + + /// + /// OTel user.hash (match: same as ECS) + /// + public const string UserHash = "user.hash"; + + /// + /// OTel user.id (match: same as ECS) + /// + public const string UserId = "user.id"; + + /// + /// OTel user.name (match: same as ECS) + /// + public const string UserName = "user.name"; + + /// + /// OTel user.roles (match: same as ECS) + /// + public const string UserRoles = "user.roles"; + + } +} diff --git a/src/Elastic.CommonSchema/Serialization/EcsDocumentJsonConverter.Generated.cs b/src/Elastic.CommonSchema/Serialization/EcsDocumentJsonConverter.Generated.cs index 50c33808..2fc50643 100644 --- a/src/Elastic.CommonSchema/Serialization/EcsDocumentJsonConverter.Generated.cs +++ b/src/Elastic.CommonSchema/Serialization/EcsDocumentJsonConverter.Generated.cs @@ -34,14 +34,14 @@ JsonSerializerOptions options { "log.level" => ReadString(ref reader, ref loglevel), "ecs.version" => ReadString(ref reader, ref ecsVersion), - "metadata" => ReadProp(ref reader, "metadata", ecsEvent, (b, v) => b.Metadata = v, options), + "metadata" => ReadMetadataIntoAttributes(ref reader, ecsEvent, options), "@timestamp" => ReadDateTime(ref reader, ref @timestamp, options), "message" => ReadProp(ref reader, "message", ecsEvent, (b, v) => b.Message = v, options), "tags" => ReadProp(ref reader, "tags", ecsEvent, (b, v) => b.Tags = v, options), "span.id" => ReadProp(ref reader, "span.id", ecsEvent, (b, v) => b.SpanId = v, options), "trace.id" => ReadProp(ref reader, "trace.id", ecsEvent, (b, v) => b.TraceId = v, options), "transaction.id" => ReadProp(ref reader, "transaction.id", ecsEvent, (b, v) => b.TransactionId = v, options), - "labels" => ReadProp(ref reader, "labels", ecsEvent, (b, v) => b.Labels = v, options), + "labels" => ReadLabelsIntoAttributes(ref reader, ecsEvent, options), "agent" => ReadProp(ref reader, "agent", EcsJsonContext.Default.Agent, ecsEvent, (b, v) => b.Agent = v), "as" => ReadProp(ref reader, "as", EcsJsonContext.Default.As, ecsEvent, (b, v) => b.As = v), "client" => ReadProp(ref reader, "client", EcsJsonContext.Default.Client, ecsEvent, (b, v) => b.Client = v), @@ -94,6 +94,7 @@ JsonSerializerOptions options "volume" => ReadProp(ref reader, "volume", EcsJsonContext.Default.Volume, ecsEvent, (b, v) => b.Volume = v), "vulnerability" => ReadProp(ref reader, "vulnerability", EcsJsonContext.Default.Vulnerability, ecsEvent, (b, v) => b.Vulnerability = v), "x509" => ReadProp(ref reader, "x509", EcsJsonContext.Default.X509, ecsEvent, (b, v) => b.X509 = v), + "attributes" => ReadOTelAttributes(ref reader, ecsEvent, options), _ => typeof(EcsDocument) == ecsEvent.GetType() ? false @@ -125,7 +126,6 @@ public override void Write(Utf8JsonWriter writer, TBase value, JsonSerializerOpt WriteProp(writer, "span.id", value.SpanId, options); WriteProp(writer, "trace.id", value.TraceId, options); WriteProp(writer, "transaction.id", value.TransactionId, options); - WriteProp(writer, "labels", value.Labels, options); // Complex types WriteProp(writer, "agent", value.Agent, EcsJsonContext.Default.Agent, options); @@ -178,7 +178,7 @@ public override void Write(Utf8JsonWriter writer, TBase value, JsonSerializerOpt WriteProp(writer, "volume", value.Volume, EcsJsonContext.Default.Volume, options); WriteProp(writer, "vulnerability", value.Vulnerability, EcsJsonContext.Default.Vulnerability, options); WriteProp(writer, "x509", value.X509, EcsJsonContext.Default.X509, options); - WriteProp(writer, "metadata", value.Metadata, options); + WriteConsolidatedAttributes(writer, value, options); if (typeof(EcsDocument) != value.GetType()) value.WriteAdditionalProperties((k, v) => WriteProp(writer, k, v, options)); diff --git a/src/Elastic.CommonSchema/Serialization/EcsDocumentJsonConverter.cs b/src/Elastic.CommonSchema/Serialization/EcsDocumentJsonConverter.cs index 8d60bac3..9eeaa1ef 100644 --- a/src/Elastic.CommonSchema/Serialization/EcsDocumentJsonConverter.cs +++ b/src/Elastic.CommonSchema/Serialization/EcsDocumentJsonConverter.cs @@ -3,6 +3,7 @@ // See the LICENSE file in the project root for more information using System; +using System.Collections.Generic; using System.Text.Json; using System.Text.Json.Serialization; @@ -97,6 +98,175 @@ private static void WriteTimestamp(Utf8JsonWriter writer, BaseFieldSet value, Js var converter = GetDateTimeOffsetConverter(options); converter.Write(writer, value.Timestamp.Value, options); } + + #pragma warning disable CS0618 // Obsolete + private static void WriteConsolidatedAttributes(Utf8JsonWriter writer, TBase value, JsonSerializerOptions options) + { + // Merge Labels, Metadata, Attributes — Attributes wins over Metadata wins over Labels + var hasLabels = value.Labels is { Count: > 0 }; + var hasMetadata = value.Metadata is { Count: > 0 }; + var hasAttributes = value.Attributes is { Count: > 0 }; + + if (!hasLabels && !hasMetadata && !hasAttributes) return; + + var merged = new MetadataDictionary(); + if (hasLabels) + foreach (var kvp in value.Labels!) merged[kvp.Key] = kvp.Value; + if (hasMetadata) + foreach (var kvp in value.Metadata!) merged[kvp.Key] = kvp.Value; + if (hasAttributes) + foreach (var kvp in value.Attributes!) merged[kvp.Key] = kvp.Value; + + WriteProp(writer, "attributes", merged, options); + } + #pragma warning restore CS0618 + + private static bool ReadLabelsIntoAttributes(ref Utf8JsonReader reader, TBase ecsEvent, JsonSerializerOptions options) + { + if (reader.TokenType == JsonTokenType.Null) + return true; + + if (reader.TokenType != JsonTokenType.StartObject) + return false; + + ecsEvent.Attributes ??= new MetadataDictionary(); + + while (reader.Read()) + { + if (reader.TokenType == JsonTokenType.EndObject) + break; + + if (reader.TokenType != JsonTokenType.PropertyName) + throw new JsonException("Expected property name in labels object"); + + var key = reader.GetString()!; + reader.Read(); + + // Labels are always strings + var value = reader.TokenType == JsonTokenType.Null ? null : reader.GetString(); + if (value != null) + ecsEvent.Attributes[key] = value; + } + return true; + } + + [System.Diagnostics.CodeAnalysis.UnconditionalSuppressMessage("ReflectionAnalysis", "IL2026:RequiresUnreferencedCode", Justification = "We always provide a static JsonTypeInfoResolver")] + [System.Diagnostics.CodeAnalysis.UnconditionalSuppressMessage("AotAnalysis", "IL3050:RequiresDynamicCode", Justification = "We always provide a static JsonTypeInfoResolver")] + private static bool ReadMetadataIntoAttributes(ref Utf8JsonReader reader, TBase ecsEvent, JsonSerializerOptions options) + { + if (reader.TokenType == JsonTokenType.Null) + return true; + + if (reader.TokenType != JsonTokenType.StartObject) + return false; + + ecsEvent.Attributes ??= new MetadataDictionary(); + + while (reader.Read()) + { + if (reader.TokenType == JsonTokenType.EndObject) + break; + + if (reader.TokenType != JsonTokenType.PropertyName) + throw new JsonException("Expected property name in metadata object"); + + var key = reader.GetString()!; + reader.Read(); + + object? value = reader.TokenType switch + { + JsonTokenType.String => reader.GetString(), + JsonTokenType.Number => reader.TryGetInt64(out var l) ? l : reader.GetDouble(), + JsonTokenType.True => true, + JsonTokenType.False => false, + JsonTokenType.Null => null, + JsonTokenType.StartObject => JsonSerializer.Deserialize(ref reader, options), + JsonTokenType.StartArray => ReadJsonArray(ref reader, options), + _ => JsonSerializer.Deserialize(ref reader, options) + }; + + if (value != null) + ecsEvent.Attributes[key] = value; + } + return true; + } + + [System.Diagnostics.CodeAnalysis.UnconditionalSuppressMessage("ReflectionAnalysis", "IL2026:RequiresUnreferencedCode", Justification = "We always provide a static JsonTypeInfoResolver")] + [System.Diagnostics.CodeAnalysis.UnconditionalSuppressMessage("AotAnalysis", "IL3050:RequiresDynamicCode", Justification = "We always provide a static JsonTypeInfoResolver")] + private static bool ReadOTelAttributes(ref Utf8JsonReader reader, TBase ecsEvent, JsonSerializerOptions options) + { + if (reader.TokenType == JsonTokenType.Null) + return true; + + if (reader.TokenType != JsonTokenType.StartObject) + return false; + + ecsEvent.Attributes ??= new MetadataDictionary(); + + while (reader.Read()) + { + if (reader.TokenType == JsonTokenType.EndObject) + break; + + if (reader.TokenType != JsonTokenType.PropertyName) + throw new JsonException("Expected property name in attributes object"); + + var attrName = reader.GetString()!; + reader.Read(); + + object? value = reader.TokenType switch + { + JsonTokenType.String => reader.GetString(), + JsonTokenType.Number => reader.TryGetInt64(out var l) ? l : reader.GetDouble(), + JsonTokenType.True => true, + JsonTokenType.False => false, + JsonTokenType.Null => null, + JsonTokenType.StartObject => JsonSerializer.Deserialize(ref reader, options), + JsonTokenType.StartArray => ReadJsonArray(ref reader, options), + _ => JsonSerializer.Deserialize(ref reader, options) + }; + + // Store in Attributes + ecsEvent.Attributes[attrName] = value; + + // If there's an OTel->ECS mapping, also set the ECS property + if (value != null) + { + if (OTelMappings.OTelToEcs.TryGetValue(attrName, out var ecsPath)) + ecsEvent.AssignField(ecsPath, value); + else + // For match relations, OTel name IS the ECS name + ecsEvent.AssignField(attrName, value); + } + } + return true; + } + + [System.Diagnostics.CodeAnalysis.UnconditionalSuppressMessage("ReflectionAnalysis", "IL2026:RequiresUnreferencedCode", Justification = "We always provide a static JsonTypeInfoResolver")] + [System.Diagnostics.CodeAnalysis.UnconditionalSuppressMessage("AotAnalysis", "IL3050:RequiresDynamicCode", Justification = "We always provide a static JsonTypeInfoResolver")] + private static List ReadJsonArray(ref Utf8JsonReader reader, JsonSerializerOptions options) + { + var list = new List(); + while (reader.Read()) + { + if (reader.TokenType == JsonTokenType.EndArray) + break; + + object? item = reader.TokenType switch + { + JsonTokenType.String => reader.GetString(), + JsonTokenType.Number => reader.TryGetInt64(out var l) ? l : reader.GetDouble(), + JsonTokenType.True => true, + JsonTokenType.False => false, + JsonTokenType.Null => null, + JsonTokenType.StartObject => JsonSerializer.Deserialize(ref reader, options), + JsonTokenType.StartArray => ReadJsonArray(ref reader, options), + _ => JsonSerializer.Deserialize(ref reader, options) + }; + list.Add(item); + } + return list; + } } /// A JsonConverter for that supports the diff --git a/src/Elastic.Extensions.Logging.Common/LogEventBuilderExtensions.cs b/src/Elastic.Extensions.Logging.Common/LogEventBuilderExtensions.cs index 54c5ddd8..64854d37 100644 --- a/src/Elastic.Extensions.Logging.Common/LogEventBuilderExtensions.cs +++ b/src/Elastic.Extensions.Logging.Common/LogEventBuilderExtensions.cs @@ -15,7 +15,9 @@ void AddScopeValue(TLocalState scope, LogEvent log) { if (scope is null) return; +#pragma warning disable CS0618 // Obsolete Labels log.Labels ??= new Labels(); +#pragma warning restore CS0618 log.Scopes ??= new List(); diff --git a/src/Specification/v9.3.0/composable/component/base.json b/src/Specification/v9.3.0/composable/component/base.json index 0a44056d..c1457fa2 100644 --- a/src/Specification/v9.3.0/composable/component/base.json +++ b/src/Specification/v9.3.0/composable/component/base.json @@ -9,6 +9,9 @@ "@timestamp": { "type": "date" }, + "attributes": { + "type": "passthrough" + }, "labels": { "type": "object" }, diff --git a/tests-integration/Elastic.Extensions.Logging.IntegrationTests/LoggingToDataStreamTests.cs b/tests-integration/Elastic.Extensions.Logging.IntegrationTests/LoggingToDataStreamTests.cs index 63d9ca3f..ea1c2cc3 100644 --- a/tests-integration/Elastic.Extensions.Logging.IntegrationTests/LoggingToDataStreamTests.cs +++ b/tests-integration/Elastic.Extensions.Logging.IntegrationTests/LoggingToDataStreamTests.cs @@ -62,8 +62,8 @@ public async Task LogsEndUpInCluster() loggedError.Ecs.Version.Should().Be(EcsDocument.Version); loggedError.Ecs.Version.Should().NotStartWith("v"); - loggedError.Labels.Should().ContainKey("Status"); - loggedError.Labels["Status"].Should().Be("Failure"); + loggedError.Attributes.Should().ContainKey("Status"); + loggedError.Attributes["Status"].Should().Be("Failure"); } [Fact] diff --git a/tests-integration/Elastic.Ingest.Elasticsearch.CommonSchema.IntegrationTests/DataStreamIngestionTests.cs b/tests-integration/Elastic.Ingest.Elasticsearch.CommonSchema.IntegrationTests/DataStreamIngestionTests.cs index 305a1d52..f19bdc1c 100644 --- a/tests-integration/Elastic.Ingest.Elasticsearch.CommonSchema.IntegrationTests/DataStreamIngestionTests.cs +++ b/tests-integration/Elastic.Ingest.Elasticsearch.CommonSchema.IntegrationTests/DataStreamIngestionTests.cs @@ -100,7 +100,7 @@ public async Task UseCustomEventWriter() { Timestamp = DateTimeOffset.Parse("2024-05-27T23:56:15.785Z"), Message = "Hello World!", - Metadata = new MetadataDictionary { { "MyEnum", MyEnum.Two } } + Attributes = new MetadataDictionary { { "MyEnum", MyEnum.Two } } }); if (!slim.WaitHandle.WaitOne(TimeSpan.FromSeconds(10))) @@ -115,6 +115,6 @@ public async Task UseCustomEventWriter() var root = searchResult.Documents.First().RootElement; root.GetProperty("@timestamp").GetString().Should().Be("2024-05-27T23:56:15.785+00:00"); root.GetProperty("message").GetString().Should().Be("Hello World!"); - root.GetProperty("metadata").GetProperty("MyEnum").GetString().Should().Be("Two"); + root.GetProperty("attributes").GetProperty("MyEnum").GetString().Should().Be("Two"); } } diff --git a/tests-integration/Elastic.Ingest.Elasticsearch.CommonSchema.IntegrationTests/IndexIngestionTests.cs b/tests-integration/Elastic.Ingest.Elasticsearch.CommonSchema.IntegrationTests/IndexIngestionTests.cs index 21621dec..5f865de6 100644 --- a/tests-integration/Elastic.Ingest.Elasticsearch.CommonSchema.IntegrationTests/IndexIngestionTests.cs +++ b/tests-integration/Elastic.Ingest.Elasticsearch.CommonSchema.IntegrationTests/IndexIngestionTests.cs @@ -110,7 +110,7 @@ public async Task UseCustomEventWriter() Created = date, Title = "Hello World!", Id = "hello-world", - Metadata = new MetadataDictionary { { "MyEnum", MyEnum.Two } } + Attributes = new MetadataDictionary { { "MyEnum", MyEnum.Two } } }); if (!slim.WaitHandle.WaitOne(TimeSpan.FromSeconds(10))) @@ -126,6 +126,6 @@ public async Task UseCustomEventWriter() root.GetProperty("created").GetString().Should().Be("2024-05-27T23:56:15.785+00:00"); root.GetProperty("title").GetString().Should().Be("Hello World!"); root.GetProperty("id").GetString().Should().Be("hello-world"); - root.GetProperty("metadata").GetProperty("MyEnum").GetString().Should().Be("Two"); + root.GetProperty("attributes").GetProperty("MyEnum").GetString().Should().Be("Two"); } } diff --git a/tests-integration/Elastic.NLog.Targets.IntegrationTests/LoggingToDataStreamTests.cs b/tests-integration/Elastic.NLog.Targets.IntegrationTests/LoggingToDataStreamTests.cs index 9dc801eb..5acce677 100644 --- a/tests-integration/Elastic.NLog.Targets.IntegrationTests/LoggingToDataStreamTests.cs +++ b/tests-integration/Elastic.NLog.Targets.IntegrationTests/LoggingToDataStreamTests.cs @@ -62,8 +62,8 @@ public async Task LogsEndUpInCluster() loggedError.Ecs.Version.Should().Be(EcsDocument.Version); loggedError.Ecs.Version.Should().NotStartWith("v"); - loggedError.Labels.Should().ContainKey("Status"); - loggedError.Labels["Status"].Should().Be("Failure"); + loggedError.Attributes.Should().ContainKey("Status"); + loggedError.Attributes["Status"].Should().Be("Failure"); } } } diff --git a/tests/Elastic.CommonSchema.Log4net.Tests/MessageTests.cs b/tests/Elastic.CommonSchema.Log4net.Tests/MessageTests.cs index 3f9f3df9..fe02576b 100644 --- a/tests/Elastic.CommonSchema.Log4net.Tests/MessageTests.cs +++ b/tests/Elastic.CommonSchema.Log4net.Tests/MessageTests.cs @@ -144,10 +144,10 @@ public void ToEcs_EventWithFormat_LabelsContainTemplateAndArgs() => TestLogger(( var (_, info) = ToEcsEvents(logEvents).First(); info.Should().NotBeNull(); - info.Labels.Should().NotBeNull(); + info.Attributes.Should().NotBeNull(); - info.Labels["MessageTemplate"].Should().Be("Log with {0}"); - info.Labels["0"].Should().Be("format"); + info.Attributes["MessageTemplate"].Should().Be("Log with {0}"); + info.Attributes["0"].Should().Be("format"); }); [Fact] @@ -160,17 +160,11 @@ public void ToEcs_AnyEvent_PopulatesMetadataAndLabelsFieldsWithoutLog4netPropert var (_, info) = ToEcsEvents(logEvents).First(); - if (info.Metadata != null) + if (info.Attributes != null) { - info.Metadata.Should().NotContainKey(LoggingEvent.IdentityProperty); - info.Metadata.Should().NotContainKey(LoggingEvent.HostNameProperty); - info.Metadata.Should().NotContainKey(LoggingEvent.UserNameProperty); - } - if (info.Labels != null) - { - info.Labels.Should().NotContainKey(LoggingEvent.IdentityProperty); - info.Labels.Should().NotContainKey(LoggingEvent.HostNameProperty); - info.Labels.Should().NotContainKey(LoggingEvent.UserNameProperty); + info.Attributes.Should().NotContainKey(LoggingEvent.IdentityProperty); + info.Attributes.Should().NotContainKey(LoggingEvent.HostNameProperty); + info.Attributes.Should().NotContainKey(LoggingEvent.UserNameProperty); } }); @@ -190,8 +184,8 @@ public void ToEcs_EventWithGlobalContextProperty_PopulatesLabelsField() => TestL var (_, info) = ToEcsEvents(logEvents).First(); - info.Labels.Should().ContainKey(property); - info.Labels[property].Should().Be(propertyValue); + info.Attributes.Should().ContainKey(property); + info.Attributes[property].Should().Be(propertyValue); } finally { @@ -213,8 +207,8 @@ public void ToEcs_EventWithThreadContextStack_PopulatesLabelsField() => TestLogg var (_, info) = ToEcsEvents(logEvents).First(); - info.Labels.Should().ContainKey(property); - info.Labels[property].Should().Be(propertyValue); + info.Attributes.Should().ContainKey(property); + info.Attributes[property].Should().Be(propertyValue); }); [Fact] @@ -233,8 +227,8 @@ public void ToEcs_EventWithThreadContextProperty_PopulatesLabelsField() => TestL var (_, info) = ToEcsEvents(logEvents).First(); - info.Labels.Should().ContainKey(property); - info.Labels[property].Should().Be(propertyValue); + info.Attributes.Should().ContainKey(property); + info.Attributes[property].Should().Be(propertyValue); } finally { @@ -256,11 +250,10 @@ public void ToEcs_EventInLogicalThreadContextStack_PopulatesLabelsField() => Tes var (_, info) = ToEcsEvents(logEvents).First(); - info.Metadata.Should().BeNull(); - info.Labels.Should().NotBeNull(); + info.Attributes.Should().NotBeNull(); - info.Labels.Should().ContainKey(property); - info.Labels[property].Should().Be(propertyValue); + info.Attributes.Should().ContainKey(property); + info.Attributes[property].Should().Be(propertyValue); }); [Fact] @@ -281,10 +274,10 @@ public void ToEcs_EventWithLogicalThreadContextProperty_PopulatesLabelsAndMetada var (_, info) = ToEcsEvents(logEvents).First(); - info.Labels.Should().ContainKey(property); - info.Labels[property].Should().Be(propertyValue); - info.Metadata.Should().ContainKey(metadataProperty); - info.Metadata[metadataProperty].Should().Be(2.0); + info.Attributes.Should().ContainKey(property); + info.Attributes[property].Should().Be(propertyValue); + info.Attributes.Should().ContainKey(metadataProperty); + info.Attributes[metadataProperty].Should().Be(2.0); } finally { @@ -307,8 +300,8 @@ public void ToEcs_EventWithProperties_PopulatesLabelsField() => TestLogger((log, var (_, info) = ToEcsEvents(logEvents).First(); - info.Labels.Should().ContainKey(property); - info.Labels[property].Should().Be(propertyValue); + info.Attributes.Should().ContainKey(property); + info.Attributes[property].Should().Be(propertyValue); }); [Fact] @@ -337,8 +330,8 @@ void TestStackValue(ILog log, Func> getLogEvents, string property, var (_, info) = ToEcsEvents(logEvents).Last(); - info.Labels.Should().ContainKey(property); - info.Labels[property].Should().Be(expectedPropertyValue); + info.Attributes.Should().ContainKey(property); + info.Attributes[property].Should().Be(expectedPropertyValue); } }); @@ -368,8 +361,8 @@ void TestStackValue(ILog log, Func> getLogEvents, string property, var (_, info) = ToEcsEvents(logEvents).Last(); - info.Labels.Should().ContainKey(property); - info.Labels[property].Should().Be(expectedPropertyValue); + info.Attributes.Should().ContainKey(property); + info.Attributes[property].Should().Be(expectedPropertyValue); } }); } diff --git a/tests/Elastic.CommonSchema.NLog.Tests/EcsFieldsInTemplateTests.cs b/tests/Elastic.CommonSchema.NLog.Tests/EcsFieldsInTemplateTests.cs index a3d1c81a..de68e656 100644 --- a/tests/Elastic.CommonSchema.NLog.Tests/EcsFieldsInTemplateTests.cs +++ b/tests/Elastic.CommonSchema.NLog.Tests/EcsFieldsInTemplateTests.cs @@ -25,8 +25,7 @@ public void CanUseEcsFieldNamesAsTemplateProperty() => TestLogger((logger, getLo var (_, info) = ecsEvents.First(); info.Message.Should().Be("Info my-trace-id: true"); - info.Labels.Should().BeNull(); - info.Metadata.Should().BeNull(); + info.Attributes.Should().BeNull(); info.TraceId.Should().Be("my-trace-id"); info.Faas.Should().NotBeNull(); diff --git a/tests/Elastic.CommonSchema.NLog.Tests/MessageTests.cs b/tests/Elastic.CommonSchema.NLog.Tests/MessageTests.cs index e51a5bf8..1ad4d50c 100644 --- a/tests/Elastic.CommonSchema.NLog.Tests/MessageTests.cs +++ b/tests/Elastic.CommonSchema.NLog.Tests/MessageTests.cs @@ -42,15 +42,14 @@ public void SeesMessageWithProp() => TestLogger((logger, getLogEvents) => var (_, info) = ecsEvents.First(); info.Message.Should().Be("Info X 2.2 42"); - info.Labels.Should().ContainKey("ValueX"); - info.Metadata.Should().ContainKey("SomeY"); - info.Metadata.Should().NotContainKey("NotX"); - info.Labels.Should().NotContainKey("NotX"); + info.Attributes.Should().ContainKey("ValueX"); + info.Attributes.Should().ContainKey("SomeY"); + info.Attributes.Should().NotContainKey("NotX"); - var x = info.Labels["ValueX"]; + var x = info.Attributes["ValueX"]; x.Should().NotBeNull().And.Be("X"); - var y = info.Metadata["SomeY"] as double?; + var y = info.Attributes["SomeY"] as double?; y.Should().HaveValue().And.Be(2.2); }); @@ -66,9 +65,9 @@ public void SeesMessageWithSafeProp() => TestLogger((logger, getLogEvents) => var (_, info) = ecsEvents.First(); info.Message.Should().Be("Info {\"ValueX\":\"X\", \"SomeY\":2.2}"); - info.Metadata.Should().ContainKey("SafeValue"); + info.Attributes.Should().ContainKey("SafeValue"); - var x = info.Metadata["SafeValue"] as Dictionary; + var x = info.Attributes["SafeValue"] as Dictionary; x.Should().NotBeNull().And.NotBeEmpty(); }); @@ -84,11 +83,10 @@ public void SeesMessageWithUnsafeProp() => TestLogger((logger, getLogEvents) => var (_, info) = ecsEvents.First(); info.Message.Should().Be("Info X=X"); - info.Metadata.Should().BeNull(); - info.Labels.Should().NotBeNull(); - info.Labels.Should().ContainKey("UnsafeValue"); + info.Attributes.Should().NotBeNull(); + info.Attributes.Should().ContainKey("UnsafeValue"); - var x = info.Labels["UnsafeValue"]; + var x = info.Attributes["UnsafeValue"]; x.Should().NotBeNull().And.Be("X=X"); }); @@ -112,9 +110,9 @@ public void SeesMessageWithStructuredProperty() => TestLogger((logger, getLogEve var (_, info) = ecsEvents.First(); info.Message.Should().Be("Info {\"ValueX\":\"X\", \"SomeY\":2.2}"); - info.Metadata.Should().ContainKey("SafeValue"); + info.Attributes.Should().ContainKey("SafeValue"); - var x = info.Metadata["SafeValue"] as Dictionary; + var x = info.Attributes["SafeValue"] as Dictionary; x.Should().NotBeNull().And.NotBeEmpty(); }); @@ -130,11 +128,10 @@ public void SeesMessageWithStructuredPropAsString() => TestLogger((logger, getLo var (_, info) = ecsEvents.First(); info.Message.Should().Be("Info X=X"); - info.Metadata.Should().BeNull(); - info.Labels.Should().NotBeNull(); - info.Labels.Should().ContainKey("StructuredValue"); + info.Attributes.Should().NotBeNull(); + info.Attributes.Should().ContainKey("StructuredValue"); - var x = info.Labels["StructuredValue"]; + var x = info.Attributes["StructuredValue"]; x.Should().NotBeNull().And.Be("X=X"); }); @@ -150,9 +147,9 @@ public void SerializesKnownBadObject() => TestLogger((logger, getLogEvents) => var (_, info) = ecsEvents.First(); info.Message.Should().StartWith("Info {\"TypeProperty"); - info.Metadata.Should().ContainKey("EvilValue"); + info.Attributes.Should().ContainKey("EvilValue"); - var x = info.Metadata["EvilValue"] as Dictionary; + var x = info.Attributes["EvilValue"] as Dictionary; x.Should().NotBeNull().And.NotBeEmpty(); }); @@ -168,10 +165,10 @@ public void SerializesObjectThatThrowsOnGetter() => TestLogger((logger, getLogEv var (_, info) = ecsEvents.First(); info.Message.Should().StartWith("Info {\"TypeProperty"); - info.Metadata.Should().NotContainKey("EvilValue"); - info.Metadata.Should().ContainKey("__failures__"); + info.Attributes.Should().NotContainKey("EvilValue"); + info.Attributes.Should().ContainKey("__failures__"); - var failures = info.Metadata["__failures__"] as List; + var failures = info.Attributes["__failures__"] as List; failures.Should().NotBeNull().And.HaveCount(1); var failure = failures![0] as MetadataDictionary; failure!["reason"].Should().NotBeNull(); @@ -240,13 +237,13 @@ public void MetadataWithSameKeys() => TestLogger((logger, getLogEvents) => var (json, info) = ecsEvents.First(); info.Message.Should().Be("Info LoggerArg"); - info.Labels.Should().Contain("DupKey", "LoggerArg"); - info.Labels.Should().Contain("DupKey_1", "Mdlc"); + info.Attributes.Should().Contain("DupKey", (object)"LoggerArg"); + info.Attributes.Should().Contain("DupKey_1", (object)"Mdlc"); - var x = info.Labels["DupKey"]; + var x = info.Attributes["DupKey"]; x.Should().NotBeNull().And.Be("LoggerArg"); - var y = info.Labels["DupKey_1"]; + var y = info.Attributes["DupKey_1"]; y.Should().NotBeNull().And.Be("Mdlc"); } }); diff --git a/tests/Elastic.CommonSchema.Serilog.Tests/EcsFieldsInTemplateTests.cs b/tests/Elastic.CommonSchema.Serilog.Tests/EcsFieldsInTemplateTests.cs index ca3e0d2d..1587157b 100644 --- a/tests/Elastic.CommonSchema.Serilog.Tests/EcsFieldsInTemplateTests.cs +++ b/tests/Elastic.CommonSchema.Serilog.Tests/EcsFieldsInTemplateTests.cs @@ -46,7 +46,6 @@ public void CanSpecifyEntityDirectly() => TestLogger((logger, getLogEvents) => info.Event.Kind.Should().Be("something"); info.As.Should().NotBeNull(); info.As.Number.Should().Be(1337); - info.Metadata.Should().BeNull(); }); [Fact] @@ -67,7 +66,7 @@ public void EntityFieldsShouldBeTypedOrTheyGoInMetaData() => TestLogger((logger, info.Event.Kind.Should().NotBe("something"); info.As.Should().BeNull(); - info.Metadata.Should().NotBeEmpty().And.ContainKey("Event").And.ContainKey("As"); + info.Attributes.Should().NotBeEmpty().And.ContainKey("Event").And.ContainKey("As"); }); @@ -85,8 +84,8 @@ public void EcsFieldsRequireType() => TestLogger((logger, getLogEvents) => info.TraceId.Should().BeNull(); info.Faas.Should().BeNull(); - info.Labels.Should().ContainKey("FaasColdstart"); - info.Metadata.Should().ContainKey("TraceId"); + info.Attributes.Should().ContainKey("FaasColdstart"); + info.Attributes.Should().ContainKey("TraceId"); }); [Fact] @@ -118,8 +117,8 @@ public void SupportsStructureCapturing() => TestLogger((logger, getLogEvents) => info.TraceId.Should().NotBeNull(); info.TraceId.Should().Be("{ x = 1 }"); info.Faas.Should().BeNull(); - info.Metadata.Should().ContainKey("FaasColdstart"); - var structured = info.Metadata["FaasColdstart"] as MetadataDictionary; + info.Attributes.Should().ContainKey("FaasColdstart"); + var structured = info.Attributes["FaasColdstart"] as MetadataDictionary; structured.Should().NotBeNull(); structured!["y"].Should().Be(2); }); diff --git a/tests/Elastic.CommonSchema.Serilog.Tests/ExtensionsLogging/SerilogWithExtensionLoggerGenerated.cs b/tests/Elastic.CommonSchema.Serilog.Tests/ExtensionsLogging/SerilogWithExtensionLoggerGenerated.cs index 83ddd07b..aef45557 100644 --- a/tests/Elastic.CommonSchema.Serilog.Tests/ExtensionsLogging/SerilogWithExtensionLoggerGenerated.cs +++ b/tests/Elastic.CommonSchema.Serilog.Tests/ExtensionsLogging/SerilogWithExtensionLoggerGenerated.cs @@ -40,7 +40,7 @@ public void CanSpecifyEventInGeneratedLogger() => TestLogger((serilogLogger, get info.Event.Should().NotBeNull(); info.Event.Timezone.Should().Be("testing"); - info.Labels.Should().HaveCount(1).And.NotContainKey("event"); + info.Attributes.Should().HaveCount(1).And.NotContainKey("event"); }); [Fact] @@ -61,8 +61,8 @@ public void CanSpecifyEventInLogger() => TestLogger((serilogLogger, getLogEvents info.Event.Should().NotBeNull(); info.Event.Timezone.Should().Be("testing"); - info.Labels.Should().HaveCount(1); - info.Labels.Should().HaveCount(1).And.NotContainKey("event"); + info.Attributes.Should().HaveCount(1); + info.Attributes.Should().HaveCount(1).And.NotContainKey("event"); }); } diff --git a/tests/Elastic.CommonSchema.Serilog.Tests/ExtensionsLogging/SerilogWithExtensionsLoggerAdapter.cs b/tests/Elastic.CommonSchema.Serilog.Tests/ExtensionsLogging/SerilogWithExtensionsLoggerAdapter.cs index 13ed9307..8b9d29ce 100644 --- a/tests/Elastic.CommonSchema.Serilog.Tests/ExtensionsLogging/SerilogWithExtensionsLoggerAdapter.cs +++ b/tests/Elastic.CommonSchema.Serilog.Tests/ExtensionsLogging/SerilogWithExtensionsLoggerAdapter.cs @@ -50,7 +50,6 @@ public void SupportsEventId() => TestLogger((serilogLogger, getLogEvents) => error.Event.Should().NotBeNull(); error.Event.Action.Should().Be("hello"); error.Event.Code.Should().Be("123"); - error.Metadata.Should().BeNull(); }); [Fact] public void SupportsStructureCapturing() => TestLogger((serilogLogger, getLogEvents) => @@ -69,8 +68,8 @@ public void SupportsStructureCapturing() => TestLogger((serilogLogger, getLogEve info.TraceId.Should().NotBeNull(); info.TraceId.Should().Be("{ x = 1 }"); info.Faas.Should().BeNull(); - info.Metadata.Should().ContainKey("FaasColdstart"); - var structured = info.Metadata["FaasColdstart"] as MetadataDictionary; + info.Attributes.Should().ContainKey("FaasColdstart"); + var structured = info.Attributes["FaasColdstart"] as MetadataDictionary; structured.Should().NotBeNull(); structured!["y"].Should().Be(2); }); diff --git a/tests/Elastic.CommonSchema.Serilog.Tests/LogEventPropFilterTests.cs b/tests/Elastic.CommonSchema.Serilog.Tests/LogEventPropFilterTests.cs index 4fd3632c..97f2771a 100644 --- a/tests/Elastic.CommonSchema.Serilog.Tests/LogEventPropFilterTests.cs +++ b/tests/Elastic.CommonSchema.Serilog.Tests/LogEventPropFilterTests.cs @@ -64,8 +64,8 @@ public void FilterLogEventProperty() => TestLogger((logger, getLogEvents) => var (_, info) = ecsEvents.First(); info.Log.Level.Should().Be("Information"); info.Error.Should().BeNull(); - info.Labels.Should().Contain("bar", "bbb"); - info.Labels.Should().NotContainKey("foo", "Should have been filtered"); + info.Attributes.Should().Contain("bar", (object)"bbb"); + info.Attributes.Should().NotContainKey("foo", "Should have been filtered"); }); /// /// Test that null does not cause any critical errors @@ -87,8 +87,8 @@ public void NullFilterLogEventProperty() => TestLogger((logger, getLogEvents) => var (_, info) = ecsEvents.First(); info.Log.Level.Should().Be("Information"); info.Error.Should().BeNull(); - info.Labels.Should().Contain("bar", "bbb"); - info.Labels.Should().Contain("foo", "aaa"); + info.Attributes.Should().Contain("bar", (object)"bbb"); + info.Attributes.Should().Contain("foo", (object)"aaa"); }); /// @@ -111,8 +111,8 @@ public void EmptyFilterLogEventProperty() => TestLogger((logger, getLogEvents) = var (_, info) = ecsEvents.First(); info.Log.Level.Should().Be("Information"); info.Error.Should().BeNull(); - info.Labels.Should().Contain("bar", "bbb"); - info.Labels.Should().Contain("foo", "aaa"); + info.Attributes.Should().Contain("bar", (object)"bbb"); + info.Attributes.Should().Contain("foo", (object)"aaa"); }); /// /// Test that can be case insensitive @@ -134,8 +134,8 @@ public void CaseInsensitiveFilterLogEventProperty() => TestLogger((logger, getLo var (_, info) = ecsEvents.First(); info.Log.Level.Should().Be("Information"); info.Error.Should().BeNull(); - info.Labels.Should().Contain("bar", "bbb"); - info.Labels.Should().NotContainKey("foo", "Should have been filtered"); + info.Attributes.Should().Contain("bar", (object)"bbb"); + info.Attributes.Should().NotContainKey("foo", "Should have been filtered"); }); /// /// Test that can be case sensitive @@ -157,8 +157,8 @@ public void CaseSensitiveFilterLogEventProperty() => TestLogger((logger, getLogE var (_, info) = ecsEvents.First(); info.Log.Level.Should().Be("Information"); info.Error.Should().BeNull(); - info.Labels.Should().Contain("bar", "bbb"); - info.Labels.Should().Contain("foo", "aaa"); + info.Attributes.Should().Contain("bar", (object)"bbb"); + info.Attributes.Should().Contain("foo", (object)"aaa"); }); } } diff --git a/tests/Elastic.CommonSchema.Serilog.Tests/MessageTests.cs b/tests/Elastic.CommonSchema.Serilog.Tests/MessageTests.cs index 8e69c2db..384d874a 100644 --- a/tests/Elastic.CommonSchema.Serilog.Tests/MessageTests.cs +++ b/tests/Elastic.CommonSchema.Serilog.Tests/MessageTests.cs @@ -48,13 +48,13 @@ public void SeesMessageWithProp() => TestLogger((logger, getLogEvents) => var (_, info) = ecsEvents.First(); info.Message.Should().Be("Info \"X\" 2.2"); - info.Labels.Should().ContainKey("ValueX"); - info.Metadata.Should().ContainKey("SomeY"); + info.Attributes.Should().ContainKey("ValueX"); + info.Attributes.Should().ContainKey("SomeY"); - var x = info.Labels["ValueX"]; + var x = info.Attributes["ValueX"]; x.Should().NotBeNull().And.Be("X"); - var y = info.Metadata["SomeY"] as double?; + var y = info.Attributes["SomeY"] as double?; y.Should().HaveValue().And.Be(2.2); }); @@ -71,17 +71,17 @@ public void SeesMessageWithDictProp() => TestLogger((logger, getLogEvents) => var (_, info) = ecsEvents.First(); info.Message.Should().Be("Info \"X\" 2.2 [(\"fieldOne\": \"value1\"), (\"fieldTwo\": \"value2\")]"); - info.Labels.Should().ContainKey("ValueX"); - info.Metadata.Should().ContainKey("SomeY"); - info.Metadata.Should().ContainKey("DictValue"); + info.Attributes.Should().ContainKey("ValueX"); + info.Attributes.Should().ContainKey("SomeY"); + info.Attributes.Should().ContainKey("DictValue"); - var x = info.Labels["ValueX"]; + var x = info.Attributes["ValueX"]; x.Should().NotBeNull().And.Be("X"); - var y = info.Metadata["SomeY"] as double?; + var y = info.Attributes["SomeY"] as double?; y.Should().HaveValue().And.Be(2.2); - var dict = info.Metadata["DictValue"] as MetadataDictionary; + var dict = info.Attributes["DictValue"] as MetadataDictionary; dict.Should().NotBeNull(); dict!["fieldOne"].Should().Be("value1"); dict["fieldTwo"].Should().Be("value2"); @@ -99,10 +99,10 @@ public void SeesMessageWithObjectProp() => TestLogger((logger, getLogEvents) => var (_, info) = ecsEvents.First(); info.Message.Should().Be("Info { TestProp: \"testing\", Child: { ChildProp: 3.3 } }"); - info.Metadata.Should().ContainKey("MyObj"); + info.Attributes.Should().ContainKey("MyObj"); - var json = info.Metadata["MyObj"] as MetadataDictionary; + var json = info.Attributes["MyObj"] as MetadataDictionary; json.Should().NotBeNull(); json!["TestProp"].Should().Be("testing"); var child = json["Child"] as MetadataDictionary; @@ -124,7 +124,6 @@ public void SeesMessageWithElapsedProp(string property) => TestLogger((logger, g var (_, info) = ecsEvents.First(); info.Event.Duration.Should().Be(2200000); - info.Metadata.Should().BeNull(); }); [Theory] @@ -141,7 +140,6 @@ public void SeesMessageWithElapsedLongProp(string property) => TestLogger((logge var (_, info) = ecsEvents.First(); info.Event.Duration.Should().Be(2000000); - info.Metadata.Should().BeNull(); }); [Theory] @@ -158,7 +156,6 @@ public void SeesMessageWithMethodProp(string property) => TestLogger((logger, ge var (_, info) = ecsEvents.First(); info.Http.RequestMethod.Should().Be("GET"); - info.Metadata.Should().BeNull(); }); [Theory] @@ -175,7 +172,6 @@ public void SeesMessageWithPathProp(string property) => TestLogger((logger, getL var (_, info) = ecsEvents.First(); info.Url.Path.Should().Be("/"); - info.Metadata.Should().BeNull(); }); [Fact] @@ -190,7 +186,6 @@ public void SeesMessageWithStatusCodeProp() => TestLogger((logger, getLogEvents) var (_, info) = ecsEvents.First(); info.Http.ResponseStatusCode.Should().Be(200); - info.Metadata.Should().BeNull(); }); [Fact] @@ -205,7 +200,6 @@ public void SeesMessageWithSchemeProp() => TestLogger((logger, getLogEvents) => var (_, info) = ecsEvents.First(); info.Url.Scheme.Should().Be("https"); - info.Metadata.Should().BeNull(); }); [Theory] @@ -226,7 +220,6 @@ public void SeesMessageWithQueryStringProp(string property, string value, object info.Url.Should().BeNull(); else ((object)info.Url.Query).Should().Be(expectedValue); - info.Metadata.Should().BeNull(); }); [Fact] @@ -241,7 +234,6 @@ public void SeesMessageWithRequestIdProp() => TestLogger((logger, getLogEvents) var (_, info) = ecsEvents.First(); info.Http.RequestId.Should().Be("34985y39y6tg95"); - info.Metadata.Should().BeNull(); }); } } diff --git a/tests/Elastic.CommonSchema.Serilog.Tests/Repro/GithubIssue225.cs b/tests/Elastic.CommonSchema.Serilog.Tests/Repro/GithubIssue225.cs index f2f1bc32..5e2b0ec8 100644 --- a/tests/Elastic.CommonSchema.Serilog.Tests/Repro/GithubIssue225.cs +++ b/tests/Elastic.CommonSchema.Serilog.Tests/Repro/GithubIssue225.cs @@ -25,11 +25,11 @@ public void Reproduce() => TestLogger((logger, getLogEvents) => var (_, info) = ecsEvents.First(); info.Message.Should().Be("Logging something with log context"); - info.Labels.Should().NotBeNull().And.ContainKey("ShipmentId"); - info.Labels["ShipmentId"].Should().Be("my-shipment-id"); + info.Attributes.Should().NotBeNull().And.ContainKey("ShipmentId"); + info.Attributes["ShipmentId"].Should().Be("my-shipment-id"); - info.Metadata.Should().NotBeNull().And.ContainKey("ShipmentAmount"); - info.Metadata["ShipmentAmount"].Should().Be(2.3); + info.Attributes.Should().ContainKey("ShipmentAmount"); + info.Attributes["ShipmentAmount"].Should().Be(2.3); }); } diff --git a/tests/Elastic.CommonSchema.Tests/EcsServiceTests.cs b/tests/Elastic.CommonSchema.Tests/EcsServiceTests.cs index 787dedfc..8441eef0 100644 --- a/tests/Elastic.CommonSchema.Tests/EcsServiceTests.cs +++ b/tests/Elastic.CommonSchema.Tests/EcsServiceTests.cs @@ -9,6 +9,7 @@ namespace Elastic.CommonSchema.Tests; +[Collection("EnvironmentVariables")] public class EcsServiceTests { public EcsServiceTests(ITestOutputHelper output) => _output = output; diff --git a/tests/Elastic.CommonSchema.Tests/OTelTests.cs b/tests/Elastic.CommonSchema.Tests/OTelTests.cs new file mode 100644 index 00000000..24489296 --- /dev/null +++ b/tests/Elastic.CommonSchema.Tests/OTelTests.cs @@ -0,0 +1,356 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System; +using System.Text.Json; +using FluentAssertions; +using Xunit; + +namespace Elastic.CommonSchema.Tests; + +[Collection("EnvironmentVariables")] +public class OTelTests +{ + private class Env : IDisposable + { + private readonly string _key; + + public Env(string key, string value) + { + _key = key; + Environment.SetEnvironmentVariable(key, value); + } + + public void Dispose() => Environment.SetEnvironmentVariable(_key, null); + } + + // ── 1. OTelMappings Static Dictionaries ── + + [Fact] + public void EcsToOTel_ContainsKnownEquivalentMappings() + { + OTelMappings.EcsToOTel.Should().ContainKey("error.message") + .WhoseValue.Should().Contain("exception.message"); + + OTelMappings.EcsToOTel.Should().ContainKey("service.environment") + .WhoseValue.Should().Contain("deployment.environment.name"); + + OTelMappings.EcsToOTel.Should().ContainKey("error.stack_trace") + .WhoseValue.Should().Contain("exception.stacktrace"); + } + + [Fact] + public void OTelToEcs_ContainsKnownEquivalentMappings() + { + OTelMappings.OTelToEcs["exception.message"].Should().Be("error.message"); + OTelMappings.OTelToEcs["deployment.environment.name"].Should().Be("service.environment"); + OTelMappings.OTelToEcs["exception.stacktrace"].Should().Be("error.stack_trace"); + } + + [Fact] + public void AllBidirectionalEcsFields_ContainsBothMatchAndEquivalent() + { + // Equivalent-relation field + OTelMappings.AllBidirectionalEcsFields.Should().Contain("error.message"); + // Match-relation fields (OTel name = ECS name) + OTelMappings.AllBidirectionalEcsFields.Should().Contain("service.name"); + OTelMappings.AllBidirectionalEcsFields.Should().Contain("host.name"); + } + + [Fact] + public void EcsToOTel_DoesNotContainMatchRelations() + { + // Match relations have identical OTel and ECS names — they should not appear in EcsToOTel + OTelMappings.EcsToOTel.Should().NotContainKey("service.name"); + OTelMappings.EcsToOTel.Should().NotContainKey("host.name"); + OTelMappings.EcsToOTel.Should().NotContainKey("container.id"); + } + + // ── 2. AssignOTelField Method ── + + [Fact] + public void AssignOTelField_EquivalentMapping_PopulatesBothAttributesAndEcsField() + { + var doc = new EcsDocument(); + doc.AssignOTelField("exception.message", "boom"); + + doc.Attributes.Should().NotBeNull(); + doc.Attributes.Should().ContainKey("exception.message"); + doc.Attributes["exception.message"].Should().Be("boom"); + + doc.Error.Should().NotBeNull(); + doc.Error.Message.Should().Be("boom"); + } + + [Fact] + public void AssignOTelField_UnknownAttribute_StoredInAttributes() + { + var doc = new EcsDocument(); + doc.AssignOTelField("custom.my_attr", "val"); + + doc.Attributes.Should().NotBeNull(); + doc.Attributes.Should().ContainKey("custom.my_attr"); + doc.Attributes["custom.my_attr"].Should().Be("val"); + } + + // ── 3. JSON Deserialization of `attributes` ── + + [Fact] + public void Deserialize_WithAttributes_PopulatesAttributesAndMappedEcsFields() + { + var json = @"{""attributes"":{""exception.message"":""test error"",""exception.stacktrace"":""at Foo.Bar()""}}"; + var doc = EcsDocument.Deserialize(json); + + doc.Attributes.Should().NotBeNull(); + doc.Attributes.Should().ContainKey("exception.message"); + doc.Attributes["exception.message"].Should().Be("test error"); + + doc.Error.Should().NotBeNull(); + doc.Error.Message.Should().Be("test error"); + doc.Error.StackTrace.Should().Be("at Foo.Bar()"); + } + + [Fact] + public void Deserialize_WithAttributes_UnknownKeys_StoredInAttributes() + { + var json = @"{""attributes"":{""my.custom.field"":""hello""}}"; + var doc = EcsDocument.Deserialize(json); + + doc.Attributes.Should().NotBeNull(); + doc.Attributes.Should().ContainKey("my.custom.field"); + doc.Attributes["my.custom.field"].Should().Be("hello"); + } + + [Fact] + public void Deserialize_WithNullAttributes_NoError() + { + var json = @"{""attributes"":null}"; + var doc = EcsDocument.Deserialize(json); + + doc.Should().NotBeNull(); + } + + [Fact] + public void Deserialize_WithEmptyAttributes_NoError() + { + var json = @"{""attributes"":{}}"; + var doc = EcsDocument.Deserialize(json); + + doc.Should().NotBeNull(); + } + + // ── 4. Serialize with Attributes ── + + [Fact] + public void Serialize_WithAttributes_WritesAttributesObject() + { + var doc = new EcsDocument + { + Timestamp = DateTimeOffset.Parse("2024-01-01T00:00:00Z"), + Attributes = new MetadataDictionary + { + [SemConv.ExceptionMessage] = "boom" + } + }; + + var json = doc.Serialize(); + json.Should().Contain("\"attributes\":"); + json.Should().Contain("\"exception.message\":\"boom\""); + } + + [Fact] + public void Serialize_EcsFieldsStayEcsFields() + { + var doc = new EcsDocument + { + Timestamp = DateTimeOffset.Parse("2024-01-01T00:00:00Z"), + Message = "hello", + Log = new Log { Level = "info" }, + Error = new Error { Message = "something failed" } + }; + + var json = doc.Serialize(); + + // ECS fields should be serialized with their ECS names, NOT renamed to OTel names + using var jsonDoc = JsonDocument.Parse(json); + var root = jsonDoc.RootElement; + + root.TryGetProperty("@timestamp", out _).Should().BeTrue(); + root.TryGetProperty("message", out _).Should().BeTrue(); + root.TryGetProperty("log.level", out _).Should().BeTrue(); + root.TryGetProperty("ecs.version", out _).Should().BeTrue(); + + // error should be a nested object at root, not inside attributes + root.TryGetProperty("error", out var errorEl).Should().BeTrue(); + errorEl.TryGetProperty("message", out _).Should().BeTrue(); + + // No attributes object when Attributes is null + root.TryGetProperty("attributes", out _).Should().BeFalse(); + } + + [Fact] + public void Serialize_RoundTrip_WithAttributes() + { + var original = new EcsDocument + { + Timestamp = DateTimeOffset.Parse("2024-01-01T00:00:00Z"), + Message = "test message", + Log = new Log { Level = "warn" }, + Attributes = new MetadataDictionary + { + [SemConv.ExceptionMessage] = "otel error", + ["custom.field"] = "custom value" + } + }; + + var json = original.Serialize(); + var deserialized = EcsDocument.Deserialize(json); + + // ECS fields round-trip + deserialized.Message.Should().Be("test message"); + deserialized.Log.Should().NotBeNull(); + deserialized.Log.Level.Should().Be("warn"); + + // Attributes round-trip and OTel-mapped attributes also set ECS fields + deserialized.Attributes.Should().NotBeNull(); + deserialized.Attributes.Should().ContainKey("exception.message"); + deserialized.Attributes["exception.message"].Should().Be("otel error"); + deserialized.Attributes.Should().ContainKey("custom.field"); + deserialized.Attributes["custom.field"].Should().Be("custom value"); + + // Mapped attribute sets the ECS property on deserialization + deserialized.Error.Should().NotBeNull(); + deserialized.Error.Message.Should().Be("otel error"); + } + + [Fact] + public void Serialize_WithoutAttributes_NoAttributesInOutput() + { + var doc = new EcsDocument + { + Timestamp = DateTimeOffset.Parse("2024-01-01T00:00:00Z"), + Message = "hello" + }; + + var json = doc.Serialize(); + json.Should().NotContain("\"attributes\""); + } + + // ── 5. SemConv Constants ── + + [Fact] + public void SemConv_HasExpectedEquivalentConstants() + { + SemConv.ExceptionMessage.Should().Be("exception.message"); + SemConv.ExceptionStacktrace.Should().Be("exception.stacktrace"); + SemConv.DeploymentEnvironmentName.Should().Be("deployment.environment.name"); + SemConv.CloudPlatform.Should().Be("cloud.platform"); + SemConv.ServiceInstanceId.Should().Be("service.instance.id"); + } + + [Fact] + public void SemConv_HasExpectedMatchConstants() + { + SemConv.ServiceName.Should().Be("service.name"); + SemConv.HostName.Should().Be("host.name"); + SemConv.ContainerId.Should().Be("container.id"); + SemConv.ErrorType.Should().Be("error.type"); + } + + [Fact] + public void AssignOTelField_WithSemConvConstant_Works() + { + var doc = new EcsDocument(); + doc.AssignOTelField(SemConv.ExceptionMessage, "boom"); + + doc.Error.Should().NotBeNull(); + doc.Error.Message.Should().Be("boom"); + doc.Attributes.Should().ContainKey(SemConv.ExceptionMessage); + } + + // ── 6. PropDispatch / Field Assignment ── + + [Fact] + public void AssignField_OTelEquivalentName_GoesToAttributes() + { + // AssignField only recognizes ECS field names. + // OTel equivalent names like "exception.message" are not in LogTemplateProperties, + // so values fall through to Attributes. Use AssignOTelField for OTel names. + var doc = new EcsDocument(); + doc.AssignField("exception.message", "dispatch test"); + + // Not set as ECS property — stored in Attributes instead + doc.Error.Should().BeNull(); + doc.Attributes.Should().NotBeNull(); + doc.Attributes.Should().ContainKey("exception.message"); + } + + [Fact] + public void AssignOTelField_EquivalentName_SetsEcsProperty() + { + // AssignOTelField handles OTel→ECS mapping + var doc = new EcsDocument(); + doc.AssignOTelField("exception.message", "dispatch test"); + + doc.Error.Should().NotBeNull(); + doc.Error.Message.Should().Be("dispatch test"); + doc.Attributes.Should().ContainKey("exception.message"); + } + + [Fact] + public void AssignField_EcsName_StillWorks() + { + var doc = new EcsDocument(); + doc.AssignField("error.message", "ecs dispatch test"); + + doc.Error.Should().NotBeNull(); + doc.Error.Message.Should().Be("ecs dispatch test"); + } + + // ── 7. Resource Attributes via CreateNewWithDefaults ── + + [Fact] + public void CreateNewWithDefaults_OTelResourceAttributes_EquivalentMapping() + { + using var env = new Env("OTEL_RESOURCE_ATTRIBUTES", "deployment.environment.name=production"); + var doc = EcsDocument.CreateNewWithDefaults(initialCache: new EcsDocumentCreationCache()); + + doc.Service.Should().NotBeNull(); + doc.Service.Environment.Should().Be("production"); + + doc.Attributes.Should().NotBeNull(); + doc.Attributes.Should().ContainKey("deployment.environment.name"); + } + + [Fact] + public void CreateNewWithDefaults_OTelResourceAttributes_UnknownAttribute() + { + using var env = new Env("OTEL_RESOURCE_ATTRIBUTES", "my.custom.thing=hello"); + var doc = EcsDocument.CreateNewWithDefaults(initialCache: new EcsDocumentCreationCache()); + + doc.Attributes.Should().NotBeNull(); + doc.Attributes.Should().ContainKey("my.custom.thing"); + doc.Attributes["my.custom.thing"].Should().Be("hello"); + } + + [Fact] + public void CreateNewWithDefaults_OTelResourceAttributes_HandledAttributes_NotInAttributes() + { + using var env = new Env("OTEL_RESOURCE_ATTRIBUTES", "service.name=my-svc,host.name=my-host"); + var doc = EcsDocument.CreateNewWithDefaults(initialCache: new EcsDocumentCreationCache()); + + // These are handled by GetService/GetHost, so should NOT appear in Attributes + doc.Service.Should().NotBeNull(); + doc.Service.Name.Should().Be("my-svc"); + doc.Host.Should().NotBeNull(); + doc.Host.Hostname.Should().Be("my-host"); + + // Attributes should be null or not contain these handled keys + if (doc.Attributes != null) + { + doc.Attributes.Should().NotContainKey("service.name"); + doc.Attributes.Should().NotContainKey("host.name"); + } + } +} diff --git a/tests/Elastic.CommonSchema.Tests/Repro/GithubIssue29.cs b/tests/Elastic.CommonSchema.Tests/Repro/GithubIssue29.cs index bdfa7fd8..0eb49597 100644 --- a/tests/Elastic.CommonSchema.Tests/Repro/GithubIssue29.cs +++ b/tests/Elastic.CommonSchema.Tests/Repro/GithubIssue29.cs @@ -11,6 +11,7 @@ public void Reproduce() { // Metadata properties with null values should be serialised var uniqueName = Guid.NewGuid().ToString(); +#pragma warning disable CS0618 // Obsolete Metadata var root = new EcsDocument { Metadata = new MetadataDictionary @@ -18,6 +19,7 @@ public void Reproduce() { uniqueName, null } } }; +#pragma warning restore CS0618 var serialised = root.Serialize(); serialised.Should().Contain($"\"{uniqueName}\":null"); diff --git a/tests/Elastic.CommonSchema.Tests/Serializes.cs b/tests/Elastic.CommonSchema.Tests/Serializes.cs index 7fc4c10a..e2411e2c 100644 --- a/tests/Elastic.CommonSchema.Tests/Serializes.cs +++ b/tests/Elastic.CommonSchema.Tests/Serializes.cs @@ -47,6 +47,7 @@ public void SerializesNestedProperties() [Fact] public void MetaDataKeysAreSerializedVerbatim() { +#pragma warning disable CS0618 // Obsolete Metadata var b = new EcsDocument { Metadata = new MetadataDictionary @@ -58,15 +59,17 @@ public void MetaDataKeysAreSerializedVerbatim() ["rule"] = "some-rule", } }; +#pragma warning restore CS0618 var serialized = b.Serialize(); var deserialized = EcsSerializerFactory.Deserialize(serialized); - deserialized.Metadata.Should().ContainKey("MessageTemplate"); - deserialized.Metadata.Should().ContainKey("WriteIO"); - deserialized.Metadata.Should().ContainKey("User_Id"); - deserialized.Metadata.Should().ContainKey("eventId"); - deserialized.Metadata.Should().ContainKey("rule"); + // Metadata is now merged into Attributes during serialization + deserialized.Attributes.Should().ContainKey("MessageTemplate"); + deserialized.Attributes.Should().ContainKey("WriteIO"); + deserialized.Attributes.Should().ContainKey("User_Id"); + deserialized.Attributes.Should().ContainKey("eventId"); + deserialized.Attributes.Should().ContainKey("rule"); } [JsonConverter(typeof(EcsDocumentJsonConverterFactory))] @@ -217,7 +220,9 @@ public void SerializesDocumentInTheReadMe() Category = new[] { "network_traffic" } }, Ecs = new Ecs { Version = "1.2.0" }, - Metadata = new MetadataDictionary { { "client", "ecs-dotnet" } } + #pragma warning disable CS0618 // Obsolete Metadata + Metadata = new MetadataDictionary { { "client", "ecs-dotnet" } } +#pragma warning restore CS0618 }; var serialized = ecsDocument.Serialize(); @@ -231,7 +236,7 @@ public void SerializesDocumentInTheReadMe() deserialized.Dns.Answers.Should().NotBeNull().And.HaveCount(2); deserialized.Dns.Answers[0].Name.Should().NotBeNull().And.Be("www.example.com"); deserialized.Dns.Answers[1].Data.Should().NotBeNull().And.EndWith(".117"); - deserialized.Metadata.Should().NotBeNull().And.HaveCount(1); + deserialized.Attributes.Should().NotBeNull().And.HaveCount(1); } private enum MyEnum { One, Two, Three } @@ -239,6 +244,7 @@ private enum MyEnum { One, Two, Three } [Fact] public void UseCustomSerializer() { +#pragma warning disable CS0618 // Obsolete Metadata var ecsDocument = new EcsDocument { Timestamp = DateTimeOffset.Parse("2024-05-27T23:56:15.785Z"), @@ -246,11 +252,12 @@ public void UseCustomSerializer() Metadata = new MetadataDictionary { { "MyEnum", MyEnum.Two } }, Ecs = new Ecs { Version = "8.11.0" } }; +#pragma warning restore CS0618 var serializerOptions = new JsonSerializerOptions { Converters = { new JsonStringEnumConverter() } }; var json = JsonSerializer.Serialize(ecsDocument, serializerOptions); - json.Should().Be("{\"@timestamp\":\"2024-05-27T23:56:15.785+00:00\",\"message\":\"Hello World!\",\"ecs.version\":\"8.11.0\",\"metadata\":{\"MyEnum\":\"Two\"}}"); + json.Should().Be("{\"@timestamp\":\"2024-05-27T23:56:15.785+00:00\",\"message\":\"Hello World!\",\"ecs.version\":\"8.11.0\",\"attributes\":{\"MyEnum\":\"Two\"}}"); } } } diff --git a/tools/Elastic.CommonSchema.Benchmarks/SerializingBase.cs b/tools/Elastic.CommonSchema.Benchmarks/SerializingBase.cs index 7aefdf3b..07b61135 100644 --- a/tools/Elastic.CommonSchema.Benchmarks/SerializingBase.cs +++ b/tools/Elastic.CommonSchema.Benchmarks/SerializingBase.cs @@ -61,7 +61,7 @@ public void GlobalSetup() { Generator = new AutoFaker() - .RuleFor(d => d.Metadata, _ => new MetadataDictionary { { "x", "y" } }) + .RuleFor(d => d.Attributes, _ => new MetadataDictionary { { "x", "y" } }) .UseSeed(1337); FullInstance = Generator.Generate(); } diff --git a/tools/Elastic.CommonSchema.Benchmarks/SerializingStringBuilderBase.cs b/tools/Elastic.CommonSchema.Benchmarks/SerializingStringBuilderBase.cs index 2acc3220..52d645c8 100644 --- a/tools/Elastic.CommonSchema.Benchmarks/SerializingStringBuilderBase.cs +++ b/tools/Elastic.CommonSchema.Benchmarks/SerializingStringBuilderBase.cs @@ -67,7 +67,7 @@ public void GlobalSetup() { Generator = new AutoFaker() - .RuleFor(d => d.Metadata, _ => new MetadataDictionary { { "x", "y" } }) + .RuleFor(d => d.Attributes, _ => new MetadataDictionary { { "x", "y" } }) .UseSeed(1337); FullInstance = Generator.Generate(); } diff --git a/tools/Elastic.CommonSchema.Generator/FileGenerator.cs b/tools/Elastic.CommonSchema.Generator/FileGenerator.cs index b82e0460..a590f493 100644 --- a/tools/Elastic.CommonSchema.Generator/FileGenerator.cs +++ b/tools/Elastic.CommonSchema.Generator/FileGenerator.cs @@ -39,6 +39,8 @@ public static void Generate(CommonSchemaTypesProjection commonSchemaTypesProject { m => Generate(m, "AssignableInterfaces"), "Assignable Interfaces" }, { m => Generate(m, "IndexTemplates"), "Elasticsearch index templates" }, { m => Generate(m, "IndexComponents"), "Elasticsearch index components" }, + { m => Generate(m, "OTelMappings"), "OTel semantic convention mappings" }, + { m => Generate(m, "SemConv"), "OTel SemConv attribute name constants" }, }; using (var progressBar = new ProgressBar(actions.Count, "Generating code", diff --git a/tools/Elastic.CommonSchema.Generator/Program.cs b/tools/Elastic.CommonSchema.Generator/Program.cs index 5707054f..ae307e07 100644 --- a/tools/Elastic.CommonSchema.Generator/Program.cs +++ b/tools/Elastic.CommonSchema.Generator/Program.cs @@ -11,37 +11,71 @@ public static class Program { private const string DefaultDownloadBranch = "v9.3.0"; + // Usage: + // dotnet run — interactive mode (prompts for download/tag) + // dotnet run -- --no-download — skip download, use cached spec, default tag + // dotnet run -- --download — download spec, default tag + // dotnet run -- --download --tag v9.3.0 — download spec with specific tag + // dotnet run -- --no-download --tag v9.3.0 — skip download, specific tag + // dotnet run -- --token — pass GitHub token for download // ReSharper disable once UnusedParameter.Local private static async Task Main(string[] args) { - var token = args.Length > 0 ? args[0] : string.Empty; - Console.WriteLine($"Running from: {Directory.GetCurrentDirectory()}"); Console.WriteLine($"Resolved codebase root to: {CodeConfiguration.Root}"); Console.WriteLine(); - var redownloadCoreSpecification = true; + var token = string.Empty; + var redownloadCoreSpecification = (bool?)null; var downloadBranch = DefaultDownloadBranch; - var answer = "invalid"; - while (answer != "y" && answer != "n" && answer != "") + // Parse CLI args + for (var i = 0; i < args.Length; i++) { - Console.Write("Download online specifications? [Y/N] (default N): "); - answer = Console.ReadLine()?.Trim().ToLowerInvariant(); - redownloadCoreSpecification = answer == "y"; + switch (args[i]) + { + case "--no-download": + redownloadCoreSpecification = false; + break; + case "--download": + redownloadCoreSpecification = true; + break; + case "--tag" when i + 1 < args.Length: + downloadBranch = args[++i]; + break; + case "--token" when i + 1 < args.Length: + token = args[++i]; + break; + default: + // Legacy: first positional arg is token + if (!args[i].StartsWith("--") && string.IsNullOrEmpty(token)) + token = args[i]; + break; + } } - Console.Write($"Tag to use (default {downloadBranch}): "); - var readBranch = Console.ReadLine()?.Trim(); - if (!string.IsNullOrEmpty(readBranch)) downloadBranch = readBranch; + // Interactive prompts only if not specified via CLI + if (redownloadCoreSpecification == null) + { + var answer = "invalid"; + while (answer != "y" && answer != "n" && answer != "") + { + Console.Write("Download online specifications? [Y/N] (default N): "); + answer = Console.ReadLine()?.Trim().ToLowerInvariant(); + redownloadCoreSpecification = answer == "y"; + } + + Console.Write($"Tag to use (default {downloadBranch}): "); + var readBranch = Console.ReadLine()?.Trim(); + if (!string.IsNullOrEmpty(readBranch)) downloadBranch = readBranch; + } if (string.IsNullOrEmpty(downloadBranch)) downloadBranch = DefaultDownloadBranch; - if (redownloadCoreSpecification) + if (redownloadCoreSpecification == true) await SpecificationDownloader.DownloadAsync(downloadBranch, token); - var ecsSchema = new EcsSchemaParser(downloadBranch).Parse(); WarnAboutSchemaValidations(ecsSchema); diff --git a/tools/Elastic.CommonSchema.Generator/Projection/OTelFieldMapping.cs b/tools/Elastic.CommonSchema.Generator/Projection/OTelFieldMapping.cs new file mode 100644 index 00000000..dc673089 --- /dev/null +++ b/tools/Elastic.CommonSchema.Generator/Projection/OTelFieldMapping.cs @@ -0,0 +1,25 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using Elastic.CommonSchema.Generator.Schema.DTO; + +namespace Elastic.CommonSchema.Generator.Projection +{ + public enum OTelMappingKind + { + Attribute, + OtlpField, + Metric, + } + + public class OTelFieldMapping + { + public string EcsFieldPath { get; set; } + public string OTelFieldName { get; set; } + public OTelRelation Relation { get; set; } + public OTelStability Stability { get; set; } + public OTelMappingKind Kind { get; set; } + public string Note { get; set; } + } +} diff --git a/tools/Elastic.CommonSchema.Generator/Projection/TypeProjector.cs b/tools/Elastic.CommonSchema.Generator/Projection/TypeProjector.cs index ca17680b..7c9ae907 100644 --- a/tools/Elastic.CommonSchema.Generator/Projection/TypeProjector.cs +++ b/tools/Elastic.CommonSchema.Generator/Projection/TypeProjector.cs @@ -33,6 +33,8 @@ public class CommonSchemaTypesProjection public List AssignableInterfaces { get; set; } public List AssignablePropDispatches { get; set; } + + public IReadOnlyCollection OTelMappings { get; set; } // ReSharper restore PropertyCanBeMadeInitOnly.Global } @@ -168,9 +170,71 @@ public CommonSchemaTypesProjection CreateProjection() propDispatches.Add(new PropDispatch(entity, a)); } Projection.AssignablePropDispatches = propDispatches; + + // Collect OTel mappings from all field sets + var otelMappings = CollectOTelMappings(); + Projection.OTelMappings = otelMappings; + + // Annotate DispatchProperties with OTel equivalent names + var otelEquivalentLookup = otelMappings + .Where(m => m.Relation == OTelRelation.Equivalent) + .ToLookup(m => m.EcsFieldPath); + + foreach (var dispatch in propDispatches) + { + foreach (var prop in dispatch.AssignableProperties) + { + foreach (var mapping in otelEquivalentLookup[prop.FullPath]) + prop.OTelNames.Add(mapping.OTelFieldName); + } + } + // Also annotate the base entity dispatch properties + foreach (var prop in Projection.Base.DispatchProperties) + { + foreach (var mapping in otelEquivalentLookup[prop.FullPath]) + prop.OTelNames.Add(mapping.OTelFieldName); + } + return Projection; } + private List CollectOTelMappings() + { + var mappings = new List(); + foreach (var fieldSet in Schema.Entities) + { + foreach (var (flatName, field) in fieldSet.Fields) + { + if (field.OTelMappings == null) continue; + foreach (var otel in field.OTelMappings) + { + if (!otel.IsBidirectional) continue; + + var otelName = otel.Relation == OTelRelation.Match + ? flatName // For match, OTel name = ECS flat_name + : otel.OTelFieldName; // For equivalent, explicit name + + if (string.IsNullOrEmpty(otelName)) continue; + + var kind = otel.Attribute != null ? OTelMappingKind.Attribute + : otel.OtlpField != null ? OTelMappingKind.OtlpField + : OTelMappingKind.Metric; + + mappings.Add(new OTelFieldMapping + { + EcsFieldPath = flatName, + OTelFieldName = otelName, + Relation = otel.Relation, + Stability = otel.Stability, + Kind = kind, + Note = otel.Note, + }); + } + } + } + return mappings; + } + private Dictionary CreateEntityTypes() { // Create concrete entity instances of base field sets to reference diff --git a/tools/Elastic.CommonSchema.Generator/Projection/Types.cs b/tools/Elastic.CommonSchema.Generator/Projection/Types.cs index f3a1a2e5..2002dac7 100644 --- a/tools/Elastic.CommonSchema.Generator/Projection/Types.cs +++ b/tools/Elastic.CommonSchema.Generator/Projection/Types.cs @@ -148,6 +148,11 @@ public class DispatchProperty public string JsonProperty { get; } public bool SelfReferential { get; } + /// + /// OTel attribute names that are equivalent to this ECS field (different name, same semantics). + /// + public List OTelNames { get; set; } = new(); + public DispatchProperty(PropertyReference property) { JsonProperty = property.JsonProperty; diff --git a/tools/Elastic.CommonSchema.Generator/Schema/DTO/Field.cs b/tools/Elastic.CommonSchema.Generator/Schema/DTO/Field.cs index a63583de..73bd32e8 100644 --- a/tools/Elastic.CommonSchema.Generator/Schema/DTO/Field.cs +++ b/tools/Elastic.CommonSchema.Generator/Schema/DTO/Field.cs @@ -170,5 +170,17 @@ public class Field [JsonProperty("pattern")] public string Pattern { get; set; } + /// + /// OTel semantic convention mappings for this field. + /// + [JsonProperty("otel")] + public List OTelMappings { get; set; } + + /// + /// Controls synthetic source keep behaviour. + /// + [JsonProperty("synthetic_source_keep")] + public string SyntheticSourceKeep { get; set; } + } } diff --git a/tools/Elastic.CommonSchema.Generator/Schema/DTO/FieldOTelMapping.cs b/tools/Elastic.CommonSchema.Generator/Schema/DTO/FieldOTelMapping.cs new file mode 100644 index 00000000..ebc0347b --- /dev/null +++ b/tools/Elastic.CommonSchema.Generator/Schema/DTO/FieldOTelMapping.cs @@ -0,0 +1,62 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System.Runtime.Serialization; +using Newtonsoft.Json; +using Newtonsoft.Json.Converters; + +namespace Elastic.CommonSchema.Generator.Schema.DTO +{ + public enum OTelRelation + { + [EnumMember(Value = "match")] Match, + [EnumMember(Value = "equivalent")] Equivalent, + [EnumMember(Value = "otlp")] Otlp, + [EnumMember(Value = "metric")] Metric, + [EnumMember(Value = "related")] Related, + [EnumMember(Value = "conflict")] Conflict, + [EnumMember(Value = "na")] Na, + } + + public enum OTelStability + { + [EnumMember(Value = "stable")] Stable, + [EnumMember(Value = "experimental")] Experimental, + [EnumMember(Value = "development")] Development, + } + + [JsonObject(MemberSerialization.OptIn)] + public class FieldOTelMapping + { + [JsonProperty("attribute")] + public string Attribute { get; set; } + + [JsonProperty("otlp_field")] + public string OtlpField { get; set; } + + [JsonProperty("metric")] + public string Metric { get; set; } + + [JsonProperty("relation")] + [JsonConverter(typeof(StringEnumConverter))] + public OTelRelation Relation { get; set; } + + [JsonProperty("stability")] + [JsonConverter(typeof(StringEnumConverter))] + public OTelStability Stability { get; set; } + + [JsonProperty("note")] + public string Note { get; set; } + + /// + /// Returns whichever source key is set: attribute, otlp_field, or metric. + /// + public string OTelFieldName => Attribute ?? OtlpField ?? Metric; + + /// + /// True for match/equivalent relations which support bidirectional mapping. + /// + public bool IsBidirectional => Relation is OTelRelation.Match or OTelRelation.Equivalent; + } +} diff --git a/tools/Elastic.CommonSchema.Generator/Schema/EcsSchemaParser.cs b/tools/Elastic.CommonSchema.Generator/Schema/EcsSchemaParser.cs index 267b95c1..03e003ad 100644 --- a/tools/Elastic.CommonSchema.Generator/Schema/EcsSchemaParser.cs +++ b/tools/Elastic.CommonSchema.Generator/Schema/EcsSchemaParser.cs @@ -6,6 +6,7 @@ using CsQuery.ExtensionMethods; using Elastic.CommonSchema.Generator.Schema.DTO; using Newtonsoft.Json; +using Newtonsoft.Json.Linq; using YamlDotNet.Serialization; namespace Elastic.CommonSchema.Generator.Schema @@ -88,11 +89,35 @@ from jsonFile in dir.EnumerateFileSystemInfos("*.json") { if (dir.Name != "component") continue; - templates.Add(jsonFile.Name.Replace(".json", ""), File.ReadAllText(jsonFile.FullName)); + var json = File.ReadAllText(jsonFile.FullName); + json = EnsurePassthroughPriority(json); + templates.Add(jsonFile.Name.Replace(".json", ""), json); } return templates; } + // Elasticsearch 9+ requires a non-negative priority on passthrough-type mappings. + // The ECS spec may omit it, so we inject a default of 10 when it's missing. + private static string EnsurePassthroughPriority(string json) + { + var root = JObject.Parse(json); + var properties = root.SelectToken("template.mappings.properties") as JObject; + if (properties == null) return json; + + var changed = false; + foreach (var prop in properties.Properties()) + { + var obj = prop.Value as JObject; + if (obj == null) continue; + if (obj["type"]?.Value() == "passthrough" && obj["priority"] == null) + { + obj["priority"] = 10; + changed = true; + } + } + return changed ? root.ToString(Formatting.Indented) : json; + } + private List SerializeParsedAndCompareWithOriginal(Dictionary spec, string asJson) { diff --git a/tools/Elastic.CommonSchema.Generator/Views/EcsDocument.Generated.cshtml b/tools/Elastic.CommonSchema.Generator/Views/EcsDocument.Generated.cshtml index 4cf6c5bc..d0b2e931 100644 --- a/tools/Elastic.CommonSchema.Generator/Views/EcsDocument.Generated.cshtml +++ b/tools/Elastic.CommonSchema.Generator/Views/EcsDocument.Generated.cshtml @@ -39,9 +39,10 @@ namespace Elastic.CommonSchema /// Container for additional metadata against this event. /// /// When working with unknown fields use .
- /// This will try to assign valid ECS fields to their respective property + /// This will try to assign valid ECS fields to their respective property /// Failing that it will assign strings to and everything else to /// + [Obsolete("Use Attributes instead. Metadata values will be merged into Attributes during serialization.")] [JsonPropertyName("metadata"), DataMember(Name = "metadata")] [JsonConverter(typeof(MetadataDictionaryConverter))] public MetadataDictionary? Metadata { get; set; } diff --git a/tools/Elastic.CommonSchema.Generator/Views/EcsDocumentJsonConverter.Generated.cshtml b/tools/Elastic.CommonSchema.Generator/Views/EcsDocumentJsonConverter.Generated.cshtml index 9e23fa31..8d890b6e 100644 --- a/tools/Elastic.CommonSchema.Generator/Views/EcsDocumentJsonConverter.Generated.cshtml +++ b/tools/Elastic.CommonSchema.Generator/Views/EcsDocumentJsonConverter.Generated.cshtml @@ -37,7 +37,7 @@ namespace Elastic.CommonSchema.Serialization { "log.level" => ReadString(ref reader, ref loglevel), "ecs.version" => ReadString(ref reader, ref ecsVersion), - "metadata" => ReadProp@(Raw(""))(ref reader, "metadata", ecsEvent, (b, v) => b.Metadata = v, options), + "metadata" => ReadMetadataIntoAttributes(ref reader, ecsEvent, options), @foreach (var property in Model.Base.BaseFieldSet.ValueProperties) { var name = property.JsonProperty; @@ -51,14 +51,19 @@ namespace Elastic.CommonSchema.Serialization @foreach (var property in Model.Base.BaseFieldSet.InlineObjectProperties) { var name = property.JsonProperty; - "@(name)" => ReadProp<@(Raw(property.InlineObject.Name))>(ref reader, "@name", ecsEvent, (b, v) => b.@(property.Name) = v, options), - + if (name == "labels") +{ "@(name)" => ReadLabelsIntoAttributes(ref reader, ecsEvent, options), +} + else +{ "@(name)" => ReadProp<@(Raw(property.InlineObject.Name))>(ref reader, "@name", ecsEvent, (b, v) => b.@(property.Name) = v, options), +} } @foreach (var entity in Model.EntityClasses) { var entityName = entity.BaseFieldSet.FieldSet.Name; "@entityName" => ReadProp<@(entity.Name)>(ref reader, "@entityName", EcsJsonContext.Default.@(entity.Name), ecsEvent, (b, v) => b.@(entity.Name) = v), } + "attributes" => ReadOTelAttributes(ref reader, ecsEvent, options), _ => typeof(@(Model.Base.Name)) == ecsEvent.GetType() ? false @@ -101,6 +106,7 @@ namespace Elastic.CommonSchema.Serialization @foreach (var property in Model.Base.BaseFieldSet.InlineObjectProperties) { var name = property.JsonProperty; + if (name == "labels") { continue; } WriteProp(writer, "@(name)", value.@(property.InlineObject.Name), options); } @@ -116,7 +122,7 @@ namespace Elastic.CommonSchema.Serialization WriteProp(writer, "@(entityName)", value.@(entity.Name), EcsJsonContext.Default.@(entity.Name), options); } - WriteProp(writer, "metadata", value.Metadata, options); + WriteConsolidatedAttributes(writer, value, options); if (typeof(@Model.Base.Name) != value.GetType()) value.WriteAdditionalProperties((k, v) => WriteProp(writer, k, v, options)); diff --git a/tools/Elastic.CommonSchema.Generator/Views/FieldSets.Generated.cshtml b/tools/Elastic.CommonSchema.Generator/Views/FieldSets.Generated.cshtml index 70fb9aad..e93ac5be 100644 --- a/tools/Elastic.CommonSchema.Generator/Views/FieldSets.Generated.cshtml +++ b/tools/Elastic.CommonSchema.Generator/Views/FieldSets.Generated.cshtml @@ -52,6 +52,9 @@ namespace Elastic.CommonSchema /// @Raw(property.Description) /// @Raw(property.Example) /// +@if (property.Name == "Labels") +{ [Obsolete("Use EcsDocument.Attributes instead. Labels values will be merged into Attributes during serialization.")] +} [JsonPropertyName("@property.JsonProperty"), DataMember(Name = "@property.JsonProperty")] public @property.ClrType? @property.Name { get; set; } diff --git a/tools/Elastic.CommonSchema.Generator/Views/OTelMappings.Generated.cshtml b/tools/Elastic.CommonSchema.Generator/Views/OTelMappings.Generated.cshtml new file mode 100644 index 00000000..853958bf --- /dev/null +++ b/tools/Elastic.CommonSchema.Generator/Views/OTelMappings.Generated.cshtml @@ -0,0 +1,89 @@ +@using System.Collections.Generic +@using System.Linq +@using Elastic.CommonSchema.Generator.Schema.DTO +@inherits Elastic.CommonSchema.Generator.Views.CodeTemplatePage +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +/* +IMPORTANT NOTE +============== +This file has been generated. +If you wish to submit a PR please modify the original csharp file and submit the PR with that change. Thanks! +*/ + +using System.Collections.Generic; +using System.Collections.ObjectModel; + +namespace Elastic.CommonSchema +{ + /// + /// Bidirectional mapping tables between ECS field paths and OTel semantic convention attribute names. + /// Only includes match and equivalent relations. + /// + public static class OTelMappings + { +@{ + // Group by ECS field path -> list of OTel names + var ecsToOTel = Model.OTelMappings + .GroupBy(m => m.EcsFieldPath) + .OrderBy(g => g.Key) + .ToDictionary(g => g.Key, g => g.Select(m => m.OTelFieldName).Distinct().OrderBy(n => n).ToArray()); + + // Reverse: OTel name -> ECS field path (first match wins if duplicates) + var otelToEcs = Model.OTelMappings + .GroupBy(m => m.OTelFieldName) + .OrderBy(g => g.Key) + .ToDictionary(g => g.Key, g => g.First().EcsFieldPath); + + // Only emit entries where OTel name differs from ECS path (equivalent) since match means they're identical + var equivalentEcsToOTel = ecsToOTel + .Where(kv => kv.Value.Any(otel => otel != kv.Key)) + .ToDictionary(kv => kv.Key, kv => kv.Value.Where(otel => otel != kv.Key).ToArray()); + + var equivalentOTelToEcs = otelToEcs + .Where(kv => kv.Key != kv.Value); +} + /// + /// ECS field path to OTel attribute name(s) for fields where the names differ (equivalent relation). + /// For match relations the names are identical and not listed here. + /// + public static readonly IReadOnlyDictionary EcsToOTel = + new ReadOnlyDictionary(new Dictionary + { +@foreach (var kv in equivalentEcsToOTel) +{ + { "@kv.Key", new[] { @Raw(string.Join(", ", kv.Value.Select(v => $"\"{v}\""))) } }, + +} + }); + + /// + /// OTel attribute name to ECS field path for fields where the names differ (equivalent relation). + /// For match relations the names are identical and not listed here. + /// + public static readonly IReadOnlyDictionary OTelToEcs = + new ReadOnlyDictionary(new Dictionary + { +@foreach (var kv in equivalentOTelToEcs) +{ + { "@kv.Key", "@kv.Value" }, + +} + }); + + /// + /// All bidirectional ECS field paths (both match and equivalent). + /// These are ECS fields that have a corresponding OTel semantic convention. + /// + public static readonly IReadOnlyCollection AllBidirectionalEcsFields = new HashSet + { +@foreach (var kv in ecsToOTel) +{ + "@kv.Key", + +} + }; + } +} diff --git a/tools/Elastic.CommonSchema.Generator/Views/PropDispatch.Generated.cshtml b/tools/Elastic.CommonSchema.Generator/Views/PropDispatch.Generated.cshtml index afdbb948..b0a22dd3 100644 --- a/tools/Elastic.CommonSchema.Generator/Views/PropDispatch.Generated.cshtml +++ b/tools/Elastic.CommonSchema.Generator/Views/PropDispatch.Generated.cshtml @@ -1,6 +1,9 @@ @* ReSharper disable once RedundantUsingDirective *@ @using System -@using System.Linq; +@using System.Collections.Generic +@using System.Linq +@using Elastic.CommonSchema.Generator.Projection +@using Elastic.CommonSchema.Generator.Schema.DTO @inherits Elastic.CommonSchema.Generator.Views.CodeTemplatePage // Licensed to Elasticsearch B.V under one or more agreements. // Elasticsearch B.V licenses this file to you under the Apache 2.0 License. @@ -9,7 +12,7 @@ /* IMPORTANT NOTE ============== -This file has been generated. +This file has been generated. If you wish to submit a PR please modify the original csharp file and submit the PR with that change. Thanks! */ @@ -28,7 +31,7 @@ using static Elastic.CommonSchema.PropDispatch; namespace Elastic.CommonSchema { /// - public partial class @Model.Base.Name : @Model.Base.BaseFieldSet.Name + public partial class @Model.Base.Name : @Model.Base.BaseFieldSet.Name { /// /// Set ECS fields by name on . @@ -47,9 +50,9 @@ namespace Elastic.CommonSchema public void AssignField(string path, object value) { var assigned = LogTemplateProperties.All.Contains(path) && TrySet(this, path, value); - if (!assigned && LogTemplateEntities.All.Contains(path)) + if (!assigned && LogTemplateEntities.All.Contains(path)) assigned = TrySetEntity(this, path, value); - if (!assigned) + if (!assigned) SetMetaOrLabel(this, path, value); } } @@ -102,9 +105,9 @@ namespace Elastic.CommonSchema } } - internal static bool TrySet(EcsDocument document, string path, object value) + internal static bool TrySet(EcsDocument document, string path, object value) { - switch (path) + switch (path) { @foreach (var prop in Model.Base.SettableProperties) { @@ -119,7 +122,7 @@ namespace Elastic.CommonSchema { continue; } - @foreach (var prop in entity.SettableProperties) + @foreach (var prop in entity.SettableProperties) { case "@prop.FullPath": case "@prop.LogTemplateAlternative": @@ -129,6 +132,19 @@ namespace Elastic.CommonSchema } default: +@{ + // Collect all OTel equivalent mappings for the OTel-aware fallback + var allOTelEquivalents = Model.OTelMappings? + .Where(m => m.Relation == Elastic.CommonSchema.Generator.Schema.DTO.OTelRelation.Equivalent && !string.IsNullOrEmpty(m.OTelFieldName)) + .ToList() ?? new List(); +} +@if (allOTelEquivalents.Any()) +{ + // OTel equivalent name fallback: look up OTel name -> ECS path, then retry + if (OTelMappings.OTelToEcs.TryGetValue(path, out var ecsPath)) + return TrySet(document, ecsPath, value); + +} return false; } } @@ -143,6 +159,11 @@ namespace Elastic.CommonSchema "@prop.FullPath" => static (e, v) => @(prop.CastFromObject)(e, v, static (ee, p) => ee.@(prop.Name) = p), "@prop.LogTemplateAlternative" => static (e, v) => @(prop.CastFromObject)(e, v, static (ee, p) => ee.@(prop.Name) = p), + foreach (var otelName in prop.OTelNames) + { + "@otelName" => static (e, v) => @(prop.CastFromObject)(e, v, static (ee, p) => ee.@(prop.Name) = p), + + } } _ => null }; @@ -164,12 +185,22 @@ namespace Elastic.CommonSchema "@prop.FullPath" => static (e, v) => @(prop.CastFromObject)(e, v, static (ee, p) => ee.@(prop.Name) = p), "@prop.LogTemplateAlternative" => static (e, v) => @(prop.CastFromObject)(e, v, static (ee, p) => ee.@(prop.Name) = p), + foreach (var otelName in prop.OTelNames) + { + "@otelName" => static (e, v) => @(prop.CastFromObject)(e, v, static (ee, p) => ee.@(prop.Name) = p), + + } } else { "@prop.FullPath" => static (e, v) => @(prop.CastFromObject)("@(prop.JsonProperty)")(e.@(prop.ContainerPath) ??= new @(prop.ContainerPathEntity)(),v), "@prop.LogTemplateAlternative" => static (e, v) => @(prop.CastFromObject)("@(prop.JsonProperty)")(e.@(prop.ContainerPath) ??= new @(prop.ContainerPathEntity)(),v), + foreach (var otelName in prop.OTelNames) + { + "@otelName" => static (e, v) => @(prop.CastFromObject)("@(prop.JsonProperty)")(e.@(prop.ContainerPath) ??= new @(prop.ContainerPathEntity)(),v), + + } } } _ => null @@ -180,7 +211,7 @@ namespace Elastic.CommonSchema { var assign = TryAssign@(dispatch.AssignEntity)(path); if (assign == null) return false; - + var entity = document.@(dispatch.AssignTarget) ?? new @(entity.Name)(); var assigned = assign(entity, value); if (assigned) document.@(dispatch.AssignTarget) = entity; diff --git a/tools/Elastic.CommonSchema.Generator/Views/SemConv.Generated.cshtml b/tools/Elastic.CommonSchema.Generator/Views/SemConv.Generated.cshtml new file mode 100644 index 00000000..a7e556f0 --- /dev/null +++ b/tools/Elastic.CommonSchema.Generator/Views/SemConv.Generated.cshtml @@ -0,0 +1,67 @@ +@using System.Collections.Generic +@using System.Linq +@using System.Globalization +@using Elastic.CommonSchema.Generator.Schema.DTO +@inherits Elastic.CommonSchema.Generator.Views.CodeTemplatePage +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +/* +IMPORTANT NOTE +============== +This file has been generated. +If you wish to submit a PR please modify the original csharp file and submit the PR with that change. Thanks! +*/ + +namespace Elastic.CommonSchema +{ + /// + /// OpenTelemetry semantic convention attribute name constants. + /// Use these with and . + /// + public static class SemConv + { +@{ + // Collect all unique OTel attribute names from bidirectional mappings + var entries = Model.OTelMappings + .Select(m => new { OTelName = m.OTelFieldName, EcsPath = m.EcsFieldPath, Relation = m.Relation }) + .GroupBy(m => m.OTelName) + .OrderBy(g => g.Key) + .Select(g => g.First()) + .ToList(); + + // Convert dotted OTel name to PascalCase constant name + // e.g. "exception.message" -> "ExceptionMessage", "deployment.environment.name" -> "DeploymentEnvironmentName" + string ToPascalCase(string dottedName) + { + return string.Join("", dottedName.Split('.', '_') + .Select(segment => segment.Length == 0 + ? "" + : char.ToUpperInvariant(segment[0]) + segment.Substring(1))); + } + + // Track used names to handle potential collisions + var usedNames = new HashSet(); +} +@foreach (var entry in entries) +{ + var constName = ToPascalCase(entry.OTelName); + // Handle potential name collisions by appending _Attr + if (!usedNames.Add(constName)) + { + constName = constName + "Attr"; + } + var relation = entry.Relation == OTelRelation.Equivalent ? "equivalent" : "match"; + var comment = entry.Relation == OTelRelation.Equivalent + ? $"OTel {entry.OTelName} ({relation}: ECS {entry.EcsPath})" + : $"OTel {entry.OTelName} ({relation}: same as ECS)"; + /// + /// @Raw(comment) + /// + public const string @constName = "@entry.OTelName"; + + +} + } +}