From abf3dace73be5af298d319575dba55e09c6f70b5 Mon Sep 17 00:00:00 2001 From: Martijn Laarman Date: Tue, 17 Feb 2026 20:58:28 +0100 Subject: [PATCH 1/4] Add OTel semantic convention mappings with bidirectional ECS field support Parse `otel` mappings from the ECS 9.x YAML spec, generate bidirectional mapping tables (ECS <-> OTel), and support reading/writing OTel attributes alongside standard ECS fields. ECS fields are always serialized as ECS fields. OTel semantic convention attributes are stored in the `Attributes` passthrough dictionary and serialized under an `attributes` object. ## Setting OTel attributes Use `SemConv` constants with `AssignOTelField` to set an OTel attribute that is stored in `Attributes` and simultaneously mapped to its ECS property: ```csharp var doc = new EcsDocument(); doc.AssignOTelField(SemConv.ExceptionMessage, "connection refused"); // doc.Error.Message == "connection refused" // doc.Attributes["exception.message"] == "connection refused" ``` Or set attributes directly on the dictionary: ```csharp var doc = new EcsDocument { Message = "Request completed", Attributes = new MetadataDictionary { [SemConv.ExceptionMessage] = "connection refused", ["custom.field"] = "custom value", } }; ``` ## Serialization The standard serializer writes ECS fields as ECS fields and the `Attributes` dictionary as the `attributes` object: ```csharp var json = doc.Serialize(); // { // "@timestamp": "...", // "message": "Request completed", // "ecs.version": "...", // "attributes": { // "exception.message": "connection refused", // "custom.field": "custom value" // } // } ``` ## Reading OTel format `EcsDocument.Deserialize()` handles JSON that includes an `attributes` object. OTel attribute names are automatically mapped to their ECS equivalents: ```csharp var json = """{"attributes":{"exception.message":"timeout"}}"""; var doc = EcsDocument.Deserialize(json); // doc.Error.Message == "timeout" // doc.Attributes["exception.message"] == "timeout" ``` ## OTEL_RESOURCE_ATTRIBUTES integration `CreateNewWithDefaults` processes `OTEL_RESOURCE_ATTRIBUTES` entries through the mapping table. Equivalent mappings (e.g. `deployment.environment.name`) set both the ECS field and `Attributes`. Unknown attributes are stored in `Attributes` only. --- src/Elastic.CommonSchema/EcsDocument.OTel.cs | 41 + src/Elastic.CommonSchema/EcsDocument.cs | 45 + .../IndexTemplates.Generated.cs | 4 +- .../OTelMappings.Generated.cs | 281 ++++++ .../PropDispatch.Generated.cs | 148 +-- src/Elastic.CommonSchema/SemConv.Generated.cs | 861 ++++++++++++++++++ .../EcsDocumentJsonConverter.Generated.cs | 2 + .../Serialization/EcsDocumentJsonConverter.cs | 49 + .../EcsServiceTests.cs | 1 + tests/Elastic.CommonSchema.Tests/OTelTests.cs | 356 ++++++++ .../FileGenerator.cs | 2 + .../Elastic.CommonSchema.Generator/Program.cs | 60 +- .../Projection/OTelFieldMapping.cs | 25 + .../Projection/TypeProjector.cs | 64 ++ .../Projection/Types.cs | 5 + .../Schema/DTO/Field.cs | 12 + .../Schema/DTO/FieldOTelMapping.cs | 62 ++ .../EcsDocumentJsonConverter.Generated.cshtml | 2 + .../Views/OTelMappings.Generated.cshtml | 89 ++ .../Views/PropDispatch.Generated.cshtml | 49 +- .../Views/SemConv.Generated.cshtml | 67 ++ 21 files changed, 2143 insertions(+), 82 deletions(-) create mode 100644 src/Elastic.CommonSchema/EcsDocument.OTel.cs create mode 100644 src/Elastic.CommonSchema/OTelMappings.Generated.cs create mode 100644 src/Elastic.CommonSchema/SemConv.Generated.cs create mode 100644 tests/Elastic.CommonSchema.Tests/OTelTests.cs create mode 100644 tools/Elastic.CommonSchema.Generator/Projection/OTelFieldMapping.cs create mode 100644 tools/Elastic.CommonSchema.Generator/Schema/DTO/FieldOTelMapping.cs create mode 100644 tools/Elastic.CommonSchema.Generator/Views/OTelMappings.Generated.cshtml create mode 100644 tools/Elastic.CommonSchema.Generator/Views/SemConv.Generated.cshtml diff --git a/src/Elastic.CommonSchema/EcsDocument.OTel.cs b/src/Elastic.CommonSchema/EcsDocument.OTel.cs new file mode 100644 index 00000000..b7df5ced --- /dev/null +++ b/src/Elastic.CommonSchema/EcsDocument.OTel.cs @@ -0,0 +1,41 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System.Collections.Generic; +using System.Runtime.Serialization; +using System.Text.Json.Serialization; + +namespace Elastic.CommonSchema; + +public partial class EcsDocument +{ + /// + /// Passthrough key-value store for OTel semantic convention attributes. + /// In Elasticsearch, attributes is a passthrough field type where the prefix can be omitted during queries. + /// In JSON source, data is nested under an attributes object. + /// + [JsonPropertyName("attributes"), DataMember(Name = "attributes")] + public MetadataDictionary? Attributes { get; set; } + + /// + /// Assigns an OTel attribute value. Always stores in . + /// If the OTel name maps to an ECS field (via ), + /// also sets the corresponding ECS property via . + /// + /// The OTel semantic convention attribute name + /// The value to assign + public void AssignOTelField(string otelName, object value) + { + // Always store in Attributes + Attributes ??= new MetadataDictionary(); + Attributes[otelName] = value; + + // If there's a mapping to an ECS field, also set it + if (OTelMappings.OTelToEcs.TryGetValue(otelName, out var ecsPath)) + AssignField(ecsPath, value); + else + // For match relations, the OTel name IS the ECS name — try direct assignment + AssignField(otelName, value); + } +} diff --git a/src/Elastic.CommonSchema/EcsDocument.cs b/src/Elastic.CommonSchema/EcsDocument.cs index a5c92774..55e515ce 100644 --- a/src/Elastic.CommonSchema/EcsDocument.cs +++ b/src/Elastic.CommonSchema/EcsDocument.cs @@ -97,6 +97,9 @@ public static TEcsDocument CreateNewWithDefaults( if (options?.IncludeActivityData is null or true) SetActivityData(doc); + // Process remaining OTel resource attributes through the mapping table + ApplyOTelResourceAttributes(doc, initialCache.OTelResourceAttributes); + return doc; } @@ -121,6 +124,48 @@ private static IDictionary ParseOTelResourceAttributes(string? r return keyValues; } + // Attributes already handled by specific code in GetService/GetHost + private static readonly HashSet HandledResourceAttributes = new() + { + "service.name", "service.version", "service.instance.id", + "deployment.environment", + "host.name", "host.type", "host.arch" + }; + + private static void ApplyOTelResourceAttributes(EcsDocument doc, IDictionary? resourceAttributes) + { + if (resourceAttributes == null || resourceAttributes.Count == 0) return; + + foreach (var kvp in resourceAttributes) + { + var attrName = kvp.Key; + var attrValue = kvp.Value; + + // Skip attributes already handled by specific code + if (HandledResourceAttributes.Contains(attrName)) continue; + + // Check if this OTel attribute name maps to an ECS field + if (OTelMappings.OTelToEcs.TryGetValue(attrName, out var ecsPath)) + { + // Equivalent mapping: set ECS property and store in Attributes + doc.AssignField(ecsPath, attrValue); + doc.Attributes ??= new MetadataDictionary(); + doc.Attributes[attrName] = attrValue; + } + else if (OTelMappings.AllBidirectionalEcsFields.Contains(attrName)) + { + // Match relation: OTel name IS the ECS name + doc.AssignField(attrName, attrValue); + } + else + { + // Unknown attribute: store in Attributes only + doc.Attributes ??= new MetadataDictionary(); + doc.Attributes[attrName] = attrValue; + } + } + } + /// /// Create an instance of that defaults to the assembly from diff --git a/src/Elastic.CommonSchema/IndexTemplates.Generated.cs b/src/Elastic.CommonSchema/IndexTemplates.Generated.cs index 669fa81f..5dfda513 100644 --- a/src/Elastic.CommonSchema/IndexTemplates.Generated.cs +++ b/src/Elastic.CommonSchema/IndexTemplates.Generated.cs @@ -110,7 +110,7 @@ public static string GetIndexTemplateForElasticsearchComposable(string indexPatt ""codec"": ""best_compression"", ""mapping"": { ""total_fields"": { - ""limit"": 3000 + ""limit"": 3212 } } } @@ -9562,7 +9562,7 @@ public static string GetIndexTemplateForElasticsearchLegacy(string indexPattern ""index"": { ""mapping"": { ""total_fields"": { - ""limit"": 10000 + ""limit"": 3212 } }, ""refresh_interval"": ""5s"" diff --git a/src/Elastic.CommonSchema/OTelMappings.Generated.cs b/src/Elastic.CommonSchema/OTelMappings.Generated.cs new file mode 100644 index 00000000..e3ba0e1d --- /dev/null +++ b/src/Elastic.CommonSchema/OTelMappings.Generated.cs @@ -0,0 +1,281 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +/* +IMPORTANT NOTE +============== +This file has been generated. +If you wish to submit a PR please modify the original csharp file and submit the PR with that change. Thanks! +*/ + +using System.Collections.Generic; +using System.Collections.ObjectModel; + +namespace Elastic.CommonSchema +{ + /// + /// Bidirectional mapping tables between ECS field paths and OTel semantic convention attribute names. + /// Only includes match and equivalent relations. + /// + public static class OTelMappings + { + /// + /// ECS field path to OTel attribute name(s) for fields where the names differ (equivalent relation). + /// For match relations the names are identical and not listed here. + /// + public static readonly IReadOnlyDictionary EcsToOTel = + new ReadOnlyDictionary(new Dictionary + { + { "cloud.service.name", new[] { "cloud.platform" } }, + { "container.image.hash.all", new[] { "container.image.repo_digests" } }, + { "container.image.tag", new[] { "container.image.tags" } }, + { "container.runtime", new[] { "container.runtime.name" } }, + { "error.message", new[] { "exception.message" } }, + { "error.stack_trace", new[] { "exception.stacktrace" } }, + { "faas.execution", new[] { "faas.invocation_id" } }, + { "faas.trigger.type", new[] { "faas.trigger" } }, + { "file.ctime", new[] { "file.changed" } }, + { "file.gid", new[] { "file.group.id" } }, + { "file.group", new[] { "file.group.name" } }, + { "file.mtime", new[] { "file.modified" } }, + { "file.owner", new[] { "file.owner.name" } }, + { "file.target_path", new[] { "file.symbolic_link.target_path" } }, + { "file.uid", new[] { "file.owner.id" } }, + { "gen_ai.system", new[] { "gen_ai.provider.name" } }, + { "geo.city_name", new[] { "geo.locality.name" } }, + { "geo.continent_code", new[] { "geo.continent.code" } }, + { "geo.country_iso_code", new[] { "geo.country.iso_code" } }, + { "geo.region_iso_code", new[] { "geo.region.iso_code" } }, + { "host.architecture", new[] { "host.arch" } }, + { "http.request.body.bytes", new[] { "http.request.body.size" } }, + { "http.request.bytes", new[] { "http.request.size" } }, + { "http.request.method", new[] { "http.request.method_original" } }, + { "http.response.body.bytes", new[] { "http.response.body.size" } }, + { "http.response.bytes", new[] { "http.response.size" } }, + { "network.protocol", new[] { "network.protocol.name" } }, + { "os.full", new[] { "os.description" } }, + { "process.args", new[] { "process.command_args" } }, + { "process.executable", new[] { "process.executable.path" } }, + { "service.environment", new[] { "deployment.environment.name" } }, + { "service.node.name", new[] { "service.instance.id" } }, + }); + + /// + /// OTel attribute name to ECS field path for fields where the names differ (equivalent relation). + /// For match relations the names are identical and not listed here. + /// + public static readonly IReadOnlyDictionary OTelToEcs = + new ReadOnlyDictionary(new Dictionary + { + { "cloud.platform", "cloud.service.name" }, + { "container.image.repo_digests", "container.image.hash.all" }, + { "container.image.tags", "container.image.tag" }, + { "container.runtime.name", "container.runtime" }, + { "deployment.environment.name", "service.environment" }, + { "exception.message", "error.message" }, + { "exception.stacktrace", "error.stack_trace" }, + { "faas.invocation_id", "faas.execution" }, + { "faas.trigger", "faas.trigger.type" }, + { "file.changed", "file.ctime" }, + { "file.group.id", "file.gid" }, + { "file.group.name", "file.group" }, + { "file.modified", "file.mtime" }, + { "file.owner.id", "file.uid" }, + { "file.owner.name", "file.owner" }, + { "file.symbolic_link.target_path", "file.target_path" }, + { "gen_ai.provider.name", "gen_ai.system" }, + { "geo.continent.code", "geo.continent_code" }, + { "geo.country.iso_code", "geo.country_iso_code" }, + { "geo.locality.name", "geo.city_name" }, + { "geo.region.iso_code", "geo.region_iso_code" }, + { "host.arch", "host.architecture" }, + { "http.request.body.size", "http.request.body.bytes" }, + { "http.request.method_original", "http.request.method" }, + { "http.request.size", "http.request.bytes" }, + { "http.response.body.size", "http.response.body.bytes" }, + { "http.response.size", "http.response.bytes" }, + { "network.protocol.name", "network.protocol" }, + { "os.description", "os.full" }, + { "process.command_args", "process.args" }, + { "process.executable.path", "process.executable" }, + { "service.instance.id", "service.node.name" }, + }); + + /// + /// All bidirectional ECS field paths (both match and equivalent). + /// These are ECS fields that have a corresponding OTel semantic convention. + /// + public static readonly IReadOnlyCollection AllBidirectionalEcsFields = new HashSet + { + "client.address", + "client.port", + "cloud.account.id", + "cloud.availability_zone", + "cloud.provider", + "cloud.region", + "cloud.service.name", + "container.id", + "container.image.hash.all", + "container.image.name", + "container.image.tag", + "container.name", + "container.runtime", + "destination.address", + "destination.port", + "device.id", + "device.manufacturer", + "device.model.identifier", + "device.model.name", + "dns.question.name", + "error.message", + "error.stack_trace", + "error.type", + "faas.coldstart", + "faas.execution", + "faas.name", + "faas.trigger.type", + "faas.version", + "file.accessed", + "file.attributes", + "file.created", + "file.ctime", + "file.directory", + "file.extension", + "file.fork_name", + "file.gid", + "file.group", + "file.inode", + "file.mode", + "file.mtime", + "file.name", + "file.owner", + "file.path", + "file.size", + "file.target_path", + "file.uid", + "gen_ai.agent.description", + "gen_ai.agent.id", + "gen_ai.agent.name", + "gen_ai.operation.name", + "gen_ai.output.type", + "gen_ai.request.choice.count", + "gen_ai.request.encoding_formats", + "gen_ai.request.frequency_penalty", + "gen_ai.request.max_tokens", + "gen_ai.request.model", + "gen_ai.request.presence_penalty", + "gen_ai.request.seed", + "gen_ai.request.stop_sequences", + "gen_ai.request.temperature", + "gen_ai.request.top_k", + "gen_ai.request.top_p", + "gen_ai.response.finish_reasons", + "gen_ai.response.id", + "gen_ai.response.model", + "gen_ai.system", + "gen_ai.token.type", + "gen_ai.tool.call.id", + "gen_ai.tool.name", + "gen_ai.tool.type", + "gen_ai.usage.input_tokens", + "gen_ai.usage.output_tokens", + "geo.city_name", + "geo.continent_code", + "geo.country_iso_code", + "geo.postal_code", + "geo.region_iso_code", + "host.architecture", + "host.id", + "host.ip", + "host.mac", + "host.name", + "host.type", + "http.request.body.bytes", + "http.request.bytes", + "http.request.method", + "http.response.body.bytes", + "http.response.bytes", + "http.response.status_code", + "log.file.path", + "network.protocol", + "network.transport", + "network.type", + "os.full", + "os.name", + "os.version", + "process.args", + "process.args_count", + "process.command_line", + "process.executable", + "process.group_leader.pid", + "process.interactive", + "process.pid", + "process.real_user.id", + "process.real_user.name", + "process.saved_user.id", + "process.saved_user.name", + "process.session_leader.pid", + "process.title", + "process.user.id", + "process.user.name", + "process.vpid", + "process.working_directory", + "server.address", + "server.port", + "service.environment", + "service.name", + "service.node.name", + "service.version", + "source.address", + "source.port", + "tls.cipher", + "tls.client.certificate", + "tls.client.certificate_chain", + "tls.client.hash.md5", + "tls.client.hash.sha1", + "tls.client.hash.sha256", + "tls.client.issuer", + "tls.client.ja3", + "tls.client.not_after", + "tls.client.not_before", + "tls.client.subject", + "tls.client.supported_ciphers", + "tls.curve", + "tls.established", + "tls.next_protocol", + "tls.resumed", + "tls.server.certificate", + "tls.server.certificate_chain", + "tls.server.hash.md5", + "tls.server.hash.sha1", + "tls.server.hash.sha256", + "tls.server.issuer", + "tls.server.ja3s", + "tls.server.not_after", + "tls.server.not_before", + "tls.server.subject", + "url.domain", + "url.extension", + "url.fragment", + "url.full", + "url.original", + "url.path", + "url.port", + "url.query", + "url.registered_domain", + "url.scheme", + "url.subdomain", + "url.top_level_domain", + "user_agent.name", + "user_agent.original", + "user_agent.version", + "user.email", + "user.full_name", + "user.hash", + "user.id", + "user.name", + "user.roles", + }; + } +} diff --git a/src/Elastic.CommonSchema/PropDispatch.Generated.cs b/src/Elastic.CommonSchema/PropDispatch.Generated.cs index 1bca65fe..3746756b 100644 --- a/src/Elastic.CommonSchema/PropDispatch.Generated.cs +++ b/src/Elastic.CommonSchema/PropDispatch.Generated.cs @@ -5,7 +5,7 @@ /* IMPORTANT NOTE ============== -This file has been generated. +This file has been generated. If you wish to submit a PR please modify the original csharp file and submit the PR with that change. Thanks! */ @@ -24,7 +24,7 @@ If you wish to submit a PR please modify the original csharp file and submit the namespace Elastic.CommonSchema { /// - public partial class EcsDocument : BaseFieldSet + public partial class EcsDocument : BaseFieldSet { /// /// Set ECS fields by name on . @@ -43,9 +43,9 @@ public partial class EcsDocument : BaseFieldSet public void AssignField(string path, object value) { var assigned = LogTemplateProperties.All.Contains(path) && TrySet(this, path, value); - if (!assigned && LogTemplateEntities.All.Contains(path)) + if (!assigned && LogTemplateEntities.All.Contains(path)) assigned = TrySetEntity(this, path, value); - if (!assigned) + if (!assigned) SetMetaOrLabel(this, path, value); } } @@ -688,9 +688,9 @@ bool TypeCheck(Dictionary templatedObject, string typeName) => } } - internal static bool TrySet(EcsDocument document, string path, object value) + internal static bool TrySet(EcsDocument document, string path, object value) { - switch (path) + switch (path) { case "@timestamp": case "Timestamp": @@ -3257,6 +3257,9 @@ internal static bool TrySet(EcsDocument document, string path, object value) case "X509VersionNumber": return TrySetX509(document, path, value); default: + // OTel equivalent name fallback: look up OTel name -> ECS path, then retry + if (OTelMappings.OTelToEcs.TryGetValue(path, out var ecsPath)) + return TrySet(document, ecsPath, value); return false; } } @@ -3304,7 +3307,7 @@ public static bool TrySetAgent(EcsDocument document, string path, object value) { var assign = TryAssignAgent(path); if (assign == null) return false; - + var entity = document.Agent ?? new Agent(); var assigned = assign(entity, value); if (assigned) document.Agent = entity; @@ -3327,7 +3330,7 @@ public static bool TrySetAs(IAs document, string path, object value) { var assign = TryAssignAs(path); if (assign == null) return false; - + var entity = document.As ?? new As(); var assigned = assign(entity, value); if (assigned) document.As = entity; @@ -3440,7 +3443,7 @@ public static bool TrySetClient(EcsDocument document, string path, object value) { var assign = TryAssignClient(path); if (assign == null) return false; - + var entity = document.Client ?? new Client(); var assigned = assign(entity, value); if (assigned) document.Client = entity; @@ -3473,6 +3476,7 @@ public static Func TryAssignCloud(string path) "CloudRegion" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Region = p), "cloud.service.name" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.ServiceName = p), "CloudServiceName" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.ServiceName = p), + "cloud.platform" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.ServiceName = p), "cloud.entity.display_name" => static (e, v) => TryAssignEntity("entity.display_name")(e.Entity ??= new Entity(),v), "CloudEntityDisplayName" => static (e, v) => TryAssignEntity("entity.display_name")(e.Entity ??= new Entity(),v), "cloud.entity.id" => static (e, v) => TryAssignEntity("entity.id")(e.Entity ??= new Entity(),v), @@ -3495,7 +3499,7 @@ public static bool TrySetCloud(EcsDocument document, string path, object value) { var assign = TryAssignCloud(path); if (assign == null) return false; - + var entity = document.Cloud ?? new Cloud(); var assigned = assign(entity, value); if (assigned) document.Cloud = entity; @@ -3536,7 +3540,7 @@ public static bool TrySetCodeSignature(ICodeSignature document, string path, obj { var assign = TryAssignCodeSignature(path); if (assign == null) return false; - + var entity = document.CodeSignature ?? new CodeSignature(); var assigned = assign(entity, value); if (assigned) document.CodeSignature = entity; @@ -3567,6 +3571,7 @@ public static Func TryAssignContainer(string path) "ContainerNetworkIngressBytes" => static (e, v) => TrySetLong(e, v, static (ee, p) => ee.NetworkIngressBytes = p), "container.runtime" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Runtime = p), "ContainerRuntime" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Runtime = p), + "container.runtime.name" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Runtime = p), "container.security_context.privileged" => static (e, v) => TrySetBool(e, v, static (ee, p) => ee.SecurityContextPrivileged = p), "ContainerSecurityContextPrivileged" => static (e, v) => TrySetBool(e, v, static (ee, p) => ee.SecurityContextPrivileged = p), _ => null @@ -3577,7 +3582,7 @@ public static bool TrySetContainer(EcsDocument document, string path, object val { var assign = TryAssignContainer(path); if (assign == null) return false; - + var entity = document.Container ?? new Container(); var assigned = assign(entity, value); if (assigned) document.Container = entity; @@ -3602,7 +3607,7 @@ public static bool TrySetDataStream(EcsDocument document, string path, object va { var assign = TryAssignDataStream(path); if (assign == null) return false; - + var entity = document.DataStream ?? new DataStream(); var assigned = assign(entity, value); if (assigned) document.DataStream = entity; @@ -3715,7 +3720,7 @@ public static bool TrySetDestination(EcsDocument document, string path, object v { var assign = TryAssignDestination(path); if (assign == null) return false; - + var entity = document.Destination ?? new Destination(); var assigned = assign(entity, value); if (assigned) document.Destination = entity; @@ -3754,7 +3759,7 @@ public static bool TrySetDevice(EcsDocument document, string path, object value) { var assign = TryAssignDevice(path); if (assign == null) return false; - + var entity = document.Device ?? new Device(); var assigned = assign(entity, value); if (assigned) document.Device = entity; @@ -3851,7 +3856,7 @@ public static bool TrySetDll(EcsDocument document, string path, object value) { var assign = TryAssignDll(path); if (assign == null) return false; - + var entity = document.Dll ?? new Dll(); var assigned = assign(entity, value); if (assigned) document.Dll = entity; @@ -3890,7 +3895,7 @@ public static bool TrySetDns(EcsDocument document, string path, object value) { var assign = TryAssignDns(path); if (assign == null) return false; - + var entity = document.Dns ?? new Dns(); var assigned = assign(entity, value); if (assigned) document.Dns = entity; @@ -3911,7 +3916,7 @@ public static bool TrySetEcs(EcsDocument document, string path, object value) { var assign = TryAssignEcs(path); if (assign == null) return false; - + var entity = document.Ecs ?? new Ecs(); var assigned = assign(entity, value); if (assigned) document.Ecs = entity; @@ -3972,7 +3977,7 @@ public static bool TrySetElf(IElf document, string path, object value) { var assign = TryAssignElf(path); if (assign == null) return false; - + var entity = document.Elf ?? new Elf(); var assigned = assign(entity, value); if (assigned) document.Elf = entity; @@ -4009,7 +4014,7 @@ public static bool TrySetEmail(EcsDocument document, string path, object value) { var assign = TryAssignEmail(path); if (assign == null) return false; - + var entity = document.Email ?? new Email(); var assigned = assign(entity, value); if (assigned) document.Email = entity; @@ -4042,7 +4047,7 @@ public static bool TrySetEntity(IEntity document, string path, object value) { var assign = TryAssignEntity(path); if (assign == null) return false; - + var entity = document.Entity ?? new Entity(); var assigned = assign(entity, value); if (assigned) document.Entity = entity; @@ -4059,8 +4064,10 @@ public static Func TryAssignError(string path) "ErrorId" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Id = p), "error.message" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Message = p), "ErrorMessage" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Message = p), + "exception.message" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Message = p), "error.stack_trace" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.StackTrace = p), "ErrorStackTrace" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.StackTrace = p), + "exception.stacktrace" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.StackTrace = p), "error.type" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Type = p), "ErrorType" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Type = p), _ => null @@ -4071,7 +4078,7 @@ public static bool TrySetError(EcsDocument document, string path, object value) { var assign = TryAssignError(path); if (assign == null) return false; - + var entity = document.Error ?? new Error(); var assigned = assign(entity, value); if (assigned) document.Error = entity; @@ -4138,7 +4145,7 @@ public static bool TrySetEvent(EcsDocument document, string path, object value) { var assign = TryAssignEvent(path); if (assign == null) return false; - + var entity = document.Event ?? new Event(); var assigned = assign(entity, value); if (assigned) document.Event = entity; @@ -4153,6 +4160,7 @@ public static Func TryAssignFaas(string path) "FaasColdstart" => static (e, v) => TrySetBool(e, v, static (ee, p) => ee.Coldstart = p), "faas.execution" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Execution = p), "FaasExecution" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Execution = p), + "faas.invocation_id" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Execution = p), "faas.id" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Id = p), "FaasId" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Id = p), "faas.name" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Name = p), @@ -4161,6 +4169,7 @@ public static Func TryAssignFaas(string path) "FaasTriggerRequestId" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.TriggerRequestId = p), "faas.trigger.type" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.TriggerType = p), "FaasTriggerType" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.TriggerType = p), + "faas.trigger" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.TriggerType = p), "faas.version" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Version = p), "FaasVersion" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Version = p), _ => null @@ -4171,7 +4180,7 @@ public static bool TrySetFaas(EcsDocument document, string path, object value) { var assign = TryAssignFaas(path); if (assign == null) return false; - + var entity = document.Faas ?? new Faas(); var assigned = assign(entity, value); if (assigned) document.Faas = entity; @@ -4188,6 +4197,7 @@ public static Func TryAssignFile(string path) "FileCreated" => static (e, v) => TrySetDateTimeOffset(e, v, static (ee, p) => ee.Created = p), "file.ctime" => static (e, v) => TrySetDateTimeOffset(e, v, static (ee, p) => ee.Ctime = p), "FileCtime" => static (e, v) => TrySetDateTimeOffset(e, v, static (ee, p) => ee.Ctime = p), + "file.changed" => static (e, v) => TrySetDateTimeOffset(e, v, static (ee, p) => ee.Ctime = p), "file.device" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Device = p), "FileDevice" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Device = p), "file.directory" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Directory = p), @@ -4200,8 +4210,10 @@ public static Func TryAssignFile(string path) "FileForkName" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.ForkName = p), "file.gid" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Gid = p), "FileGid" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Gid = p), + "file.group.id" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Gid = p), "file.group" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Group = p), "FileGroup" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Group = p), + "file.group.name" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Group = p), "file.inode" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Inode = p), "FileInode" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Inode = p), "file.mime_type" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.MimeType = p), @@ -4210,6 +4222,7 @@ public static Func TryAssignFile(string path) "FileMode" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Mode = p), "file.mtime" => static (e, v) => TrySetDateTimeOffset(e, v, static (ee, p) => ee.Mtime = p), "FileMtime" => static (e, v) => TrySetDateTimeOffset(e, v, static (ee, p) => ee.Mtime = p), + "file.modified" => static (e, v) => TrySetDateTimeOffset(e, v, static (ee, p) => ee.Mtime = p), "file.name" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Name = p), "FileName" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Name = p), "file.origin_referrer_url" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.OriginReferrerUrl = p), @@ -4218,16 +4231,19 @@ public static Func TryAssignFile(string path) "FileOriginUrl" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.OriginUrl = p), "file.owner" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Owner = p), "FileOwner" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Owner = p), + "file.owner.name" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Owner = p), "file.path" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Path = p), "FilePath" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Path = p), "file.size" => static (e, v) => TrySetLong(e, v, static (ee, p) => ee.Size = p), "FileSize" => static (e, v) => TrySetLong(e, v, static (ee, p) => ee.Size = p), "file.target_path" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.TargetPath = p), "FileTargetPath" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.TargetPath = p), + "file.symbolic_link.target_path" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.TargetPath = p), "file.type" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Type = p), "FileType" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Type = p), "file.uid" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Uid = p), "FileUid" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Uid = p), + "file.owner.id" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Uid = p), "file.hash.cdhash" => static (e, v) => TryAssignHash("hash.cdhash")(e.Hash ??= new Hash(),v), "FileHashCdhash" => static (e, v) => TryAssignHash("hash.cdhash")(e.Hash ??= new Hash(),v), "file.hash.md5" => static (e, v) => TryAssignHash("hash.md5")(e.Hash ??= new Hash(),v), @@ -4388,7 +4404,7 @@ public static bool TrySetFile(EcsDocument document, string path, object value) { var assign = TryAssignFile(path); if (assign == null) return false; - + var entity = document.File ?? new File(); var assigned = assign(entity, value); if (assigned) document.File = entity; @@ -4433,6 +4449,7 @@ public static Func TryAssignGenAi(string path) "GenAiResponseModel" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.ResponseModel = p), "gen_ai.system" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.System = p), "GenAiSystem" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.System = p), + "gen_ai.provider.name" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.System = p), "gen_ai.token.type" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.TokenType = p), "GenAiTokenType" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.TokenType = p), "gen_ai.tool.call.id" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.ToolCallId = p), @@ -4453,7 +4470,7 @@ public static bool TrySetGenAi(EcsDocument document, string path, object value) { var assign = TryAssignGenAi(path); if (assign == null) return false; - + var entity = document.GenAi ?? new GenAi(); var assigned = assign(entity, value); if (assigned) document.GenAi = entity; @@ -4466,12 +4483,15 @@ public static Func TryAssignGeo(string path) { "geo.city_name" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.CityName = p), "GeoCityName" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.CityName = p), + "geo.locality.name" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.CityName = p), "geo.continent_code" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.ContinentCode = p), "GeoContinentCode" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.ContinentCode = p), + "geo.continent.code" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.ContinentCode = p), "geo.continent_name" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.ContinentName = p), "GeoContinentName" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.ContinentName = p), "geo.country_iso_code" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.CountryIsoCode = p), "GeoCountryIsoCode" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.CountryIsoCode = p), + "geo.country.iso_code" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.CountryIsoCode = p), "geo.country_name" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.CountryName = p), "GeoCountryName" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.CountryName = p), "geo.name" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Name = p), @@ -4480,6 +4500,7 @@ public static Func TryAssignGeo(string path) "GeoPostalCode" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.PostalCode = p), "geo.region_iso_code" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.RegionIsoCode = p), "GeoRegionIsoCode" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.RegionIsoCode = p), + "geo.region.iso_code" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.RegionIsoCode = p), "geo.region_name" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.RegionName = p), "GeoRegionName" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.RegionName = p), "geo.timezone" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Timezone = p), @@ -4492,7 +4513,7 @@ public static bool TrySetGeo(IGeo document, string path, object value) { var assign = TryAssignGeo(path); if (assign == null) return false; - + var entity = document.Geo ?? new Geo(); var assigned = assign(entity, value); if (assigned) document.Geo = entity; @@ -4517,7 +4538,7 @@ public static bool TrySetGroup(IGroup document, string path, object value) { var assign = TryAssignGroup(path); if (assign == null) return false; - + var entity = document.Group ?? new Group(); var assigned = assign(entity, value); if (assigned) document.Group = entity; @@ -4552,7 +4573,7 @@ public static bool TrySetHash(IHash document, string path, object value) { var assign = TryAssignHash(path); if (assign == null) return false; - + var entity = document.Hash ?? new Hash(); var assigned = assign(entity, value); if (assigned) document.Hash = entity; @@ -4565,6 +4586,7 @@ public static Func TryAssignHost(string path) { "host.architecture" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Architecture = p), "HostArchitecture" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Architecture = p), + "host.arch" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Architecture = p), "host.boot.id" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.BootId = p), "HostBootId" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.BootId = p), "host.cpu.usage" => static (e, v) => TrySetFloat(e, v, static (ee, p) => ee.CpuUsage = p), @@ -4665,7 +4687,7 @@ public static bool TrySetHost(EcsDocument document, string path, object value) { var assign = TryAssignHost(path); if (assign == null) return false; - + var entity = document.Host ?? new Host(); var assigned = assign(entity, value); if (assigned) document.Host = entity; @@ -4678,24 +4700,29 @@ public static Func TryAssignHttp(string path) { "http.request.body.bytes" => static (e, v) => TrySetLong(e, v, static (ee, p) => ee.RequestBodyBytes = p), "HttpRequestBodyBytes" => static (e, v) => TrySetLong(e, v, static (ee, p) => ee.RequestBodyBytes = p), + "http.request.body.size" => static (e, v) => TrySetLong(e, v, static (ee, p) => ee.RequestBodyBytes = p), "http.request.body.content" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.RequestBodyContent = p), "HttpRequestBodyContent" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.RequestBodyContent = p), "http.request.bytes" => static (e, v) => TrySetLong(e, v, static (ee, p) => ee.RequestBytes = p), "HttpRequestBytes" => static (e, v) => TrySetLong(e, v, static (ee, p) => ee.RequestBytes = p), + "http.request.size" => static (e, v) => TrySetLong(e, v, static (ee, p) => ee.RequestBytes = p), "http.request.id" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.RequestId = p), "HttpRequestId" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.RequestId = p), "http.request.method" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.RequestMethod = p), "HttpRequestMethod" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.RequestMethod = p), + "http.request.method_original" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.RequestMethod = p), "http.request.mime_type" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.RequestMimeType = p), "HttpRequestMimeType" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.RequestMimeType = p), "http.request.referrer" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.RequestReferrer = p), "HttpRequestReferrer" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.RequestReferrer = p), "http.response.body.bytes" => static (e, v) => TrySetLong(e, v, static (ee, p) => ee.ResponseBodyBytes = p), "HttpResponseBodyBytes" => static (e, v) => TrySetLong(e, v, static (ee, p) => ee.ResponseBodyBytes = p), + "http.response.body.size" => static (e, v) => TrySetLong(e, v, static (ee, p) => ee.ResponseBodyBytes = p), "http.response.body.content" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.ResponseBodyContent = p), "HttpResponseBodyContent" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.ResponseBodyContent = p), "http.response.bytes" => static (e, v) => TrySetLong(e, v, static (ee, p) => ee.ResponseBytes = p), "HttpResponseBytes" => static (e, v) => TrySetLong(e, v, static (ee, p) => ee.ResponseBytes = p), + "http.response.size" => static (e, v) => TrySetLong(e, v, static (ee, p) => ee.ResponseBytes = p), "http.response.mime_type" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.ResponseMimeType = p), "HttpResponseMimeType" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.ResponseMimeType = p), "http.response.status_code" => static (e, v) => TrySetLong(e, v, static (ee, p) => ee.ResponseStatusCode = p), @@ -4710,7 +4737,7 @@ public static bool TrySetHttp(EcsDocument document, string path, object value) { var assign = TryAssignHttp(path); if (assign == null) return false; - + var entity = document.Http ?? new Http(); var assigned = assign(entity, value); if (assigned) document.Http = entity; @@ -4735,7 +4762,7 @@ public static bool TrySetInterface(EcsDocument document, string path, object val { var assign = TryAssignInterface(path); if (assign == null) return false; - + var entity = document.Interface ?? new Interface(); var assigned = assign(entity, value); if (assigned) document.Interface = entity; @@ -4766,7 +4793,7 @@ public static bool TrySetLog(EcsDocument document, string path, object value) { var assign = TryAssignLog(path); if (assign == null) return false; - + var entity = document.Log ?? new Log(); var assigned = assign(entity, value); if (assigned) document.Log = entity; @@ -4803,7 +4830,7 @@ public static bool TrySetMacho(IMacho document, string path, object value) { var assign = TryAssignMacho(path); if (assign == null) return false; - + var entity = document.Macho ?? new Macho(); var assigned = assign(entity, value); if (assigned) document.Macho = entity; @@ -4832,6 +4859,7 @@ public static Func TryAssignNetwork(string path) "NetworkPackets" => static (e, v) => TrySetLong(e, v, static (ee, p) => ee.Packets = p), "network.protocol" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Protocol = p), "NetworkProtocol" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Protocol = p), + "network.protocol.name" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Protocol = p), "network.transport" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Transport = p), "NetworkTransport" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Transport = p), "network.type" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Type = p), @@ -4848,7 +4876,7 @@ public static bool TrySetNetwork(EcsDocument document, string path, object value { var assign = TryAssignNetwork(path); if (assign == null) return false; - + var entity = document.Network ?? new Network(); var assigned = assign(entity, value); if (assigned) document.Network = entity; @@ -4917,7 +4945,7 @@ public static bool TrySetObserver(EcsDocument document, string path, object valu { var assign = TryAssignObserver(path); if (assign == null) return false; - + var entity = document.Observer ?? new Observer(); var assigned = assign(entity, value); if (assigned) document.Observer = entity; @@ -4974,7 +5002,7 @@ public static bool TrySetOrchestrator(EcsDocument document, string path, object { var assign = TryAssignOrchestrator(path); if (assign == null) return false; - + var entity = document.Orchestrator ?? new Orchestrator(); var assigned = assign(entity, value); if (assigned) document.Orchestrator = entity; @@ -4997,7 +5025,7 @@ public static bool TrySetOrganization(EcsDocument document, string path, object { var assign = TryAssignOrganization(path); if (assign == null) return false; - + var entity = document.Organization ?? new Organization(); var assigned = assign(entity, value); if (assigned) document.Organization = entity; @@ -5012,6 +5040,7 @@ public static Func TryAssignOs(string path) "OsFamily" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Family = p), "os.full" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Full = p), "OsFull" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Full = p), + "os.description" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Full = p), "os.kernel" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Kernel = p), "OsKernel" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Kernel = p), "os.name" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Name = p), @@ -5030,7 +5059,7 @@ public static bool TrySetOs(IOs document, string path, object value) { var assign = TryAssignOs(path); if (assign == null) return false; - + var entity = document.Os ?? new Os(); var assigned = assign(entity, value); if (assigned) document.Os = entity; @@ -5075,7 +5104,7 @@ public static bool TrySetPackage(EcsDocument document, string path, object value { var assign = TryAssignPackage(path); if (assign == null) return false; - + var entity = document.Package ?? new Package(); var assigned = assign(entity, value); if (assigned) document.Package = entity; @@ -5126,7 +5155,7 @@ public static bool TrySetPe(IPe document, string path, object value) { var assign = TryAssignPe(path); if (assign == null) return false; - + var entity = document.Pe ?? new Pe(); var assigned = assign(entity, value); if (assigned) document.Pe = entity; @@ -5147,6 +5176,7 @@ public static Func TryAssignProcess(string path) "ProcessEntityId" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.EntityId = p), "process.executable" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Executable = p), "ProcessExecutable" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Executable = p), + "process.executable.path" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Executable = p), "process.exit_code" => static (e, v) => TrySetLong(e, v, static (ee, p) => ee.ExitCode = p), "ProcessExitCode" => static (e, v) => TrySetLong(e, v, static (ee, p) => ee.ExitCode = p), "process.interactive" => static (e, v) => TrySetBool(e, v, static (ee, p) => ee.Interactive = p), @@ -5595,7 +5625,7 @@ public static bool TrySetProcess(EcsDocument document, string path, object value { var assign = TryAssignProcess(path); if (assign == null) return false; - + var entity = document.Process ?? new Process(); var assigned = assign(entity, value); if (assigned) document.Process = entity; @@ -5626,7 +5656,7 @@ public static bool TrySetRegistry(EcsDocument document, string path, object valu { var assign = TryAssignRegistry(path); if (assign == null) return false; - + var entity = document.Registry ?? new Registry(); var assigned = assign(entity, value); if (assigned) document.Registry = entity; @@ -5645,7 +5675,7 @@ public static bool TrySetRelated(EcsDocument document, string path, object value { var assign = TryAssignRelated(path); if (assign == null) return false; - + var entity = document.Related ?? new Related(); var assigned = assign(entity, value); if (assigned) document.Related = entity; @@ -5676,7 +5706,7 @@ public static bool TrySetRisk(IRisk document, string path, object value) { var assign = TryAssignRisk(path); if (assign == null) return false; - + var entity = document.Risk ?? new Risk(); var assigned = assign(entity, value); if (assigned) document.Risk = entity; @@ -5713,7 +5743,7 @@ public static bool TrySetRule(EcsDocument document, string path, object value) { var assign = TryAssignRule(path); if (assign == null) return false; - + var entity = document.Rule ?? new Rule(); var assigned = assign(entity, value); if (assigned) document.Rule = entity; @@ -5826,7 +5856,7 @@ public static bool TrySetServer(EcsDocument document, string path, object value) { var assign = TryAssignServer(path); if (assign == null) return false; - + var entity = document.Server ?? new Server(); var assigned = assign(entity, value); if (assigned) document.Server = entity; @@ -5841,6 +5871,7 @@ public static Func TryAssignService(string path) "ServiceAddress" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Address = p), "service.environment" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Environment = p), "ServiceEnvironment" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Environment = p), + "deployment.environment.name" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Environment = p), "service.ephemeral_id" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.EphemeralId = p), "ServiceEphemeralId" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.EphemeralId = p), "service.id" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Id = p), @@ -5849,6 +5880,7 @@ public static Func TryAssignService(string path) "ServiceName" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.Name = p), "service.node.name" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.NodeName = p), "ServiceNodeName" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.NodeName = p), + "service.instance.id" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.NodeName = p), "service.node.role" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.NodeRole = p), "ServiceNodeRole" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.NodeRole = p), "service.state" => static (e, v) => TrySetString(e, v, static (ee, p) => ee.State = p), @@ -5879,7 +5911,7 @@ public static bool TrySetService(EcsDocument document, string path, object value { var assign = TryAssignService(path); if (assign == null) return false; - + var entity = document.Service ?? new Service(); var assigned = assign(entity, value); if (assigned) document.Service = entity; @@ -5992,7 +6024,7 @@ public static bool TrySetSource(EcsDocument document, string path, object value) { var assign = TryAssignSource(path); if (assign == null) return false; - + var entity = document.Source ?? new Source(); var assigned = assign(entity, value); if (assigned) document.Source = entity; @@ -6353,7 +6385,7 @@ public static bool TrySetThreat(EcsDocument document, string path, object value) { var assign = TryAssignThreat(path); if (assign == null) return false; - + var entity = document.Threat ?? new Threat(); var assigned = assign(entity, value); if (assigned) document.Threat = entity; @@ -6446,7 +6478,7 @@ public static bool TrySetTls(EcsDocument document, string path, object value) { var assign = TryAssignTls(path); if (assign == null) return false; - + var entity = document.Tls ?? new Tls(); var assigned = assign(entity, value); if (assigned) document.Tls = entity; @@ -6493,7 +6525,7 @@ public static bool TrySetUrl(EcsDocument document, string path, object value) { var assign = TryAssignUrl(path); if (assign == null) return false; - + var entity = document.Url ?? new Url(); var assigned = assign(entity, value); if (assigned) document.Url = entity; @@ -6556,7 +6588,7 @@ public static bool TrySetUser(IUser document, string path, object value) { var assign = TryAssignUser(path); if (assign == null) return false; - + var entity = document.User ?? new User(); var assigned = assign(entity, value); if (assigned) document.User = entity; @@ -6597,7 +6629,7 @@ public static bool TrySetUserAgent(EcsDocument document, string path, object val { var assign = TryAssignUserAgent(path); if (assign == null) return false; - + var entity = document.UserAgent ?? new UserAgent(); var assigned = assign(entity, value); if (assigned) document.UserAgent = entity; @@ -6620,7 +6652,7 @@ public static bool TrySetVlan(IVlan document, string path, object value) { var assign = TryAssignVlan(path); if (assign == null) return false; - + var entity = document.Vlan ?? new Vlan(); var assigned = assign(entity, value); if (assigned) document.Vlan = entity; @@ -6671,7 +6703,7 @@ public static bool TrySetVolume(EcsDocument document, string path, object value) { var assign = TryAssignVolume(path); if (assign == null) return false; - + var entity = document.Volume ?? new Volume(); var assigned = assign(entity, value); if (assigned) document.Volume = entity; @@ -6714,7 +6746,7 @@ public static bool TrySetVulnerability(EcsDocument document, string path, object { var assign = TryAssignVulnerability(path); if (assign == null) return false; - + var entity = document.Vulnerability ?? new Vulnerability(); var assigned = assign(entity, value); if (assigned) document.Vulnerability = entity; @@ -6755,7 +6787,7 @@ public static bool TrySetX509(IX509 document, string path, object value) { var assign = TryAssignX509(path); if (assign == null) return false; - + var entity = document.X509 ?? new X509(); var assigned = assign(entity, value); if (assigned) document.X509 = entity; diff --git a/src/Elastic.CommonSchema/SemConv.Generated.cs b/src/Elastic.CommonSchema/SemConv.Generated.cs new file mode 100644 index 00000000..8663c5ec --- /dev/null +++ b/src/Elastic.CommonSchema/SemConv.Generated.cs @@ -0,0 +1,861 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +/* +IMPORTANT NOTE +============== +This file has been generated. +If you wish to submit a PR please modify the original csharp file and submit the PR with that change. Thanks! +*/ + +namespace Elastic.CommonSchema +{ + /// + /// OpenTelemetry semantic convention attribute name constants. + /// Use these with and . + /// + public static class SemConv + { + /// + /// OTel client.address (match: same as ECS) + /// + public const string ClientAddress = "client.address"; + + /// + /// OTel client.port (match: same as ECS) + /// + public const string ClientPort = "client.port"; + + /// + /// OTel cloud.account.id (match: same as ECS) + /// + public const string CloudAccountId = "cloud.account.id"; + + /// + /// OTel cloud.availability_zone (match: same as ECS) + /// + public const string CloudAvailabilityZone = "cloud.availability_zone"; + + /// + /// OTel cloud.platform (equivalent: ECS cloud.service.name) + /// + public const string CloudPlatform = "cloud.platform"; + + /// + /// OTel cloud.provider (match: same as ECS) + /// + public const string CloudProvider = "cloud.provider"; + + /// + /// OTel cloud.region (match: same as ECS) + /// + public const string CloudRegion = "cloud.region"; + + /// + /// OTel container.id (match: same as ECS) + /// + public const string ContainerId = "container.id"; + + /// + /// OTel container.image.name (match: same as ECS) + /// + public const string ContainerImageName = "container.image.name"; + + /// + /// OTel container.image.repo_digests (equivalent: ECS container.image.hash.all) + /// + public const string ContainerImageRepoDigests = "container.image.repo_digests"; + + /// + /// OTel container.image.tags (equivalent: ECS container.image.tag) + /// + public const string ContainerImageTags = "container.image.tags"; + + /// + /// OTel container.name (match: same as ECS) + /// + public const string ContainerName = "container.name"; + + /// + /// OTel container.runtime.name (equivalent: ECS container.runtime) + /// + public const string ContainerRuntimeName = "container.runtime.name"; + + /// + /// OTel deployment.environment.name (equivalent: ECS service.environment) + /// + public const string DeploymentEnvironmentName = "deployment.environment.name"; + + /// + /// OTel destination.address (match: same as ECS) + /// + public const string DestinationAddress = "destination.address"; + + /// + /// OTel destination.port (match: same as ECS) + /// + public const string DestinationPort = "destination.port"; + + /// + /// OTel device.id (match: same as ECS) + /// + public const string DeviceId = "device.id"; + + /// + /// OTel device.manufacturer (match: same as ECS) + /// + public const string DeviceManufacturer = "device.manufacturer"; + + /// + /// OTel device.model.identifier (match: same as ECS) + /// + public const string DeviceModelIdentifier = "device.model.identifier"; + + /// + /// OTel device.model.name (match: same as ECS) + /// + public const string DeviceModelName = "device.model.name"; + + /// + /// OTel dns.question.name (match: same as ECS) + /// + public const string DnsQuestionName = "dns.question.name"; + + /// + /// OTel error.type (match: same as ECS) + /// + public const string ErrorType = "error.type"; + + /// + /// OTel exception.message (equivalent: ECS error.message) + /// + public const string ExceptionMessage = "exception.message"; + + /// + /// OTel exception.stacktrace (equivalent: ECS error.stack_trace) + /// + public const string ExceptionStacktrace = "exception.stacktrace"; + + /// + /// OTel faas.coldstart (match: same as ECS) + /// + public const string FaasColdstart = "faas.coldstart"; + + /// + /// OTel faas.invocation_id (equivalent: ECS faas.execution) + /// + public const string FaasInvocationId = "faas.invocation_id"; + + /// + /// OTel faas.name (match: same as ECS) + /// + public const string FaasName = "faas.name"; + + /// + /// OTel faas.trigger (equivalent: ECS faas.trigger.type) + /// + public const string FaasTrigger = "faas.trigger"; + + /// + /// OTel faas.version (match: same as ECS) + /// + public const string FaasVersion = "faas.version"; + + /// + /// OTel file.accessed (match: same as ECS) + /// + public const string FileAccessed = "file.accessed"; + + /// + /// OTel file.attributes (match: same as ECS) + /// + public const string FileAttributes = "file.attributes"; + + /// + /// OTel file.changed (equivalent: ECS file.ctime) + /// + public const string FileChanged = "file.changed"; + + /// + /// OTel file.created (match: same as ECS) + /// + public const string FileCreated = "file.created"; + + /// + /// OTel file.directory (match: same as ECS) + /// + public const string FileDirectory = "file.directory"; + + /// + /// OTel file.extension (match: same as ECS) + /// + public const string FileExtension = "file.extension"; + + /// + /// OTel file.fork_name (match: same as ECS) + /// + public const string FileForkName = "file.fork_name"; + + /// + /// OTel file.group.id (equivalent: ECS file.gid) + /// + public const string FileGroupId = "file.group.id"; + + /// + /// OTel file.group.name (equivalent: ECS file.group) + /// + public const string FileGroupName = "file.group.name"; + + /// + /// OTel file.inode (match: same as ECS) + /// + public const string FileInode = "file.inode"; + + /// + /// OTel file.mode (match: same as ECS) + /// + public const string FileMode = "file.mode"; + + /// + /// OTel file.modified (equivalent: ECS file.mtime) + /// + public const string FileModified = "file.modified"; + + /// + /// OTel file.name (match: same as ECS) + /// + public const string FileName = "file.name"; + + /// + /// OTel file.owner.id (equivalent: ECS file.uid) + /// + public const string FileOwnerId = "file.owner.id"; + + /// + /// OTel file.owner.name (equivalent: ECS file.owner) + /// + public const string FileOwnerName = "file.owner.name"; + + /// + /// OTel file.path (match: same as ECS) + /// + public const string FilePath = "file.path"; + + /// + /// OTel file.size (match: same as ECS) + /// + public const string FileSize = "file.size"; + + /// + /// OTel file.symbolic_link.target_path (equivalent: ECS file.target_path) + /// + public const string FileSymbolicLinkTargetPath = "file.symbolic_link.target_path"; + + /// + /// OTel gen_ai.agent.description (match: same as ECS) + /// + public const string GenAiAgentDescription = "gen_ai.agent.description"; + + /// + /// OTel gen_ai.agent.id (match: same as ECS) + /// + public const string GenAiAgentId = "gen_ai.agent.id"; + + /// + /// OTel gen_ai.agent.name (match: same as ECS) + /// + public const string GenAiAgentName = "gen_ai.agent.name"; + + /// + /// OTel gen_ai.operation.name (match: same as ECS) + /// + public const string GenAiOperationName = "gen_ai.operation.name"; + + /// + /// OTel gen_ai.output.type (match: same as ECS) + /// + public const string GenAiOutputType = "gen_ai.output.type"; + + /// + /// OTel gen_ai.provider.name (equivalent: ECS gen_ai.system) + /// + public const string GenAiProviderName = "gen_ai.provider.name"; + + /// + /// OTel gen_ai.request.choice.count (match: same as ECS) + /// + public const string GenAiRequestChoiceCount = "gen_ai.request.choice.count"; + + /// + /// OTel gen_ai.request.encoding_formats (match: same as ECS) + /// + public const string GenAiRequestEncodingFormats = "gen_ai.request.encoding_formats"; + + /// + /// OTel gen_ai.request.frequency_penalty (match: same as ECS) + /// + public const string GenAiRequestFrequencyPenalty = "gen_ai.request.frequency_penalty"; + + /// + /// OTel gen_ai.request.max_tokens (match: same as ECS) + /// + public const string GenAiRequestMaxTokens = "gen_ai.request.max_tokens"; + + /// + /// OTel gen_ai.request.model (match: same as ECS) + /// + public const string GenAiRequestModel = "gen_ai.request.model"; + + /// + /// OTel gen_ai.request.presence_penalty (match: same as ECS) + /// + public const string GenAiRequestPresencePenalty = "gen_ai.request.presence_penalty"; + + /// + /// OTel gen_ai.request.seed (match: same as ECS) + /// + public const string GenAiRequestSeed = "gen_ai.request.seed"; + + /// + /// OTel gen_ai.request.stop_sequences (match: same as ECS) + /// + public const string GenAiRequestStopSequences = "gen_ai.request.stop_sequences"; + + /// + /// OTel gen_ai.request.temperature (match: same as ECS) + /// + public const string GenAiRequestTemperature = "gen_ai.request.temperature"; + + /// + /// OTel gen_ai.request.top_k (match: same as ECS) + /// + public const string GenAiRequestTopK = "gen_ai.request.top_k"; + + /// + /// OTel gen_ai.request.top_p (match: same as ECS) + /// + public const string GenAiRequestTopP = "gen_ai.request.top_p"; + + /// + /// OTel gen_ai.response.finish_reasons (match: same as ECS) + /// + public const string GenAiResponseFinishReasons = "gen_ai.response.finish_reasons"; + + /// + /// OTel gen_ai.response.id (match: same as ECS) + /// + public const string GenAiResponseId = "gen_ai.response.id"; + + /// + /// OTel gen_ai.response.model (match: same as ECS) + /// + public const string GenAiResponseModel = "gen_ai.response.model"; + + /// + /// OTel gen_ai.token.type (match: same as ECS) + /// + public const string GenAiTokenType = "gen_ai.token.type"; + + /// + /// OTel gen_ai.tool.call.id (match: same as ECS) + /// + public const string GenAiToolCallId = "gen_ai.tool.call.id"; + + /// + /// OTel gen_ai.tool.name (match: same as ECS) + /// + public const string GenAiToolName = "gen_ai.tool.name"; + + /// + /// OTel gen_ai.tool.type (match: same as ECS) + /// + public const string GenAiToolType = "gen_ai.tool.type"; + + /// + /// OTel gen_ai.usage.input_tokens (match: same as ECS) + /// + public const string GenAiUsageInputTokens = "gen_ai.usage.input_tokens"; + + /// + /// OTel gen_ai.usage.output_tokens (match: same as ECS) + /// + public const string GenAiUsageOutputTokens = "gen_ai.usage.output_tokens"; + + /// + /// OTel geo.continent.code (equivalent: ECS geo.continent_code) + /// + public const string GeoContinentCode = "geo.continent.code"; + + /// + /// OTel geo.country.iso_code (equivalent: ECS geo.country_iso_code) + /// + public const string GeoCountryIsoCode = "geo.country.iso_code"; + + /// + /// OTel geo.locality.name (equivalent: ECS geo.city_name) + /// + public const string GeoLocalityName = "geo.locality.name"; + + /// + /// OTel geo.postal_code (match: same as ECS) + /// + public const string GeoPostalCode = "geo.postal_code"; + + /// + /// OTel geo.region.iso_code (equivalent: ECS geo.region_iso_code) + /// + public const string GeoRegionIsoCode = "geo.region.iso_code"; + + /// + /// OTel host.arch (equivalent: ECS host.architecture) + /// + public const string HostArch = "host.arch"; + + /// + /// OTel host.id (match: same as ECS) + /// + public const string HostId = "host.id"; + + /// + /// OTel host.ip (match: same as ECS) + /// + public const string HostIp = "host.ip"; + + /// + /// OTel host.mac (match: same as ECS) + /// + public const string HostMac = "host.mac"; + + /// + /// OTel host.name (match: same as ECS) + /// + public const string HostName = "host.name"; + + /// + /// OTel host.type (match: same as ECS) + /// + public const string HostType = "host.type"; + + /// + /// OTel http.request.body.size (equivalent: ECS http.request.body.bytes) + /// + public const string HttpRequestBodySize = "http.request.body.size"; + + /// + /// OTel http.request.method_original (equivalent: ECS http.request.method) + /// + public const string HttpRequestMethodOriginal = "http.request.method_original"; + + /// + /// OTel http.request.size (equivalent: ECS http.request.bytes) + /// + public const string HttpRequestSize = "http.request.size"; + + /// + /// OTel http.response.body.size (equivalent: ECS http.response.body.bytes) + /// + public const string HttpResponseBodySize = "http.response.body.size"; + + /// + /// OTel http.response.size (equivalent: ECS http.response.bytes) + /// + public const string HttpResponseSize = "http.response.size"; + + /// + /// OTel http.response.status_code (match: same as ECS) + /// + public const string HttpResponseStatusCode = "http.response.status_code"; + + /// + /// OTel log.file.path (match: same as ECS) + /// + public const string LogFilePath = "log.file.path"; + + /// + /// OTel network.protocol.name (equivalent: ECS network.protocol) + /// + public const string NetworkProtocolName = "network.protocol.name"; + + /// + /// OTel network.transport (match: same as ECS) + /// + public const string NetworkTransport = "network.transport"; + + /// + /// OTel network.type (match: same as ECS) + /// + public const string NetworkType = "network.type"; + + /// + /// OTel os.description (equivalent: ECS os.full) + /// + public const string OsDescription = "os.description"; + + /// + /// OTel os.name (match: same as ECS) + /// + public const string OsName = "os.name"; + + /// + /// OTel os.version (match: same as ECS) + /// + public const string OsVersion = "os.version"; + + /// + /// OTel process.args_count (match: same as ECS) + /// + public const string ProcessArgsCount = "process.args_count"; + + /// + /// OTel process.command_args (equivalent: ECS process.args) + /// + public const string ProcessCommandArgs = "process.command_args"; + + /// + /// OTel process.command_line (match: same as ECS) + /// + public const string ProcessCommandLine = "process.command_line"; + + /// + /// OTel process.executable.path (equivalent: ECS process.executable) + /// + public const string ProcessExecutablePath = "process.executable.path"; + + /// + /// OTel process.group_leader.pid (match: same as ECS) + /// + public const string ProcessGroupLeaderPid = "process.group_leader.pid"; + + /// + /// OTel process.interactive (match: same as ECS) + /// + public const string ProcessInteractive = "process.interactive"; + + /// + /// OTel process.pid (match: same as ECS) + /// + public const string ProcessPid = "process.pid"; + + /// + /// OTel process.real_user.id (match: same as ECS) + /// + public const string ProcessRealUserId = "process.real_user.id"; + + /// + /// OTel process.real_user.name (match: same as ECS) + /// + public const string ProcessRealUserName = "process.real_user.name"; + + /// + /// OTel process.saved_user.id (match: same as ECS) + /// + public const string ProcessSavedUserId = "process.saved_user.id"; + + /// + /// OTel process.saved_user.name (match: same as ECS) + /// + public const string ProcessSavedUserName = "process.saved_user.name"; + + /// + /// OTel process.session_leader.pid (match: same as ECS) + /// + public const string ProcessSessionLeaderPid = "process.session_leader.pid"; + + /// + /// OTel process.title (match: same as ECS) + /// + public const string ProcessTitle = "process.title"; + + /// + /// OTel process.user.id (match: same as ECS) + /// + public const string ProcessUserId = "process.user.id"; + + /// + /// OTel process.user.name (match: same as ECS) + /// + public const string ProcessUserName = "process.user.name"; + + /// + /// OTel process.vpid (match: same as ECS) + /// + public const string ProcessVpid = "process.vpid"; + + /// + /// OTel process.working_directory (match: same as ECS) + /// + public const string ProcessWorkingDirectory = "process.working_directory"; + + /// + /// OTel server.address (match: same as ECS) + /// + public const string ServerAddress = "server.address"; + + /// + /// OTel server.port (match: same as ECS) + /// + public const string ServerPort = "server.port"; + + /// + /// OTel service.instance.id (equivalent: ECS service.node.name) + /// + public const string ServiceInstanceId = "service.instance.id"; + + /// + /// OTel service.name (match: same as ECS) + /// + public const string ServiceName = "service.name"; + + /// + /// OTel service.version (match: same as ECS) + /// + public const string ServiceVersion = "service.version"; + + /// + /// OTel source.address (match: same as ECS) + /// + public const string SourceAddress = "source.address"; + + /// + /// OTel source.port (match: same as ECS) + /// + public const string SourcePort = "source.port"; + + /// + /// OTel tls.cipher (match: same as ECS) + /// + public const string TlsCipher = "tls.cipher"; + + /// + /// OTel tls.client.certificate (match: same as ECS) + /// + public const string TlsClientCertificate = "tls.client.certificate"; + + /// + /// OTel tls.client.certificate_chain (match: same as ECS) + /// + public const string TlsClientCertificateChain = "tls.client.certificate_chain"; + + /// + /// OTel tls.client.hash.md5 (match: same as ECS) + /// + public const string TlsClientHashMd5 = "tls.client.hash.md5"; + + /// + /// OTel tls.client.hash.sha1 (match: same as ECS) + /// + public const string TlsClientHashSha1 = "tls.client.hash.sha1"; + + /// + /// OTel tls.client.hash.sha256 (match: same as ECS) + /// + public const string TlsClientHashSha256 = "tls.client.hash.sha256"; + + /// + /// OTel tls.client.issuer (match: same as ECS) + /// + public const string TlsClientIssuer = "tls.client.issuer"; + + /// + /// OTel tls.client.ja3 (match: same as ECS) + /// + public const string TlsClientJa3 = "tls.client.ja3"; + + /// + /// OTel tls.client.not_after (match: same as ECS) + /// + public const string TlsClientNotAfter = "tls.client.not_after"; + + /// + /// OTel tls.client.not_before (match: same as ECS) + /// + public const string TlsClientNotBefore = "tls.client.not_before"; + + /// + /// OTel tls.client.subject (match: same as ECS) + /// + public const string TlsClientSubject = "tls.client.subject"; + + /// + /// OTel tls.client.supported_ciphers (match: same as ECS) + /// + public const string TlsClientSupportedCiphers = "tls.client.supported_ciphers"; + + /// + /// OTel tls.curve (match: same as ECS) + /// + public const string TlsCurve = "tls.curve"; + + /// + /// OTel tls.established (match: same as ECS) + /// + public const string TlsEstablished = "tls.established"; + + /// + /// OTel tls.next_protocol (match: same as ECS) + /// + public const string TlsNextProtocol = "tls.next_protocol"; + + /// + /// OTel tls.resumed (match: same as ECS) + /// + public const string TlsResumed = "tls.resumed"; + + /// + /// OTel tls.server.certificate (match: same as ECS) + /// + public const string TlsServerCertificate = "tls.server.certificate"; + + /// + /// OTel tls.server.certificate_chain (match: same as ECS) + /// + public const string TlsServerCertificateChain = "tls.server.certificate_chain"; + + /// + /// OTel tls.server.hash.md5 (match: same as ECS) + /// + public const string TlsServerHashMd5 = "tls.server.hash.md5"; + + /// + /// OTel tls.server.hash.sha1 (match: same as ECS) + /// + public const string TlsServerHashSha1 = "tls.server.hash.sha1"; + + /// + /// OTel tls.server.hash.sha256 (match: same as ECS) + /// + public const string TlsServerHashSha256 = "tls.server.hash.sha256"; + + /// + /// OTel tls.server.issuer (match: same as ECS) + /// + public const string TlsServerIssuer = "tls.server.issuer"; + + /// + /// OTel tls.server.ja3s (match: same as ECS) + /// + public const string TlsServerJa3s = "tls.server.ja3s"; + + /// + /// OTel tls.server.not_after (match: same as ECS) + /// + public const string TlsServerNotAfter = "tls.server.not_after"; + + /// + /// OTel tls.server.not_before (match: same as ECS) + /// + public const string TlsServerNotBefore = "tls.server.not_before"; + + /// + /// OTel tls.server.subject (match: same as ECS) + /// + public const string TlsServerSubject = "tls.server.subject"; + + /// + /// OTel url.domain (match: same as ECS) + /// + public const string UrlDomain = "url.domain"; + + /// + /// OTel url.extension (match: same as ECS) + /// + public const string UrlExtension = "url.extension"; + + /// + /// OTel url.fragment (match: same as ECS) + /// + public const string UrlFragment = "url.fragment"; + + /// + /// OTel url.full (match: same as ECS) + /// + public const string UrlFull = "url.full"; + + /// + /// OTel url.original (match: same as ECS) + /// + public const string UrlOriginal = "url.original"; + + /// + /// OTel url.path (match: same as ECS) + /// + public const string UrlPath = "url.path"; + + /// + /// OTel url.port (match: same as ECS) + /// + public const string UrlPort = "url.port"; + + /// + /// OTel url.query (match: same as ECS) + /// + public const string UrlQuery = "url.query"; + + /// + /// OTel url.registered_domain (match: same as ECS) + /// + public const string UrlRegisteredDomain = "url.registered_domain"; + + /// + /// OTel url.scheme (match: same as ECS) + /// + public const string UrlScheme = "url.scheme"; + + /// + /// OTel url.subdomain (match: same as ECS) + /// + public const string UrlSubdomain = "url.subdomain"; + + /// + /// OTel url.top_level_domain (match: same as ECS) + /// + public const string UrlTopLevelDomain = "url.top_level_domain"; + + /// + /// OTel user_agent.name (match: same as ECS) + /// + public const string UserAgentName = "user_agent.name"; + + /// + /// OTel user_agent.original (match: same as ECS) + /// + public const string UserAgentOriginal = "user_agent.original"; + + /// + /// OTel user_agent.version (match: same as ECS) + /// + public const string UserAgentVersion = "user_agent.version"; + + /// + /// OTel user.email (match: same as ECS) + /// + public const string UserEmail = "user.email"; + + /// + /// OTel user.full_name (match: same as ECS) + /// + public const string UserFullName = "user.full_name"; + + /// + /// OTel user.hash (match: same as ECS) + /// + public const string UserHash = "user.hash"; + + /// + /// OTel user.id (match: same as ECS) + /// + public const string UserId = "user.id"; + + /// + /// OTel user.name (match: same as ECS) + /// + public const string UserName = "user.name"; + + /// + /// OTel user.roles (match: same as ECS) + /// + public const string UserRoles = "user.roles"; + + } +} diff --git a/src/Elastic.CommonSchema/Serialization/EcsDocumentJsonConverter.Generated.cs b/src/Elastic.CommonSchema/Serialization/EcsDocumentJsonConverter.Generated.cs index 50c33808..40d80dfd 100644 --- a/src/Elastic.CommonSchema/Serialization/EcsDocumentJsonConverter.Generated.cs +++ b/src/Elastic.CommonSchema/Serialization/EcsDocumentJsonConverter.Generated.cs @@ -94,6 +94,7 @@ JsonSerializerOptions options "volume" => ReadProp(ref reader, "volume", EcsJsonContext.Default.Volume, ecsEvent, (b, v) => b.Volume = v), "vulnerability" => ReadProp(ref reader, "vulnerability", EcsJsonContext.Default.Vulnerability, ecsEvent, (b, v) => b.Vulnerability = v), "x509" => ReadProp(ref reader, "x509", EcsJsonContext.Default.X509, ecsEvent, (b, v) => b.X509 = v), + "attributes" => ReadOTelAttributes(ref reader, ecsEvent, options), _ => typeof(EcsDocument) == ecsEvent.GetType() ? false @@ -179,6 +180,7 @@ public override void Write(Utf8JsonWriter writer, TBase value, JsonSerializerOpt WriteProp(writer, "vulnerability", value.Vulnerability, EcsJsonContext.Default.Vulnerability, options); WriteProp(writer, "x509", value.X509, EcsJsonContext.Default.X509, options); WriteProp(writer, "metadata", value.Metadata, options); + WriteProp(writer, "attributes", value.Attributes, options); if (typeof(EcsDocument) != value.GetType()) value.WriteAdditionalProperties((k, v) => WriteProp(writer, k, v, options)); diff --git a/src/Elastic.CommonSchema/Serialization/EcsDocumentJsonConverter.cs b/src/Elastic.CommonSchema/Serialization/EcsDocumentJsonConverter.cs index 8d60bac3..fbeadfa8 100644 --- a/src/Elastic.CommonSchema/Serialization/EcsDocumentJsonConverter.cs +++ b/src/Elastic.CommonSchema/Serialization/EcsDocumentJsonConverter.cs @@ -97,6 +97,55 @@ private static void WriteTimestamp(Utf8JsonWriter writer, BaseFieldSet value, Js var converter = GetDateTimeOffsetConverter(options); converter.Write(writer, value.Timestamp.Value, options); } + + [System.Diagnostics.CodeAnalysis.UnconditionalSuppressMessage("ReflectionAnalysis", "IL2026:RequiresUnreferencedCode", Justification = "We always provide a static JsonTypeInfoResolver")] + [System.Diagnostics.CodeAnalysis.UnconditionalSuppressMessage("AotAnalysis", "IL3050:RequiresDynamicCode", Justification = "We always provide a static JsonTypeInfoResolver")] + private static bool ReadOTelAttributes(ref Utf8JsonReader reader, TBase ecsEvent, JsonSerializerOptions options) + { + if (reader.TokenType == JsonTokenType.Null) + return true; + + if (reader.TokenType != JsonTokenType.StartObject) + return false; + + ecsEvent.Attributes ??= new MetadataDictionary(); + + while (reader.Read()) + { + if (reader.TokenType == JsonTokenType.EndObject) + break; + + if (reader.TokenType != JsonTokenType.PropertyName) + throw new JsonException("Expected property name in attributes object"); + + var attrName = reader.GetString()!; + reader.Read(); + + object? value = reader.TokenType switch + { + JsonTokenType.String => reader.GetString(), + JsonTokenType.Number => reader.TryGetInt64(out var l) ? l : reader.GetDouble(), + JsonTokenType.True => true, + JsonTokenType.False => false, + JsonTokenType.Null => null, + _ => JsonSerializer.Deserialize(ref reader, options) + }; + + // Store in Attributes + ecsEvent.Attributes[attrName] = value; + + // If there's an OTel->ECS mapping, also set the ECS property + if (value != null) + { + if (OTelMappings.OTelToEcs.TryGetValue(attrName, out var ecsPath)) + ecsEvent.AssignField(ecsPath, value); + else + // For match relations, OTel name IS the ECS name + ecsEvent.AssignField(attrName, value); + } + } + return true; + } } /// A JsonConverter for that supports the diff --git a/tests/Elastic.CommonSchema.Tests/EcsServiceTests.cs b/tests/Elastic.CommonSchema.Tests/EcsServiceTests.cs index 787dedfc..8441eef0 100644 --- a/tests/Elastic.CommonSchema.Tests/EcsServiceTests.cs +++ b/tests/Elastic.CommonSchema.Tests/EcsServiceTests.cs @@ -9,6 +9,7 @@ namespace Elastic.CommonSchema.Tests; +[Collection("EnvironmentVariables")] public class EcsServiceTests { public EcsServiceTests(ITestOutputHelper output) => _output = output; diff --git a/tests/Elastic.CommonSchema.Tests/OTelTests.cs b/tests/Elastic.CommonSchema.Tests/OTelTests.cs new file mode 100644 index 00000000..684c296e --- /dev/null +++ b/tests/Elastic.CommonSchema.Tests/OTelTests.cs @@ -0,0 +1,356 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System; +using System.Text.Json; +using FluentAssertions; +using Xunit; + +namespace Elastic.CommonSchema.Tests; + +[Collection("EnvironmentVariables")] +public class OTelTests +{ + private class Env : IDisposable + { + private readonly string _key; + + public Env(string key, string value) + { + _key = key; + Environment.SetEnvironmentVariable(key, value); + } + + public void Dispose() => Environment.SetEnvironmentVariable(_key, null); + } + + // ── 1. OTelMappings Static Dictionaries ── + + [Fact] + public void EcsToOTel_ContainsKnownEquivalentMappings() + { + OTelMappings.EcsToOTel.Should().ContainKey("error.message") + .WhoseValue.Should().Contain("exception.message"); + + OTelMappings.EcsToOTel.Should().ContainKey("service.environment") + .WhoseValue.Should().Contain("deployment.environment.name"); + + OTelMappings.EcsToOTel.Should().ContainKey("error.stack_trace") + .WhoseValue.Should().Contain("exception.stacktrace"); + } + + [Fact] + public void OTelToEcs_ContainsKnownEquivalentMappings() + { + OTelMappings.OTelToEcs["exception.message"].Should().Be("error.message"); + OTelMappings.OTelToEcs["deployment.environment.name"].Should().Be("service.environment"); + OTelMappings.OTelToEcs["exception.stacktrace"].Should().Be("error.stack_trace"); + } + + [Fact] + public void AllBidirectionalEcsFields_ContainsBothMatchAndEquivalent() + { + // Equivalent-relation field + OTelMappings.AllBidirectionalEcsFields.Should().Contain("error.message"); + // Match-relation fields (OTel name = ECS name) + OTelMappings.AllBidirectionalEcsFields.Should().Contain("service.name"); + OTelMappings.AllBidirectionalEcsFields.Should().Contain("host.name"); + } + + [Fact] + public void EcsToOTel_DoesNotContainMatchRelations() + { + // Match relations have identical OTel and ECS names — they should not appear in EcsToOTel + OTelMappings.EcsToOTel.Should().NotContainKey("service.name"); + OTelMappings.EcsToOTel.Should().NotContainKey("host.name"); + OTelMappings.EcsToOTel.Should().NotContainKey("container.id"); + } + + // ── 2. AssignOTelField Method ── + + [Fact] + public void AssignOTelField_EquivalentMapping_PopulatesBothAttributesAndEcsField() + { + var doc = new EcsDocument(); + doc.AssignOTelField("exception.message", "boom"); + + doc.Attributes.Should().NotBeNull(); + doc.Attributes.Should().ContainKey("exception.message"); + doc.Attributes["exception.message"].Should().Be("boom"); + + doc.Error.Should().NotBeNull(); + doc.Error.Message.Should().Be("boom"); + } + + [Fact] + public void AssignOTelField_UnknownAttribute_StoredInAttributes() + { + var doc = new EcsDocument(); + doc.AssignOTelField("custom.my_attr", "val"); + + doc.Attributes.Should().NotBeNull(); + doc.Attributes.Should().ContainKey("custom.my_attr"); + doc.Attributes["custom.my_attr"].Should().Be("val"); + } + + // ── 3. JSON Deserialization of `attributes` ── + + [Fact] + public void Deserialize_WithAttributes_PopulatesAttributesAndMappedEcsFields() + { + var json = @"{""attributes"":{""exception.message"":""test error"",""exception.stacktrace"":""at Foo.Bar()""}}"; + var doc = EcsDocument.Deserialize(json); + + doc.Attributes.Should().NotBeNull(); + doc.Attributes.Should().ContainKey("exception.message"); + doc.Attributes["exception.message"].Should().Be("test error"); + + doc.Error.Should().NotBeNull(); + doc.Error.Message.Should().Be("test error"); + doc.Error.StackTrace.Should().Be("at Foo.Bar()"); + } + + [Fact] + public void Deserialize_WithAttributes_UnknownKeys_StoredInAttributes() + { + var json = @"{""attributes"":{""my.custom.field"":""hello""}}"; + var doc = EcsDocument.Deserialize(json); + + doc.Attributes.Should().NotBeNull(); + doc.Attributes.Should().ContainKey("my.custom.field"); + doc.Attributes["my.custom.field"].Should().Be("hello"); + } + + [Fact] + public void Deserialize_WithNullAttributes_NoError() + { + var json = @"{""attributes"":null}"; + var doc = EcsDocument.Deserialize(json); + + doc.Should().NotBeNull(); + } + + [Fact] + public void Deserialize_WithEmptyAttributes_NoError() + { + var json = @"{""attributes"":{}}"; + var doc = EcsDocument.Deserialize(json); + + doc.Should().NotBeNull(); + } + + // ── 4. Serialize with Attributes ── + + [Fact] + public void Serialize_WithAttributes_WritesAttributesObject() + { + var doc = new EcsDocument + { + Timestamp = DateTimeOffset.Parse("2024-01-01T00:00:00Z"), + Attributes = new MetadataDictionary + { + [SemConv.ExceptionMessage] = "boom" + } + }; + + var json = doc.Serialize(); + json.Should().Contain("\"attributes\":"); + json.Should().Contain("\"exception.message\":\"boom\""); + } + + [Fact] + public void Serialize_EcsFieldsStayEcsFields() + { + var doc = new EcsDocument + { + Timestamp = DateTimeOffset.Parse("2024-01-01T00:00:00Z"), + Message = "hello", + Log = new Log { Level = "info" }, + Error = new Error { Message = "something failed" } + }; + + var json = doc.Serialize(); + + // ECS fields should be serialized with their ECS names, NOT renamed to OTel names + using var jsonDoc = JsonDocument.Parse(json); + var root = jsonDoc.RootElement; + + root.TryGetProperty("@timestamp", out _).Should().BeTrue(); + root.TryGetProperty("message", out _).Should().BeTrue(); + root.TryGetProperty("log.level", out _).Should().BeTrue(); + root.TryGetProperty("ecs.version", out _).Should().BeTrue(); + + // error should be a nested object at root, not inside attributes + root.TryGetProperty("error", out var errorEl).Should().BeTrue(); + errorEl.TryGetProperty("message", out _).Should().BeTrue(); + + // No attributes object when Attributes is null + root.TryGetProperty("attributes", out _).Should().BeFalse(); + } + + [Fact] + public void Serialize_RoundTrip_WithAttributes() + { + var original = new EcsDocument + { + Timestamp = DateTimeOffset.Parse("2024-01-01T00:00:00Z"), + Message = "test message", + Log = new Log { Level = "warn" }, + Attributes = new MetadataDictionary + { + [SemConv.ExceptionMessage] = "otel error", + ["custom.field"] = "custom value" + } + }; + + var json = original.Serialize(); + var deserialized = EcsDocument.Deserialize(json); + + // ECS fields round-trip + deserialized.Message.Should().Be("test message"); + deserialized.Log.Should().NotBeNull(); + deserialized.Log.Level.Should().Be("warn"); + + // Attributes round-trip and OTel-mapped attributes also set ECS fields + deserialized.Attributes.Should().NotBeNull(); + deserialized.Attributes.Should().ContainKey("exception.message"); + deserialized.Attributes["exception.message"].Should().Be("otel error"); + deserialized.Attributes.Should().ContainKey("custom.field"); + deserialized.Attributes["custom.field"].Should().Be("custom value"); + + // Mapped attribute sets the ECS property on deserialization + deserialized.Error.Should().NotBeNull(); + deserialized.Error.Message.Should().Be("otel error"); + } + + [Fact] + public void Serialize_WithoutAttributes_NoAttributesInOutput() + { + var doc = new EcsDocument + { + Timestamp = DateTimeOffset.Parse("2024-01-01T00:00:00Z"), + Message = "hello" + }; + + var json = doc.Serialize(); + json.Should().NotContain("\"attributes\""); + } + + // ── 5. SemConv Constants ── + + [Fact] + public void SemConv_HasExpectedEquivalentConstants() + { + SemConv.ExceptionMessage.Should().Be("exception.message"); + SemConv.ExceptionStacktrace.Should().Be("exception.stacktrace"); + SemConv.DeploymentEnvironmentName.Should().Be("deployment.environment.name"); + SemConv.CloudPlatform.Should().Be("cloud.platform"); + SemConv.ServiceInstanceId.Should().Be("service.instance.id"); + } + + [Fact] + public void SemConv_HasExpectedMatchConstants() + { + SemConv.ServiceName.Should().Be("service.name"); + SemConv.HostName.Should().Be("host.name"); + SemConv.ContainerId.Should().Be("container.id"); + SemConv.ErrorType.Should().Be("error.type"); + } + + [Fact] + public void AssignOTelField_WithSemConvConstant_Works() + { + var doc = new EcsDocument(); + doc.AssignOTelField(SemConv.ExceptionMessage, "boom"); + + doc.Error.Should().NotBeNull(); + doc.Error.Message.Should().Be("boom"); + doc.Attributes.Should().ContainKey(SemConv.ExceptionMessage); + } + + // ── 6. PropDispatch / Field Assignment ── + + [Fact] + public void AssignField_OTelEquivalentName_GoesToLabels() + { + // AssignField only recognizes ECS field names. + // OTel equivalent names like "exception.message" are not in LogTemplateProperties, + // so string values fall through to Labels. Use AssignOTelField for OTel names. + var doc = new EcsDocument(); + doc.AssignField("exception.message", "dispatch test"); + + // Not set as ECS property — stored as label instead (strings go to Labels) + doc.Error.Should().BeNull(); + doc.Labels.Should().NotBeNull(); + doc.Labels.Should().ContainKey("exception.message"); + } + + [Fact] + public void AssignOTelField_EquivalentName_SetsEcsProperty() + { + // AssignOTelField handles OTel→ECS mapping + var doc = new EcsDocument(); + doc.AssignOTelField("exception.message", "dispatch test"); + + doc.Error.Should().NotBeNull(); + doc.Error.Message.Should().Be("dispatch test"); + doc.Attributes.Should().ContainKey("exception.message"); + } + + [Fact] + public void AssignField_EcsName_StillWorks() + { + var doc = new EcsDocument(); + doc.AssignField("error.message", "ecs dispatch test"); + + doc.Error.Should().NotBeNull(); + doc.Error.Message.Should().Be("ecs dispatch test"); + } + + // ── 7. Resource Attributes via CreateNewWithDefaults ── + + [Fact] + public void CreateNewWithDefaults_OTelResourceAttributes_EquivalentMapping() + { + using var env = new Env("OTEL_RESOURCE_ATTRIBUTES", "deployment.environment.name=production"); + var doc = EcsDocument.CreateNewWithDefaults(initialCache: new EcsDocumentCreationCache()); + + doc.Service.Should().NotBeNull(); + doc.Service.Environment.Should().Be("production"); + + doc.Attributes.Should().NotBeNull(); + doc.Attributes.Should().ContainKey("deployment.environment.name"); + } + + [Fact] + public void CreateNewWithDefaults_OTelResourceAttributes_UnknownAttribute() + { + using var env = new Env("OTEL_RESOURCE_ATTRIBUTES", "my.custom.thing=hello"); + var doc = EcsDocument.CreateNewWithDefaults(initialCache: new EcsDocumentCreationCache()); + + doc.Attributes.Should().NotBeNull(); + doc.Attributes.Should().ContainKey("my.custom.thing"); + doc.Attributes["my.custom.thing"].Should().Be("hello"); + } + + [Fact] + public void CreateNewWithDefaults_OTelResourceAttributes_HandledAttributes_NotInAttributes() + { + using var env = new Env("OTEL_RESOURCE_ATTRIBUTES", "service.name=my-svc,host.name=my-host"); + var doc = EcsDocument.CreateNewWithDefaults(initialCache: new EcsDocumentCreationCache()); + + // These are handled by GetService/GetHost, so should NOT appear in Attributes + doc.Service.Should().NotBeNull(); + doc.Service.Name.Should().Be("my-svc"); + doc.Host.Should().NotBeNull(); + doc.Host.Hostname.Should().Be("my-host"); + + // Attributes should be null or not contain these handled keys + if (doc.Attributes != null) + { + doc.Attributes.Should().NotContainKey("service.name"); + doc.Attributes.Should().NotContainKey("host.name"); + } + } +} diff --git a/tools/Elastic.CommonSchema.Generator/FileGenerator.cs b/tools/Elastic.CommonSchema.Generator/FileGenerator.cs index b82e0460..a590f493 100644 --- a/tools/Elastic.CommonSchema.Generator/FileGenerator.cs +++ b/tools/Elastic.CommonSchema.Generator/FileGenerator.cs @@ -39,6 +39,8 @@ public static void Generate(CommonSchemaTypesProjection commonSchemaTypesProject { m => Generate(m, "AssignableInterfaces"), "Assignable Interfaces" }, { m => Generate(m, "IndexTemplates"), "Elasticsearch index templates" }, { m => Generate(m, "IndexComponents"), "Elasticsearch index components" }, + { m => Generate(m, "OTelMappings"), "OTel semantic convention mappings" }, + { m => Generate(m, "SemConv"), "OTel SemConv attribute name constants" }, }; using (var progressBar = new ProgressBar(actions.Count, "Generating code", diff --git a/tools/Elastic.CommonSchema.Generator/Program.cs b/tools/Elastic.CommonSchema.Generator/Program.cs index 5707054f..ae307e07 100644 --- a/tools/Elastic.CommonSchema.Generator/Program.cs +++ b/tools/Elastic.CommonSchema.Generator/Program.cs @@ -11,37 +11,71 @@ public static class Program { private const string DefaultDownloadBranch = "v9.3.0"; + // Usage: + // dotnet run — interactive mode (prompts for download/tag) + // dotnet run -- --no-download — skip download, use cached spec, default tag + // dotnet run -- --download — download spec, default tag + // dotnet run -- --download --tag v9.3.0 — download spec with specific tag + // dotnet run -- --no-download --tag v9.3.0 — skip download, specific tag + // dotnet run -- --token — pass GitHub token for download // ReSharper disable once UnusedParameter.Local private static async Task Main(string[] args) { - var token = args.Length > 0 ? args[0] : string.Empty; - Console.WriteLine($"Running from: {Directory.GetCurrentDirectory()}"); Console.WriteLine($"Resolved codebase root to: {CodeConfiguration.Root}"); Console.WriteLine(); - var redownloadCoreSpecification = true; + var token = string.Empty; + var redownloadCoreSpecification = (bool?)null; var downloadBranch = DefaultDownloadBranch; - var answer = "invalid"; - while (answer != "y" && answer != "n" && answer != "") + // Parse CLI args + for (var i = 0; i < args.Length; i++) { - Console.Write("Download online specifications? [Y/N] (default N): "); - answer = Console.ReadLine()?.Trim().ToLowerInvariant(); - redownloadCoreSpecification = answer == "y"; + switch (args[i]) + { + case "--no-download": + redownloadCoreSpecification = false; + break; + case "--download": + redownloadCoreSpecification = true; + break; + case "--tag" when i + 1 < args.Length: + downloadBranch = args[++i]; + break; + case "--token" when i + 1 < args.Length: + token = args[++i]; + break; + default: + // Legacy: first positional arg is token + if (!args[i].StartsWith("--") && string.IsNullOrEmpty(token)) + token = args[i]; + break; + } } - Console.Write($"Tag to use (default {downloadBranch}): "); - var readBranch = Console.ReadLine()?.Trim(); - if (!string.IsNullOrEmpty(readBranch)) downloadBranch = readBranch; + // Interactive prompts only if not specified via CLI + if (redownloadCoreSpecification == null) + { + var answer = "invalid"; + while (answer != "y" && answer != "n" && answer != "") + { + Console.Write("Download online specifications? [Y/N] (default N): "); + answer = Console.ReadLine()?.Trim().ToLowerInvariant(); + redownloadCoreSpecification = answer == "y"; + } + + Console.Write($"Tag to use (default {downloadBranch}): "); + var readBranch = Console.ReadLine()?.Trim(); + if (!string.IsNullOrEmpty(readBranch)) downloadBranch = readBranch; + } if (string.IsNullOrEmpty(downloadBranch)) downloadBranch = DefaultDownloadBranch; - if (redownloadCoreSpecification) + if (redownloadCoreSpecification == true) await SpecificationDownloader.DownloadAsync(downloadBranch, token); - var ecsSchema = new EcsSchemaParser(downloadBranch).Parse(); WarnAboutSchemaValidations(ecsSchema); diff --git a/tools/Elastic.CommonSchema.Generator/Projection/OTelFieldMapping.cs b/tools/Elastic.CommonSchema.Generator/Projection/OTelFieldMapping.cs new file mode 100644 index 00000000..dc673089 --- /dev/null +++ b/tools/Elastic.CommonSchema.Generator/Projection/OTelFieldMapping.cs @@ -0,0 +1,25 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using Elastic.CommonSchema.Generator.Schema.DTO; + +namespace Elastic.CommonSchema.Generator.Projection +{ + public enum OTelMappingKind + { + Attribute, + OtlpField, + Metric, + } + + public class OTelFieldMapping + { + public string EcsFieldPath { get; set; } + public string OTelFieldName { get; set; } + public OTelRelation Relation { get; set; } + public OTelStability Stability { get; set; } + public OTelMappingKind Kind { get; set; } + public string Note { get; set; } + } +} diff --git a/tools/Elastic.CommonSchema.Generator/Projection/TypeProjector.cs b/tools/Elastic.CommonSchema.Generator/Projection/TypeProjector.cs index ca17680b..7c9ae907 100644 --- a/tools/Elastic.CommonSchema.Generator/Projection/TypeProjector.cs +++ b/tools/Elastic.CommonSchema.Generator/Projection/TypeProjector.cs @@ -33,6 +33,8 @@ public class CommonSchemaTypesProjection public List AssignableInterfaces { get; set; } public List AssignablePropDispatches { get; set; } + + public IReadOnlyCollection OTelMappings { get; set; } // ReSharper restore PropertyCanBeMadeInitOnly.Global } @@ -168,9 +170,71 @@ public CommonSchemaTypesProjection CreateProjection() propDispatches.Add(new PropDispatch(entity, a)); } Projection.AssignablePropDispatches = propDispatches; + + // Collect OTel mappings from all field sets + var otelMappings = CollectOTelMappings(); + Projection.OTelMappings = otelMappings; + + // Annotate DispatchProperties with OTel equivalent names + var otelEquivalentLookup = otelMappings + .Where(m => m.Relation == OTelRelation.Equivalent) + .ToLookup(m => m.EcsFieldPath); + + foreach (var dispatch in propDispatches) + { + foreach (var prop in dispatch.AssignableProperties) + { + foreach (var mapping in otelEquivalentLookup[prop.FullPath]) + prop.OTelNames.Add(mapping.OTelFieldName); + } + } + // Also annotate the base entity dispatch properties + foreach (var prop in Projection.Base.DispatchProperties) + { + foreach (var mapping in otelEquivalentLookup[prop.FullPath]) + prop.OTelNames.Add(mapping.OTelFieldName); + } + return Projection; } + private List CollectOTelMappings() + { + var mappings = new List(); + foreach (var fieldSet in Schema.Entities) + { + foreach (var (flatName, field) in fieldSet.Fields) + { + if (field.OTelMappings == null) continue; + foreach (var otel in field.OTelMappings) + { + if (!otel.IsBidirectional) continue; + + var otelName = otel.Relation == OTelRelation.Match + ? flatName // For match, OTel name = ECS flat_name + : otel.OTelFieldName; // For equivalent, explicit name + + if (string.IsNullOrEmpty(otelName)) continue; + + var kind = otel.Attribute != null ? OTelMappingKind.Attribute + : otel.OtlpField != null ? OTelMappingKind.OtlpField + : OTelMappingKind.Metric; + + mappings.Add(new OTelFieldMapping + { + EcsFieldPath = flatName, + OTelFieldName = otelName, + Relation = otel.Relation, + Stability = otel.Stability, + Kind = kind, + Note = otel.Note, + }); + } + } + } + return mappings; + } + private Dictionary CreateEntityTypes() { // Create concrete entity instances of base field sets to reference diff --git a/tools/Elastic.CommonSchema.Generator/Projection/Types.cs b/tools/Elastic.CommonSchema.Generator/Projection/Types.cs index f3a1a2e5..2002dac7 100644 --- a/tools/Elastic.CommonSchema.Generator/Projection/Types.cs +++ b/tools/Elastic.CommonSchema.Generator/Projection/Types.cs @@ -148,6 +148,11 @@ public class DispatchProperty public string JsonProperty { get; } public bool SelfReferential { get; } + /// + /// OTel attribute names that are equivalent to this ECS field (different name, same semantics). + /// + public List OTelNames { get; set; } = new(); + public DispatchProperty(PropertyReference property) { JsonProperty = property.JsonProperty; diff --git a/tools/Elastic.CommonSchema.Generator/Schema/DTO/Field.cs b/tools/Elastic.CommonSchema.Generator/Schema/DTO/Field.cs index a63583de..73bd32e8 100644 --- a/tools/Elastic.CommonSchema.Generator/Schema/DTO/Field.cs +++ b/tools/Elastic.CommonSchema.Generator/Schema/DTO/Field.cs @@ -170,5 +170,17 @@ public class Field [JsonProperty("pattern")] public string Pattern { get; set; } + /// + /// OTel semantic convention mappings for this field. + /// + [JsonProperty("otel")] + public List OTelMappings { get; set; } + + /// + /// Controls synthetic source keep behaviour. + /// + [JsonProperty("synthetic_source_keep")] + public string SyntheticSourceKeep { get; set; } + } } diff --git a/tools/Elastic.CommonSchema.Generator/Schema/DTO/FieldOTelMapping.cs b/tools/Elastic.CommonSchema.Generator/Schema/DTO/FieldOTelMapping.cs new file mode 100644 index 00000000..ebc0347b --- /dev/null +++ b/tools/Elastic.CommonSchema.Generator/Schema/DTO/FieldOTelMapping.cs @@ -0,0 +1,62 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System.Runtime.Serialization; +using Newtonsoft.Json; +using Newtonsoft.Json.Converters; + +namespace Elastic.CommonSchema.Generator.Schema.DTO +{ + public enum OTelRelation + { + [EnumMember(Value = "match")] Match, + [EnumMember(Value = "equivalent")] Equivalent, + [EnumMember(Value = "otlp")] Otlp, + [EnumMember(Value = "metric")] Metric, + [EnumMember(Value = "related")] Related, + [EnumMember(Value = "conflict")] Conflict, + [EnumMember(Value = "na")] Na, + } + + public enum OTelStability + { + [EnumMember(Value = "stable")] Stable, + [EnumMember(Value = "experimental")] Experimental, + [EnumMember(Value = "development")] Development, + } + + [JsonObject(MemberSerialization.OptIn)] + public class FieldOTelMapping + { + [JsonProperty("attribute")] + public string Attribute { get; set; } + + [JsonProperty("otlp_field")] + public string OtlpField { get; set; } + + [JsonProperty("metric")] + public string Metric { get; set; } + + [JsonProperty("relation")] + [JsonConverter(typeof(StringEnumConverter))] + public OTelRelation Relation { get; set; } + + [JsonProperty("stability")] + [JsonConverter(typeof(StringEnumConverter))] + public OTelStability Stability { get; set; } + + [JsonProperty("note")] + public string Note { get; set; } + + /// + /// Returns whichever source key is set: attribute, otlp_field, or metric. + /// + public string OTelFieldName => Attribute ?? OtlpField ?? Metric; + + /// + /// True for match/equivalent relations which support bidirectional mapping. + /// + public bool IsBidirectional => Relation is OTelRelation.Match or OTelRelation.Equivalent; + } +} diff --git a/tools/Elastic.CommonSchema.Generator/Views/EcsDocumentJsonConverter.Generated.cshtml b/tools/Elastic.CommonSchema.Generator/Views/EcsDocumentJsonConverter.Generated.cshtml index 9e23fa31..63372c59 100644 --- a/tools/Elastic.CommonSchema.Generator/Views/EcsDocumentJsonConverter.Generated.cshtml +++ b/tools/Elastic.CommonSchema.Generator/Views/EcsDocumentJsonConverter.Generated.cshtml @@ -59,6 +59,7 @@ namespace Elastic.CommonSchema.Serialization var entityName = entity.BaseFieldSet.FieldSet.Name; "@entityName" => ReadProp<@(entity.Name)>(ref reader, "@entityName", EcsJsonContext.Default.@(entity.Name), ecsEvent, (b, v) => b.@(entity.Name) = v), } + "attributes" => ReadOTelAttributes(ref reader, ecsEvent, options), _ => typeof(@(Model.Base.Name)) == ecsEvent.GetType() ? false @@ -117,6 +118,7 @@ namespace Elastic.CommonSchema.Serialization } WriteProp(writer, "metadata", value.Metadata, options); + WriteProp(writer, "attributes", value.Attributes, options); if (typeof(@Model.Base.Name) != value.GetType()) value.WriteAdditionalProperties((k, v) => WriteProp(writer, k, v, options)); diff --git a/tools/Elastic.CommonSchema.Generator/Views/OTelMappings.Generated.cshtml b/tools/Elastic.CommonSchema.Generator/Views/OTelMappings.Generated.cshtml new file mode 100644 index 00000000..853958bf --- /dev/null +++ b/tools/Elastic.CommonSchema.Generator/Views/OTelMappings.Generated.cshtml @@ -0,0 +1,89 @@ +@using System.Collections.Generic +@using System.Linq +@using Elastic.CommonSchema.Generator.Schema.DTO +@inherits Elastic.CommonSchema.Generator.Views.CodeTemplatePage +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +/* +IMPORTANT NOTE +============== +This file has been generated. +If you wish to submit a PR please modify the original csharp file and submit the PR with that change. Thanks! +*/ + +using System.Collections.Generic; +using System.Collections.ObjectModel; + +namespace Elastic.CommonSchema +{ + /// + /// Bidirectional mapping tables between ECS field paths and OTel semantic convention attribute names. + /// Only includes match and equivalent relations. + /// + public static class OTelMappings + { +@{ + // Group by ECS field path -> list of OTel names + var ecsToOTel = Model.OTelMappings + .GroupBy(m => m.EcsFieldPath) + .OrderBy(g => g.Key) + .ToDictionary(g => g.Key, g => g.Select(m => m.OTelFieldName).Distinct().OrderBy(n => n).ToArray()); + + // Reverse: OTel name -> ECS field path (first match wins if duplicates) + var otelToEcs = Model.OTelMappings + .GroupBy(m => m.OTelFieldName) + .OrderBy(g => g.Key) + .ToDictionary(g => g.Key, g => g.First().EcsFieldPath); + + // Only emit entries where OTel name differs from ECS path (equivalent) since match means they're identical + var equivalentEcsToOTel = ecsToOTel + .Where(kv => kv.Value.Any(otel => otel != kv.Key)) + .ToDictionary(kv => kv.Key, kv => kv.Value.Where(otel => otel != kv.Key).ToArray()); + + var equivalentOTelToEcs = otelToEcs + .Where(kv => kv.Key != kv.Value); +} + /// + /// ECS field path to OTel attribute name(s) for fields where the names differ (equivalent relation). + /// For match relations the names are identical and not listed here. + /// + public static readonly IReadOnlyDictionary EcsToOTel = + new ReadOnlyDictionary(new Dictionary + { +@foreach (var kv in equivalentEcsToOTel) +{ + { "@kv.Key", new[] { @Raw(string.Join(", ", kv.Value.Select(v => $"\"{v}\""))) } }, + +} + }); + + /// + /// OTel attribute name to ECS field path for fields where the names differ (equivalent relation). + /// For match relations the names are identical and not listed here. + /// + public static readonly IReadOnlyDictionary OTelToEcs = + new ReadOnlyDictionary(new Dictionary + { +@foreach (var kv in equivalentOTelToEcs) +{ + { "@kv.Key", "@kv.Value" }, + +} + }); + + /// + /// All bidirectional ECS field paths (both match and equivalent). + /// These are ECS fields that have a corresponding OTel semantic convention. + /// + public static readonly IReadOnlyCollection AllBidirectionalEcsFields = new HashSet + { +@foreach (var kv in ecsToOTel) +{ + "@kv.Key", + +} + }; + } +} diff --git a/tools/Elastic.CommonSchema.Generator/Views/PropDispatch.Generated.cshtml b/tools/Elastic.CommonSchema.Generator/Views/PropDispatch.Generated.cshtml index afdbb948..b0a22dd3 100644 --- a/tools/Elastic.CommonSchema.Generator/Views/PropDispatch.Generated.cshtml +++ b/tools/Elastic.CommonSchema.Generator/Views/PropDispatch.Generated.cshtml @@ -1,6 +1,9 @@ @* ReSharper disable once RedundantUsingDirective *@ @using System -@using System.Linq; +@using System.Collections.Generic +@using System.Linq +@using Elastic.CommonSchema.Generator.Projection +@using Elastic.CommonSchema.Generator.Schema.DTO @inherits Elastic.CommonSchema.Generator.Views.CodeTemplatePage // Licensed to Elasticsearch B.V under one or more agreements. // Elasticsearch B.V licenses this file to you under the Apache 2.0 License. @@ -9,7 +12,7 @@ /* IMPORTANT NOTE ============== -This file has been generated. +This file has been generated. If you wish to submit a PR please modify the original csharp file and submit the PR with that change. Thanks! */ @@ -28,7 +31,7 @@ using static Elastic.CommonSchema.PropDispatch; namespace Elastic.CommonSchema { /// - public partial class @Model.Base.Name : @Model.Base.BaseFieldSet.Name + public partial class @Model.Base.Name : @Model.Base.BaseFieldSet.Name { /// /// Set ECS fields by name on . @@ -47,9 +50,9 @@ namespace Elastic.CommonSchema public void AssignField(string path, object value) { var assigned = LogTemplateProperties.All.Contains(path) && TrySet(this, path, value); - if (!assigned && LogTemplateEntities.All.Contains(path)) + if (!assigned && LogTemplateEntities.All.Contains(path)) assigned = TrySetEntity(this, path, value); - if (!assigned) + if (!assigned) SetMetaOrLabel(this, path, value); } } @@ -102,9 +105,9 @@ namespace Elastic.CommonSchema } } - internal static bool TrySet(EcsDocument document, string path, object value) + internal static bool TrySet(EcsDocument document, string path, object value) { - switch (path) + switch (path) { @foreach (var prop in Model.Base.SettableProperties) { @@ -119,7 +122,7 @@ namespace Elastic.CommonSchema { continue; } - @foreach (var prop in entity.SettableProperties) + @foreach (var prop in entity.SettableProperties) { case "@prop.FullPath": case "@prop.LogTemplateAlternative": @@ -129,6 +132,19 @@ namespace Elastic.CommonSchema } default: +@{ + // Collect all OTel equivalent mappings for the OTel-aware fallback + var allOTelEquivalents = Model.OTelMappings? + .Where(m => m.Relation == Elastic.CommonSchema.Generator.Schema.DTO.OTelRelation.Equivalent && !string.IsNullOrEmpty(m.OTelFieldName)) + .ToList() ?? new List(); +} +@if (allOTelEquivalents.Any()) +{ + // OTel equivalent name fallback: look up OTel name -> ECS path, then retry + if (OTelMappings.OTelToEcs.TryGetValue(path, out var ecsPath)) + return TrySet(document, ecsPath, value); + +} return false; } } @@ -143,6 +159,11 @@ namespace Elastic.CommonSchema "@prop.FullPath" => static (e, v) => @(prop.CastFromObject)(e, v, static (ee, p) => ee.@(prop.Name) = p), "@prop.LogTemplateAlternative" => static (e, v) => @(prop.CastFromObject)(e, v, static (ee, p) => ee.@(prop.Name) = p), + foreach (var otelName in prop.OTelNames) + { + "@otelName" => static (e, v) => @(prop.CastFromObject)(e, v, static (ee, p) => ee.@(prop.Name) = p), + + } } _ => null }; @@ -164,12 +185,22 @@ namespace Elastic.CommonSchema "@prop.FullPath" => static (e, v) => @(prop.CastFromObject)(e, v, static (ee, p) => ee.@(prop.Name) = p), "@prop.LogTemplateAlternative" => static (e, v) => @(prop.CastFromObject)(e, v, static (ee, p) => ee.@(prop.Name) = p), + foreach (var otelName in prop.OTelNames) + { + "@otelName" => static (e, v) => @(prop.CastFromObject)(e, v, static (ee, p) => ee.@(prop.Name) = p), + + } } else { "@prop.FullPath" => static (e, v) => @(prop.CastFromObject)("@(prop.JsonProperty)")(e.@(prop.ContainerPath) ??= new @(prop.ContainerPathEntity)(),v), "@prop.LogTemplateAlternative" => static (e, v) => @(prop.CastFromObject)("@(prop.JsonProperty)")(e.@(prop.ContainerPath) ??= new @(prop.ContainerPathEntity)(),v), + foreach (var otelName in prop.OTelNames) + { + "@otelName" => static (e, v) => @(prop.CastFromObject)("@(prop.JsonProperty)")(e.@(prop.ContainerPath) ??= new @(prop.ContainerPathEntity)(),v), + + } } } _ => null @@ -180,7 +211,7 @@ namespace Elastic.CommonSchema { var assign = TryAssign@(dispatch.AssignEntity)(path); if (assign == null) return false; - + var entity = document.@(dispatch.AssignTarget) ?? new @(entity.Name)(); var assigned = assign(entity, value); if (assigned) document.@(dispatch.AssignTarget) = entity; diff --git a/tools/Elastic.CommonSchema.Generator/Views/SemConv.Generated.cshtml b/tools/Elastic.CommonSchema.Generator/Views/SemConv.Generated.cshtml new file mode 100644 index 00000000..a7e556f0 --- /dev/null +++ b/tools/Elastic.CommonSchema.Generator/Views/SemConv.Generated.cshtml @@ -0,0 +1,67 @@ +@using System.Collections.Generic +@using System.Linq +@using System.Globalization +@using Elastic.CommonSchema.Generator.Schema.DTO +@inherits Elastic.CommonSchema.Generator.Views.CodeTemplatePage +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +/* +IMPORTANT NOTE +============== +This file has been generated. +If you wish to submit a PR please modify the original csharp file and submit the PR with that change. Thanks! +*/ + +namespace Elastic.CommonSchema +{ + /// + /// OpenTelemetry semantic convention attribute name constants. + /// Use these with and . + /// + public static class SemConv + { +@{ + // Collect all unique OTel attribute names from bidirectional mappings + var entries = Model.OTelMappings + .Select(m => new { OTelName = m.OTelFieldName, EcsPath = m.EcsFieldPath, Relation = m.Relation }) + .GroupBy(m => m.OTelName) + .OrderBy(g => g.Key) + .Select(g => g.First()) + .ToList(); + + // Convert dotted OTel name to PascalCase constant name + // e.g. "exception.message" -> "ExceptionMessage", "deployment.environment.name" -> "DeploymentEnvironmentName" + string ToPascalCase(string dottedName) + { + return string.Join("", dottedName.Split('.', '_') + .Select(segment => segment.Length == 0 + ? "" + : char.ToUpperInvariant(segment[0]) + segment.Substring(1))); + } + + // Track used names to handle potential collisions + var usedNames = new HashSet(); +} +@foreach (var entry in entries) +{ + var constName = ToPascalCase(entry.OTelName); + // Handle potential name collisions by appending _Attr + if (!usedNames.Add(constName)) + { + constName = constName + "Attr"; + } + var relation = entry.Relation == OTelRelation.Equivalent ? "equivalent" : "match"; + var comment = entry.Relation == OTelRelation.Equivalent + ? $"OTel {entry.OTelName} ({relation}: ECS {entry.EcsPath})" + : $"OTel {entry.OTelName} ({relation}: same as ECS)"; + /// + /// @Raw(comment) + /// + public const string @constName = "@entry.OTelName"; + + +} + } +} From c0888802e435b97ec8f3a0fbed0edf06895afecf Mon Sep 17 00:00:00 2001 From: Martijn Laarman Date: Wed, 18 Feb 2026 14:45:35 +0100 Subject: [PATCH 2/4] Consolidate Labels, Metadata, and Attributes into single Attributes passthrough Labels and Metadata are now [Obsolete] and route to Attributes. Serialization merges all three into a single "attributes" JSON key (Attributes wins over Metadata wins over Labels). Deserialization reads "labels", "metadata", and "attributes" JSON keys all into the Attributes property. The ES base component now includes an "attributes" passthrough mapping. --- src/Elastic.CommonSchema.NLog/EcsLayout.cs | 2 + .../EcsDocument.Generated.cs | 3 +- .../FieldSets.Generated.cs | 2 + .../IndexComponents.Generated.cs | 3 + src/Elastic.CommonSchema/PropDispatch.cs | 17 +-- .../EcsDocumentJsonConverter.Generated.cs | 8 +- .../Serialization/EcsDocumentJsonConverter.cs | 121 ++++++++++++++++++ .../LogEventBuilderExtensions.cs | 2 + .../v9.3.0/composable/component/base.json | 3 + .../MessageTests.cs | 59 ++++----- .../EcsFieldsInTemplateTests.cs | 3 +- .../MessageTests.cs | 51 ++++---- .../EcsFieldsInTemplateTests.cs | 11 +- .../SerilogWithExtensionLoggerGenerated.cs | 6 +- .../SerilogWithExtensionsLoggerAdapter.cs | 5 +- .../LogEventPropFilterTests.cs | 20 +-- .../MessageTests.cs | 32 ++--- .../Repro/GithubIssue225.cs | 8 +- tests/Elastic.CommonSchema.Tests/OTelTests.cs | 10 +- .../Repro/GithubIssue29.cs | 2 + .../Elastic.CommonSchema.Tests/Serializes.cs | 23 ++-- .../Views/EcsDocument.Generated.cshtml | 3 +- .../EcsDocumentJsonConverter.Generated.cshtml | 14 +- .../Views/FieldSets.Generated.cshtml | 3 + 24 files changed, 263 insertions(+), 148 deletions(-) diff --git a/src/Elastic.CommonSchema.NLog/EcsLayout.cs b/src/Elastic.CommonSchema.NLog/EcsLayout.cs index c7583220..6fe3ced0 100644 --- a/src/Elastic.CommonSchema.NLog/EcsLayout.cs +++ b/src/Elastic.CommonSchema.NLog/EcsLayout.cs @@ -304,7 +304,9 @@ public NLogEcsDocument RenderEcsDocument(LogEventInfo logEvent) ecsEvent.Event = GetEvent(logEvent); ecsEvent.Process = GetProcess(logEvent); ecsEvent.Tags = GetTags(logEvent); +#pragma warning disable CS0618 // Obsolete Labels ecsEvent.Labels = GetLabels(logEvent); +#pragma warning restore CS0618 ecsEvent.Http = GetHttp(logEvent); ecsEvent.Url = GetUrl(logEvent); diff --git a/src/Elastic.CommonSchema/EcsDocument.Generated.cs b/src/Elastic.CommonSchema/EcsDocument.Generated.cs index fbdefb92..08951555 100644 --- a/src/Elastic.CommonSchema/EcsDocument.Generated.cs +++ b/src/Elastic.CommonSchema/EcsDocument.Generated.cs @@ -36,9 +36,10 @@ public partial class EcsDocument : BaseFieldSet , IAs, ICodeSignature, IElf, IEn /// Container for additional metadata against this event. /// /// When working with unknown fields use .
- /// This will try to assign valid ECS fields to their respective property + /// This will try to assign valid ECS fields to their respective property /// Failing that it will assign strings to and everything else to /// + [Obsolete("Use Attributes instead. Metadata values will be merged into Attributes during serialization.")] [JsonPropertyName("metadata"), DataMember(Name = "metadata")] [JsonConverter(typeof(MetadataDictionaryConverter))] public MetadataDictionary? Metadata { get; set; } diff --git a/src/Elastic.CommonSchema/FieldSets.Generated.cs b/src/Elastic.CommonSchema/FieldSets.Generated.cs index e7e46e1f..71d4e35e 100644 --- a/src/Elastic.CommonSchema/FieldSets.Generated.cs +++ b/src/Elastic.CommonSchema/FieldSets.Generated.cs @@ -173,6 +173,7 @@ public abstract class BaseFieldSet { /// Example: `docker` and `k8s` labels. /// {"application": "foo-bar", "env": "production"} /// + [Obsolete("Use EcsDocument.Attributes instead. Labels values will be merged into Attributes during serialization.")] [JsonPropertyName("labels"), DataMember(Name = "labels")] public Labels? Labels { get; set; } } @@ -605,6 +606,7 @@ public abstract class ContainerFieldSet { /// Image labels. /// /// + [Obsolete("Use EcsDocument.Attributes instead. Labels values will be merged into Attributes during serialization.")] [JsonPropertyName("labels"), DataMember(Name = "labels")] public ContainerLabels? Labels { get; set; } } diff --git a/src/Elastic.CommonSchema/IndexComponents.Generated.cs b/src/Elastic.CommonSchema/IndexComponents.Generated.cs index def12d37..e196dab0 100644 --- a/src/Elastic.CommonSchema/IndexComponents.Generated.cs +++ b/src/Elastic.CommonSchema/IndexComponents.Generated.cs @@ -345,6 +345,9 @@ public static class IndexComponents ""@timestamp"": { ""type"": ""date"" }, + ""attributes"": { + ""type"": ""passthrough"" + }, ""labels"": { ""type"": ""object"" }, diff --git a/src/Elastic.CommonSchema/PropDispatch.cs b/src/Elastic.CommonSchema/PropDispatch.cs index 68806606..6aa98d72 100644 --- a/src/Elastic.CommonSchema/PropDispatch.cs +++ b/src/Elastic.CommonSchema/PropDispatch.cs @@ -30,21 +30,8 @@ internal static partial class PropDispatch { public static void SetMetaOrLabel(EcsDocument document, string path, object value) { - switch (value) - { - case string s: - document.Labels ??= new Labels(); - document.Labels[path] = s; - break; - case bool b: - document.Labels ??= new Labels(); - document.Labels[path] = b.ToString(CultureInfo.InvariantCulture).ToLowerInvariant(); - break; - default: - document.Metadata ??= new MetadataDictionary(); - document.Metadata[path] = value; - break; - } + document.Attributes ??= new MetadataDictionary(); + document.Attributes[path] = value; } private static bool TrySetLong(T target, object value, Action set) diff --git a/src/Elastic.CommonSchema/Serialization/EcsDocumentJsonConverter.Generated.cs b/src/Elastic.CommonSchema/Serialization/EcsDocumentJsonConverter.Generated.cs index 40d80dfd..2fc50643 100644 --- a/src/Elastic.CommonSchema/Serialization/EcsDocumentJsonConverter.Generated.cs +++ b/src/Elastic.CommonSchema/Serialization/EcsDocumentJsonConverter.Generated.cs @@ -34,14 +34,14 @@ JsonSerializerOptions options { "log.level" => ReadString(ref reader, ref loglevel), "ecs.version" => ReadString(ref reader, ref ecsVersion), - "metadata" => ReadProp(ref reader, "metadata", ecsEvent, (b, v) => b.Metadata = v, options), + "metadata" => ReadMetadataIntoAttributes(ref reader, ecsEvent, options), "@timestamp" => ReadDateTime(ref reader, ref @timestamp, options), "message" => ReadProp(ref reader, "message", ecsEvent, (b, v) => b.Message = v, options), "tags" => ReadProp(ref reader, "tags", ecsEvent, (b, v) => b.Tags = v, options), "span.id" => ReadProp(ref reader, "span.id", ecsEvent, (b, v) => b.SpanId = v, options), "trace.id" => ReadProp(ref reader, "trace.id", ecsEvent, (b, v) => b.TraceId = v, options), "transaction.id" => ReadProp(ref reader, "transaction.id", ecsEvent, (b, v) => b.TransactionId = v, options), - "labels" => ReadProp(ref reader, "labels", ecsEvent, (b, v) => b.Labels = v, options), + "labels" => ReadLabelsIntoAttributes(ref reader, ecsEvent, options), "agent" => ReadProp(ref reader, "agent", EcsJsonContext.Default.Agent, ecsEvent, (b, v) => b.Agent = v), "as" => ReadProp(ref reader, "as", EcsJsonContext.Default.As, ecsEvent, (b, v) => b.As = v), "client" => ReadProp(ref reader, "client", EcsJsonContext.Default.Client, ecsEvent, (b, v) => b.Client = v), @@ -126,7 +126,6 @@ public override void Write(Utf8JsonWriter writer, TBase value, JsonSerializerOpt WriteProp(writer, "span.id", value.SpanId, options); WriteProp(writer, "trace.id", value.TraceId, options); WriteProp(writer, "transaction.id", value.TransactionId, options); - WriteProp(writer, "labels", value.Labels, options); // Complex types WriteProp(writer, "agent", value.Agent, EcsJsonContext.Default.Agent, options); @@ -179,8 +178,7 @@ public override void Write(Utf8JsonWriter writer, TBase value, JsonSerializerOpt WriteProp(writer, "volume", value.Volume, EcsJsonContext.Default.Volume, options); WriteProp(writer, "vulnerability", value.Vulnerability, EcsJsonContext.Default.Vulnerability, options); WriteProp(writer, "x509", value.X509, EcsJsonContext.Default.X509, options); - WriteProp(writer, "metadata", value.Metadata, options); - WriteProp(writer, "attributes", value.Attributes, options); + WriteConsolidatedAttributes(writer, value, options); if (typeof(EcsDocument) != value.GetType()) value.WriteAdditionalProperties((k, v) => WriteProp(writer, k, v, options)); diff --git a/src/Elastic.CommonSchema/Serialization/EcsDocumentJsonConverter.cs b/src/Elastic.CommonSchema/Serialization/EcsDocumentJsonConverter.cs index fbeadfa8..9eeaa1ef 100644 --- a/src/Elastic.CommonSchema/Serialization/EcsDocumentJsonConverter.cs +++ b/src/Elastic.CommonSchema/Serialization/EcsDocumentJsonConverter.cs @@ -3,6 +3,7 @@ // See the LICENSE file in the project root for more information using System; +using System.Collections.Generic; using System.Text.Json; using System.Text.Json.Serialization; @@ -98,6 +99,98 @@ private static void WriteTimestamp(Utf8JsonWriter writer, BaseFieldSet value, Js converter.Write(writer, value.Timestamp.Value, options); } + #pragma warning disable CS0618 // Obsolete + private static void WriteConsolidatedAttributes(Utf8JsonWriter writer, TBase value, JsonSerializerOptions options) + { + // Merge Labels, Metadata, Attributes — Attributes wins over Metadata wins over Labels + var hasLabels = value.Labels is { Count: > 0 }; + var hasMetadata = value.Metadata is { Count: > 0 }; + var hasAttributes = value.Attributes is { Count: > 0 }; + + if (!hasLabels && !hasMetadata && !hasAttributes) return; + + var merged = new MetadataDictionary(); + if (hasLabels) + foreach (var kvp in value.Labels!) merged[kvp.Key] = kvp.Value; + if (hasMetadata) + foreach (var kvp in value.Metadata!) merged[kvp.Key] = kvp.Value; + if (hasAttributes) + foreach (var kvp in value.Attributes!) merged[kvp.Key] = kvp.Value; + + WriteProp(writer, "attributes", merged, options); + } + #pragma warning restore CS0618 + + private static bool ReadLabelsIntoAttributes(ref Utf8JsonReader reader, TBase ecsEvent, JsonSerializerOptions options) + { + if (reader.TokenType == JsonTokenType.Null) + return true; + + if (reader.TokenType != JsonTokenType.StartObject) + return false; + + ecsEvent.Attributes ??= new MetadataDictionary(); + + while (reader.Read()) + { + if (reader.TokenType == JsonTokenType.EndObject) + break; + + if (reader.TokenType != JsonTokenType.PropertyName) + throw new JsonException("Expected property name in labels object"); + + var key = reader.GetString()!; + reader.Read(); + + // Labels are always strings + var value = reader.TokenType == JsonTokenType.Null ? null : reader.GetString(); + if (value != null) + ecsEvent.Attributes[key] = value; + } + return true; + } + + [System.Diagnostics.CodeAnalysis.UnconditionalSuppressMessage("ReflectionAnalysis", "IL2026:RequiresUnreferencedCode", Justification = "We always provide a static JsonTypeInfoResolver")] + [System.Diagnostics.CodeAnalysis.UnconditionalSuppressMessage("AotAnalysis", "IL3050:RequiresDynamicCode", Justification = "We always provide a static JsonTypeInfoResolver")] + private static bool ReadMetadataIntoAttributes(ref Utf8JsonReader reader, TBase ecsEvent, JsonSerializerOptions options) + { + if (reader.TokenType == JsonTokenType.Null) + return true; + + if (reader.TokenType != JsonTokenType.StartObject) + return false; + + ecsEvent.Attributes ??= new MetadataDictionary(); + + while (reader.Read()) + { + if (reader.TokenType == JsonTokenType.EndObject) + break; + + if (reader.TokenType != JsonTokenType.PropertyName) + throw new JsonException("Expected property name in metadata object"); + + var key = reader.GetString()!; + reader.Read(); + + object? value = reader.TokenType switch + { + JsonTokenType.String => reader.GetString(), + JsonTokenType.Number => reader.TryGetInt64(out var l) ? l : reader.GetDouble(), + JsonTokenType.True => true, + JsonTokenType.False => false, + JsonTokenType.Null => null, + JsonTokenType.StartObject => JsonSerializer.Deserialize(ref reader, options), + JsonTokenType.StartArray => ReadJsonArray(ref reader, options), + _ => JsonSerializer.Deserialize(ref reader, options) + }; + + if (value != null) + ecsEvent.Attributes[key] = value; + } + return true; + } + [System.Diagnostics.CodeAnalysis.UnconditionalSuppressMessage("ReflectionAnalysis", "IL2026:RequiresUnreferencedCode", Justification = "We always provide a static JsonTypeInfoResolver")] [System.Diagnostics.CodeAnalysis.UnconditionalSuppressMessage("AotAnalysis", "IL3050:RequiresDynamicCode", Justification = "We always provide a static JsonTypeInfoResolver")] private static bool ReadOTelAttributes(ref Utf8JsonReader reader, TBase ecsEvent, JsonSerializerOptions options) @@ -128,6 +221,8 @@ private static bool ReadOTelAttributes(ref Utf8JsonReader reader, TBase ecsEvent JsonTokenType.True => true, JsonTokenType.False => false, JsonTokenType.Null => null, + JsonTokenType.StartObject => JsonSerializer.Deserialize(ref reader, options), + JsonTokenType.StartArray => ReadJsonArray(ref reader, options), _ => JsonSerializer.Deserialize(ref reader, options) }; @@ -146,6 +241,32 @@ private static bool ReadOTelAttributes(ref Utf8JsonReader reader, TBase ecsEvent } return true; } + + [System.Diagnostics.CodeAnalysis.UnconditionalSuppressMessage("ReflectionAnalysis", "IL2026:RequiresUnreferencedCode", Justification = "We always provide a static JsonTypeInfoResolver")] + [System.Diagnostics.CodeAnalysis.UnconditionalSuppressMessage("AotAnalysis", "IL3050:RequiresDynamicCode", Justification = "We always provide a static JsonTypeInfoResolver")] + private static List ReadJsonArray(ref Utf8JsonReader reader, JsonSerializerOptions options) + { + var list = new List(); + while (reader.Read()) + { + if (reader.TokenType == JsonTokenType.EndArray) + break; + + object? item = reader.TokenType switch + { + JsonTokenType.String => reader.GetString(), + JsonTokenType.Number => reader.TryGetInt64(out var l) ? l : reader.GetDouble(), + JsonTokenType.True => true, + JsonTokenType.False => false, + JsonTokenType.Null => null, + JsonTokenType.StartObject => JsonSerializer.Deserialize(ref reader, options), + JsonTokenType.StartArray => ReadJsonArray(ref reader, options), + _ => JsonSerializer.Deserialize(ref reader, options) + }; + list.Add(item); + } + return list; + } } /// A JsonConverter for that supports the diff --git a/src/Elastic.Extensions.Logging.Common/LogEventBuilderExtensions.cs b/src/Elastic.Extensions.Logging.Common/LogEventBuilderExtensions.cs index 54c5ddd8..64854d37 100644 --- a/src/Elastic.Extensions.Logging.Common/LogEventBuilderExtensions.cs +++ b/src/Elastic.Extensions.Logging.Common/LogEventBuilderExtensions.cs @@ -15,7 +15,9 @@ void AddScopeValue(TLocalState scope, LogEvent log) { if (scope is null) return; +#pragma warning disable CS0618 // Obsolete Labels log.Labels ??= new Labels(); +#pragma warning restore CS0618 log.Scopes ??= new List(); diff --git a/src/Specification/v9.3.0/composable/component/base.json b/src/Specification/v9.3.0/composable/component/base.json index 0a44056d..c1457fa2 100644 --- a/src/Specification/v9.3.0/composable/component/base.json +++ b/src/Specification/v9.3.0/composable/component/base.json @@ -9,6 +9,9 @@ "@timestamp": { "type": "date" }, + "attributes": { + "type": "passthrough" + }, "labels": { "type": "object" }, diff --git a/tests/Elastic.CommonSchema.Log4net.Tests/MessageTests.cs b/tests/Elastic.CommonSchema.Log4net.Tests/MessageTests.cs index 3f9f3df9..fe02576b 100644 --- a/tests/Elastic.CommonSchema.Log4net.Tests/MessageTests.cs +++ b/tests/Elastic.CommonSchema.Log4net.Tests/MessageTests.cs @@ -144,10 +144,10 @@ public void ToEcs_EventWithFormat_LabelsContainTemplateAndArgs() => TestLogger(( var (_, info) = ToEcsEvents(logEvents).First(); info.Should().NotBeNull(); - info.Labels.Should().NotBeNull(); + info.Attributes.Should().NotBeNull(); - info.Labels["MessageTemplate"].Should().Be("Log with {0}"); - info.Labels["0"].Should().Be("format"); + info.Attributes["MessageTemplate"].Should().Be("Log with {0}"); + info.Attributes["0"].Should().Be("format"); }); [Fact] @@ -160,17 +160,11 @@ public void ToEcs_AnyEvent_PopulatesMetadataAndLabelsFieldsWithoutLog4netPropert var (_, info) = ToEcsEvents(logEvents).First(); - if (info.Metadata != null) + if (info.Attributes != null) { - info.Metadata.Should().NotContainKey(LoggingEvent.IdentityProperty); - info.Metadata.Should().NotContainKey(LoggingEvent.HostNameProperty); - info.Metadata.Should().NotContainKey(LoggingEvent.UserNameProperty); - } - if (info.Labels != null) - { - info.Labels.Should().NotContainKey(LoggingEvent.IdentityProperty); - info.Labels.Should().NotContainKey(LoggingEvent.HostNameProperty); - info.Labels.Should().NotContainKey(LoggingEvent.UserNameProperty); + info.Attributes.Should().NotContainKey(LoggingEvent.IdentityProperty); + info.Attributes.Should().NotContainKey(LoggingEvent.HostNameProperty); + info.Attributes.Should().NotContainKey(LoggingEvent.UserNameProperty); } }); @@ -190,8 +184,8 @@ public void ToEcs_EventWithGlobalContextProperty_PopulatesLabelsField() => TestL var (_, info) = ToEcsEvents(logEvents).First(); - info.Labels.Should().ContainKey(property); - info.Labels[property].Should().Be(propertyValue); + info.Attributes.Should().ContainKey(property); + info.Attributes[property].Should().Be(propertyValue); } finally { @@ -213,8 +207,8 @@ public void ToEcs_EventWithThreadContextStack_PopulatesLabelsField() => TestLogg var (_, info) = ToEcsEvents(logEvents).First(); - info.Labels.Should().ContainKey(property); - info.Labels[property].Should().Be(propertyValue); + info.Attributes.Should().ContainKey(property); + info.Attributes[property].Should().Be(propertyValue); }); [Fact] @@ -233,8 +227,8 @@ public void ToEcs_EventWithThreadContextProperty_PopulatesLabelsField() => TestL var (_, info) = ToEcsEvents(logEvents).First(); - info.Labels.Should().ContainKey(property); - info.Labels[property].Should().Be(propertyValue); + info.Attributes.Should().ContainKey(property); + info.Attributes[property].Should().Be(propertyValue); } finally { @@ -256,11 +250,10 @@ public void ToEcs_EventInLogicalThreadContextStack_PopulatesLabelsField() => Tes var (_, info) = ToEcsEvents(logEvents).First(); - info.Metadata.Should().BeNull(); - info.Labels.Should().NotBeNull(); + info.Attributes.Should().NotBeNull(); - info.Labels.Should().ContainKey(property); - info.Labels[property].Should().Be(propertyValue); + info.Attributes.Should().ContainKey(property); + info.Attributes[property].Should().Be(propertyValue); }); [Fact] @@ -281,10 +274,10 @@ public void ToEcs_EventWithLogicalThreadContextProperty_PopulatesLabelsAndMetada var (_, info) = ToEcsEvents(logEvents).First(); - info.Labels.Should().ContainKey(property); - info.Labels[property].Should().Be(propertyValue); - info.Metadata.Should().ContainKey(metadataProperty); - info.Metadata[metadataProperty].Should().Be(2.0); + info.Attributes.Should().ContainKey(property); + info.Attributes[property].Should().Be(propertyValue); + info.Attributes.Should().ContainKey(metadataProperty); + info.Attributes[metadataProperty].Should().Be(2.0); } finally { @@ -307,8 +300,8 @@ public void ToEcs_EventWithProperties_PopulatesLabelsField() => TestLogger((log, var (_, info) = ToEcsEvents(logEvents).First(); - info.Labels.Should().ContainKey(property); - info.Labels[property].Should().Be(propertyValue); + info.Attributes.Should().ContainKey(property); + info.Attributes[property].Should().Be(propertyValue); }); [Fact] @@ -337,8 +330,8 @@ void TestStackValue(ILog log, Func> getLogEvents, string property, var (_, info) = ToEcsEvents(logEvents).Last(); - info.Labels.Should().ContainKey(property); - info.Labels[property].Should().Be(expectedPropertyValue); + info.Attributes.Should().ContainKey(property); + info.Attributes[property].Should().Be(expectedPropertyValue); } }); @@ -368,8 +361,8 @@ void TestStackValue(ILog log, Func> getLogEvents, string property, var (_, info) = ToEcsEvents(logEvents).Last(); - info.Labels.Should().ContainKey(property); - info.Labels[property].Should().Be(expectedPropertyValue); + info.Attributes.Should().ContainKey(property); + info.Attributes[property].Should().Be(expectedPropertyValue); } }); } diff --git a/tests/Elastic.CommonSchema.NLog.Tests/EcsFieldsInTemplateTests.cs b/tests/Elastic.CommonSchema.NLog.Tests/EcsFieldsInTemplateTests.cs index a3d1c81a..de68e656 100644 --- a/tests/Elastic.CommonSchema.NLog.Tests/EcsFieldsInTemplateTests.cs +++ b/tests/Elastic.CommonSchema.NLog.Tests/EcsFieldsInTemplateTests.cs @@ -25,8 +25,7 @@ public void CanUseEcsFieldNamesAsTemplateProperty() => TestLogger((logger, getLo var (_, info) = ecsEvents.First(); info.Message.Should().Be("Info my-trace-id: true"); - info.Labels.Should().BeNull(); - info.Metadata.Should().BeNull(); + info.Attributes.Should().BeNull(); info.TraceId.Should().Be("my-trace-id"); info.Faas.Should().NotBeNull(); diff --git a/tests/Elastic.CommonSchema.NLog.Tests/MessageTests.cs b/tests/Elastic.CommonSchema.NLog.Tests/MessageTests.cs index e51a5bf8..1ad4d50c 100644 --- a/tests/Elastic.CommonSchema.NLog.Tests/MessageTests.cs +++ b/tests/Elastic.CommonSchema.NLog.Tests/MessageTests.cs @@ -42,15 +42,14 @@ public void SeesMessageWithProp() => TestLogger((logger, getLogEvents) => var (_, info) = ecsEvents.First(); info.Message.Should().Be("Info X 2.2 42"); - info.Labels.Should().ContainKey("ValueX"); - info.Metadata.Should().ContainKey("SomeY"); - info.Metadata.Should().NotContainKey("NotX"); - info.Labels.Should().NotContainKey("NotX"); + info.Attributes.Should().ContainKey("ValueX"); + info.Attributes.Should().ContainKey("SomeY"); + info.Attributes.Should().NotContainKey("NotX"); - var x = info.Labels["ValueX"]; + var x = info.Attributes["ValueX"]; x.Should().NotBeNull().And.Be("X"); - var y = info.Metadata["SomeY"] as double?; + var y = info.Attributes["SomeY"] as double?; y.Should().HaveValue().And.Be(2.2); }); @@ -66,9 +65,9 @@ public void SeesMessageWithSafeProp() => TestLogger((logger, getLogEvents) => var (_, info) = ecsEvents.First(); info.Message.Should().Be("Info {\"ValueX\":\"X\", \"SomeY\":2.2}"); - info.Metadata.Should().ContainKey("SafeValue"); + info.Attributes.Should().ContainKey("SafeValue"); - var x = info.Metadata["SafeValue"] as Dictionary; + var x = info.Attributes["SafeValue"] as Dictionary; x.Should().NotBeNull().And.NotBeEmpty(); }); @@ -84,11 +83,10 @@ public void SeesMessageWithUnsafeProp() => TestLogger((logger, getLogEvents) => var (_, info) = ecsEvents.First(); info.Message.Should().Be("Info X=X"); - info.Metadata.Should().BeNull(); - info.Labels.Should().NotBeNull(); - info.Labels.Should().ContainKey("UnsafeValue"); + info.Attributes.Should().NotBeNull(); + info.Attributes.Should().ContainKey("UnsafeValue"); - var x = info.Labels["UnsafeValue"]; + var x = info.Attributes["UnsafeValue"]; x.Should().NotBeNull().And.Be("X=X"); }); @@ -112,9 +110,9 @@ public void SeesMessageWithStructuredProperty() => TestLogger((logger, getLogEve var (_, info) = ecsEvents.First(); info.Message.Should().Be("Info {\"ValueX\":\"X\", \"SomeY\":2.2}"); - info.Metadata.Should().ContainKey("SafeValue"); + info.Attributes.Should().ContainKey("SafeValue"); - var x = info.Metadata["SafeValue"] as Dictionary; + var x = info.Attributes["SafeValue"] as Dictionary; x.Should().NotBeNull().And.NotBeEmpty(); }); @@ -130,11 +128,10 @@ public void SeesMessageWithStructuredPropAsString() => TestLogger((logger, getLo var (_, info) = ecsEvents.First(); info.Message.Should().Be("Info X=X"); - info.Metadata.Should().BeNull(); - info.Labels.Should().NotBeNull(); - info.Labels.Should().ContainKey("StructuredValue"); + info.Attributes.Should().NotBeNull(); + info.Attributes.Should().ContainKey("StructuredValue"); - var x = info.Labels["StructuredValue"]; + var x = info.Attributes["StructuredValue"]; x.Should().NotBeNull().And.Be("X=X"); }); @@ -150,9 +147,9 @@ public void SerializesKnownBadObject() => TestLogger((logger, getLogEvents) => var (_, info) = ecsEvents.First(); info.Message.Should().StartWith("Info {\"TypeProperty"); - info.Metadata.Should().ContainKey("EvilValue"); + info.Attributes.Should().ContainKey("EvilValue"); - var x = info.Metadata["EvilValue"] as Dictionary; + var x = info.Attributes["EvilValue"] as Dictionary; x.Should().NotBeNull().And.NotBeEmpty(); }); @@ -168,10 +165,10 @@ public void SerializesObjectThatThrowsOnGetter() => TestLogger((logger, getLogEv var (_, info) = ecsEvents.First(); info.Message.Should().StartWith("Info {\"TypeProperty"); - info.Metadata.Should().NotContainKey("EvilValue"); - info.Metadata.Should().ContainKey("__failures__"); + info.Attributes.Should().NotContainKey("EvilValue"); + info.Attributes.Should().ContainKey("__failures__"); - var failures = info.Metadata["__failures__"] as List; + var failures = info.Attributes["__failures__"] as List; failures.Should().NotBeNull().And.HaveCount(1); var failure = failures![0] as MetadataDictionary; failure!["reason"].Should().NotBeNull(); @@ -240,13 +237,13 @@ public void MetadataWithSameKeys() => TestLogger((logger, getLogEvents) => var (json, info) = ecsEvents.First(); info.Message.Should().Be("Info LoggerArg"); - info.Labels.Should().Contain("DupKey", "LoggerArg"); - info.Labels.Should().Contain("DupKey_1", "Mdlc"); + info.Attributes.Should().Contain("DupKey", (object)"LoggerArg"); + info.Attributes.Should().Contain("DupKey_1", (object)"Mdlc"); - var x = info.Labels["DupKey"]; + var x = info.Attributes["DupKey"]; x.Should().NotBeNull().And.Be("LoggerArg"); - var y = info.Labels["DupKey_1"]; + var y = info.Attributes["DupKey_1"]; y.Should().NotBeNull().And.Be("Mdlc"); } }); diff --git a/tests/Elastic.CommonSchema.Serilog.Tests/EcsFieldsInTemplateTests.cs b/tests/Elastic.CommonSchema.Serilog.Tests/EcsFieldsInTemplateTests.cs index ca3e0d2d..1587157b 100644 --- a/tests/Elastic.CommonSchema.Serilog.Tests/EcsFieldsInTemplateTests.cs +++ b/tests/Elastic.CommonSchema.Serilog.Tests/EcsFieldsInTemplateTests.cs @@ -46,7 +46,6 @@ public void CanSpecifyEntityDirectly() => TestLogger((logger, getLogEvents) => info.Event.Kind.Should().Be("something"); info.As.Should().NotBeNull(); info.As.Number.Should().Be(1337); - info.Metadata.Should().BeNull(); }); [Fact] @@ -67,7 +66,7 @@ public void EntityFieldsShouldBeTypedOrTheyGoInMetaData() => TestLogger((logger, info.Event.Kind.Should().NotBe("something"); info.As.Should().BeNull(); - info.Metadata.Should().NotBeEmpty().And.ContainKey("Event").And.ContainKey("As"); + info.Attributes.Should().NotBeEmpty().And.ContainKey("Event").And.ContainKey("As"); }); @@ -85,8 +84,8 @@ public void EcsFieldsRequireType() => TestLogger((logger, getLogEvents) => info.TraceId.Should().BeNull(); info.Faas.Should().BeNull(); - info.Labels.Should().ContainKey("FaasColdstart"); - info.Metadata.Should().ContainKey("TraceId"); + info.Attributes.Should().ContainKey("FaasColdstart"); + info.Attributes.Should().ContainKey("TraceId"); }); [Fact] @@ -118,8 +117,8 @@ public void SupportsStructureCapturing() => TestLogger((logger, getLogEvents) => info.TraceId.Should().NotBeNull(); info.TraceId.Should().Be("{ x = 1 }"); info.Faas.Should().BeNull(); - info.Metadata.Should().ContainKey("FaasColdstart"); - var structured = info.Metadata["FaasColdstart"] as MetadataDictionary; + info.Attributes.Should().ContainKey("FaasColdstart"); + var structured = info.Attributes["FaasColdstart"] as MetadataDictionary; structured.Should().NotBeNull(); structured!["y"].Should().Be(2); }); diff --git a/tests/Elastic.CommonSchema.Serilog.Tests/ExtensionsLogging/SerilogWithExtensionLoggerGenerated.cs b/tests/Elastic.CommonSchema.Serilog.Tests/ExtensionsLogging/SerilogWithExtensionLoggerGenerated.cs index 83ddd07b..aef45557 100644 --- a/tests/Elastic.CommonSchema.Serilog.Tests/ExtensionsLogging/SerilogWithExtensionLoggerGenerated.cs +++ b/tests/Elastic.CommonSchema.Serilog.Tests/ExtensionsLogging/SerilogWithExtensionLoggerGenerated.cs @@ -40,7 +40,7 @@ public void CanSpecifyEventInGeneratedLogger() => TestLogger((serilogLogger, get info.Event.Should().NotBeNull(); info.Event.Timezone.Should().Be("testing"); - info.Labels.Should().HaveCount(1).And.NotContainKey("event"); + info.Attributes.Should().HaveCount(1).And.NotContainKey("event"); }); [Fact] @@ -61,8 +61,8 @@ public void CanSpecifyEventInLogger() => TestLogger((serilogLogger, getLogEvents info.Event.Should().NotBeNull(); info.Event.Timezone.Should().Be("testing"); - info.Labels.Should().HaveCount(1); - info.Labels.Should().HaveCount(1).And.NotContainKey("event"); + info.Attributes.Should().HaveCount(1); + info.Attributes.Should().HaveCount(1).And.NotContainKey("event"); }); } diff --git a/tests/Elastic.CommonSchema.Serilog.Tests/ExtensionsLogging/SerilogWithExtensionsLoggerAdapter.cs b/tests/Elastic.CommonSchema.Serilog.Tests/ExtensionsLogging/SerilogWithExtensionsLoggerAdapter.cs index 13ed9307..8b9d29ce 100644 --- a/tests/Elastic.CommonSchema.Serilog.Tests/ExtensionsLogging/SerilogWithExtensionsLoggerAdapter.cs +++ b/tests/Elastic.CommonSchema.Serilog.Tests/ExtensionsLogging/SerilogWithExtensionsLoggerAdapter.cs @@ -50,7 +50,6 @@ public void SupportsEventId() => TestLogger((serilogLogger, getLogEvents) => error.Event.Should().NotBeNull(); error.Event.Action.Should().Be("hello"); error.Event.Code.Should().Be("123"); - error.Metadata.Should().BeNull(); }); [Fact] public void SupportsStructureCapturing() => TestLogger((serilogLogger, getLogEvents) => @@ -69,8 +68,8 @@ public void SupportsStructureCapturing() => TestLogger((serilogLogger, getLogEve info.TraceId.Should().NotBeNull(); info.TraceId.Should().Be("{ x = 1 }"); info.Faas.Should().BeNull(); - info.Metadata.Should().ContainKey("FaasColdstart"); - var structured = info.Metadata["FaasColdstart"] as MetadataDictionary; + info.Attributes.Should().ContainKey("FaasColdstart"); + var structured = info.Attributes["FaasColdstart"] as MetadataDictionary; structured.Should().NotBeNull(); structured!["y"].Should().Be(2); }); diff --git a/tests/Elastic.CommonSchema.Serilog.Tests/LogEventPropFilterTests.cs b/tests/Elastic.CommonSchema.Serilog.Tests/LogEventPropFilterTests.cs index 4fd3632c..97f2771a 100644 --- a/tests/Elastic.CommonSchema.Serilog.Tests/LogEventPropFilterTests.cs +++ b/tests/Elastic.CommonSchema.Serilog.Tests/LogEventPropFilterTests.cs @@ -64,8 +64,8 @@ public void FilterLogEventProperty() => TestLogger((logger, getLogEvents) => var (_, info) = ecsEvents.First(); info.Log.Level.Should().Be("Information"); info.Error.Should().BeNull(); - info.Labels.Should().Contain("bar", "bbb"); - info.Labels.Should().NotContainKey("foo", "Should have been filtered"); + info.Attributes.Should().Contain("bar", (object)"bbb"); + info.Attributes.Should().NotContainKey("foo", "Should have been filtered"); }); /// /// Test that null does not cause any critical errors @@ -87,8 +87,8 @@ public void NullFilterLogEventProperty() => TestLogger((logger, getLogEvents) => var (_, info) = ecsEvents.First(); info.Log.Level.Should().Be("Information"); info.Error.Should().BeNull(); - info.Labels.Should().Contain("bar", "bbb"); - info.Labels.Should().Contain("foo", "aaa"); + info.Attributes.Should().Contain("bar", (object)"bbb"); + info.Attributes.Should().Contain("foo", (object)"aaa"); }); /// @@ -111,8 +111,8 @@ public void EmptyFilterLogEventProperty() => TestLogger((logger, getLogEvents) = var (_, info) = ecsEvents.First(); info.Log.Level.Should().Be("Information"); info.Error.Should().BeNull(); - info.Labels.Should().Contain("bar", "bbb"); - info.Labels.Should().Contain("foo", "aaa"); + info.Attributes.Should().Contain("bar", (object)"bbb"); + info.Attributes.Should().Contain("foo", (object)"aaa"); }); /// /// Test that can be case insensitive @@ -134,8 +134,8 @@ public void CaseInsensitiveFilterLogEventProperty() => TestLogger((logger, getLo var (_, info) = ecsEvents.First(); info.Log.Level.Should().Be("Information"); info.Error.Should().BeNull(); - info.Labels.Should().Contain("bar", "bbb"); - info.Labels.Should().NotContainKey("foo", "Should have been filtered"); + info.Attributes.Should().Contain("bar", (object)"bbb"); + info.Attributes.Should().NotContainKey("foo", "Should have been filtered"); }); /// /// Test that can be case sensitive @@ -157,8 +157,8 @@ public void CaseSensitiveFilterLogEventProperty() => TestLogger((logger, getLogE var (_, info) = ecsEvents.First(); info.Log.Level.Should().Be("Information"); info.Error.Should().BeNull(); - info.Labels.Should().Contain("bar", "bbb"); - info.Labels.Should().Contain("foo", "aaa"); + info.Attributes.Should().Contain("bar", (object)"bbb"); + info.Attributes.Should().Contain("foo", (object)"aaa"); }); } } diff --git a/tests/Elastic.CommonSchema.Serilog.Tests/MessageTests.cs b/tests/Elastic.CommonSchema.Serilog.Tests/MessageTests.cs index 8e69c2db..384d874a 100644 --- a/tests/Elastic.CommonSchema.Serilog.Tests/MessageTests.cs +++ b/tests/Elastic.CommonSchema.Serilog.Tests/MessageTests.cs @@ -48,13 +48,13 @@ public void SeesMessageWithProp() => TestLogger((logger, getLogEvents) => var (_, info) = ecsEvents.First(); info.Message.Should().Be("Info \"X\" 2.2"); - info.Labels.Should().ContainKey("ValueX"); - info.Metadata.Should().ContainKey("SomeY"); + info.Attributes.Should().ContainKey("ValueX"); + info.Attributes.Should().ContainKey("SomeY"); - var x = info.Labels["ValueX"]; + var x = info.Attributes["ValueX"]; x.Should().NotBeNull().And.Be("X"); - var y = info.Metadata["SomeY"] as double?; + var y = info.Attributes["SomeY"] as double?; y.Should().HaveValue().And.Be(2.2); }); @@ -71,17 +71,17 @@ public void SeesMessageWithDictProp() => TestLogger((logger, getLogEvents) => var (_, info) = ecsEvents.First(); info.Message.Should().Be("Info \"X\" 2.2 [(\"fieldOne\": \"value1\"), (\"fieldTwo\": \"value2\")]"); - info.Labels.Should().ContainKey("ValueX"); - info.Metadata.Should().ContainKey("SomeY"); - info.Metadata.Should().ContainKey("DictValue"); + info.Attributes.Should().ContainKey("ValueX"); + info.Attributes.Should().ContainKey("SomeY"); + info.Attributes.Should().ContainKey("DictValue"); - var x = info.Labels["ValueX"]; + var x = info.Attributes["ValueX"]; x.Should().NotBeNull().And.Be("X"); - var y = info.Metadata["SomeY"] as double?; + var y = info.Attributes["SomeY"] as double?; y.Should().HaveValue().And.Be(2.2); - var dict = info.Metadata["DictValue"] as MetadataDictionary; + var dict = info.Attributes["DictValue"] as MetadataDictionary; dict.Should().NotBeNull(); dict!["fieldOne"].Should().Be("value1"); dict["fieldTwo"].Should().Be("value2"); @@ -99,10 +99,10 @@ public void SeesMessageWithObjectProp() => TestLogger((logger, getLogEvents) => var (_, info) = ecsEvents.First(); info.Message.Should().Be("Info { TestProp: \"testing\", Child: { ChildProp: 3.3 } }"); - info.Metadata.Should().ContainKey("MyObj"); + info.Attributes.Should().ContainKey("MyObj"); - var json = info.Metadata["MyObj"] as MetadataDictionary; + var json = info.Attributes["MyObj"] as MetadataDictionary; json.Should().NotBeNull(); json!["TestProp"].Should().Be("testing"); var child = json["Child"] as MetadataDictionary; @@ -124,7 +124,6 @@ public void SeesMessageWithElapsedProp(string property) => TestLogger((logger, g var (_, info) = ecsEvents.First(); info.Event.Duration.Should().Be(2200000); - info.Metadata.Should().BeNull(); }); [Theory] @@ -141,7 +140,6 @@ public void SeesMessageWithElapsedLongProp(string property) => TestLogger((logge var (_, info) = ecsEvents.First(); info.Event.Duration.Should().Be(2000000); - info.Metadata.Should().BeNull(); }); [Theory] @@ -158,7 +156,6 @@ public void SeesMessageWithMethodProp(string property) => TestLogger((logger, ge var (_, info) = ecsEvents.First(); info.Http.RequestMethod.Should().Be("GET"); - info.Metadata.Should().BeNull(); }); [Theory] @@ -175,7 +172,6 @@ public void SeesMessageWithPathProp(string property) => TestLogger((logger, getL var (_, info) = ecsEvents.First(); info.Url.Path.Should().Be("/"); - info.Metadata.Should().BeNull(); }); [Fact] @@ -190,7 +186,6 @@ public void SeesMessageWithStatusCodeProp() => TestLogger((logger, getLogEvents) var (_, info) = ecsEvents.First(); info.Http.ResponseStatusCode.Should().Be(200); - info.Metadata.Should().BeNull(); }); [Fact] @@ -205,7 +200,6 @@ public void SeesMessageWithSchemeProp() => TestLogger((logger, getLogEvents) => var (_, info) = ecsEvents.First(); info.Url.Scheme.Should().Be("https"); - info.Metadata.Should().BeNull(); }); [Theory] @@ -226,7 +220,6 @@ public void SeesMessageWithQueryStringProp(string property, string value, object info.Url.Should().BeNull(); else ((object)info.Url.Query).Should().Be(expectedValue); - info.Metadata.Should().BeNull(); }); [Fact] @@ -241,7 +234,6 @@ public void SeesMessageWithRequestIdProp() => TestLogger((logger, getLogEvents) var (_, info) = ecsEvents.First(); info.Http.RequestId.Should().Be("34985y39y6tg95"); - info.Metadata.Should().BeNull(); }); } } diff --git a/tests/Elastic.CommonSchema.Serilog.Tests/Repro/GithubIssue225.cs b/tests/Elastic.CommonSchema.Serilog.Tests/Repro/GithubIssue225.cs index f2f1bc32..5e2b0ec8 100644 --- a/tests/Elastic.CommonSchema.Serilog.Tests/Repro/GithubIssue225.cs +++ b/tests/Elastic.CommonSchema.Serilog.Tests/Repro/GithubIssue225.cs @@ -25,11 +25,11 @@ public void Reproduce() => TestLogger((logger, getLogEvents) => var (_, info) = ecsEvents.First(); info.Message.Should().Be("Logging something with log context"); - info.Labels.Should().NotBeNull().And.ContainKey("ShipmentId"); - info.Labels["ShipmentId"].Should().Be("my-shipment-id"); + info.Attributes.Should().NotBeNull().And.ContainKey("ShipmentId"); + info.Attributes["ShipmentId"].Should().Be("my-shipment-id"); - info.Metadata.Should().NotBeNull().And.ContainKey("ShipmentAmount"); - info.Metadata["ShipmentAmount"].Should().Be(2.3); + info.Attributes.Should().ContainKey("ShipmentAmount"); + info.Attributes["ShipmentAmount"].Should().Be(2.3); }); } diff --git a/tests/Elastic.CommonSchema.Tests/OTelTests.cs b/tests/Elastic.CommonSchema.Tests/OTelTests.cs index 684c296e..24489296 100644 --- a/tests/Elastic.CommonSchema.Tests/OTelTests.cs +++ b/tests/Elastic.CommonSchema.Tests/OTelTests.cs @@ -272,18 +272,18 @@ public void AssignOTelField_WithSemConvConstant_Works() // ── 6. PropDispatch / Field Assignment ── [Fact] - public void AssignField_OTelEquivalentName_GoesToLabels() + public void AssignField_OTelEquivalentName_GoesToAttributes() { // AssignField only recognizes ECS field names. // OTel equivalent names like "exception.message" are not in LogTemplateProperties, - // so string values fall through to Labels. Use AssignOTelField for OTel names. + // so values fall through to Attributes. Use AssignOTelField for OTel names. var doc = new EcsDocument(); doc.AssignField("exception.message", "dispatch test"); - // Not set as ECS property — stored as label instead (strings go to Labels) + // Not set as ECS property — stored in Attributes instead doc.Error.Should().BeNull(); - doc.Labels.Should().NotBeNull(); - doc.Labels.Should().ContainKey("exception.message"); + doc.Attributes.Should().NotBeNull(); + doc.Attributes.Should().ContainKey("exception.message"); } [Fact] diff --git a/tests/Elastic.CommonSchema.Tests/Repro/GithubIssue29.cs b/tests/Elastic.CommonSchema.Tests/Repro/GithubIssue29.cs index bdfa7fd8..0eb49597 100644 --- a/tests/Elastic.CommonSchema.Tests/Repro/GithubIssue29.cs +++ b/tests/Elastic.CommonSchema.Tests/Repro/GithubIssue29.cs @@ -11,6 +11,7 @@ public void Reproduce() { // Metadata properties with null values should be serialised var uniqueName = Guid.NewGuid().ToString(); +#pragma warning disable CS0618 // Obsolete Metadata var root = new EcsDocument { Metadata = new MetadataDictionary @@ -18,6 +19,7 @@ public void Reproduce() { uniqueName, null } } }; +#pragma warning restore CS0618 var serialised = root.Serialize(); serialised.Should().Contain($"\"{uniqueName}\":null"); diff --git a/tests/Elastic.CommonSchema.Tests/Serializes.cs b/tests/Elastic.CommonSchema.Tests/Serializes.cs index 7fc4c10a..e2411e2c 100644 --- a/tests/Elastic.CommonSchema.Tests/Serializes.cs +++ b/tests/Elastic.CommonSchema.Tests/Serializes.cs @@ -47,6 +47,7 @@ public void SerializesNestedProperties() [Fact] public void MetaDataKeysAreSerializedVerbatim() { +#pragma warning disable CS0618 // Obsolete Metadata var b = new EcsDocument { Metadata = new MetadataDictionary @@ -58,15 +59,17 @@ public void MetaDataKeysAreSerializedVerbatim() ["rule"] = "some-rule", } }; +#pragma warning restore CS0618 var serialized = b.Serialize(); var deserialized = EcsSerializerFactory.Deserialize(serialized); - deserialized.Metadata.Should().ContainKey("MessageTemplate"); - deserialized.Metadata.Should().ContainKey("WriteIO"); - deserialized.Metadata.Should().ContainKey("User_Id"); - deserialized.Metadata.Should().ContainKey("eventId"); - deserialized.Metadata.Should().ContainKey("rule"); + // Metadata is now merged into Attributes during serialization + deserialized.Attributes.Should().ContainKey("MessageTemplate"); + deserialized.Attributes.Should().ContainKey("WriteIO"); + deserialized.Attributes.Should().ContainKey("User_Id"); + deserialized.Attributes.Should().ContainKey("eventId"); + deserialized.Attributes.Should().ContainKey("rule"); } [JsonConverter(typeof(EcsDocumentJsonConverterFactory))] @@ -217,7 +220,9 @@ public void SerializesDocumentInTheReadMe() Category = new[] { "network_traffic" } }, Ecs = new Ecs { Version = "1.2.0" }, - Metadata = new MetadataDictionary { { "client", "ecs-dotnet" } } + #pragma warning disable CS0618 // Obsolete Metadata + Metadata = new MetadataDictionary { { "client", "ecs-dotnet" } } +#pragma warning restore CS0618 }; var serialized = ecsDocument.Serialize(); @@ -231,7 +236,7 @@ public void SerializesDocumentInTheReadMe() deserialized.Dns.Answers.Should().NotBeNull().And.HaveCount(2); deserialized.Dns.Answers[0].Name.Should().NotBeNull().And.Be("www.example.com"); deserialized.Dns.Answers[1].Data.Should().NotBeNull().And.EndWith(".117"); - deserialized.Metadata.Should().NotBeNull().And.HaveCount(1); + deserialized.Attributes.Should().NotBeNull().And.HaveCount(1); } private enum MyEnum { One, Two, Three } @@ -239,6 +244,7 @@ private enum MyEnum { One, Two, Three } [Fact] public void UseCustomSerializer() { +#pragma warning disable CS0618 // Obsolete Metadata var ecsDocument = new EcsDocument { Timestamp = DateTimeOffset.Parse("2024-05-27T23:56:15.785Z"), @@ -246,11 +252,12 @@ public void UseCustomSerializer() Metadata = new MetadataDictionary { { "MyEnum", MyEnum.Two } }, Ecs = new Ecs { Version = "8.11.0" } }; +#pragma warning restore CS0618 var serializerOptions = new JsonSerializerOptions { Converters = { new JsonStringEnumConverter() } }; var json = JsonSerializer.Serialize(ecsDocument, serializerOptions); - json.Should().Be("{\"@timestamp\":\"2024-05-27T23:56:15.785+00:00\",\"message\":\"Hello World!\",\"ecs.version\":\"8.11.0\",\"metadata\":{\"MyEnum\":\"Two\"}}"); + json.Should().Be("{\"@timestamp\":\"2024-05-27T23:56:15.785+00:00\",\"message\":\"Hello World!\",\"ecs.version\":\"8.11.0\",\"attributes\":{\"MyEnum\":\"Two\"}}"); } } } diff --git a/tools/Elastic.CommonSchema.Generator/Views/EcsDocument.Generated.cshtml b/tools/Elastic.CommonSchema.Generator/Views/EcsDocument.Generated.cshtml index 4cf6c5bc..d0b2e931 100644 --- a/tools/Elastic.CommonSchema.Generator/Views/EcsDocument.Generated.cshtml +++ b/tools/Elastic.CommonSchema.Generator/Views/EcsDocument.Generated.cshtml @@ -39,9 +39,10 @@ namespace Elastic.CommonSchema /// Container for additional metadata against this event. /// /// When working with unknown fields use .
- /// This will try to assign valid ECS fields to their respective property + /// This will try to assign valid ECS fields to their respective property /// Failing that it will assign strings to and everything else to /// + [Obsolete("Use Attributes instead. Metadata values will be merged into Attributes during serialization.")] [JsonPropertyName("metadata"), DataMember(Name = "metadata")] [JsonConverter(typeof(MetadataDictionaryConverter))] public MetadataDictionary? Metadata { get; set; } diff --git a/tools/Elastic.CommonSchema.Generator/Views/EcsDocumentJsonConverter.Generated.cshtml b/tools/Elastic.CommonSchema.Generator/Views/EcsDocumentJsonConverter.Generated.cshtml index 63372c59..8d890b6e 100644 --- a/tools/Elastic.CommonSchema.Generator/Views/EcsDocumentJsonConverter.Generated.cshtml +++ b/tools/Elastic.CommonSchema.Generator/Views/EcsDocumentJsonConverter.Generated.cshtml @@ -37,7 +37,7 @@ namespace Elastic.CommonSchema.Serialization { "log.level" => ReadString(ref reader, ref loglevel), "ecs.version" => ReadString(ref reader, ref ecsVersion), - "metadata" => ReadProp@(Raw(""))(ref reader, "metadata", ecsEvent, (b, v) => b.Metadata = v, options), + "metadata" => ReadMetadataIntoAttributes(ref reader, ecsEvent, options), @foreach (var property in Model.Base.BaseFieldSet.ValueProperties) { var name = property.JsonProperty; @@ -51,8 +51,12 @@ namespace Elastic.CommonSchema.Serialization @foreach (var property in Model.Base.BaseFieldSet.InlineObjectProperties) { var name = property.JsonProperty; - "@(name)" => ReadProp<@(Raw(property.InlineObject.Name))>(ref reader, "@name", ecsEvent, (b, v) => b.@(property.Name) = v, options), - + if (name == "labels") +{ "@(name)" => ReadLabelsIntoAttributes(ref reader, ecsEvent, options), +} + else +{ "@(name)" => ReadProp<@(Raw(property.InlineObject.Name))>(ref reader, "@name", ecsEvent, (b, v) => b.@(property.Name) = v, options), +} } @foreach (var entity in Model.EntityClasses) { @@ -102,6 +106,7 @@ namespace Elastic.CommonSchema.Serialization @foreach (var property in Model.Base.BaseFieldSet.InlineObjectProperties) { var name = property.JsonProperty; + if (name == "labels") { continue; } WriteProp(writer, "@(name)", value.@(property.InlineObject.Name), options); } @@ -117,8 +122,7 @@ namespace Elastic.CommonSchema.Serialization WriteProp(writer, "@(entityName)", value.@(entity.Name), EcsJsonContext.Default.@(entity.Name), options); } - WriteProp(writer, "metadata", value.Metadata, options); - WriteProp(writer, "attributes", value.Attributes, options); + WriteConsolidatedAttributes(writer, value, options); if (typeof(@Model.Base.Name) != value.GetType()) value.WriteAdditionalProperties((k, v) => WriteProp(writer, k, v, options)); diff --git a/tools/Elastic.CommonSchema.Generator/Views/FieldSets.Generated.cshtml b/tools/Elastic.CommonSchema.Generator/Views/FieldSets.Generated.cshtml index 70fb9aad..e93ac5be 100644 --- a/tools/Elastic.CommonSchema.Generator/Views/FieldSets.Generated.cshtml +++ b/tools/Elastic.CommonSchema.Generator/Views/FieldSets.Generated.cshtml @@ -52,6 +52,9 @@ namespace Elastic.CommonSchema /// @Raw(property.Description) /// @Raw(property.Example) /// +@if (property.Name == "Labels") +{ [Obsolete("Use EcsDocument.Attributes instead. Labels values will be merged into Attributes during serialization.")] +} [JsonPropertyName("@property.JsonProperty"), DataMember(Name = "@property.JsonProperty")] public @property.ClrType? @property.Name { get; set; } From 2183c38c850292fbb40131a6091b9498b41ab330 Mon Sep 17 00:00:00 2001 From: Martijn Laarman Date: Wed, 18 Feb 2026 15:49:51 +0100 Subject: [PATCH 3/4] Fix CS0618 build errors in benchmarks and integration tests --- .../LoggingToDataStreamTests.cs | 4 ++-- .../DataStreamIngestionTests.cs | 4 ++-- .../IndexIngestionTests.cs | 4 ++-- .../LoggingToDataStreamTests.cs | 4 ++-- tools/Elastic.CommonSchema.Benchmarks/SerializingBase.cs | 2 +- .../SerializingStringBuilderBase.cs | 2 +- 6 files changed, 10 insertions(+), 10 deletions(-) diff --git a/tests-integration/Elastic.Extensions.Logging.IntegrationTests/LoggingToDataStreamTests.cs b/tests-integration/Elastic.Extensions.Logging.IntegrationTests/LoggingToDataStreamTests.cs index 63d9ca3f..ea1c2cc3 100644 --- a/tests-integration/Elastic.Extensions.Logging.IntegrationTests/LoggingToDataStreamTests.cs +++ b/tests-integration/Elastic.Extensions.Logging.IntegrationTests/LoggingToDataStreamTests.cs @@ -62,8 +62,8 @@ public async Task LogsEndUpInCluster() loggedError.Ecs.Version.Should().Be(EcsDocument.Version); loggedError.Ecs.Version.Should().NotStartWith("v"); - loggedError.Labels.Should().ContainKey("Status"); - loggedError.Labels["Status"].Should().Be("Failure"); + loggedError.Attributes.Should().ContainKey("Status"); + loggedError.Attributes["Status"].Should().Be("Failure"); } [Fact] diff --git a/tests-integration/Elastic.Ingest.Elasticsearch.CommonSchema.IntegrationTests/DataStreamIngestionTests.cs b/tests-integration/Elastic.Ingest.Elasticsearch.CommonSchema.IntegrationTests/DataStreamIngestionTests.cs index 305a1d52..f19bdc1c 100644 --- a/tests-integration/Elastic.Ingest.Elasticsearch.CommonSchema.IntegrationTests/DataStreamIngestionTests.cs +++ b/tests-integration/Elastic.Ingest.Elasticsearch.CommonSchema.IntegrationTests/DataStreamIngestionTests.cs @@ -100,7 +100,7 @@ public async Task UseCustomEventWriter() { Timestamp = DateTimeOffset.Parse("2024-05-27T23:56:15.785Z"), Message = "Hello World!", - Metadata = new MetadataDictionary { { "MyEnum", MyEnum.Two } } + Attributes = new MetadataDictionary { { "MyEnum", MyEnum.Two } } }); if (!slim.WaitHandle.WaitOne(TimeSpan.FromSeconds(10))) @@ -115,6 +115,6 @@ public async Task UseCustomEventWriter() var root = searchResult.Documents.First().RootElement; root.GetProperty("@timestamp").GetString().Should().Be("2024-05-27T23:56:15.785+00:00"); root.GetProperty("message").GetString().Should().Be("Hello World!"); - root.GetProperty("metadata").GetProperty("MyEnum").GetString().Should().Be("Two"); + root.GetProperty("attributes").GetProperty("MyEnum").GetString().Should().Be("Two"); } } diff --git a/tests-integration/Elastic.Ingest.Elasticsearch.CommonSchema.IntegrationTests/IndexIngestionTests.cs b/tests-integration/Elastic.Ingest.Elasticsearch.CommonSchema.IntegrationTests/IndexIngestionTests.cs index 21621dec..5f865de6 100644 --- a/tests-integration/Elastic.Ingest.Elasticsearch.CommonSchema.IntegrationTests/IndexIngestionTests.cs +++ b/tests-integration/Elastic.Ingest.Elasticsearch.CommonSchema.IntegrationTests/IndexIngestionTests.cs @@ -110,7 +110,7 @@ public async Task UseCustomEventWriter() Created = date, Title = "Hello World!", Id = "hello-world", - Metadata = new MetadataDictionary { { "MyEnum", MyEnum.Two } } + Attributes = new MetadataDictionary { { "MyEnum", MyEnum.Two } } }); if (!slim.WaitHandle.WaitOne(TimeSpan.FromSeconds(10))) @@ -126,6 +126,6 @@ public async Task UseCustomEventWriter() root.GetProperty("created").GetString().Should().Be("2024-05-27T23:56:15.785+00:00"); root.GetProperty("title").GetString().Should().Be("Hello World!"); root.GetProperty("id").GetString().Should().Be("hello-world"); - root.GetProperty("metadata").GetProperty("MyEnum").GetString().Should().Be("Two"); + root.GetProperty("attributes").GetProperty("MyEnum").GetString().Should().Be("Two"); } } diff --git a/tests-integration/Elastic.NLog.Targets.IntegrationTests/LoggingToDataStreamTests.cs b/tests-integration/Elastic.NLog.Targets.IntegrationTests/LoggingToDataStreamTests.cs index 9dc801eb..5acce677 100644 --- a/tests-integration/Elastic.NLog.Targets.IntegrationTests/LoggingToDataStreamTests.cs +++ b/tests-integration/Elastic.NLog.Targets.IntegrationTests/LoggingToDataStreamTests.cs @@ -62,8 +62,8 @@ public async Task LogsEndUpInCluster() loggedError.Ecs.Version.Should().Be(EcsDocument.Version); loggedError.Ecs.Version.Should().NotStartWith("v"); - loggedError.Labels.Should().ContainKey("Status"); - loggedError.Labels["Status"].Should().Be("Failure"); + loggedError.Attributes.Should().ContainKey("Status"); + loggedError.Attributes["Status"].Should().Be("Failure"); } } } diff --git a/tools/Elastic.CommonSchema.Benchmarks/SerializingBase.cs b/tools/Elastic.CommonSchema.Benchmarks/SerializingBase.cs index 7aefdf3b..07b61135 100644 --- a/tools/Elastic.CommonSchema.Benchmarks/SerializingBase.cs +++ b/tools/Elastic.CommonSchema.Benchmarks/SerializingBase.cs @@ -61,7 +61,7 @@ public void GlobalSetup() { Generator = new AutoFaker() - .RuleFor(d => d.Metadata, _ => new MetadataDictionary { { "x", "y" } }) + .RuleFor(d => d.Attributes, _ => new MetadataDictionary { { "x", "y" } }) .UseSeed(1337); FullInstance = Generator.Generate(); } diff --git a/tools/Elastic.CommonSchema.Benchmarks/SerializingStringBuilderBase.cs b/tools/Elastic.CommonSchema.Benchmarks/SerializingStringBuilderBase.cs index 2acc3220..52d645c8 100644 --- a/tools/Elastic.CommonSchema.Benchmarks/SerializingStringBuilderBase.cs +++ b/tools/Elastic.CommonSchema.Benchmarks/SerializingStringBuilderBase.cs @@ -67,7 +67,7 @@ public void GlobalSetup() { Generator = new AutoFaker() - .RuleFor(d => d.Metadata, _ => new MetadataDictionary { { "x", "y" } }) + .RuleFor(d => d.Attributes, _ => new MetadataDictionary { { "x", "y" } }) .UseSeed(1337); FullInstance = Generator.Generate(); } From 4be6a3936dd8d70717a5d2070ff7c405e322ae99 Mon Sep 17 00:00:00 2001 From: Martijn Laarman Date: Wed, 18 Feb 2026 16:23:56 +0100 Subject: [PATCH 4/4] Add priority to passthrough mapping for Elasticsearch 9+ compatibility Elasticsearch 9 requires a non-negative priority on passthrough-type mappings. The ECS 9.3.0 base component template defined attributes as passthrough without a priority, causing a mapper_parsing_exception when bootstrapping the ecs_9.3.0_base component template. Fix the generator to inject priority: 10 when reading passthrough-type component mappings that lack one, and regenerate. --- .../IndexComponents.Generated.cs | 6 ++--- .../Schema/EcsSchemaParser.cs | 27 ++++++++++++++++++- 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/src/Elastic.CommonSchema/IndexComponents.Generated.cs b/src/Elastic.CommonSchema/IndexComponents.Generated.cs index e196dab0..5255fbfc 100644 --- a/src/Elastic.CommonSchema/IndexComponents.Generated.cs +++ b/src/Elastic.CommonSchema/IndexComponents.Generated.cs @@ -346,7 +346,8 @@ public static class IndexComponents ""type"": ""date"" }, ""attributes"": { - ""type"": ""passthrough"" + ""type"": ""passthrough"", + ""priority"": 10 }, ""labels"": { ""type"": ""object"" @@ -362,8 +363,7 @@ public static class IndexComponents } } } -} -" +}" }, { "ecs_9.3.0_client", diff --git a/tools/Elastic.CommonSchema.Generator/Schema/EcsSchemaParser.cs b/tools/Elastic.CommonSchema.Generator/Schema/EcsSchemaParser.cs index 267b95c1..03e003ad 100644 --- a/tools/Elastic.CommonSchema.Generator/Schema/EcsSchemaParser.cs +++ b/tools/Elastic.CommonSchema.Generator/Schema/EcsSchemaParser.cs @@ -6,6 +6,7 @@ using CsQuery.ExtensionMethods; using Elastic.CommonSchema.Generator.Schema.DTO; using Newtonsoft.Json; +using Newtonsoft.Json.Linq; using YamlDotNet.Serialization; namespace Elastic.CommonSchema.Generator.Schema @@ -88,11 +89,35 @@ from jsonFile in dir.EnumerateFileSystemInfos("*.json") { if (dir.Name != "component") continue; - templates.Add(jsonFile.Name.Replace(".json", ""), File.ReadAllText(jsonFile.FullName)); + var json = File.ReadAllText(jsonFile.FullName); + json = EnsurePassthroughPriority(json); + templates.Add(jsonFile.Name.Replace(".json", ""), json); } return templates; } + // Elasticsearch 9+ requires a non-negative priority on passthrough-type mappings. + // The ECS spec may omit it, so we inject a default of 10 when it's missing. + private static string EnsurePassthroughPriority(string json) + { + var root = JObject.Parse(json); + var properties = root.SelectToken("template.mappings.properties") as JObject; + if (properties == null) return json; + + var changed = false; + foreach (var prop in properties.Properties()) + { + var obj = prop.Value as JObject; + if (obj == null) continue; + if (obj["type"]?.Value() == "passthrough" && obj["priority"] == null) + { + obj["priority"] = 10; + changed = true; + } + } + return changed ? root.ToString(Formatting.Indented) : json; + } + private List SerializeParsedAndCompareWithOriginal(Dictionary spec, string asJson) {