From b09df3b84e8571f4f9860e7eefd9fe425587988b Mon Sep 17 00:00:00 2001 From: Stephen Wakely Date: Wed, 25 Jun 2025 16:45:05 +0100 Subject: [PATCH 01/35] Add origin detection --- Gemfile | 1 + lib/datadog/statsd.rb | 33 +- lib/datadog/statsd/origin_detection.rb | 138 +++++ .../statsd/serialization/serializer.rb | 4 +- .../statsd/serialization/stat_serializer.rb | 21 +- spec/integrations/allocation_spec.rb | 5 + spec/integrations/buffering_spec.rb | 2 + spec/integrations/delay_serialization_spec.rb | 10 +- spec/integrations/telemetry_spec.rb | 5 +- spec/statsd/origin_detection_spec.rb | 532 ++++++++++++++++++ .../serialization/stat_serializer_spec.rb | 41 +- spec/statsd_spec.rb | 1 + 12 files changed, 780 insertions(+), 13 deletions(-) create mode 100644 lib/datadog/statsd/origin_detection.rb create mode 100644 spec/statsd/origin_detection_spec.rb diff --git a/Gemfile b/Gemfile index 28e6ad45..be8f42ef 100644 --- a/Gemfile +++ b/Gemfile @@ -27,4 +27,5 @@ end group :test do gem 'mocha' + gem 'fakefs', '~> 3.0' end diff --git a/lib/datadog/statsd.rb b/lib/datadog/statsd.rb index 32f8d1a6..b7c31731 100644 --- a/lib/datadog/statsd.rb +++ b/lib/datadog/statsd.rb @@ -6,6 +6,7 @@ require_relative 'statsd/udp_connection' require_relative 'statsd/uds_connection' require_relative 'statsd/connection_cfg' +require_relative 'statsd/origin_detection' require_relative 'statsd/message_buffer' require_relative 'statsd/serialization' require_relative 'statsd/sender' @@ -82,6 +83,8 @@ def tags # @option [Float] default sample rate if not overridden # @option [Boolean] single_thread flushes the metrics on the main thread instead of in a companion thread # @option [Boolean] delay_serialization delays stat serialization + # @option [Boolean] origin_detection is origin detection enabled + # @option [String] container_id the container ID field, used for origin detection def initialize( host = nil, port = nil, @@ -104,7 +107,10 @@ def initialize( delay_serialization: false, telemetry_enable: true, - telemetry_flush_interval: DEFAULT_TELEMETRY_FLUSH_INTERVAL + telemetry_flush_interval: DEFAULT_TELEMETRY_FLUSH_INTERVAL, + + origin_detection: true, + container_id: nil ) unless tags.nil? || tags.is_a?(Array) || tags.is_a?(Hash) raise ArgumentError, 'tags must be an array of string tags or a Hash' @@ -112,7 +118,12 @@ def initialize( @namespace = namespace @prefix = @namespace ? "#{@namespace}.".freeze : nil - @serializer = Serialization::Serializer.new(prefix: @prefix, global_tags: tags) + + origin_detection_enabled = is_origin_detection_enabled(origin_detection) + container_id = OriginDetection + .new() + .get_container_id(container_id, origin_detection_enabled) + @serializer = Serialization::Serializer.new(prefix: @prefix, container_id: container_id, global_tags: tags) @sample_rate = sample_rate @delay_serialization = delay_serialization @@ -439,5 +450,23 @@ def send_stats(stat, delta, type, opts = EMPTY_OPTIONS) forwarder.send_message(full_stat) end end + + def is_origin_detection_enabled(origin_detection) + if !origin_detection.nil? && !origin_detection + return false + end + + if ENV['DD_ORIGIN_DETECTION_ENABLED'] + return ![ + '0', + 'f', + 'false' + ].include?( + ENV['DD_ORIGIN_DETECTION_ENABLED'].downcase + ) + end + + return true + end end end diff --git a/lib/datadog/statsd/origin_detection.rb b/lib/datadog/statsd/origin_detection.rb new file mode 100644 index 00000000..a3e97bca --- /dev/null +++ b/lib/datadog/statsd/origin_detection.rb @@ -0,0 +1,138 @@ +module Datadog + class Statsd + class OriginDetection + CGROUPV1BASECONTROLLER = "memory" + HOSTCGROUPNAMESPACEINODE = 0xEFFFFFFB + + def get_filepaths + { + cgroup_path: "/proc/self/cgroup", + self_mount_info_path: "/proc/self/mountinfo", + default_cgroup_mount_path: "/sys/fs/cgroup" + } + end + + def is_host_cgroup_namespace? + stat = File.stat("/proc/self/ns/cgroup") rescue nil + return false unless stat + stat.ino == HOSTCGROUPNAMESPACEINODE + end + + def parse_cgroup_node_path(lines) + res = {} + lines.split("\n").each do |line| + tokens = line.split(':') + next unless tokens.length == 3 + + controller = tokens[1] + path = tokens[2] + + if controller == CGROUPV1BASECONTROLLER || controller == '' + res[controller] = path + end + end + + res + end + + def get_cgroup_inode(cgroup_mount_path, proc_self_cgroup_path) + content = File.read(proc_self_cgroup_path) + controllers = parse_cgroup_node_path(content) + + [CGROUPV1BASECONTROLLER, ''].each do |controller| + next unless controllers[controller] + + segments = [ + cgroup_mount_path.chomp('/'), + controller.strip, + controllers[controller].sub(/^\//, '') + ] + path = segments.reject(&:empty?).join("/") + inode = inode_for_path(path) + return inode unless inode.nil? + end + + nil + end + + private + + def inode_for_path(path) + stat = File.stat(path) rescue nil + return nil unless stat + "in-#{stat.ino}" + end + + def parse_container_id(handle) + exp_line = /^\d+:[^:]*:(.+)$/ + uuid = /[0-9a-f]{8}[-_][0-9a-f]{4}[-_][0-9a-f]{4}[-_][0-9a-f]{4}[-_][0-9a-f]{12}/ + container = /[0-9a-f]{64}/ + task = /[0-9a-f]{32}-\d+/ + exp_container_id = /(#{uuid}|#{container}|#{task})(?:\.scope)?$/ + + handle.each_line do |line| + match = line.match(exp_line) + next unless match && match[1] + id_match = match[1].match(exp_container_id) + + return id_match[1] if id_match && id_match[1] + end + + nil + end + + public + + def read_container_id(fpath) + handle = File.open(fpath, 'r') rescue nil + return nil unless handle + + id = parse_container_id(handle) + handle.close + id + end + + def parse_mount_info(handle) + container_regexp = '([0-9a-f]{64})|([0-9a-f]{32}-\d+)|([0-9a-f]{8}(-[0-9a-f]{4}){4}$)' + cid_mount_info_regexp = %r{.*/([^\s/]+)/(?:#{container_regexp})/[\S]*hostname} + + handle.each_line do |line| + matches = line.scan(cid_mount_info_regexp) + next if matches.empty? + + match = matches.last + containerd_sandbox_prefix = "sandboxes" + if match && match[0] != containerd_sandbox_prefix + return match[1] + end + end + + nil + end + + def read_mount_info(path) + handle = File.open(path, 'r') rescue nil + return nil unless handle + + info = parse_mount_info(handle) + handle.close + info + end + + def get_container_id(user_provided_id, cgroup_fallback) + return user_provided_id unless user_provided_id.nil? + return nil unless cgroup_fallback + + container_id = read_container_id("/proc/self/cgroup") + return container_id unless container_id.nil? + + container_id = read_mount_info("/proc/self/mountinfo") + return container_id unless container_id.nil? + + return nil if is_host_cgroup_namespace? + + get_cgroup_inode("/sys/fs/cgroup", "/proc/self/cgroup") + end + end + end +end diff --git a/lib/datadog/statsd/serialization/serializer.rb b/lib/datadog/statsd/serialization/serializer.rb index e7ed23df..f0b2bcfd 100644 --- a/lib/datadog/statsd/serialization/serializer.rb +++ b/lib/datadog/statsd/serialization/serializer.rb @@ -6,8 +6,8 @@ module Datadog class Statsd module Serialization class Serializer - def initialize(prefix: nil, global_tags: []) - @stat_serializer = StatSerializer.new(prefix, global_tags: global_tags) + def initialize(prefix: nil, container_id: nil, global_tags: []) + @stat_serializer = StatSerializer.new(prefix, container_id, global_tags: global_tags) @service_check_serializer = ServiceCheckSerializer.new(global_tags: global_tags) @event_serializer = EventSerializer.new(global_tags: global_tags) end diff --git a/lib/datadog/statsd/serialization/stat_serializer.rb b/lib/datadog/statsd/serialization/stat_serializer.rb index c8e9bcb3..4d5f09e9 100644 --- a/lib/datadog/statsd/serialization/stat_serializer.rb +++ b/lib/datadog/statsd/serialization/stat_serializer.rb @@ -4,26 +4,37 @@ module Datadog class Statsd module Serialization class StatSerializer - def initialize(prefix, global_tags: []) + def initialize(prefix, container_id, global_tags: []) @prefix = prefix @prefix_str = prefix.to_s + @container_id = container_id @tag_serializer = TagSerializer.new(global_tags) end + def format_fields + field = String.new + unless @container_id.nil? + field << "|c:#{@container_id}" + end + + field + end + def format(metric_name, delta, type, tags: [], sample_rate: 1) metric_name = formatted_metric_name(metric_name) + fields = format_fields if sample_rate != 1 if tags_list = tag_serializer.format(tags) - "#{@prefix_str}#{metric_name}:#{delta}|#{type}|@#{sample_rate}|##{tags_list}" + "#{@prefix_str}#{metric_name}:#{delta}|#{type}|@#{sample_rate}#{fields}|##{tags_list}" else - "#{@prefix_str}#{metric_name}:#{delta}|#{type}|@#{sample_rate}" + "#{@prefix_str}#{metric_name}:#{delta}|#{type}|@#{sample_rate}#{fields}" end else if tags_list = tag_serializer.format(tags) - "#{@prefix_str}#{metric_name}:#{delta}|#{type}|##{tags_list}" + "#{@prefix_str}#{metric_name}:#{delta}|#{type}|##{tags_list}#{fields}" else - "#{@prefix_str}#{metric_name}:#{delta}|#{type}" + "#{@prefix_str}#{metric_name}:#{delta}|#{type}#{fields}" end end end diff --git a/spec/integrations/allocation_spec.rb b/spec/integrations/allocation_spec.rb index 8751e6e7..9fcbceca 100644 --- a/spec/integrations/allocation_spec.rb +++ b/spec/integrations/allocation_spec.rb @@ -16,6 +16,7 @@ tags: tags, logger: logger, telemetry_flush_interval: -1, + origin_detection: false, ) end @@ -67,6 +68,7 @@ tags: tags, logger: logger, telemetry_enable: false, + origin_detection: false, ) end @@ -146,6 +148,7 @@ tags: tags, logger: logger, telemetry_enable: false, + origin_detection: false, ) end @@ -225,6 +228,7 @@ tags: tags, logger: logger, telemetry_enable: false, + origin_detection: false, ) end @@ -304,6 +308,7 @@ tags: tags, logger: logger, telemetry_enable: false, + origin_detection: false, ) end diff --git a/spec/integrations/buffering_spec.rb b/spec/integrations/buffering_spec.rb index 2b767010..a5d8a3aa 100644 --- a/spec/integrations/buffering_spec.rb +++ b/spec/integrations/buffering_spec.rb @@ -9,6 +9,7 @@ telemetry_enable: false, single_thread: single_thread, buffer_max_pool_size: buffer_max_pool_size, + origin_detection: false, ) end let(:socket) { FakeUDPSocket.new(copy_message: true) } @@ -137,6 +138,7 @@ telemetry_flush_interval: 60, buffer_max_pool_size: buffer_max_pool_size, single_thread: single_thread, + origin_detection: false, ) end diff --git a/spec/integrations/delay_serialization_spec.rb b/spec/integrations/delay_serialization_spec.rb index 18b2cfc3..4a14a9f8 100644 --- a/spec/integrations/delay_serialization_spec.rb +++ b/spec/integrations/delay_serialization_spec.rb @@ -13,7 +13,10 @@ .to receive(:flush) allow(Datadog::Statsd::MessageBuffer).to receive(:new).and_return(buffer) - dogstats = Datadog::Statsd.new("localhost", 1234, delay_serialization: true) + dogstats = Datadog::Statsd.new("localhost", 1234, + delay_serialization: true, + origin_detection: false, + ) dogstats.increment("boo") dogstats.flush(sync: true) @@ -22,7 +25,10 @@ it "serializes messages normally" do socket = FakeUDPSocket.new(copy_message: true) allow(UDPSocket).to receive(:new).and_return(socket) - dogstats = Datadog::Statsd.new("localhost", 1234, delay_serialization: true) + dogstats = Datadog::Statsd.new("localhost", 1234, + delay_serialization: true, + origin_detection: false, + ) dogstats.increment("boo") dogstats.increment("oob", tags: {tag1: "val1"}) diff --git a/spec/integrations/telemetry_spec.rb b/spec/integrations/telemetry_spec.rb index 5703a66a..24166775 100644 --- a/spec/integrations/telemetry_spec.rb +++ b/spec/integrations/telemetry_spec.rb @@ -7,7 +7,8 @@ subject do Datadog::Statsd.new('localhost', 1234, - telemetry_flush_interval: -1 + telemetry_flush_interval: -1, + origin_detection: false, ) end @@ -26,6 +27,7 @@ subject do Datadog::Statsd.new('localhost', 1234, telemetry_enable: false, + origin_detection: false, ) end @@ -59,6 +61,7 @@ subject do Datadog::Statsd.new('localhost', 1234, telemetry_flush_interval: 2, + origin_detection: false, ) end diff --git a/spec/statsd/origin_detection_spec.rb b/spec/statsd/origin_detection_spec.rb new file mode 100644 index 00000000..aa6a3f52 --- /dev/null +++ b/spec/statsd/origin_detection_spec.rb @@ -0,0 +1,532 @@ +require 'spec_helper' +require 'fakefs/safe' + +describe Datadog::Statsd::OriginDetection do + subject do + described_class.new() + end + + it 'writes a file in the fake filesystem' do + read = nil + FakeFS.with_fresh do + FileUtils.mkdir_p('/tmp') + File.write('/tmp/test.txt', 'hello') + read = File.read('/tmp/test.txt') + end + + expect(read).to eq('hello') + end + + it 'extracts the container ID from the cgroup file' do + container_id = nil + FakeFS.with_fresh do + FileUtils.mkdir_p('/proc/self') + File.write('/proc/self/cgroup', <<~CGROUP) + 4:blkio:/kubepods/burstable/podfd52ef25-a87d-11e9-9423-0800271a638e/8c046cb0b72cd4c99f51b5591cd5b095967f58ee003710a45280c28ee1a9c7fa + CGROUP + + container_id = subject.get_container_id(nil, true) + end + + expect(container_id).to eq('8c046cb0b72cd4c99f51b5591cd5b095967f58ee003710a45280c28ee1a9c7fa') + end + + describe 'read container id' do + [ + { + input: <<~CGROUP, + other_line + 10:hugetlb:/kubepods/burstable/podfd52ef25-a87d-11e9-9423-0800271a638e/8c046cb0b72cd4c99f51b5591cd5b095967f58ee003710a45280c28ee1a9c7fa + 9:cpuset:/kubepods/burstable/podfd52ef25-a87d-11e9-9423-0800271a638e/8c046cb0b72cd4c99f51b5591cd5b095967f58ee003710a45280c28ee1a9c7fa + 8:pids:/kubepods/burstable/podfd52ef25-a87d-11e9-9423-0800271a638e/8c046cb0b72cd4c99f51b5591cd5b095967f58ee003710a45280c28ee1a9c7fa + 7:freezer:/kubepods/burstable/podfd52ef25-a87d-11e9-9423-0800271a638e/8c046cb0b72cd4c99f51b5591cd5b095967f58ee003710a45280c28ee1a9c7fa + 6:cpu,cpuacct:/kubepods/burstable/podfd52ef25-a87d-11e9-9423-0800271a638e/8c046cb0b72cd4c99f51b5591cd5b095967f58ee003710a45280c28ee1a9c7fa + 5:perf_event:/kubepods/burstable/podfd52ef25-a87d-11e9-9423-0800271a638e/8c046cb0b72cd4c99f51b5591cd5b095967f58ee003710a45280c28ee1a9c7fa + 4:blkio:/kubepods/burstable/podfd52ef25-a87d-11e9-9423-0800271a638e/8c046cb0b72cd4c99f51b5591cd5b095967f58ee003710a45280c28ee1a9c7fa + 3:devices:/kubepods/burstable/podfd52ef25-a87d-11e9-9423-0800271a638e/8c046cb0b72cd4c99f51b5591cd5b095967f58ee003710a45280c28ee1a9c7fa + 2:net_cls,net_prio:/kubepods/burstable/podfd52ef25-a87d-11e9-9423-0800271a638e/8c046cb0b72cd4c99f51b5591cd5b095967f58ee003710a45280c28ee1a9c7fa + CGROUP + expected: '8c046cb0b72cd4c99f51b5591cd5b095967f58ee003710a45280c28ee1a9c7fa' + }, + { + input: "10:hugetlb:/kubepods/burstable/podfd52ef25-a87d-11e9-9423-0800271a638e/8c046cb0b72cd4c99f51b5591cd5b095967f58ee003710a45280c28ee1a9c7fa", + expected: '8c046cb0b72cd4c99f51b5591cd5b095967f58ee003710a45280c28ee1a9c7fa' + }, + { + input: "10:hugetlb:/kubepods", + expected: nil + }, + { + input: "11:hugetlb:/ecs/55091c13-b8cf-4801-b527-f4601742204d/432624d2150b349fe35ba397284dea788c2bf66b885d14dfc1569b01890ca7da", + expected: '432624d2150b349fe35ba397284dea788c2bf66b885d14dfc1569b01890ca7da' + }, + { + input: "1:name=systemd:/docker/34dc0b5e626f2c5c4c5170e34b10e7654ce36f0fcd532739f4445baabea03376", + expected: '34dc0b5e626f2c5c4c5170e34b10e7654ce36f0fcd532739f4445baabea03376' + }, + { + input: "1:name=systemd:/uuid/34dc0b5e-626f-2c5c-4c51-70e34b10e765", + expected: '34dc0b5e-626f-2c5c-4c51-70e34b10e765' + }, + { + input: "1:name=systemd:/ecs/34dc0b5e626f2c5c4c5170e34b10e765-1234567890", + expected: '34dc0b5e626f2c5c4c5170e34b10e765-1234567890' + }, + { + input: "1:name=systemd:/docker/34dc0b5e626f2c5c4c5170e34b10e7654ce36f0fcd532739f4445baabea03376.scope", + expected: '34dc0b5e626f2c5c4c5170e34b10e7654ce36f0fcd532739f4445baabea03376' + }, + { + input: <<~CGROUP, + 1:name=systemd:/nope + 2:pids:/docker/34dc0b5e626f2c5c4c5170e34b10e7654ce36f0fcd532739f4445baabea03376 + 3:cpu:/invalid + CGROUP + expected: '34dc0b5e626f2c5c4c5170e34b10e7654ce36f0fcd532739f4445baabea03376' + } + ].each_with_index do |test_case, index| + it "extracts container ID from case #{index + 1}" do + result = nil + + FakeFS.with_fresh do + FileUtils.mkdir_p('/proc/self') + File.write('/proc/self/cgroup', test_case[:input]) + + result = subject.read_container_id('/proc/self/cgroup') + end + expect(result).to eq(test_case[:expected]) + end + end + end + + it 'extracts container ID from mountinfo' do + container_id = nil + FakeFS.with_fresh do + # Set up fake /proc/self directory + FileUtils.mkdir_p('/proc/self') + + # Write only mountinfo file content (cgroup file unused here but still required for consistency) + File.write('/proc/self/mountinfo', <<~MOUNTINFO) + 2282 2269 8:1 /var/lib/containerd/io.containerd.grpc.v1.cri/sandboxes/c0a82a3506b0366c9666f6dbe71c783abeb26ba65e312e918a49e10a277196d0/hostname /host/var/run/containerd/io.containerd.runtime.v2.task/k8s.io/fc7038bc73a8d3850c66ddbfb0b2901afa378bfcbb942cc384b051767e4ac6b0/rootfs/etc/hostname rw,nosuid,nodev,relatime - ext4 /dev/sda1 rw,commit=30 + MOUNTINFO + + File.write('/proc/self/cgroup', "") + + container_id = subject.get_container_id(nil, true) + end + + expect(container_id).to eq("fc7038bc73a8d3850c66ddbfb0b2901afa378bfcbb942cc384b051767e4ac6b0") + end + + describe 'Parse mount info' do + test_cases = [ + { + input: <<~MOUNTINFO, +608 554 0:42 / / rw,relatime master:289 - overlay overlay rw,lowerdir=/var/lib/docker/overlay2/l/RQH52YWGJKBXL6THNS7EASIUNY:/var/lib/docker/overlay2/l/EHJME4ZW2BP2W7SGOCNTKY76NI,upperdir=/var/lib/docker/overlay2/241a77e9a6a048e54d5b5700afaeab6071cb5ffef1fd2acbebbf19935a429897/diff,workdir=/var/lib/docker/overlay2/241a77e9a6a048e54d5b5700afaeab6071cb5ffef1fd2acbebbf19935a429897/work +609 608 0:46 / /proc rw,nosuid,nodev,noexec,relatime - proc proc rw +610 608 0:47 / /dev rw,nosuid - tmpfs tmpfs rw,size=65536k,mode=755,inode64 +611 610 0:48 / /dev/pts rw,nosuid,noexec,relatime - devpts devpts rw,gid=5,mode=620,ptmxmode=666 +615 608 0:49 / /sys ro,nosuid,nodev,noexec,relatime - sysfs sysfs ro +623 615 0:28 / /sys/fs/cgroup ro,nosuid,nodev,noexec,relatime - cgroup2 cgroup rw,nsdelegate,memory_recursiveprot +624 610 0:45 / /dev/mqueue rw,nosuid,nodev,noexec,relatime - mqueue mqueue rw +625 610 0:50 / /dev/shm rw,nosuid,nodev,noexec,relatime - tmpfs shm rw,size=65536k,inode64 +626 608 259:1 /var/lib/docker/containers/0cfa82bf3ab29da271548d6a044e95c948c6fd2f7578fb41833a44ca23da425f/resolv.conf /etc/resolv.conf rw,relatime - ext4 /dev/root rw,discard,errors=remount-ro +627 608 259:1 /var/lib/docker/containers/0cfa82bf3ab29da271548d6a044e95c948c6fd2f7578fb41833a44ca23da425f/hostname /etc/hostname rw,relatime - ext4 /dev/root rw,discard,errors=remount-ro +628 608 259:1 /var/lib/docker/containers/0cfa82bf3ab29da271548d6a044e95c948c6fd2f7578fb41833a44ca23da425f/hosts /etc/hosts rw,relatime - ext4 /dev/root rw,discard,errors=remount-ro +555 609 0:46 /bus /proc/bus ro,nosuid,nodev,noexec,relatime - proc proc rw +556 609 0:46 /fs /proc/fs ro,nosuid,nodev,noexec,relatime - proc proc rw +557 609 0:46 /irq /proc/irq ro,nosuid,nodev,noexec,relatime - proc proc rw +558 609 0:46 /sys /proc/sys ro,nosuid,nodev,noexec,relatime - proc proc rw +559 609 0:46 /sysrq-trigger /proc/sysrq-trigger ro,nosuid,nodev,noexec,relatime - proc proc rw +560 609 0:51 / /proc/acpi ro,relatime - tmpfs tmpfs ro,inode64 +561 609 0:47 /null /proc/kcore rw,nosuid - tmpfs tmpfs rw,size=65536k,mode=755,inode64 +562 609 0:47 /null /proc/keys rw,nosuid - tmpfs tmpfs rw,size=65536k,mode=755,inode64 +563 609 0:47 /null /proc/timer_list rw,nosuid - tmpfs tmpfs rw,size=65536k,mode=755,inode64 +564 609 0:52 / /proc/scsi ro,relatime - tmpfs tmpfs ro,inode64 +565 615 0:53 / /sys/firmware ro,relatime - tmpfs tmpfs ro,inode64 + MOUNTINFO + expected: '0cfa82bf3ab29da271548d6a044e95c948c6fd2f7578fb41833a44ca23da425f' + }, + { + input: <<~MOUNTINFO, +2775 2588 0:310 / / rw,relatime master:760 - overlay overlay rw,lowerdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/163/fs,upperdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/164/fs,workdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/164/work +2776 2775 0:312 / /proc rw,nosuid,nodev,noexec,relatime - proc proc rw +2777 2775 0:313 / /dev rw,nosuid - tmpfs tmpfs rw,size=65536k,mode=755 +2778 2777 0:314 / /dev/pts rw,nosuid,noexec,relatime - devpts devpts rw,gid=5,mode=620,ptmxmode=666 +2779 2777 0:301 / /dev/mqueue rw,nosuid,nodev,noexec,relatime - mqueue mqueue rw +2780 2775 0:306 / /sys ro,nosuid,nodev,noexec,relatime - sysfs sysfs ro +2781 2780 0:24 / /sys/fs/cgroup ro,nosuid,nodev,noexec,relatime - cgroup2 cgroup rw +2782 2775 8:1 /var/lib/kubelet/pods/e50f2933-b36b-4e3b-89b1-280707f7bc53/etc-hosts /etc/hosts rw,relatime - ext4 /dev/sda1 rw,commit=30 +2783 2777 8:1 /var/lib/kubelet/pods/e50f2933-b36b-4e3b-89b1-280707f7bc53/containers/bash2/3404dd90 /dev/termination-log rw,relatime - ext4 /dev/sda1 rw,commit=30 +2784 2775 8:1 /var/lib/containerd/io.containerd.grpc.v1.cri/sandboxes/c5866f3bfef020a8c045540b4a05dd9d193443740be76437a6130c7759ab2076/hostname /etc/hostname rw,nosuid,nodev,relatime - ext4 /dev/sda1 rw,commit=30 +2785 2775 8:1 /var/lib/containerd/io.containerd.grpc.v1.cri/sandboxes/c5866f3bfef020a8c045540b4a05dd9d193443740be76437a6130c7759ab2076/resolv.conf /etc/resolv.conf rw,nosuid,nodev,relatime - ext4 /dev/sda1 rw,commit=30 +2786 2777 0:298 / /dev/shm rw,nosuid,nodev,noexec,relatime - tmpfs shm rw,size=65536k +2787 2775 0:297 / /var/run/secrets/kubernetes.io/serviceaccount ro,relatime - tmpfs tmpfs rw,size=2880088k +2602 2777 0:314 /0 /dev/console rw,nosuid,noexec,relatime - devpts devpts rw,gid=5,mode=620,ptmxmode=666 +2603 2776 0:312 /bus /proc/bus ro,nosuid,nodev,noexec,relatime - proc proc rw +2604 2776 0:312 /fs /proc/fs ro,nosuid,nodev,noexec,relatime - proc proc rw +2605 2776 0:312 /irq /proc/irq ro,nosuid,nodev,noexec,relatime - proc proc rw +2606 2776 0:312 /sys /proc/sys ro,nosuid,nodev,noexec,relatime - proc proc rw +2607 2776 0:312 /sysrq-trigger /proc/sysrq-trigger ro,nosuid,nodev,noexec,relatime - proc proc rw +2608 2776 0:315 / /proc/acpi ro,relatime - tmpfs tmpfs ro +2609 2776 0:313 /null /proc/kcore rw,nosuid - tmpfs tmpfs rw,size=65536k,mode=755 +2610 2776 0:313 /null /proc/keys rw,nosuid - tmpfs tmpfs rw,size=65536k,mode=755 +2611 2776 0:313 /null /proc/timer_list rw,nosuid - tmpfs tmpfs rw,size=65536k,mode=755 +2612 2776 0:316 / /proc/scsi ro,relatime - tmpfs tmpfs ro +2613 2780 0:317 / /sys/firmware ro,relatime - tmpfs tmpfs ro + MOUNTINFO + expected: nil + }, + { + input: <<~MOUNTINFO, +2208 2025 0:249 / / rw,relatime master:691 - overlay overlay rw,lowerdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/133/fs,upperdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/134/fs,workdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/134/work +2209 2208 0:251 / /proc rw,nosuid,nodev,noexec,relatime - proc proc rw +2210 2208 0:252 / /dev rw,nosuid - tmpfs tmpfs rw,size=65536k,mode=755 +2211 2210 0:253 / /dev/pts rw,nosuid,noexec,relatime - devpts devpts rw,gid=5,mode=620,ptmxmode=666 +2212 2210 0:91 / /dev/mqueue rw,nosuid,nodev,noexec,relatime - mqueue mqueue rw +2213 2208 0:96 / /sys ro,nosuid,nodev,noexec,relatime - sysfs sysfs ro +2214 2213 0:24 / /sys/fs/cgroup ro,nosuid,nodev,noexec,relatime - cgroup2 cgroup rw +2215 2208 8:1 /var/lib/kubelet/pods/ce225bf3-5dbb-44d7-9591-29d1cc65cdc6/volumes/kubernetes.io~empty-dir/tmpdir /tmp rw,relatime - ext4 /dev/sda1 rw,commit=30 +2216 2210 0:88 / /dev/shm rw,nosuid,nodev,noexec,relatime - tmpfs shm rw,size=65536k +2217 2210 8:1 /var/lib/kubelet/pods/ce225bf3-5dbb-44d7-9591-29d1cc65cdc6/containers/agent/22381dc2 /dev/termination-log rw,relatime - ext4 /dev/sda1 rw,commit=30 +2218 2208 8:1 /var/lib/kubelet/pods/ce225bf3-5dbb-44d7-9591-29d1cc65cdc6/etc-hosts /etc/hosts rw,relatime - ext4 /dev/sda1 rw,commit=30 +2219 2208 8:1 /var/lib/kubelet/pods/ce225bf3-5dbb-44d7-9591-29d1cc65cdc6/volumes/kubernetes.io~empty-dir/config /etc/datadog-agent rw,relatime - ext4 /dev/sda1 rw,commit=30 +2220 2208 8:1 /var/lib/containerd/io.containerd.grpc.v1.cri/sandboxes/c0a82a3506b0366c9666f6dbe71c783abeb26ba65e312e918a49e10a277196d0/hostname /etc/hostname rw,nosuid,nodev,relatime - ext4 /dev/sda1 rw,commit=30 +2221 2208 8:1 /var/lib/containerd/io.containerd.grpc.v1.cri/sandboxes/c0a82a3506b0366c9666f6dbe71c783abeb26ba65e312e918a49e10a277196d0/resolv.conf /etc/resolv.conf rw,nosuid,nodev,relatime - ext4 /dev/sda1 rw,commit=30 +2222 2208 0:19 / /host/proc ro,relatime - proc proc rw +2223 2222 0:27 / /host/proc/sys/fs/binfmt_misc rw,relatime - autofs systemd-1 rw,fd=30,pgrp=0,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=1284 +2224 2223 0:45 / /host/proc/sys/fs/binfmt_misc rw,nosuid,nodev,noexec,relatime - binfmt_misc binfmt_misc rw +2225 2219 8:1 /var/lib/kubelet/pods/ce225bf3-5dbb-44d7-9591-29d1cc65cdc6/volumes/kubernetes.io~configmap/installinfo/..2022_10_25_08_56_17.196723275/install_info /etc/datadog-agent/install_info ro,relatime - ext4 /dev/sda1 rw,commit=30 +2226 2208 8:1 /var/lib/datadog-agent/logs /opt/datadog-agent/run rw,nosuid,nodev,noexec,relatime - ext4 /dev/sda1 rw,commit=30 +2227 2208 8:1 /var/log/pods /var/log/pods ro,relatime - ext4 /dev/sda1 rw,commit=30 +2228 2208 8:1 /var/log/containers /var/log/containers ro,relatime - ext4 /dev/sda1 rw,commit=30 +2229 2208 0:23 /datadog /var/run/datadog rw,nosuid,nodev - tmpfs tmpfs rw,size=805192k,nr_inodes=819200,mode=755 +2230 2208 0:23 / /host/var/run ro - tmpfs tmpfs rw,size=805192k,nr_inodes=819200,mode=755 +2231 2230 0:42 / /host/var/run/containerd/io.containerd.grpc.v1.cri/sandboxes/188cedbba84c09ef9084420db548c00fadeb590e6dcd955cad3413f8339bce84/shm rw,nosuid,nodev,noexec,relatime - tmpfs shm rw,size=65536k +2232 2230 0:44 / /host/var/run/containerd/io.containerd.runtime.v2.task/k8s.io/188cedbba84c09ef9084420db548c00fadeb590e6dcd955cad3413f8339bce84/rootfs rw,relatime - overlay overlay rw,lowerdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/42/fs,upperdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/43/fs,workdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/43/work +2233 2230 0:56 / /host/var/run/containerd/io.containerd.runtime.v2.task/k8s.io/f0d7bcb99d31a341fbcc1c17e7a91c223a16997168c4ed756f61448de95ec0ab/rootfs rw,relatime - overlay overlay rw,lowerdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/3/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/2/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/1/fs,upperdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/44/fs,workdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/44/work +2234 2230 0:63 / /host/var/run/containerd/io.containerd.grpc.v1.cri/sandboxes/67d04649323de52cee959f9cd3172469cc29b914569bf1b33761ba75b344b2e0/shm rw,nosuid,nodev,noexec,relatime - tmpfs shm rw,size=65536k +2235 2230 0:64 / /host/var/run/containerd/io.containerd.runtime.v2.task/k8s.io/67d04649323de52cee959f9cd3172469cc29b914569bf1b33761ba75b344b2e0/rootfs rw,relatime - overlay overlay rw,lowerdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/42/fs,upperdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/45/fs,workdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/45/work +2236 2230 0:74 / /host/var/run/containerd/io.containerd.grpc.v1.cri/sandboxes/82c2fc7d2c974f108c8ed035393ba4864881fca51d34d68ab65adf9b3ecb1e39/shm rw,nosuid,nodev,noexec,relatime - tmpfs shm rw,size=65536k +2237 2230 0:75 / /host/var/run/containerd/io.containerd.runtime.v2.task/k8s.io/82c2fc7d2c974f108c8ed035393ba4864881fca51d34d68ab65adf9b3ecb1e39/rootfs rw,relatime - overlay overlay rw,lowerdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/42/fs,upperdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/46/fs,workdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/46/work +2238 2230 0:4 net:[4026532317] /host/var/run/netns/cni-a6d26b34-6006-776c-6297-d43cbd377e1a rw - nsfs nsfs rw +2239 2230 0:88 / /host/var/run/containerd/io.containerd.grpc.v1.cri/sandboxes/c0a82a3506b0366c9666f6dbe71c783abeb26ba65e312e918a49e10a277196d0/shm rw,nosuid,nodev,noexec,relatime - tmpfs shm rw,size=65536k +2240 2230 0:89 / /host/var/run/containerd/io.containerd.runtime.v2.task/k8s.io/c0a82a3506b0366c9666f6dbe71c783abeb26ba65e312e918a49e10a277196d0/rootfs rw,relatime - overlay overlay rw,lowerdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/42/fs,upperdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/47/fs,workdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/47/work +2241 2230 0:4 net:[4026532392] /host/var/run/netns/cni-b9ddcb41-083c-d886-cc1b-ad856fae244a rw - nsfs nsfs rw +2242 2230 0:100 / /host/var/run/containerd/io.containerd.grpc.v1.cri/sandboxes/006c472d636bbaa9bce73084d7545085d4b6127f8e6ffc39245ebc1f65284f1a/shm rw,nosuid,nodev,noexec,relatime - tmpfs shm rw,size=65536k +2243 2230 0:101 / /host/var/run/containerd/io.containerd.runtime.v2.task/k8s.io/006c472d636bbaa9bce73084d7545085d4b6127f8e6ffc39245ebc1f65284f1a/rootfs rw,relatime - overlay overlay rw,lowerdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/42/fs,upperdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/48/fs,workdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/48/work +2244 2230 0:112 / /host/var/run/containerd/io.containerd.runtime.v2.task/k8s.io/1c2200d42c9b200327a9f369b680ef0fabd20ae1fd322abb5f37e480037c40fe/rootfs rw,relatime - overlay overlay rw,lowerdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/51/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/50/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/49/fs,upperdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/52/fs,workdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/52/work +2245 2230 0:120 / /host/var/run/containerd/io.containerd.runtime.v2.task/k8s.io/3aaa22321201b33a7630c48471a3caa31b14e2d71011333751f174fb44b16944/rootfs rw,relatime - overlay overlay rw,lowerdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/91/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/90/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/89/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/88/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/87/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/86/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/85/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/84/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/83/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/82/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/81/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/80/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/79/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/78/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/77/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/76/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/75/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/74/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/73/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/72/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/71/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/70/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/69/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/68/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/67/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/66/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/65/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/64/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/63/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/62/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/61/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/60/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/59/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/58/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/57/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/56/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/55/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/54/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/53/fs,upperdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/92/fs,workdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/92/work +2246 2230 0:128 / /host/var/run/containerd/io.containerd.runtime.v2.task/k8s.io/feecdc0737367e535ad3111be3570aa9d8f3efab4db569a9c93dff3b82fb749b/rootfs rw,relatime - overlay overlay rw,lowerdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/97/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/96/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/95/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/94/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/93/fs,upperdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/98/fs,workdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/98/work +2247 2230 0:4 net:[4026532475] /host/var/run/netns/cni-e418c801-72a5-c80e-541c-5b7a706f8180 rw - nsfs nsfs rw +2248 2230 0:4 net:[4026532543] /host/var/run/netns/cni-d616515e-36c5-4fd5-3de5-3e0d01ca1299 rw - nsfs nsfs rw +2249 2230 0:4 net:[4026532615] /host/var/run/netns/cni-01c3425d-10c5-ecd0-b7f0-4dcb9b514038 rw - nsfs nsfs rw +2250 2230 0:139 / /host/var/run/containerd/io.containerd.grpc.v1.cri/sandboxes/71ff5735b57177a4f24269bee3e4961a248ecf495c1ff63fa8f95c74bc9206b4/shm rw,nosuid,nodev,noexec,relatime - tmpfs shm rw,size=65536k +2251 2230 0:140 / /host/var/run/containerd/io.containerd.grpc.v1.cri/sandboxes/5bbba3a7d2259ada29c791ad8ae1a5f5dc52f176792e92f78695bde2da6a6f63/shm rw,nosuid,nodev,noexec,relatime - tmpfs shm rw,size=65536k +2252 2230 0:141 / /host/var/run/containerd/io.containerd.grpc.v1.cri/sandboxes/a16f90bbb1340ea0fc60e4c94b2adca5acb7a2d191020de17b99800d9942aec5/shm rw,nosuid,nodev,noexec,relatime - tmpfs shm rw,size=65536k +2253 2230 0:142 / /host/var/run/containerd/io.containerd.runtime.v2.task/k8s.io/5bbba3a7d2259ada29c791ad8ae1a5f5dc52f176792e92f78695bde2da6a6f63/rootfs rw,relatime - overlay overlay rw,lowerdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/42/fs,upperdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/101/fs,workdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/101/work +2254 2230 0:144 / /host/var/run/containerd/io.containerd.runtime.v2.task/k8s.io/71ff5735b57177a4f24269bee3e4961a248ecf495c1ff63fa8f95c74bc9206b4/rootfs rw,relatime - overlay overlay rw,lowerdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/42/fs,upperdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/100/fs,workdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/100/work +2255 2230 0:146 / /host/var/run/containerd/io.containerd.runtime.v2.task/k8s.io/a16f90bbb1340ea0fc60e4c94b2adca5acb7a2d191020de17b99800d9942aec5/rootfs rw,relatime - overlay overlay rw,lowerdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/42/fs,upperdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/102/fs,workdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/102/work +2256 2230 0:4 net:[4026532767] /host/var/run/netns/cni-635fdd5c-891b-7dce-6359-3f22711c901f rw - nsfs nsfs rw +2257 2230 0:180 / /host/var/run/containerd/io.containerd.grpc.v1.cri/sandboxes/3380ce37d9657a68c2be7e2d1a5f3a7c6b6434d98c512650efa81489008e16e8/shm rw,nosuid,nodev,noexec,relatime - tmpfs shm rw,size=65536k +2258 2230 0:183 / /host/var/run/containerd/io.containerd.runtime.v2.task/k8s.io/3380ce37d9657a68c2be7e2d1a5f3a7c6b6434d98c512650efa81489008e16e8/rootfs rw,relatime - overlay overlay rw,lowerdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/42/fs,upperdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/104/fs,workdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/104/work +2259 2230 0:4 net:[4026532845] /host/var/run/netns/cni-caed5c60-e9fd-55e4-fd26-9d3522d7eeda rw - nsfs nsfs rw +2260 2230 0:203 / /host/var/run/containerd/io.containerd.grpc.v1.cri/sandboxes/4e0303bc6d20caffed804b11c0d01dd2c8c4e9309d8ef632757b009a58aa089b/shm rw,nosuid,nodev,noexec,relatime - tmpfs shm rw,size=65536k +2261 2230 0:204 / /host/var/run/containerd/io.containerd.runtime.v2.task/k8s.io/4e0303bc6d20caffed804b11c0d01dd2c8c4e9309d8ef632757b009a58aa089b/rootfs rw,relatime - overlay overlay rw,lowerdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/42/fs,upperdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/105/fs,workdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/105/work +2262 2230 0:223 / /host/var/run/containerd/io.containerd.runtime.v2.task/k8s.io/d89d74b92f8548ad9b8ac4299736b6f4f4665dee36c392fb6956acb956ede47d/rootfs rw,relatime - overlay overlay rw,lowerdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/20/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/19/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/18/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/17/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/16/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/4/fs,upperdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/107/fs,workdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/107/work +2263 2230 0:148 / /host/var/run/containerd/io.containerd.runtime.v2.task/k8s.io/4a9b64ac877dbf02470a508c34bdc48d9aa3377c2f4ed33f66fb66c34d8421d4/rootfs rw,relatime - overlay overlay rw,lowerdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/117/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/116/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/115/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/114/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/113/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/112/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/111/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/109/fs,upperdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/118/fs,workdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/118/work +2264 2230 0:188 / /host/var/run/containerd/io.containerd.runtime.v2.task/k8s.io/876c755819a777a720e6683114bbd0372bca83e7a547e401b63ae1a7cb754b79/rootfs rw,relatime - overlay overlay rw,lowerdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/124/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/123/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/122/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/121/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/120/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/119/fs,upperdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/125/fs,workdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/125/work +2265 2230 0:217 / /host/var/run/containerd/io.containerd.runtime.v2.task/k8s.io/faf62a9d55405c0ca0e9becbd6e02b22ecfab2c8f466294d188e6a3d34299032/rootfs rw,relatime - overlay overlay rw,lowerdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/99/fs,upperdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/129/fs,workdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/129/work +2266 2230 0:233 / /host/var/run/containerd/io.containerd.runtime.v2.task/k8s.io/79fc04c9a48dae0de5ac951860bbf811d660f67a96c84d62162cc10972579d3f/rootfs rw,relatime - overlay overlay rw,lowerdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/128/fs,upperdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/130/fs,workdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/130/work +2267 2230 0:241 / /host/var/run/containerd/io.containerd.runtime.v2.task/k8s.io/84952b0516872d00333667f6f132dcacb1a935f0d19862b54c5e045d45f4c642/rootfs rw,relatime - overlay overlay rw,lowerdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/131/fs,upperdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/132/fs,workdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/132/work +2268 2230 0:249 / /host/var/run/containerd/io.containerd.runtime.v2.task/k8s.io/fc7038bc73a8d3850c66ddbfb0b2901afa378bfcbb942cc384b051767e4ac6b0/rootfs rw,relatime - overlay overlay rw,lowerdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/133/fs,upperdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/134/fs,workdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/134/work +2269 2268 0:249 / /host/var/run/containerd/io.containerd.runtime.v2.task/k8s.io/fc7038bc73a8d3850c66ddbfb0b2901afa378bfcbb942cc384b051767e4ac6b0/rootfs rw,relatime - overlay overlay rw,lowerdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/133/fs,upperdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/134/fs,workdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/134/work +2270 2269 0:251 / /host/var/run/containerd/io.containerd.runtime.v2.task/k8s.io/fc7038bc73a8d3850c66ddbfb0b2901afa378bfcbb942cc384b051767e4ac6b0/rootfs/proc rw,nosuid,nodev,noexec,relatime - proc proc rw +2271 2269 0:252 / /host/var/run/containerd/io.containerd.runtime.v2.task/k8s.io/fc7038bc73a8d3850c66ddbfb0b2901afa378bfcbb942cc384b051767e4ac6b0/rootfs/dev rw,nosuid - tmpfs tmpfs rw,size=65536k,mode=755 +2272 2271 0:253 / /host/var/run/containerd/io.containerd.runtime.v2.task/k8s.io/fc7038bc73a8d3850c66ddbfb0b2901afa378bfcbb942cc384b051767e4ac6b0/rootfs/dev/pts rw,nosuid,noexec,relatime - devpts devpts rw,gid=5,mode=620,ptmxmode=666 +2273 2271 0:91 / /host/var/run/containerd/io.containerd.runtime.v2.task/k8s.io/fc7038bc73a8d3850c66ddbfb0b2901afa378bfcbb942cc384b051767e4ac6b0/rootfs/dev/mqueue rw,nosuid,nodev,noexec,relatime - mqueue mqueue rw +2274 2271 0:88 / /host/var/run/containerd/io.containerd.runtime.v2.task/k8s.io/fc7038bc73a8d3850c66ddbfb0b2901afa378bfcbb942cc384b051767e4ac6b0/rootfs/dev/shm rw,nosuid,nodev,noexec,relatime - tmpfs shm rw,size=65536k +2275 2271 8:1 /var/lib/kubelet/pods/ce225bf3-5dbb-44d7-9591-29d1cc65cdc6/containers/agent/22381dc2 /host/var/run/containerd/io.containerd.runtime.v2.task/k8s.io/fc7038bc73a8d3850c66ddbfb0b2901afa378bfcbb942cc384b051767e4ac6b0/rootfs/dev/termination-log rw,relatime - ext4 /dev/sda1 rw,commit=30 +2276 2269 0:96 / /host/var/run/containerd/io.containerd.runtime.v2.task/k8s.io/fc7038bc73a8d3850c66ddbfb0b2901afa378bfcbb942cc384b051767e4ac6b0/rootfs/sys ro,nosuid,nodev,noexec,relatime - sysfs sysfs ro +2277 2276 0:24 / /host/var/run/containerd/io.containerd.runtime.v2.task/k8s.io/fc7038bc73a8d3850c66ddbfb0b2901afa378bfcbb942cc384b051767e4ac6b0/rootfs/sys/fs/cgroup ro,nosuid,nodev,noexec,relatime - cgroup2 cgroup rw +2278 2269 8:1 /var/lib/kubelet/pods/ce225bf3-5dbb-44d7-9591-29d1cc65cdc6/volumes/kubernetes.io~empty-dir/tmpdir /host/var/run/containerd/io.containerd.runtime.v2.task/k8s.io/fc7038bc73a8d3850c66ddbfb0b2901afa378bfcbb942cc384b051767e4ac6b0/rootfs/tmp rw,relatime - ext4 /dev/sda1 rw,commit=30 +2279 2269 8:1 /var/lib/kubelet/pods/ce225bf3-5dbb-44d7-9591-29d1cc65cdc6/etc-hosts /host/var/run/containerd/io.containerd.runtime.v2.task/k8s.io/fc7038bc73a8d3850c66ddbfb0b2901afa378bfcbb942cc384b051767e4ac6b0/rootfs/etc/hosts rw,relatime - ext4 /dev/sda1 rw,commit=30 +2280 2269 8:1 /var/lib/kubelet/pods/ce225bf3-5dbb-44d7-9591-29d1cc65cdc6/volumes/kubernetes.io~empty-dir/config /host/var/run/containerd/io.containerd.runtime.v2.task/k8s.io/fc7038bc73a8d3850c66ddbfb0b2901afa378bfcbb942cc384b051767e4ac6b0/rootfs/etc/datadog-agent rw,relatime - ext4 /dev/sda1 rw,commit=30 +2281 2280 8:1 /var/lib/kubelet/pods/ce225bf3-5dbb-44d7-9591-29d1cc65cdc6/volumes/kubernetes.io~configmap/installinfo/..2022_10_25_08_56_17.196723275/install_info /host/var/run/containerd/io.containerd.runtime.v2.task/k8s.io/fc7038bc73a8d3850c66ddbfb0b2901afa378bfcbb942cc384b051767e4ac6b0/rootfs/etc/datadog-agent/install_info ro,relatime - ext4 /dev/sda1 rw,commit=30 +2282 2269 8:1 /var/lib/containerd/io.containerd.grpc.v1.cri/sandboxes/c0a82a3506b0366c9666f6dbe71c783abeb26ba65e312e918a49e10a277196d0/hostname /host/var/run/containerd/io.containerd.runtime.v2.task/k8s.io/fc7038bc73a8d3850c66ddbfb0b2901afa378bfcbb942cc384b051767e4ac6b0/rootfs/etc/hostname rw,nosuid,nodev,relatime - ext4 /dev/sda1 rw,commit=30 +2283 2269 8:1 /var/lib/containerd/io.containerd.grpc.v1.cri/sandboxes/c0a82a3506b0366c9666f6dbe71c783abeb26ba65e312e918a49e10a277196d0/resolv.conf /host/var/run/containerd/io.containerd.runtime.v2.task/k8s.io/fc7038bc73a8d3850c66ddbfb0b2901afa378bfcbb942cc384b051767e4ac6b0/rootfs/etc/resolv.conf rw,nosuid,nodev,relatime - ext4 /dev/sda1 rw,commit=30 +2284 2269 0:19 / /host/var/run/containerd/io.containerd.runtime.v2.task/k8s.io/fc7038bc73a8d3850c66ddbfb0b2901afa378bfcbb942cc384b051767e4ac6b0/rootfs/host/proc ro,relatime - proc proc rw +2285 2284 0:27 / /host/var/run/containerd/io.containerd.runtime.v2.task/k8s.io/fc7038bc73a8d3850c66ddbfb0b2901afa378bfcbb942cc384b051767e4ac6b0/rootfs/host/proc/sys/fs/binfmt_misc rw,relatime - autofs systemd-1 rw,fd=30,pgrp=0,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=1284 +2286 2285 0:45 / /host/var/run/containerd/io.containerd.runtime.v2.task/k8s.io/fc7038bc73a8d3850c66ddbfb0b2901afa378bfcbb942cc384b051767e4ac6b0/rootfs/host/proc/sys/fs/binfmt_misc rw,nosuid,nodev,noexec,relatime - binfmt_misc binfmt_misc rw +2287 2269 8:1 /var/lib/datadog-agent/logs /host/var/run/containerd/io.containerd.runtime.v2.task/k8s.io/fc7038bc73a8d3850c66ddbfb0b2901afa378bfcbb942cc384b051767e4ac6b0/rootfs/opt/datadog-agent/run rw,nosuid,nodev,noexec,relatime - ext4 /dev/sda1 rw,commit=30 +2288 2269 8:1 /var/log/pods /host/var/run/containerd/io.containerd.runtime.v2.task/k8s.io/fc7038bc73a8d3850c66ddbfb0b2901afa378bfcbb942cc384b051767e4ac6b0/rootfs/var/log/pods ro,relatime - ext4 /dev/sda1 rw,commit=30 +2289 2269 8:1 /var/log/containers /host/var/run/containerd/io.containerd.runtime.v2.task/k8s.io/fc7038bc73a8d3850c66ddbfb0b2901afa378bfcbb942cc384b051767e4ac6b0/rootfs/var/log/containers ro,relatime - ext4 /dev/sda1 rw,commit=30 +2290 2269 0:23 /datadog /host/var/run/containerd/io.containerd.runtime.v2.task/k8s.io/fc7038bc73a8d3850c66ddbfb0b2901afa378bfcbb942cc384b051767e4ac6b0/rootfs/var/run/datadog rw,nosuid,nodev - tmpfs tmpfs rw,size=805192k,nr_inodes=819200,mode=755 +2291 2208 0:34 /os-release /host/etc/os-release ro,relatime - overlay overlayfs rw,lowerdir=/etc,upperdir=/tmp/etc_overlay/etc,workdir=/tmp/etc_overlay/.work +2292 2208 8:1 /var/lib/kubelet/pods/ce225bf3-5dbb-44d7-9591-29d1cc65cdc6/volumes/kubernetes.io~empty-dir/logdatadog /var/log/datadog rw,relatime - ext4 /dev/sda1 rw,commit=30 +2293 2208 8:1 /var/lib/containerd/io.containerd.grpc.v1.cri/containers/fc7038bc73a8d3850c66ddbfb0b2901afa378bfcbb942cc384b051767e4ac6b0/volumes/33a073eeb8406d18d9007e3dcf8690235726a4f5aa81dd293650f75fc2c2a620 /var/run/s6 rw,nosuid,nodev,relatime - ext4 /dev/sda1 rw,commit=30 +2294 2208 8:1 /var/lib/docker/containers /var/lib/docker/containers ro,relatime - ext4 /dev/sda1 rw,commit=30 +2295 2208 0:24 /../../../.. /host/sys/fs/cgroup ro,relatime - cgroup2 cgroup2 rw +2296 2208 0:85 / /var/run/secrets/kubernetes.io/serviceaccount ro,relatime - tmpfs tmpfs rw,size=2880088k +2026 2209 0:251 /bus /proc/bus ro,nosuid,nodev,noexec,relatime - proc proc rw +2027 2209 0:251 /fs /proc/fs ro,nosuid,nodev,noexec,relatime - proc proc rw +2028 2209 0:251 /irq /proc/irq ro,nosuid,nodev,noexec,relatime - proc proc rw +2029 2209 0:251 /sys /proc/sys ro,nosuid,nodev,noexec,relatime - proc proc rw +2030 2209 0:251 /sysrq-trigger /proc/sysrq-trigger ro,nosuid,nodev,noexec,relatime - proc proc rw +2031 2209 0:254 / /proc/acpi ro,relatime - tmpfs tmpfs ro +2032 2209 0:252 /null /proc/kcore rw,nosuid - tmpfs tmpfs rw,size=65536k,mode=755 +2033 2209 0:252 /null /proc/keys rw,nosuid - tmpfs tmpfs rw,size=65536k,mode=755 +2034 2209 0:252 /null /proc/timer_list rw,nosuid - tmpfs tmpfs rw,size=65536k,mode=755 +2035 2209 0:255 / /proc/scsi ro,relatime - tmpfs tmpfs ro +2036 2213 0:256 / /sys/firmware ro,relatime - tmpfs tmpfs ro + MOUNTINFO + expected: 'fc7038bc73a8d3850c66ddbfb0b2901afa378bfcbb942cc384b051767e4ac6b0' + }, + { + input: '1258 1249 254:1 /docker/volumes/0919c2d87ec8ba99f3c85fdada5fe26eca73b2fce73a5974d6030f30bf91cbaf/_data/lib/containerd/io.containerd.grpc.v1.cri/sandboxes/ca30bb64884083e29b1dc08a1081dd2df123f13f045dadb64dc346e56c0b6871/hostname /etc/hostname rw,relatime - ext4 /dev/vda1 rw,discard', + expected: nil + } + ] + + test_cases.each_with_index do |test_case, index| + it "extracts container ID from mountinfo case #{index + 1}" do + result = "" + FakeFS.with_fresh do + FileUtils.mkdir_p('/proc/self') + File.write('/proc/self/mountinfo', test_case[:input]) + result = subject.read_mount_info('/proc/self/mountinfo') + end + + expect(result).to eq(test_case[:expected]) + end + end + end + + it 'extracts the container ID when found in mountinfo path' do + cid = "0cfa82bf3ab29da271548d6a044e95c948c6fd2f7578fb41833a44ca23da425f" + + mountinfo_contents = <<~MOUNTINFO + 608 554 0:42 / / rw,relatime master:289 - overlay overlay rw,lowerdir=/var/lib/docker/overlay2/l/RQH52YWGJKBXL6THNS7EASIUNY:/var/lib/docker/overlay2/l/EHJME4ZW2BP2W7SGOCNTKY76NI,upperdir=/var/lib/docker/overlay2/241a77e9a6a048e54d5b5700afaeab6071cb5ffef1fd2acbebbf19935a429897/diff,workdir=/var/lib/docker/overlay2/241a77e9a6a048e54d5b5700afaeab6071cb5ffef1fd2acbebbf19935a429897/work + 609 608 0:46 / /proc rw,nosuid,nodev,noexec,relatime - proc proc rw + 610 608 0:47 / /dev rw,nosuid - tmpfs tmpfs rw,size=65536k,mode=755,inode64 + 611 610 0:48 / /dev/pts rw,nosuid,noexec,relatime - devpts devpts rw,gid=5,mode=620,ptmxmode=666 + 615 608 0:49 / /sys ro,nosuid,nodev,noexec,relatime - sysfs sysfs ro + 623 615 0:28 / /sys/fs/cgroup ro,nosuid,nodev,noexec,relatime - cgroup2 cgroup rw,nsdelegate,memory_recursiveprot + 624 610 0:45 / /dev/mqueue rw,nosuid,nodev,noexec,relatime - mqueue mqueue rw + 625 610 0:50 / /dev/shm rw,nosuid,nodev,noexec,relatime - tmpfs shm rw,size=65536k,inode64 + 626 608 259:1 /var/lib/docker/containers/#{cid}/resolv.conf /etc/resolv.conf rw,relatime - ext4 /dev/root rw,discard,errors=remount-ro + 627 608 259:1 /var/lib/docker/containers/#{cid}/hostname /etc/hostname rw,relatime - ext4 /dev/root rw,discard,errors=remount-ro + 628 608 259:1 /var/lib/docker/containers/#{cid}/hosts /etc/hosts rw,relatime - ext4 /dev/root rw,discard,errors=remount-ro + 555 609 0:46 /bus /proc/bus ro,nosuid,nodev,noexec,relatime - proc proc rw + 556 609 0:46 /fs /proc/fs ro,nosuid,nodev,noexec,relatime - proc proc rw + 557 609 0:46 /irq /proc/irq ro,nosuid,nodev,noexec,relatime - proc proc rw + 558 609 0:46 /sys /proc/sys ro,nosuid,nodev,noexec,relatime - proc proc rw + 559 609 0:46 /sysrq-trigger /proc/sysrq-trigger ro,nosuid,nodev,noexec,relatime - proc proc rw + 560 609 0:51 / /proc/acpi ro,relatime - tmpfs tmpfs ro,inode64 + 561 609 0:47 /null /proc/kcore rw,nosuid - tmpfs tmpfs rw,size=65536k,mode=755,inode64 + 562 609 0:47 /null /proc/keys rw,nosuid - tmpfs tmpfs rw,size=65536k,mode=755,inode64 + 563 609 0:47 /null /proc/timer_list rw,nosuid - tmpfs tmpfs rw,size=65536k,mode=755,inode64 + 564 609 0:52 / /proc/scsi ro,relatime - tmpfs tmpfs ro,inode64 + 565 615 0:53 / /sys/firmware ro,relatime - tmpfs tmpfs ro,inode64 + MOUNTINFO + + container_id = "" + FakeFS.with_fresh do + FileUtils.mkdir_p('/proc/self') + File.write('/proc/self/mountinfo', mountinfo_contents) + File.write('/proc/self/cgroup', '') # Needed if get_container_id uses it + + container_id = subject.get_container_id(nil, true) + end + + expect(container_id).to eq(cid) + end + + describe '#parse_cgroup_node_path' do + test_cases = [ + { + name: 'cgroup2 normal case', + content: '0::/', + expected: { + '' => '/', + } + }, + { + name: 'hybrid', + content: "other_line\n0::/\n1:memory:/docker/abc123", + expected: { + '' => '/', + 'memory' => '/docker/abc123', + } + }, + { + name: 'with other controllers', + content: "other_line\n12:pids:/docker/abc123\n11:hugetlb:/docker/abc123\n10:net_cls,net_prio:/docker/abc123\n0::/docker/abc123\n", + expected: { + '' => '/docker/abc123' + } + }, + { + name: 'no controller', + content: 'empty', + expected: {} + } + ] + + test_cases.each do |test_case| + it "parses correctly for: #{test_case[:name]}" do + result = subject.parse_cgroup_node_path(test_case[:content]) + expect(result).to eq(test_case[:expected]) + end + end + end + + describe '#get_cgroup_inode' do + test_cases = [ + { + description: 'matching entry in /proc/self/cgroup and /proc/mounts - cgroup2 only', + cgroup_node_dir: 'system.slice/docker-abcdef0123456789abcdef0123456789.scope', + proc_self_cgroup_content: "0::/system.slice/docker-abcdef0123456789abcdef0123456789.scope\n", + controller: nil, + expected_result: 'in-{inode}' + }, + { + description: 'matching entry in /proc/self/cgroup and /proc/mounts - cgroup/hybrid only', + cgroup_node_dir: 'system.slice/docker-abcdef0123456789abcdef0123456789.scope', + proc_self_cgroup_content: <<~CG, + 3:memory:/system.slice/docker-abcdef0123456789abcdef0123456789.scope + 2:net_cls,net_prio:c + 1:name=systemd:b + 0::a + CG + controller: 'memory', + expected_result: 'in-{inode}' + }, + { + description: 'non memory or empty controller', + cgroup_node_dir: 'system.slice/docker-abcdef0123456789abcdef0123456789.scope', + proc_self_cgroup_content: <<~CG, + 3:cpu:/system.slice/docker-abcdef0123456789abcdef0123456789.scope + 2:net_cls,net_prio:c + 1:name=systemd:b + 0::a + CG + controller: 'cpu', + expected_result: nil + }, + { + description: 'path does not exist', + cgroup_node_dir: 'dummy.scope', + proc_self_cgroup_content: <<~CG, + 3:memory:/system.slice/docker-abcdef0123456789abcdef0123456789.scope + 2:net_cls,net_prio:c + 1:name=systemd:b + 0::a + CG + controller: nil, + expected_result: nil + }, + { + description: 'no entry in /proc/self/cgroup', + cgroup_node_dir: 'system.slice/docker-abcdef0123456789abcdef0123456789.scope', + proc_self_cgroup_content: 'nothing', + controller: nil, + expected_result: nil + } + ] + + test_cases.each_with_index do |test_case, index| + it "handles: #{test_case[:description]}" do + result = "" + FakeFS.with_fresh do + FileUtils.mkdir_p('/proc/self') + File.write('/proc/self/cgroup', test_case[:proc_self_cgroup_content]) + + controller = test_case[:controller] + segments = ['/sys/fs/cgroup', controller, test_case[:cgroup_node_dir]].compact + cgroup_node_path = File.join(*segments) + FileUtils.mkdir_p(File.dirname(cgroup_node_path)) + File.write(cgroup_node_path, '') + inode = File.stat(cgroup_node_path).ino + unless test_case[:expected_result].nil? + test_case[:expected_result] = test_case[:expected_result].gsub("{inode}", inode.to_s) + end + + result = subject.get_cgroup_inode('/sys/fs/cgroup', '/proc/self/cgroup') + end + + expect(result).to eq(test_case[:expected_result]) + end + end + end + + describe '#get_container_id' do + test_cases = [ + { + description: 'extract container-id from /proc/self/cgroup', + proc_self_cgroup_content: "4:blkio:/kubepods/burstable/podfd52ef25-a87d-11e9-9423-0800271a638e/8c046cb0b72cd4c99f51b5591cd5b095967f58ee003710a45280c28ee1a9c7fa\n", + is_host_cgroup_ns: true, + expected_result: '8c046cb0b72cd4c99f51b5591cd5b095967f58ee003710a45280c28ee1a9c7fa' + }, + { + description: 'extract container-id from /proc/self/cgroup in private cgroup ns', + proc_self_cgroup_content: "4:blkio:/kubepods/burstable/podfd52ef25-a87d-11e9-9423-0800271a638e/8c046cb0b72cd4c99f51b5591cd5b095967f58ee003710a45280c28ee1a9c7fa\n", + expected_result: '8c046cb0b72cd4c99f51b5591cd5b095967f58ee003710a45280c28ee1a9c7fa' + }, + { + description: 'extract container-id from mountinfo in private cgroup ns', + mount_info_content: "2282 2269 8:1 /var/lib/containerd/io.containerd.grpc.v1.cri/sandboxes/c0a82a3506b0366c9666f6dbe71c783abeb26ba65e312e918a49e10a277196d0/hostname /host/var/run/containerd/io.containerd.runtime.v2.task/k8s.io/fc7038bc73a8d3850c66ddbfb0b2901afa378bfcbb942cc384b051767e4ac6b0/rootfs/etc/hostname rw,nosuid,nodev,relatime - ext4 /dev/sda1 rw,commit=30\n", + expected_result: 'fc7038bc73a8d3850c66ddbfb0b2901afa378bfcbb942cc384b051767e4ac6b0' + }, + { + description: 'extract container-id from mountinfo', + mount_info_content: "2282 2269 8:1 /var/lib/containerd/io.containerd.grpc.v1.cri/sandboxes/c0a82a3506b0366c9666f6dbe71c783abeb26ba65e312e918a49e10a277196d0/hostname /host/var/run/containerd/io.containerd.runtime.v2.task/k8s.io/fc7038bc73a8d3850c66ddbfb0b2901afa378bfcbb942cc384b051767e4ac6b0/rootfs/etc/hostname rw,nosuid,nodev,relatime - ext4 /dev/sda1 rw,commit=30\n", + expected_result: 'fc7038bc73a8d3850c66ddbfb0b2901afa378bfcbb942cc384b051767e4ac6b0', + is_host_cgroup_ns: true + }, + { + description: 'extract inode only in private cgroup ns', + cgroup_node_dir: 'system.slice/docker-abcdef0123456789abcdef0123456789.scope', + proc_self_cgroup_content: "0::/system.slice/docker-abcdef0123456789abcdef0123456789.scope\n", + expected_result: 'in-{inode}' + }, + { + description: 'do not extract inode in host cgroup ns', + cgroup_node_dir: 'system.slice/docker-abcdef0123456789abcdef0123456789.scope', + proc_self_cgroup_content: "0::/system.slice/docker-abcdef0123456789abcdef0123456789.scope\n", + is_host_cgroup_ns: true, + expected_result: nil + } + ] + + test_cases.each_with_index do |test_case, index| + it "handles: #{test_case[:description]}" do + container_id = "" + FakeFS.with_fresh do + FileUtils.mkdir_p('/proc/self') + File.write('/proc/self/cgroup', test_case[:proc_self_cgroup_content].to_s) + File.write('/proc/self/mountinfo', test_case[:mount_info_content].to_s) + + allow(subject).to receive(:is_host_cgroup_namespace?) + .and_return(test_case[:is_host_cgroup_ns] || false) + + if test_case[:cgroup_node_dir] + cgroup_path = File.join('/sys/fs/cgroup', test_case[:cgroup_node_dir]) + FileUtils.mkdir_p(File.dirname(cgroup_path)) + File.write(cgroup_path, '') + + inode = File.stat(cgroup_path).ino + unless test_case[:expected_result].nil? + test_case[:expected_result] = test_case[:expected_result].gsub("{inode}", inode.to_s) + end + end + + container_id = subject.get_container_id(nil, true) + end + + expect(container_id).to eq(test_case[:expected_result]) + end + end + end + +end diff --git a/spec/statsd/serialization/stat_serializer_spec.rb b/spec/statsd/serialization/stat_serializer_spec.rb index 894e79d6..9fde1efb 100644 --- a/spec/statsd/serialization/stat_serializer_spec.rb +++ b/spec/statsd/serialization/stat_serializer_spec.rb @@ -2,7 +2,11 @@ describe Datadog::Statsd::Serialization::StatSerializer do subject do - described_class.new(prefix, global_tags: global_tags) + described_class.new(prefix, container_id, global_tags: global_tags) + end + + let(:container_id) do + nil end let(:prefix) do @@ -93,6 +97,41 @@ end end + context 'when having a container id' do + let(:container_id) do + 'in-23' + end + + it 'adds the container id field correctly' do + expect(subject.format('somecount', 42, 'c')).to eq 'somecount:42|c|c:in-23' + end + end + + context 'when having a prefix, tags, sample_rate and container id' do + let(:prefix) do + 'swag.' + end + + let(:message_tags) do + double('message tags') + end + + let(:container_id) do + 'in-23' + end + + before do + allow(tag_serializer) + .to receive(:format) + .with(message_tags) + .and_return('globaltag1:value1,msgtag2:value2') + end + + it 'adds the tags to the stat correctly' do + expect(subject.format('somecount', 42, 'c', tags: message_tags, sample_rate: 0.5)).to eq 'swag.somecount:42|c|@0.5|c:in-23|#globaltag1:value1,msgtag2:value2' + end + end + context "metric name contains unsupported characters" do it 'does not alter the provided metric name when containing ::' do input = 'somecount::test' diff --git a/spec/statsd_spec.rb b/spec/statsd_spec.rb index 1c659b21..f159818a 100644 --- a/spec/statsd_spec.rb +++ b/spec/statsd_spec.rb @@ -10,6 +10,7 @@ tags: tags, logger: logger, telemetry_flush_interval: -1, + origin_detection: false, ) end From cf25b165033761aa0b9b01bfcf5b21feb3103720 Mon Sep 17 00:00:00 2001 From: Stephen Wakely Date: Thu, 26 Jun 2025 15:33:37 +0100 Subject: [PATCH 02/35] Add external env --- lib/datadog/statsd.rb | 17 ++++++++++++++++- lib/datadog/statsd/serialization/serializer.rb | 4 ++-- .../statsd/serialization/stat_serializer.rb | 7 ++++++- .../serialization/stat_serializer_spec.rb | 16 +++++++++++++++- 4 files changed, 39 insertions(+), 5 deletions(-) diff --git a/lib/datadog/statsd.rb b/lib/datadog/statsd.rb index b7c31731..bb712f7d 100644 --- a/lib/datadog/statsd.rb +++ b/lib/datadog/statsd.rb @@ -123,7 +123,15 @@ def initialize( container_id = OriginDetection .new() .get_container_id(container_id, origin_detection_enabled) - @serializer = Serialization::Serializer.new(prefix: @prefix, container_id: container_id, global_tags: tags) + + external_data = sanitize(ENV['DD_EXTERNAL_ENV']) + + @serializer = Serialization::Serializer.new(prefix: @prefix, + container_id: container_id, + external_data: external_data, + global_tags: tags, + ) + @sample_rate = sample_rate @delay_serialization = delay_serialization @@ -468,5 +476,12 @@ def is_origin_detection_enabled(origin_detection) return true end + + # Sanitize the DD_EXTERNAL_ENV input to ensure it doesn't contain invalid characters + # that may break the protocol. + # Removing any non-printable characters and `|`. + def sanitize(external_data) + external_data.gsub(/[^[:print:]]|`\|/, '') unless external_data.nil? + end end end diff --git a/lib/datadog/statsd/serialization/serializer.rb b/lib/datadog/statsd/serialization/serializer.rb index f0b2bcfd..79923296 100644 --- a/lib/datadog/statsd/serialization/serializer.rb +++ b/lib/datadog/statsd/serialization/serializer.rb @@ -6,8 +6,8 @@ module Datadog class Statsd module Serialization class Serializer - def initialize(prefix: nil, container_id: nil, global_tags: []) - @stat_serializer = StatSerializer.new(prefix, container_id, global_tags: global_tags) + def initialize(prefix: nil, container_id: nil, external_data: nil, global_tags: []) + @stat_serializer = StatSerializer.new(prefix, container_id, external_data, global_tags: global_tags) @service_check_serializer = ServiceCheckSerializer.new(global_tags: global_tags) @event_serializer = EventSerializer.new(global_tags: global_tags) end diff --git a/lib/datadog/statsd/serialization/stat_serializer.rb b/lib/datadog/statsd/serialization/stat_serializer.rb index 4d5f09e9..1841bd03 100644 --- a/lib/datadog/statsd/serialization/stat_serializer.rb +++ b/lib/datadog/statsd/serialization/stat_serializer.rb @@ -4,10 +4,11 @@ module Datadog class Statsd module Serialization class StatSerializer - def initialize(prefix, container_id, global_tags: []) + def initialize(prefix, container_id, external_data, global_tags: []) @prefix = prefix @prefix_str = prefix.to_s @container_id = container_id + @external_data = external_data @tag_serializer = TagSerializer.new(global_tags) end @@ -17,6 +18,10 @@ def format_fields field << "|c:#{@container_id}" end + unless @external_data.nil? + field << "|e:#{@external_data}" + end + field end diff --git a/spec/statsd/serialization/stat_serializer_spec.rb b/spec/statsd/serialization/stat_serializer_spec.rb index 9fde1efb..c4dd9928 100644 --- a/spec/statsd/serialization/stat_serializer_spec.rb +++ b/spec/statsd/serialization/stat_serializer_spec.rb @@ -2,7 +2,11 @@ describe Datadog::Statsd::Serialization::StatSerializer do subject do - described_class.new(prefix, container_id, global_tags: global_tags) + described_class.new(prefix, container_id, external_data, global_tags: global_tags) + end + + let(:external_data) do + nil end let(:container_id) do @@ -132,6 +136,16 @@ end end + context 'when having a external data' do + let(:external_data) do + 'cn-SomeKindOfContainerName' + end + + it 'adds the external data field correctly' do + expect(subject.format('somecount', 42, 'c')).to eq 'somecount:42|c|e:cn-SomeKindOfContainerName' + end + end + context "metric name contains unsupported characters" do it 'does not alter the provided metric name when containing ::' do input = 'somecount::test' From 7dee16ab1c8eadba1b7cce741d459d6e41db9ba2 Mon Sep 17 00:00:00 2001 From: Stephen Wakely Date: Fri, 27 Jun 2025 12:05:29 +0100 Subject: [PATCH 03/35] Add tag cardinality option --- lib/datadog/statsd.rb | 24 +++++++++++++++---- .../statsd/serialization/serializer.rb | 4 ++-- .../statsd/serialization/stat_serializer.rb | 16 ++++++++++--- .../serialization/stat_serializer_spec.rb | 17 ++++++++++++- 4 files changed, 51 insertions(+), 10 deletions(-) diff --git a/lib/datadog/statsd.rb b/lib/datadog/statsd.rb index bb712f7d..a37f327f 100644 --- a/lib/datadog/statsd.rb +++ b/lib/datadog/statsd.rb @@ -85,6 +85,7 @@ def tags # @option [Boolean] delay_serialization delays stat serialization # @option [Boolean] origin_detection is origin detection enabled # @option [String] container_id the container ID field, used for origin detection + # @option [String] cardinality the default tag cardinality to use def initialize( host = nil, port = nil, @@ -110,7 +111,8 @@ def initialize( telemetry_flush_interval: DEFAULT_TELEMETRY_FLUSH_INTERVAL, origin_detection: true, - container_id: nil + container_id: nil, + cardinality: nil ) unless tags.nil? || tags.is_a?(Array) || tags.is_a?(Hash) raise ArgumentError, 'tags must be an array of string tags or a Hash' @@ -119,7 +121,7 @@ def initialize( @namespace = namespace @prefix = @namespace ? "#{@namespace}.".freeze : nil - origin_detection_enabled = is_origin_detection_enabled(origin_detection) + origin_detection_enabled = origin_detection_enabled?(origin_detection) container_id = OriginDetection .new() .get_container_id(container_id, origin_detection_enabled) @@ -132,6 +134,8 @@ def initialize( global_tags: tags, ) + @cardinality = cardinality || ENV['DD_CARDINALITY'] || ENV['DATADOG_CARDINALITY'] + @sample_rate = sample_rate @delay_serialization = delay_serialization @@ -178,6 +182,7 @@ def self.open(*args, **kwargs) # @option opts [Boolean] :pre_sampled If true, the client assumes the caller has already sampled metrics at :sample_rate, and doesn't perform sampling. # @option opts [Array] :tags An array of tags # @option opts [Numeric] :by increment value, default 1 + # @option opts [String] :cardinality The tag cardinality to use # @see #count def increment(stat, opts = EMPTY_OPTIONS) opts = { sample_rate: opts } if opts.is_a?(Numeric) @@ -193,6 +198,7 @@ def increment(stat, opts = EMPTY_OPTIONS) # @option opts [Boolean] :pre_sampled If true, the client assumes the caller has already sampled metrics at :sample_rate, and doesn't perform sampling. # @option opts [Array] :tags An array of tags # @option opts [Numeric] :by decrement value, default 1 + # @option opts [String] :cardinality The tag cardinality to use # @see #count def decrement(stat, opts = EMPTY_OPTIONS) opts = { sample_rate: opts } if opts.is_a?(Numeric) @@ -208,6 +214,7 @@ def decrement(stat, opts = EMPTY_OPTIONS) # @option opts [Numeric] :sample_rate sample rate, 1 for always # @option opts [Boolean] :pre_sampled If true, the client assumes the caller has already sampled metrics at :sample_rate, and doesn't perform sampling. # @option opts [Array] :tags An array of tags + # @option opts [String] :cardinality The tag cardinality to use def count(stat, count, opts = EMPTY_OPTIONS) opts = { sample_rate: opts } if opts.is_a?(Numeric) send_stats(stat, count, COUNTER_TYPE, opts) @@ -225,6 +232,7 @@ def count(stat, count, opts = EMPTY_OPTIONS) # @option opts [Numeric] :sample_rate sample rate, 1 for always # @option opts [Boolean] :pre_sampled If true, the client assumes the caller has already sampled metrics at :sample_rate, and doesn't perform sampling. # @option opts [Array] :tags An array of tags + # @option opts [String] :cardinality The tag cardinality to use # @example Report the current user count: # $statsd.gauge('user.count', User.count) def gauge(stat, value, opts = EMPTY_OPTIONS) @@ -240,6 +248,7 @@ def gauge(stat, value, opts = EMPTY_OPTIONS) # @option opts [Numeric] :sample_rate sample rate, 1 for always # @option opts [Boolean] :pre_sampled If true, the client assumes the caller has already sampled metrics at :sample_rate, and doesn't perform sampling. # @option opts [Array] :tags An array of tags + # @option opts [String] :cardinality The tag cardinality to use # @example Report the current user count: # $statsd.histogram('user.count', User.count) def histogram(stat, value, opts = EMPTY_OPTIONS) @@ -254,6 +263,7 @@ def histogram(stat, value, opts = EMPTY_OPTIONS) # @option opts [Numeric] :sample_rate sample rate, 1 for always # @option opts [Boolean] :pre_sampled If true, the client assumes the caller has already sampled metrics at :sample_rate, and doesn't perform sampling. # @option opts [Array] :tags An array of tags + # @option opts [String] :cardinality The tag cardinality to use # @example Report the current user count: # $statsd.distribution('user.count', User.count) def distribution(stat, value, opts = EMPTY_OPTIONS) @@ -270,6 +280,7 @@ def distribution(stat, value, opts = EMPTY_OPTIONS) # @param [Hash] opts the options to create the metric with # @option opts [Numeric] :sample_rate sample rate, 1 for always # @option opts [Array] :tags An array of tags + # @option opts [String] :cardinality The tag cardinality to use # @example Report the time (in ms) taken to activate an account # $statsd.distribution_time('account.activate') { @account.activate! } def distribution_time(stat, opts = EMPTY_OPTIONS) @@ -291,6 +302,7 @@ def distribution_time(stat, opts = EMPTY_OPTIONS) # @option opts [Numeric] :sample_rate sample rate, 1 for always # @option opts [Boolean] :pre_sampled If true, the client assumes the caller has already sampled metrics at :sample_rate, and doesn't perform sampling. # @option opts [Array] :tags An array of tags + # @option opts [String] :cardinality The tag cardinality to use def timing(stat, ms, opts = EMPTY_OPTIONS) opts = { sample_rate: opts } if opts.is_a?(Numeric) send_stats(stat, ms, TIMING_TYPE, opts) @@ -306,6 +318,7 @@ def timing(stat, ms, opts = EMPTY_OPTIONS) # @option opts [Numeric] :sample_rate sample rate, 1 for always # @option opts [Boolean] :pre_sampled If true, the client assumes the caller has already sampled metrics at :sample_rate, and doesn't perform sampling. # @option opts [Array] :tags An array of tags + # @option opts [String] :cardinality The tag cardinality to use # @yield The operation to be timed # @see #timing # @example Report the time (in ms) taken to activate an account @@ -326,6 +339,7 @@ def time(stat, opts = EMPTY_OPTIONS) # @option opts [Numeric] :sample_rate sample rate, 1 for always # @option opts [Boolean] :pre_sampled If true, the client assumes the caller has already sampled metrics at :sample_rate, and doesn't perform sampling. # @option opts [Array] :tags An array of tags + # @option opts [String] :cardinality The tag cardinality to use # @example Record a unique visitory by id: # $statsd.set('visitors.uniques', User.id) def set(stat, value, opts = EMPTY_OPTIONS) @@ -367,6 +381,7 @@ def service_check(name, status, opts = EMPTY_OPTIONS) # @option opts [String, nil] :alert_type ('info') Can be "error", "warning", "info" or "success". # @option opts [Boolean, false] :truncate_if_too_long (false) Truncate the event if it is too long # @option opts [Array] :tags tags to be added to every metric + # @option opts [String] :cardinality The tag cardinality to use # @example Report an awful event: # $statsd.event('Something terrible happened', 'The end is near if we do nothing', :alert_type=>'warning', :tags=>['end_of_times','urgent']) def event(title, text, opts = EMPTY_OPTIONS) @@ -446,20 +461,21 @@ def send_stats(stat, delta, type, opts = EMPTY_OPTIONS) telemetry.sent(metrics: 1) if telemetry sample_rate = opts[:sample_rate] || @sample_rate || 1 + cardinality = opts[:cardinality] || @cardinality if sample_rate == 1 || opts[:pre_sampled] || rand <= sample_rate full_stat = if @delay_serialization [stat, delta, type, opts[:tags], sample_rate] else - serializer.to_stat(stat, delta, type, tags: opts[:tags], sample_rate: sample_rate) + serializer.to_stat(stat, delta, type, tags: opts[:tags], sample_rate: sample_rate, cardinality: cardinality) end forwarder.send_message(full_stat) end end - def is_origin_detection_enabled(origin_detection) + def origin_detection_enabled?(origin_detection) if !origin_detection.nil? && !origin_detection return false end diff --git a/lib/datadog/statsd/serialization/serializer.rb b/lib/datadog/statsd/serialization/serializer.rb index 79923296..651c9a7d 100644 --- a/lib/datadog/statsd/serialization/serializer.rb +++ b/lib/datadog/statsd/serialization/serializer.rb @@ -13,8 +13,8 @@ def initialize(prefix: nil, container_id: nil, external_data: nil, global_tags: end # using *args would make new allocations - def to_stat(name, delta, type, tags: [], sample_rate: 1) - stat_serializer.format(name, delta, type, tags: tags, sample_rate: sample_rate) + def to_stat(name, delta, type, tags: [], sample_rate: 1, cardinality: nil) + stat_serializer.format(name, delta, type, tags: tags, sample_rate: sample_rate, cardinality: cardinality) end # using *args would make new allocations diff --git a/lib/datadog/statsd/serialization/stat_serializer.rb b/lib/datadog/statsd/serialization/stat_serializer.rb index 1841bd03..56ef2502 100644 --- a/lib/datadog/statsd/serialization/stat_serializer.rb +++ b/lib/datadog/statsd/serialization/stat_serializer.rb @@ -4,6 +4,8 @@ module Datadog class Statsd module Serialization class StatSerializer + VALID_CARDINALITY = [:none, :low, :orchestrator, :high] + def initialize(prefix, container_id, external_data, global_tags: []) @prefix = prefix @prefix_str = prefix.to_s @@ -12,7 +14,7 @@ def initialize(prefix, container_id, external_data, global_tags: []) @tag_serializer = TagSerializer.new(global_tags) end - def format_fields + def format_fields(cardinality) field = String.new unless @container_id.nil? field << "|c:#{@container_id}" @@ -22,12 +24,20 @@ def format_fields field << "|e:#{@external_data}" end + unless cardinality.nil? + unless VALID_CARDINALITY.include?(cardinality.to_sym) + raise ArgumentError, "Invalid cardinality #{cardinality}. Valid options are #{VALID_CARDINALITY.join(', ')}." + end + + field << "|card:#{cardinality}" + end + field end - def format(metric_name, delta, type, tags: [], sample_rate: 1) + def format(metric_name, delta, type, tags: [], sample_rate: 1, cardinality: nil) metric_name = formatted_metric_name(metric_name) - fields = format_fields + fields = format_fields(cardinality) if sample_rate != 1 if tags_list = tag_serializer.format(tags) diff --git a/spec/statsd/serialization/stat_serializer_spec.rb b/spec/statsd/serialization/stat_serializer_spec.rb index c4dd9928..ecd6617b 100644 --- a/spec/statsd/serialization/stat_serializer_spec.rb +++ b/spec/statsd/serialization/stat_serializer_spec.rb @@ -136,7 +136,7 @@ end end - context 'when having a external data' do + context 'when having external data' do let(:external_data) do 'cn-SomeKindOfContainerName' end @@ -162,6 +162,21 @@ end end + context 'when having cardinality' do + it 'adds the cardinality field correctly' do + expect(subject.format('somecount', 42, 'c', cardinality: :high)).to eq 'somecount:42|c|card:high' + expect(subject.format('somecount', 42, 'c', cardinality: 'none')).to eq 'somecount:42|c|card:none' + end + + it 'validates the cardinality is correct' do + expect { + subject.format('somecount', 42, 'c', cardinality: :potato) + }.to raise_error(ArgumentError) do |e| + expect(e.message).to eq 'Invalid cardinality potato. Valid options are none, low, orchestrator, high.' + end + end + end + context "when metric name is a symbol" do it 'formats correctly without error' do input = :'somecount::test' From c28b06a89904556e12c2d836402cdf3e99e7a503 Mon Sep 17 00:00:00 2001 From: Stephen Wakely Date: Wed, 2 Jul 2025 17:10:24 +0100 Subject: [PATCH 04/35] Serialize fields --- lib/datadog/statsd/serialization.rb | 1 + .../statsd/serialization/event_serializer.rb | 8 +++- .../statsd/serialization/field_serializer.rb | 37 +++++++++++++++++++ .../statsd/serialization/stat_serializer.rb | 27 ++------------ .../serialization/event_serializer_spec.rb | 26 ++++++++++++- 5 files changed, 73 insertions(+), 26 deletions(-) create mode 100644 lib/datadog/statsd/serialization/field_serializer.rb diff --git a/lib/datadog/statsd/serialization.rb b/lib/datadog/statsd/serialization.rb index b0717fcc..53de39e1 100644 --- a/lib/datadog/statsd/serialization.rb +++ b/lib/datadog/statsd/serialization.rb @@ -10,6 +10,7 @@ module Serialization require_relative 'serialization/tag_serializer' require_relative 'serialization/service_check_serializer' require_relative 'serialization/event_serializer' +require_relative 'serialization/field_serializer' require_relative 'serialization/stat_serializer' require_relative 'serialization/serializer' diff --git a/lib/datadog/statsd/serialization/event_serializer.rb b/lib/datadog/statsd/serialization/event_serializer.rb index 960589e2..c0611a93 100644 --- a/lib/datadog/statsd/serialization/event_serializer.rb +++ b/lib/datadog/statsd/serialization/event_serializer.rb @@ -13,8 +13,9 @@ class EventSerializer alert_type: 't:', }.freeze - def initialize(global_tags: []) + def initialize(container_id, external_data, global_tags: []) @tag_serializer = TagSerializer.new(global_tags) + @field_serializer = FieldSerializer.new(container_id, external_data) end def format(title, text, options = EMPTY_OPTIONS) @@ -47,6 +48,10 @@ def format(title, text, options = EMPTY_OPTIONS) event << tags end + if fields = field_serializer.format(options[:cardinality]) + event << fields + end + if event.bytesize > MAX_EVENT_SIZE if options[:truncate_if_too_long] event.slice!(MAX_EVENT_SIZE..event.length) @@ -59,6 +64,7 @@ def format(title, text, options = EMPTY_OPTIONS) protected attr_reader :tag_serializer + attr_reader :field_serializer def escape(text) text.delete('|').tap do |t| diff --git a/lib/datadog/statsd/serialization/field_serializer.rb b/lib/datadog/statsd/serialization/field_serializer.rb new file mode 100644 index 00000000..fe7ac34c --- /dev/null +++ b/lib/datadog/statsd/serialization/field_serializer.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +module Datadog + class Statsd + module Serialization + class FieldSerializer + VALID_CARDINALITY = [:none, :low, :orchestrator, :high] + + def initialize(container_id, external_data) + @container_id = container_id + @external_data = external_data + end + + def format(cardinality) + field = String.new + unless @container_id.nil? + field << "|c:#{@container_id}" + end + + unless @external_data.nil? + field << "|e:#{@external_data}" + end + + unless cardinality.nil? + unless VALID_CARDINALITY.include?(cardinality.to_sym) + raise ArgumentError, "Invalid cardinality #{cardinality}. Valid options are #{VALID_CARDINALITY.join(', ')}." + end + + field << "|card:#{cardinality}" + end + + field + end + end + end + end +end diff --git a/lib/datadog/statsd/serialization/stat_serializer.rb b/lib/datadog/statsd/serialization/stat_serializer.rb index 56ef2502..c6e3195e 100644 --- a/lib/datadog/statsd/serialization/stat_serializer.rb +++ b/lib/datadog/statsd/serialization/stat_serializer.rb @@ -4,40 +4,18 @@ module Datadog class Statsd module Serialization class StatSerializer - VALID_CARDINALITY = [:none, :low, :orchestrator, :high] - def initialize(prefix, container_id, external_data, global_tags: []) @prefix = prefix @prefix_str = prefix.to_s @container_id = container_id @external_data = external_data @tag_serializer = TagSerializer.new(global_tags) - end - - def format_fields(cardinality) - field = String.new - unless @container_id.nil? - field << "|c:#{@container_id}" - end - - unless @external_data.nil? - field << "|e:#{@external_data}" - end - - unless cardinality.nil? - unless VALID_CARDINALITY.include?(cardinality.to_sym) - raise ArgumentError, "Invalid cardinality #{cardinality}. Valid options are #{VALID_CARDINALITY.join(', ')}." - end - - field << "|card:#{cardinality}" - end - - field + @field_serializer = FieldSerializer.new(container_id, external_data) end def format(metric_name, delta, type, tags: [], sample_rate: 1, cardinality: nil) metric_name = formatted_metric_name(metric_name) - fields = format_fields(cardinality) + fields = field_serializer.format(cardinality) if sample_rate != 1 if tags_list = tag_serializer.format(tags) @@ -62,6 +40,7 @@ def global_tags attr_reader :prefix attr_reader :tag_serializer + attr_reader :field_serializer if RUBY_VERSION < '3' def metric_name_to_string(metric_name) diff --git a/spec/statsd/serialization/event_serializer_spec.rb b/spec/statsd/serialization/event_serializer_spec.rb index e675d162..256fe99e 100644 --- a/spec/statsd/serialization/event_serializer_spec.rb +++ b/spec/statsd/serialization/event_serializer_spec.rb @@ -2,7 +2,15 @@ describe Datadog::Statsd::Serialization::EventSerializer do subject do - described_class.new(global_tags: global_tags) + described_class.new(container_id, external_data, global_tags: global_tags) + end + + let(:external_data) do + nil + end + + let(:container_id) do + nil end let(:global_tags) do @@ -145,6 +153,22 @@ end end + context 'when having a container id' do + let(:container_id) do + 'in-23' + end + + it 'adds the container id field correctly' do + expect(subject.format('this is a title', 'this is a longer text')) + .to eq '_e{15,21}:this is a title|this is a longer text|c:in-23' + end + + it 'adds the cardinality field correctly' do + expect(subject.format('this is a title', 'this is a longer text', cardinality: :low)) + .to eq '_e{15,21}:this is a title|this is a longer text|c:in-23|card:low' + end + end + context 'when having several parameters (hostname, alert_type, priority, source_type, timestamp, tags)' do let(:message_tags) do double('message tags') From 4df07b8f4480c3b2a2bc26b47d2e2989b49b5f6a Mon Sep 17 00:00:00 2001 From: Stephen Wakely Date: Thu, 3 Jul 2025 14:32:47 +0100 Subject: [PATCH 05/35] Update event serializer params --- lib/datadog/statsd/serialization/serializer.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/datadog/statsd/serialization/serializer.rb b/lib/datadog/statsd/serialization/serializer.rb index 651c9a7d..9d800448 100644 --- a/lib/datadog/statsd/serialization/serializer.rb +++ b/lib/datadog/statsd/serialization/serializer.rb @@ -9,7 +9,7 @@ class Serializer def initialize(prefix: nil, container_id: nil, external_data: nil, global_tags: []) @stat_serializer = StatSerializer.new(prefix, container_id, external_data, global_tags: global_tags) @service_check_serializer = ServiceCheckSerializer.new(global_tags: global_tags) - @event_serializer = EventSerializer.new(global_tags: global_tags) + @event_serializer = EventSerializer.new(container_id, external_data, global_tags: global_tags) end # using *args would make new allocations From 5b48dd25c7ca67084c121878983e3088f7e0f974 Mon Sep 17 00:00:00 2001 From: Stephen Wakely Date: Fri, 4 Jul 2025 11:25:19 +0100 Subject: [PATCH 06/35] Add fields to service check --- .../statsd/serialization/serializer.rb | 2 +- .../serialization/service_check_serializer.rb | 8 +++++- spec/integrations/allocation_spec.rb | 24 ++++++++--------- .../service_check_serializer_spec.rb | 26 ++++++++++++++++++- 4 files changed, 45 insertions(+), 15 deletions(-) diff --git a/lib/datadog/statsd/serialization/serializer.rb b/lib/datadog/statsd/serialization/serializer.rb index 9d800448..f4bdcfed 100644 --- a/lib/datadog/statsd/serialization/serializer.rb +++ b/lib/datadog/statsd/serialization/serializer.rb @@ -8,7 +8,7 @@ module Serialization class Serializer def initialize(prefix: nil, container_id: nil, external_data: nil, global_tags: []) @stat_serializer = StatSerializer.new(prefix, container_id, external_data, global_tags: global_tags) - @service_check_serializer = ServiceCheckSerializer.new(global_tags: global_tags) + @service_check_serializer = ServiceCheckSerializer.new(container_id, external_data, global_tags: global_tags) @event_serializer = EventSerializer.new(container_id, external_data, global_tags: global_tags) end diff --git a/lib/datadog/statsd/serialization/service_check_serializer.rb b/lib/datadog/statsd/serialization/service_check_serializer.rb index 828098c4..8248f86f 100644 --- a/lib/datadog/statsd/serialization/service_check_serializer.rb +++ b/lib/datadog/statsd/serialization/service_check_serializer.rb @@ -9,8 +9,9 @@ class ServiceCheckSerializer hostname: 'h:', }.freeze - def initialize(global_tags: []) + def initialize(container_id, external_data, global_tags: []) @tag_serializer = TagSerializer.new(global_tags) + @field_serializer = FieldSerializer.new(container_id, external_data) end def format(name, status, options = EMPTY_OPTIONS) @@ -42,11 +43,16 @@ def format(name, status, options = EMPTY_OPTIONS) service_check << '|#' service_check << tags end + + if fields = field_serializer.format(options[:cardinality]) + service_check << fields + end end end protected attr_reader :tag_serializer + attr_reader :field_serializer def escape_message(message) message.delete('|').tap do |m| diff --git a/spec/integrations/allocation_spec.rb b/spec/integrations/allocation_spec.rb index 9fcbceca..c7a7491c 100644 --- a/spec/integrations/allocation_spec.rb +++ b/spec/integrations/allocation_spec.rb @@ -203,13 +203,13 @@ let(:expected_allocations) do if RUBY_VERSION < '2.4.0' - 22 + 23 elsif RUBY_VERSION < '2.5.0' - 21 + 22 elsif RUBY_VERSION < '2.6.0' - 20 + 21 else - 19 + 20 end end @@ -234,13 +234,13 @@ let(:expected_allocations) do if RUBY_VERSION < '2.4.0' - 14 + 15 elsif RUBY_VERSION < '2.5.0' - 13 + 14 elsif RUBY_VERSION < '2.6.0' - 12 + 13 else - 11 + 12 end end @@ -283,13 +283,13 @@ let(:expected_allocations) do if RUBY_VERSION < '2.4.0' - 18 + 19 elsif RUBY_VERSION < '2.5.0' - 17 + 18 elsif RUBY_VERSION < '2.6.0' - 16 + 17 else - 15 + 16 end end diff --git a/spec/statsd/serialization/service_check_serializer_spec.rb b/spec/statsd/serialization/service_check_serializer_spec.rb index ee755a44..522b1eb3 100644 --- a/spec/statsd/serialization/service_check_serializer_spec.rb +++ b/spec/statsd/serialization/service_check_serializer_spec.rb @@ -2,7 +2,15 @@ describe Datadog::Statsd::Serialization::ServiceCheckSerializer do subject do - described_class.new(global_tags: global_tags) + described_class.new(container_id, external_data, global_tags: global_tags) + end + + let(:external_data) do + nil + end + + let(:container_id) do + nil end let(:global_tags) do @@ -87,6 +95,22 @@ end end + context 'when having a container id' do + let(:container_id) do + 'in-23' + end + + it 'adds the container id field correctly' do + expect(subject.format('windmill', 'grinding')) + .to eq '_sc|windmill|grinding|c:in-23' + end + + it 'adds the cardinality field correctly' do + expect(subject.format('windmill', 'grinding', cardinality: :low)) + .to eq '_sc|windmill|grinding|c:in-23|card:low' + end + end + context 'when having several parameters (hostname, message, timestamp, tags)' do let(:message_tags) do double('message tags') From abb3b790b27d30407f7d852bb350262d4d6bac06 Mon Sep 17 00:00:00 2001 From: Stephen Wakely Date: Mon, 7 Jul 2025 10:29:27 +0100 Subject: [PATCH 07/35] Use older fakefs version to test against Ruby v2 --- Gemfile | 2 +- spec/statsd/origin_detection_spec.rb | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/Gemfile b/Gemfile index be8f42ef..b1e5d449 100644 --- a/Gemfile +++ b/Gemfile @@ -27,5 +27,5 @@ end group :test do gem 'mocha' - gem 'fakefs', '~> 3.0' + gem 'fakefs', '~> 0.13.3' end diff --git a/spec/statsd/origin_detection_spec.rb b/spec/statsd/origin_detection_spec.rb index aa6a3f52..57d4ba7b 100644 --- a/spec/statsd/origin_detection_spec.rb +++ b/spec/statsd/origin_detection_spec.rb @@ -1,6 +1,15 @@ require 'spec_helper' require 'fakefs/safe' +# To ensure that we can against Ruby v2 we need to use quite an old +# version of fakefs that doesn't provide the ino function. +# Monkey patch a version here. +class FakeFS::File::Stat + def ino + 42 + end +end + describe Datadog::Statsd::OriginDetection do subject do described_class.new() From a50fafbaa1aeaf090f69923c6ff4db613d2d7284 Mon Sep 17 00:00:00 2001 From: Stephen Wakely Date: Mon, 7 Jul 2025 11:32:17 +0100 Subject: [PATCH 08/35] Ruby 2.1 compatible multiline strings --- spec/statsd/origin_detection_spec.rb | 134 +++++++++++++-------------- 1 file changed, 67 insertions(+), 67 deletions(-) diff --git a/spec/statsd/origin_detection_spec.rb b/spec/statsd/origin_detection_spec.rb index 57d4ba7b..99dc3fba 100644 --- a/spec/statsd/origin_detection_spec.rb +++ b/spec/statsd/origin_detection_spec.rb @@ -30,9 +30,9 @@ def ino container_id = nil FakeFS.with_fresh do FileUtils.mkdir_p('/proc/self') - File.write('/proc/self/cgroup', <<~CGROUP) - 4:blkio:/kubepods/burstable/podfd52ef25-a87d-11e9-9423-0800271a638e/8c046cb0b72cd4c99f51b5591cd5b095967f58ee003710a45280c28ee1a9c7fa - CGROUP + File.write('/proc/self/cgroup', < Date: Mon, 7 Jul 2025 11:52:59 +0100 Subject: [PATCH 09/35] Only use old fakefs on old versions --- Gemfile | 6 +++++- spec/statsd/origin_detection_spec.rb | 8 +++++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/Gemfile b/Gemfile index b1e5d449..65690b55 100644 --- a/Gemfile +++ b/Gemfile @@ -27,5 +27,9 @@ end group :test do gem 'mocha' - gem 'fakefs', '~> 0.13.3' + if RUBY_VERSION < '3.0' + gem 'fakefs', '~> 0.13.3' + else + gem 'fakefs', '~> 3.0' + end end diff --git a/spec/statsd/origin_detection_spec.rb b/spec/statsd/origin_detection_spec.rb index 99dc3fba..8446ea92 100644 --- a/spec/statsd/origin_detection_spec.rb +++ b/spec/statsd/origin_detection_spec.rb @@ -4,9 +4,11 @@ # To ensure that we can against Ruby v2 we need to use quite an old # version of fakefs that doesn't provide the ino function. # Monkey patch a version here. -class FakeFS::File::Stat - def ino - 42 +if RUBY_VERSION < '3.0' + class FakeFS::File::Stat + def ino + 42 + end end end From deee15437322dc58f1fbc23cf0bff2c65f0edb3c Mon Sep 17 00:00:00 2001 From: Stephen Wakely Date: Mon, 7 Jul 2025 11:56:59 +0100 Subject: [PATCH 10/35] Missed multiline --- spec/statsd/origin_detection_spec.rb | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/spec/statsd/origin_detection_spec.rb b/spec/statsd/origin_detection_spec.rb index 8446ea92..65ec8bbc 100644 --- a/spec/statsd/origin_detection_spec.rb +++ b/spec/statsd/origin_detection_spec.rb @@ -88,11 +88,11 @@ def ino expected: '34dc0b5e626f2c5c4c5170e34b10e7654ce36f0fcd532739f4445baabea03376' }, { - input: <<~CGROUP, - 1:name=systemd:/nope - 2:pids:/docker/34dc0b5e626f2c5c4c5170e34b10e7654ce36f0fcd532739f4445baabea03376 - 3:cpu:/invalid - CGROUP + input: < Date: Mon, 7 Jul 2025 12:07:46 +0100 Subject: [PATCH 11/35] Further fixes for windows --- lib/datadog/statsd/origin_detection.rb | 4 +++- spec/integrations/allocation_spec.rb | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/datadog/statsd/origin_detection.rb b/lib/datadog/statsd/origin_detection.rb index a3e97bca..37c3d3d5 100644 --- a/lib/datadog/statsd/origin_detection.rb +++ b/lib/datadog/statsd/origin_detection.rb @@ -36,7 +36,9 @@ def parse_cgroup_node_path(lines) end def get_cgroup_inode(cgroup_mount_path, proc_self_cgroup_path) - content = File.read(proc_self_cgroup_path) + content = File.read(proc_self_cgroup_path) rescue nil + return nil unless content.nil? + controllers = parse_cgroup_node_path(content) [CGROUPV1BASECONTROLLER, ''].each do |controller| diff --git a/spec/integrations/allocation_spec.rb b/spec/integrations/allocation_spec.rb index c7a7491c..d4e917a6 100644 --- a/spec/integrations/allocation_spec.rb +++ b/spec/integrations/allocation_spec.rb @@ -101,7 +101,7 @@ elsif RUBY_VERSION < '2.6.0' 26 else - 25 + 26 end end From 9c182b25d96404471218aa0d240d359a46b22941 Mon Sep 17 00:00:00 2001 From: Stephen Wakely Date: Mon, 7 Jul 2025 12:23:31 +0100 Subject: [PATCH 12/35] Fix content logic --- lib/datadog/statsd/origin_detection.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/datadog/statsd/origin_detection.rb b/lib/datadog/statsd/origin_detection.rb index 37c3d3d5..5cfcdbdf 100644 --- a/lib/datadog/statsd/origin_detection.rb +++ b/lib/datadog/statsd/origin_detection.rb @@ -37,7 +37,7 @@ def parse_cgroup_node_path(lines) def get_cgroup_inode(cgroup_mount_path, proc_self_cgroup_path) content = File.read(proc_self_cgroup_path) rescue nil - return nil unless content.nil? + return nil unless content controllers = parse_cgroup_node_path(content) From 61117426eae94a910b730e9b9f277c2809ff9855 Mon Sep 17 00:00:00 2001 From: Stephen Wakely Date: Mon, 7 Jul 2025 12:55:34 +0100 Subject: [PATCH 13/35] Fudge allocation test --- spec/integrations/allocation_spec.rb | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/spec/integrations/allocation_spec.rb b/spec/integrations/allocation_spec.rb index d4e917a6..b92056da 100644 --- a/spec/integrations/allocation_spec.rb +++ b/spec/integrations/allocation_spec.rb @@ -259,9 +259,17 @@ elsif RUBY_VERSION < '2.5.0' 29 elsif RUBY_VERSION < '2.6.0' - 28 + if RUBY_PLATFORM.include?('linux') + 28 + else + 29 + end else - 27 + if RUBY_PLATFORM.include?('linux') + 27 + else + 28 + end end end @@ -339,9 +347,17 @@ elsif RUBY_VERSION < '2.5.0' 25 elsif RUBY_VERSION < '2.6.0' - 24 + if RUBY_PLATFORM.include?('linux') + 24 + else + 25 + end else - 23 + if RUBY_PLATFORM.include?('linux') + 23 + else + 24 + end end end From 6282d13ac3d519a365657d64d33706a96f81b2d0 Mon Sep 17 00:00:00 2001 From: Stephen Wakely Date: Mon, 7 Jul 2025 12:59:32 +0100 Subject: [PATCH 14/35] Increase timing for macos --- spec/statsd/timer_spec.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spec/statsd/timer_spec.rb b/spec/statsd/timer_spec.rb index 08aa7c05..56ef8fb4 100644 --- a/spec/statsd/timer_spec.rb +++ b/spec/statsd/timer_spec.rb @@ -50,8 +50,8 @@ second_call_time = call_times.pop # the third call is made immediatelly after the second call third_call_time = call_times.pop - expect(second_call_time - first_call_time).to be_within(0.02).of(interval * 2) - expect(third_call_time - second_call_time).to be_within(0.02).of(0) + expect(second_call_time - first_call_time).to be_within(0.03).of(interval * 2) + expect(third_call_time - second_call_time).to be_within(0.03).of(0) end end end From 280b919c5a3c995c219866de13b38e7f6a0d9210 Mon Sep 17 00:00:00 2001 From: Stephen Wakely Date: Mon, 7 Jul 2025 15:51:22 +0100 Subject: [PATCH 15/35] Try avoiding allocation --- lib/datadog/statsd/serialization/field_serializer.rb | 12 ++++++------ spec/integrations/allocation_spec.rb | 1 + 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/lib/datadog/statsd/serialization/field_serializer.rb b/lib/datadog/statsd/serialization/field_serializer.rb index fe7ac34c..9fab6ff7 100644 --- a/lib/datadog/statsd/serialization/field_serializer.rb +++ b/lib/datadog/statsd/serialization/field_serializer.rb @@ -12,14 +12,14 @@ def initialize(container_id, external_data) end def format(cardinality) - field = String.new - unless @container_id.nil? - field << "|c:#{@container_id}" + if @container_id.nil? && @external_data.nil? && cardinality.nil? + # Avoid the allocation unless needed. + return nil end - unless @external_data.nil? - field << "|e:#{@external_data}" - end + field = String.new + field << "|c:#{@container_id}" unless @container_id.nil? + field << "|e:#{@external_data}" unless @external_data.nil? unless cardinality.nil? unless VALID_CARDINALITY.include?(cardinality.to_sym) diff --git a/spec/integrations/allocation_spec.rb b/spec/integrations/allocation_spec.rb index b92056da..42ac37b8 100644 --- a/spec/integrations/allocation_spec.rb +++ b/spec/integrations/allocation_spec.rb @@ -1,6 +1,7 @@ # frozen_string_literal: true require 'spec_helper' +require 'objspace' describe 'Allocations and garbage collection' do before do From 571458d69531deaaa95750f294696f7bfeaf6843 Mon Sep 17 00:00:00 2001 From: Stephen Wakely Date: Mon, 7 Jul 2025 16:11:46 +0100 Subject: [PATCH 16/35] Trace the allocations --- spec/integrations/allocation_spec.rb | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/spec/integrations/allocation_spec.rb b/spec/integrations/allocation_spec.rb index 42ac37b8..a39db595 100644 --- a/spec/integrations/allocation_spec.rb +++ b/spec/integrations/allocation_spec.rb @@ -276,8 +276,23 @@ it 'produces low amounts of garbage' do expect do + ObjectSpace.trace_object_allocations_start + subject.event('foobar', 'happening', tags: { something: 'a value' }) subject.flush(sync: true) + + ObjectSpace.trace_object_allocations_stop + + c = 0 + ObjectSpace.each_object(String) do |str| + file = ObjectSpace.allocation_sourcefile(str) + unless !file.nil? && file.include?("allocation_spec") + line = ObjectSpace.allocation_sourceline(str) + c += 1 if file + puts "(#{c}):#{file}:#{line} - #{str.inspect}" if file + end + end + end.to make_allocations(expected_allocations) end end From b7ed78a49660b6141655992870e76b549c89a7b9 Mon Sep 17 00:00:00 2001 From: Stephen Wakely Date: Mon, 7 Jul 2025 16:30:58 +0100 Subject: [PATCH 17/35] Trace initialization --- spec/integrations/allocation_spec.rb | 55 ++++++++++++++++------------ 1 file changed, 31 insertions(+), 24 deletions(-) diff --git a/spec/integrations/allocation_spec.rb b/spec/integrations/allocation_spec.rb index a39db595..9ba80f55 100644 --- a/spec/integrations/allocation_spec.rb +++ b/spec/integrations/allocation_spec.rb @@ -3,6 +3,24 @@ require 'spec_helper' require 'objspace' +def trace_alloctions + ObjectSpace.trace_object_allocations_start + + yield + + ObjectSpace.trace_object_allocations_stop + + c = 0 + ObjectSpace.each_object(String) do |str| + file = ObjectSpace.allocation_sourcefile(str) + unless !file.nil? && file.include?("allocation_spec") + line = ObjectSpace.allocation_sourceline(str) + c += 1 if file + puts "(#{c}):#{file}:#{line} - #{str.inspect}" if file + end + end +end + describe 'Allocations and garbage collection' do before do skip 'Ruby too old' if RUBY_VERSION < '2.3.0' @@ -11,14 +29,16 @@ let(:socket) { FakeUDPSocket.new } subject do - Datadog::Statsd.new('localhost', 1234, - namespace: namespace, - sample_rate: sample_rate, - tags: tags, - logger: logger, - telemetry_flush_interval: -1, - origin_detection: false, - ) + trace_alloctions do + Datadog::Statsd.new('localhost', 1234, + namespace: namespace, + sample_rate: sample_rate, + tags: tags, + logger: logger, + telemetry_flush_interval: -1, + origin_detection: false, + ) + end end let(:namespace) { 'sample_ns' } @@ -276,23 +296,10 @@ it 'produces low amounts of garbage' do expect do - ObjectSpace.trace_object_allocations_start - - subject.event('foobar', 'happening', tags: { something: 'a value' }) - subject.flush(sync: true) - - ObjectSpace.trace_object_allocations_stop - - c = 0 - ObjectSpace.each_object(String) do |str| - file = ObjectSpace.allocation_sourcefile(str) - unless !file.nil? && file.include?("allocation_spec") - line = ObjectSpace.allocation_sourceline(str) - c += 1 if file - puts "(#{c}):#{file}:#{line} - #{str.inspect}" if file - end + trace_alloctions do + subject.event('foobar', 'happening', tags: { something: 'a value' }) + subject.flush(sync: true) end - end.to make_allocations(expected_allocations) end end From afb698bc91a838a01189426dce36569cd3268d48 Mon Sep 17 00:00:00 2001 From: Stephen Wakely Date: Mon, 7 Jul 2025 16:41:18 +0100 Subject: [PATCH 18/35] Return block result --- spec/integrations/allocation_spec.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/spec/integrations/allocation_spec.rb b/spec/integrations/allocation_spec.rb index 9ba80f55..5e16a5dc 100644 --- a/spec/integrations/allocation_spec.rb +++ b/spec/integrations/allocation_spec.rb @@ -6,7 +6,7 @@ def trace_alloctions ObjectSpace.trace_object_allocations_start - yield + result = yield ObjectSpace.trace_object_allocations_stop @@ -19,6 +19,8 @@ def trace_alloctions puts "(#{c}):#{file}:#{line} - #{str.inspect}" if file end end + + result end describe 'Allocations and garbage collection' do From 035cb8881a4d21ebb7d38f4e056097f4a75b3c9c Mon Sep 17 00:00:00 2001 From: Stephen Wakely Date: Tue, 8 Jul 2025 10:48:50 +0100 Subject: [PATCH 19/35] Try warming up --- spec/integrations/allocation_spec.rb | 36 +++++++++++++++++----------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/spec/integrations/allocation_spec.rb b/spec/integrations/allocation_spec.rb index 5e16a5dc..01695662 100644 --- a/spec/integrations/allocation_spec.rb +++ b/spec/integrations/allocation_spec.rb @@ -31,16 +31,14 @@ def trace_alloctions let(:socket) { FakeUDPSocket.new } subject do - trace_alloctions do - Datadog::Statsd.new('localhost', 1234, - namespace: namespace, - sample_rate: sample_rate, - tags: tags, - logger: logger, - telemetry_flush_interval: -1, - origin_detection: false, - ) - end + Datadog::Statsd.new('localhost', 1234, + namespace: namespace, + sample_rate: sample_rate, + tags: tags, + logger: logger, + telemetry_flush_interval: -1, + origin_detection: false, + ) end let(:namespace) { 'sample_ns' } @@ -276,6 +274,12 @@ def trace_alloctions end context 'with tags' do + before do + # warmup + subject.event('foobar', 'happening', tags: { something: 'a value' }) + subject.flush(sync: true) + end + let(:expected_allocations) do if RUBY_VERSION < '2.4.0' 31 @@ -298,10 +302,8 @@ def trace_alloctions it 'produces low amounts of garbage' do expect do - trace_alloctions do - subject.event('foobar', 'happening', tags: { something: 'a value' }) - subject.flush(sync: true) - end + subject.event('foobar', 'happening', tags: { something: 'a value' }) + subject.flush(sync: true) end.to make_allocations(expected_allocations) end end @@ -366,6 +368,12 @@ def trace_alloctions end context 'with tags' do + before do + # warmup + subject.service_check('foobar', 'happening', tags: { something: 'a value' }) + subject.flush(sync: true) + end + let(:expected_allocations) do if RUBY_VERSION < '2.4.0' 27 From 4289183304cbdf10b585462f8e9e4cc266aea738 Mon Sep 17 00:00:00 2001 From: Stephen Wakely Date: Tue, 8 Jul 2025 11:18:48 +0100 Subject: [PATCH 20/35] Origin detection doesn't need to be a class --- lib/datadog/statsd.rb | 4 +- lib/datadog/statsd/origin_detection.rb | 192 ++++++++++++------------- spec/statsd/origin_detection_spec.rb | 2 +- 3 files changed, 93 insertions(+), 105 deletions(-) diff --git a/lib/datadog/statsd.rb b/lib/datadog/statsd.rb index a37f327f..e1829cc8 100644 --- a/lib/datadog/statsd.rb +++ b/lib/datadog/statsd.rb @@ -122,9 +122,7 @@ def initialize( @prefix = @namespace ? "#{@namespace}.".freeze : nil origin_detection_enabled = origin_detection_enabled?(origin_detection) - container_id = OriginDetection - .new() - .get_container_id(container_id, origin_detection_enabled) + container_id = get_container_id(container_id, origin_detection_enabled) external_data = sanitize(ENV['DD_EXTERNAL_ENV']) diff --git a/lib/datadog/statsd/origin_detection.rb b/lib/datadog/statsd/origin_detection.rb index 5cfcdbdf..b6943b43 100644 --- a/lib/datadog/statsd/origin_detection.rb +++ b/lib/datadog/statsd/origin_detection.rb @@ -1,140 +1,130 @@ module Datadog class Statsd - class OriginDetection - CGROUPV1BASECONTROLLER = "memory" - HOSTCGROUPNAMESPACEINODE = 0xEFFFFFFB - - def get_filepaths - { - cgroup_path: "/proc/self/cgroup", - self_mount_info_path: "/proc/self/mountinfo", - default_cgroup_mount_path: "/sys/fs/cgroup" - } - end + CGROUPV1BASECONTROLLER = "memory" + HOSTCGROUPNAMESPACEINODE = 0xEFFFFFFB - def is_host_cgroup_namespace? - stat = File.stat("/proc/self/ns/cgroup") rescue nil - return false unless stat - stat.ino == HOSTCGROUPNAMESPACEINODE - end + def is_host_cgroup_namespace? + stat = File.stat("/proc/self/ns/cgroup") rescue nil + return false unless stat + stat.ino == HOSTCGROUPNAMESPACEINODE + end - def parse_cgroup_node_path(lines) - res = {} - lines.split("\n").each do |line| - tokens = line.split(':') - next unless tokens.length == 3 + def parse_cgroup_node_path(lines) + res = {} + lines.split("\n").each do |line| + tokens = line.split(':') + next unless tokens.length == 3 - controller = tokens[1] - path = tokens[2] + controller = tokens[1] + path = tokens[2] - if controller == CGROUPV1BASECONTROLLER || controller == '' - res[controller] = path - end + if controller == CGROUPV1BASECONTROLLER || controller == '' + res[controller] = path end - - res end - def get_cgroup_inode(cgroup_mount_path, proc_self_cgroup_path) - content = File.read(proc_self_cgroup_path) rescue nil - return nil unless content + res + end - controllers = parse_cgroup_node_path(content) + def get_cgroup_inode(cgroup_mount_path, proc_self_cgroup_path) + content = File.read(proc_self_cgroup_path) rescue nil + return nil unless content - [CGROUPV1BASECONTROLLER, ''].each do |controller| - next unless controllers[controller] + controllers = parse_cgroup_node_path(content) - segments = [ - cgroup_mount_path.chomp('/'), - controller.strip, - controllers[controller].sub(/^\//, '') - ] - path = segments.reject(&:empty?).join("/") - inode = inode_for_path(path) - return inode unless inode.nil? - end + [CGROUPV1BASECONTROLLER, ''].each do |controller| + next unless controllers[controller] - nil + segments = [ + cgroup_mount_path.chomp('/'), + controller.strip, + controllers[controller].sub(/^\//, '') + ] + path = segments.reject(&:empty?).join("/") + inode = inode_for_path(path) + return inode unless inode.nil? end - private + nil + end - def inode_for_path(path) - stat = File.stat(path) rescue nil - return nil unless stat - "in-#{stat.ino}" - end + private - def parse_container_id(handle) - exp_line = /^\d+:[^:]*:(.+)$/ - uuid = /[0-9a-f]{8}[-_][0-9a-f]{4}[-_][0-9a-f]{4}[-_][0-9a-f]{4}[-_][0-9a-f]{12}/ - container = /[0-9a-f]{64}/ - task = /[0-9a-f]{32}-\d+/ - exp_container_id = /(#{uuid}|#{container}|#{task})(?:\.scope)?$/ + def inode_for_path(path) + stat = File.stat(path) rescue nil + return nil unless stat + "in-#{stat.ino}" + end - handle.each_line do |line| - match = line.match(exp_line) - next unless match && match[1] - id_match = match[1].match(exp_container_id) + def parse_container_id(handle) + exp_line = /^\d+:[^:]*:(.+)$/ + uuid = /[0-9a-f]{8}[-_][0-9a-f]{4}[-_][0-9a-f]{4}[-_][0-9a-f]{4}[-_][0-9a-f]{12}/ + container = /[0-9a-f]{64}/ + task = /[0-9a-f]{32}-\d+/ + exp_container_id = /(#{uuid}|#{container}|#{task})(?:\.scope)?$/ - return id_match[1] if id_match && id_match[1] - end + handle.each_line do |line| + match = line.match(exp_line) + next unless match && match[1] + id_match = match[1].match(exp_container_id) - nil + return id_match[1] if id_match && id_match[1] end - public + nil + end - def read_container_id(fpath) - handle = File.open(fpath, 'r') rescue nil - return nil unless handle + public - id = parse_container_id(handle) - handle.close - id - end + def read_container_id(fpath) + handle = File.open(fpath, 'r') rescue nil + return nil unless handle - def parse_mount_info(handle) - container_regexp = '([0-9a-f]{64})|([0-9a-f]{32}-\d+)|([0-9a-f]{8}(-[0-9a-f]{4}){4}$)' - cid_mount_info_regexp = %r{.*/([^\s/]+)/(?:#{container_regexp})/[\S]*hostname} + id = parse_container_id(handle) + handle.close + id + end - handle.each_line do |line| - matches = line.scan(cid_mount_info_regexp) - next if matches.empty? + def parse_mount_info(handle) + container_regexp = '([0-9a-f]{64})|([0-9a-f]{32}-\d+)|([0-9a-f]{8}(-[0-9a-f]{4}){4}$)' + cid_mount_info_regexp = %r{.*/([^\s/]+)/(?:#{container_regexp})/[\S]*hostname} - match = matches.last - containerd_sandbox_prefix = "sandboxes" - if match && match[0] != containerd_sandbox_prefix - return match[1] - end - end + handle.each_line do |line| + matches = line.scan(cid_mount_info_regexp) + next if matches.empty? - nil + match = matches.last + containerd_sandbox_prefix = "sandboxes" + if match && match[0] != containerd_sandbox_prefix + return match[1] + end end - def read_mount_info(path) - handle = File.open(path, 'r') rescue nil - return nil unless handle + nil + end - info = parse_mount_info(handle) - handle.close - info - end + def read_mount_info(path) + handle = File.open(path, 'r') rescue nil + return nil unless handle - def get_container_id(user_provided_id, cgroup_fallback) - return user_provided_id unless user_provided_id.nil? - return nil unless cgroup_fallback + info = parse_mount_info(handle) + handle.close + info + end - container_id = read_container_id("/proc/self/cgroup") - return container_id unless container_id.nil? + def get_container_id(user_provided_id, cgroup_fallback) + return user_provided_id unless user_provided_id.nil? + return nil unless cgroup_fallback - container_id = read_mount_info("/proc/self/mountinfo") - return container_id unless container_id.nil? + container_id = read_container_id("/proc/self/cgroup") + return container_id unless container_id.nil? - return nil if is_host_cgroup_namespace? + container_id = read_mount_info("/proc/self/mountinfo") + return container_id unless container_id.nil? - get_cgroup_inode("/sys/fs/cgroup", "/proc/self/cgroup") - end + return nil if is_host_cgroup_namespace? + + get_cgroup_inode("/sys/fs/cgroup", "/proc/self/cgroup") end end end diff --git a/spec/statsd/origin_detection_spec.rb b/spec/statsd/origin_detection_spec.rb index 65ec8bbc..d12ad3bc 100644 --- a/spec/statsd/origin_detection_spec.rb +++ b/spec/statsd/origin_detection_spec.rb @@ -12,7 +12,7 @@ def ino end end -describe Datadog::Statsd::OriginDetection do +describe Datadog::Statsd do subject do described_class.new() end From e7d35dd812101fe43905418e0d02363c4d2db976 Mon Sep 17 00:00:00 2001 From: Stephen Wakely Date: Tue, 8 Jul 2025 11:36:02 +0100 Subject: [PATCH 21/35] Remove field serializer from event --- lib/datadog/statsd/serialization/event_serializer.rb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/datadog/statsd/serialization/event_serializer.rb b/lib/datadog/statsd/serialization/event_serializer.rb index c0611a93..dac67d54 100644 --- a/lib/datadog/statsd/serialization/event_serializer.rb +++ b/lib/datadog/statsd/serialization/event_serializer.rb @@ -15,7 +15,7 @@ class EventSerializer def initialize(container_id, external_data, global_tags: []) @tag_serializer = TagSerializer.new(global_tags) - @field_serializer = FieldSerializer.new(container_id, external_data) + #@field_serializer = FieldSerializer.new(container_id, external_data) end def format(title, text, options = EMPTY_OPTIONS) @@ -48,9 +48,9 @@ def format(title, text, options = EMPTY_OPTIONS) event << tags end - if fields = field_serializer.format(options[:cardinality]) - event << fields - end + #if fields = field_serializer.format(options[:cardinality]) + # event << fields + #end if event.bytesize > MAX_EVENT_SIZE if options[:truncate_if_too_long] From 922cd2ee401e4c35f05c28982dbdb6636c1ceeca Mon Sep 17 00:00:00 2001 From: Stephen Wakely Date: Tue, 8 Jul 2025 11:45:31 +0100 Subject: [PATCH 22/35] Remove windows specific --- lib/datadog/statsd.rb | 4 ++-- lib/datadog/statsd/serialization/event_serializer.rb | 8 ++++---- spec/integrations/allocation_spec.rb | 12 ++---------- 3 files changed, 8 insertions(+), 16 deletions(-) diff --git a/lib/datadog/statsd.rb b/lib/datadog/statsd.rb index e1829cc8..9ca27a8b 100644 --- a/lib/datadog/statsd.rb +++ b/lib/datadog/statsd.rb @@ -121,8 +121,8 @@ def initialize( @namespace = namespace @prefix = @namespace ? "#{@namespace}.".freeze : nil - origin_detection_enabled = origin_detection_enabled?(origin_detection) - container_id = get_container_id(container_id, origin_detection_enabled) + #origin_detection_enabled = origin_detection_enabled?(origin_detection) + #container_id = get_container_id(container_id, origin_detection_enabled) external_data = sanitize(ENV['DD_EXTERNAL_ENV']) diff --git a/lib/datadog/statsd/serialization/event_serializer.rb b/lib/datadog/statsd/serialization/event_serializer.rb index dac67d54..c0611a93 100644 --- a/lib/datadog/statsd/serialization/event_serializer.rb +++ b/lib/datadog/statsd/serialization/event_serializer.rb @@ -15,7 +15,7 @@ class EventSerializer def initialize(container_id, external_data, global_tags: []) @tag_serializer = TagSerializer.new(global_tags) - #@field_serializer = FieldSerializer.new(container_id, external_data) + @field_serializer = FieldSerializer.new(container_id, external_data) end def format(title, text, options = EMPTY_OPTIONS) @@ -48,9 +48,9 @@ def format(title, text, options = EMPTY_OPTIONS) event << tags end - #if fields = field_serializer.format(options[:cardinality]) - # event << fields - #end + if fields = field_serializer.format(options[:cardinality]) + event << fields + end if event.bytesize > MAX_EVENT_SIZE if options[:truncate_if_too_long] diff --git a/spec/integrations/allocation_spec.rb b/spec/integrations/allocation_spec.rb index 01695662..0b393ed9 100644 --- a/spec/integrations/allocation_spec.rb +++ b/spec/integrations/allocation_spec.rb @@ -286,17 +286,9 @@ def trace_alloctions elsif RUBY_VERSION < '2.5.0' 29 elsif RUBY_VERSION < '2.6.0' - if RUBY_PLATFORM.include?('linux') - 28 - else - 29 - end + 28 else - if RUBY_PLATFORM.include?('linux') - 27 - else - 28 - end + 27 end end From 7e1bb67696130e494d90bf283db2145e51404908 Mon Sep 17 00:00:00 2001 From: Stephen Wakely Date: Tue, 8 Jul 2025 11:59:42 +0100 Subject: [PATCH 23/35] Return origin detection --- lib/datadog/statsd.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/datadog/statsd.rb b/lib/datadog/statsd.rb index 9ca27a8b..e1829cc8 100644 --- a/lib/datadog/statsd.rb +++ b/lib/datadog/statsd.rb @@ -121,8 +121,8 @@ def initialize( @namespace = namespace @prefix = @namespace ? "#{@namespace}.".freeze : nil - #origin_detection_enabled = origin_detection_enabled?(origin_detection) - #container_id = get_container_id(container_id, origin_detection_enabled) + origin_detection_enabled = origin_detection_enabled?(origin_detection) + container_id = get_container_id(container_id, origin_detection_enabled) external_data = sanitize(ENV['DD_EXTERNAL_ENV']) From dd1eb6a91ad5a51485dfd0b84eb55339c52ec09a Mon Sep 17 00:00:00 2001 From: Stephen Wakely Date: Tue, 8 Jul 2025 12:03:30 +0100 Subject: [PATCH 24/35] Remove all linux specific alloction test --- spec/integrations/allocation_spec.rb | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/spec/integrations/allocation_spec.rb b/spec/integrations/allocation_spec.rb index 0b393ed9..b012bcea 100644 --- a/spec/integrations/allocation_spec.rb +++ b/spec/integrations/allocation_spec.rb @@ -372,17 +372,9 @@ def trace_alloctions elsif RUBY_VERSION < '2.5.0' 25 elsif RUBY_VERSION < '2.6.0' - if RUBY_PLATFORM.include?('linux') - 24 - else - 25 - end + 24 else - if RUBY_PLATFORM.include?('linux') - 23 - else - 24 - end + 23 end end From 76486f8c6eff2ea2f3aaac12b7fe00ceccf3e07d Mon Sep 17 00:00:00 2001 From: Stephen Wakely Date: Tue, 8 Jul 2025 13:59:46 +0100 Subject: [PATCH 25/35] Attempt to bound regex --- lib/datadog/statsd/origin_detection.rb | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/lib/datadog/statsd/origin_detection.rb b/lib/datadog/statsd/origin_detection.rb index b6943b43..b0782ea8 100644 --- a/lib/datadog/statsd/origin_detection.rb +++ b/lib/datadog/statsd/origin_detection.rb @@ -87,16 +87,21 @@ def read_container_id(fpath) def parse_mount_info(handle) container_regexp = '([0-9a-f]{64})|([0-9a-f]{32}-\d+)|([0-9a-f]{8}(-[0-9a-f]{4}){4}$)' - cid_mount_info_regexp = %r{.*/([^\s/]+)/(?:#{container_regexp})/[\S]*hostname} + cid_mount_info_regexp = %r{[^\s]*/([^\s/]+)/(?:#{container_regexp})/[\S]*hostname$} handle.each_line do |line| - matches = line.scan(cid_mount_info_regexp) - next if matches.empty? - - match = matches.last - containerd_sandbox_prefix = "sandboxes" - if match && match[0] != containerd_sandbox_prefix - return match[1] + split = line.split(" ") + mnt1 = split[3] + mnt2 = split[4] + [mnt1, mnt2].each do |line| + matches = line.scan(cid_mount_info_regexp) + next if matches.empty? + + match = matches.last + containerd_sandbox_prefix = "sandboxes" + if match && match[0] != containerd_sandbox_prefix + return match[1] + end end end From 166fd8baea1f37d7d1d95029b4575c14952e5a4b Mon Sep 17 00:00:00 2001 From: Stephen Wakely Date: Tue, 8 Jul 2025 14:27:46 +0100 Subject: [PATCH 26/35] Take substring of path. --- lib/datadog/statsd/origin_detection.rb | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/datadog/statsd/origin_detection.rb b/lib/datadog/statsd/origin_detection.rb index b0782ea8..70f2f039 100644 --- a/lib/datadog/statsd/origin_detection.rb +++ b/lib/datadog/statsd/origin_detection.rb @@ -91,8 +91,9 @@ def parse_mount_info(handle) handle.each_line do |line| split = line.split(" ") - mnt1 = split[3] - mnt2 = split[4] + # Take the first 4,096 bytes of the path to avoid the regex exploding. + mnt1 = split[3][0, 4096] + mnt2 = split[4][0, 4096] [mnt1, mnt2].each do |line| matches = line.scan(cid_mount_info_regexp) next if matches.empty? From cd89b8cf057e61ecb6b866441019eb9bc084504d Mon Sep 17 00:00:00 2001 From: Stephen Wakely Date: Tue, 8 Jul 2025 16:58:19 +0100 Subject: [PATCH 27/35] Replace regex with parser --- lib/datadog/statsd/origin_detection.rb | 48 ++++++++++++++++++-------- 1 file changed, 34 insertions(+), 14 deletions(-) diff --git a/lib/datadog/statsd/origin_detection.rb b/lib/datadog/statsd/origin_detection.rb index 70f2f039..bdb6730b 100644 --- a/lib/datadog/statsd/origin_detection.rb +++ b/lib/datadog/statsd/origin_detection.rb @@ -85,24 +85,44 @@ def read_container_id(fpath) id end - def parse_mount_info(handle) - container_regexp = '([0-9a-f]{64})|([0-9a-f]{32}-\d+)|([0-9a-f]{8}(-[0-9a-f]{4}){4}$)' - cid_mount_info_regexp = %r{[^\s]*/([^\s/]+)/(?:#{container_regexp})/[\S]*hostname$} + # Extracts the final container info from a line in mount info + def extract_container_info(line) + parts = line.strip.split("/") + return nil unless parts.last == "hostname" + + # Expected structure: [..., , , ..., "hostname"] + container_id = nil + group = nil + + parts.each_with_index do |part, idx| + # Match the container id and include the section prior to it. + if part.length == 64 && part.match?(/\A[0-9a-f]{64}\z/) + group = parts[idx - 1] if idx >= 1 + container_id = part + elsif part.length > 32 && part.match?(/\A[0-9a-f]{32}-\d+\z/) + group = parts[idx - 1] if idx >= 1 + container_id = part + elsif part.match?(/\A[0-9a-f]{8}(-[0-9a-f]{4}){4}\z/) + group = parts[idx - 1] if idx >= 1 + container_id = part + end + end + return container_id unless group == "sandboxes" + end + + # Parse /proc/self/mountinfo to extract the container id. + # Often container runtimes embed the container id in the mount paths. + # We parse the mount with a final `hostname` component, which is part of + # the containers `etc/hostname` bind mount. + def parse_mount_info(handle) handle.each_line do |line| split = line.split(" ") - # Take the first 4,096 bytes of the path to avoid the regex exploding. - mnt1 = split[3][0, 4096] - mnt2 = split[4][0, 4096] + mnt1 = split[3] + mnt2 = split[4] [mnt1, mnt2].each do |line| - matches = line.scan(cid_mount_info_regexp) - next if matches.empty? - - match = matches.last - containerd_sandbox_prefix = "sandboxes" - if match && match[0] != containerd_sandbox_prefix - return match[1] - end + container_id = extract_container_info(line) + return container_id unless container_id.nil? end end From 556bfe15114440b86bd696eeafc40c0793f0428f Mon Sep 17 00:00:00 2001 From: Stephen Wakely Date: Tue, 8 Jul 2025 17:09:20 +0100 Subject: [PATCH 28/35] Ruby <= 2.3 compliance --- lib/datadog/statsd/origin_detection.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/datadog/statsd/origin_detection.rb b/lib/datadog/statsd/origin_detection.rb index bdb6730b..aef443ce 100644 --- a/lib/datadog/statsd/origin_detection.rb +++ b/lib/datadog/statsd/origin_detection.rb @@ -96,13 +96,13 @@ def extract_container_info(line) parts.each_with_index do |part, idx| # Match the container id and include the section prior to it. - if part.length == 64 && part.match?(/\A[0-9a-f]{64}\z/) + if part.length == 64 && !!(part =~ /\A[0-9a-f]{64}\z/) group = parts[idx - 1] if idx >= 1 container_id = part - elsif part.length > 32 && part.match?(/\A[0-9a-f]{32}-\d+\z/) + elsif part.length > 32 && !!(part =~ /\A[0-9a-f]{32}-\d+\z/) group = parts[idx - 1] if idx >= 1 container_id = part - elsif part.match?(/\A[0-9a-f]{8}(-[0-9a-f]{4}){4}\z/) + elsif !!(part =~ /\A[0-9a-f]{8}(-[0-9a-f]{4}){4}\z/) group = parts[idx - 1] if idx >= 1 container_id = part end From fcbd7f7256e00e1ee210020862ae7008ba4206fa Mon Sep 17 00:00:00 2001 From: Stephen Wakely Date: Tue, 8 Jul 2025 17:18:50 +0100 Subject: [PATCH 29/35] Remove trace allocations --- spec/integrations/allocation_spec.rb | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/spec/integrations/allocation_spec.rb b/spec/integrations/allocation_spec.rb index b012bcea..3ddf90e5 100644 --- a/spec/integrations/allocation_spec.rb +++ b/spec/integrations/allocation_spec.rb @@ -1,27 +1,6 @@ # frozen_string_literal: true require 'spec_helper' -require 'objspace' - -def trace_alloctions - ObjectSpace.trace_object_allocations_start - - result = yield - - ObjectSpace.trace_object_allocations_stop - - c = 0 - ObjectSpace.each_object(String) do |str| - file = ObjectSpace.allocation_sourcefile(str) - unless !file.nil? && file.include?("allocation_spec") - line = ObjectSpace.allocation_sourceline(str) - c += 1 if file - puts "(#{c}):#{file}:#{line} - #{str.inspect}" if file - end - end - - result -end describe 'Allocations and garbage collection' do before do From 29f4f083c88c63ea50e72deebcc07952b545f3a6 Mon Sep 17 00:00:00 2001 From: Stephen Wakely Date: Tue, 8 Jul 2025 17:26:06 +0100 Subject: [PATCH 30/35] Only turn origin detection off for linux --- spec/statsd_spec.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/spec/statsd_spec.rb b/spec/statsd_spec.rb index f159818a..f56d28a6 100644 --- a/spec/statsd_spec.rb +++ b/spec/statsd_spec.rb @@ -10,7 +10,9 @@ tags: tags, logger: logger, telemetry_flush_interval: -1, - origin_detection: false, + # Only turn off origin detection for linux, other + # platforms should work as before even with it enabled. + origin_detection: !RUBY_PLATFORM.include?("linux"), ) end From a4bd5995bc420eebf29071fdb6f41663eec32c68 Mon Sep 17 00:00:00 2001 From: Stephen Wakely Date: Wed, 9 Jul 2025 12:41:50 +0100 Subject: [PATCH 31/35] Make origin detection methods private --- lib/datadog/statsd/origin_detection.rb | 10 ++++------ spec/statsd/origin_detection_spec.rb | 18 +++++++++--------- 2 files changed, 13 insertions(+), 15 deletions(-) diff --git a/lib/datadog/statsd/origin_detection.rb b/lib/datadog/statsd/origin_detection.rb index aef443ce..b025fe75 100644 --- a/lib/datadog/statsd/origin_detection.rb +++ b/lib/datadog/statsd/origin_detection.rb @@ -1,9 +1,11 @@ module Datadog class Statsd + private + CGROUPV1BASECONTROLLER = "memory" HOSTCGROUPNAMESPACEINODE = 0xEFFFFFFB - def is_host_cgroup_namespace? + def host_cgroup_namespace? stat = File.stat("/proc/self/ns/cgroup") rescue nil return false unless stat stat.ino == HOSTCGROUPNAMESPACEINODE @@ -48,8 +50,6 @@ def get_cgroup_inode(cgroup_mount_path, proc_self_cgroup_path) nil end - private - def inode_for_path(path) stat = File.stat(path) rescue nil return nil unless stat @@ -74,8 +74,6 @@ def parse_container_id(handle) nil end - public - def read_container_id(fpath) handle = File.open(fpath, 'r') rescue nil return nil unless handle @@ -148,7 +146,7 @@ def get_container_id(user_provided_id, cgroup_fallback) container_id = read_mount_info("/proc/self/mountinfo") return container_id unless container_id.nil? - return nil if is_host_cgroup_namespace? + return nil if host_cgroup_namespace? get_cgroup_inode("/sys/fs/cgroup", "/proc/self/cgroup") end diff --git a/spec/statsd/origin_detection_spec.rb b/spec/statsd/origin_detection_spec.rb index d12ad3bc..b840b153 100644 --- a/spec/statsd/origin_detection_spec.rb +++ b/spec/statsd/origin_detection_spec.rb @@ -36,7 +36,7 @@ def ino 4:blkio:/kubepods/burstable/podfd52ef25-a87d-11e9-9423-0800271a638e/8c046cb0b72cd4c99f51b5591cd5b095967f58ee003710a45280c28ee1a9c7fa CGROUP - container_id = subject.get_container_id(nil, true) + container_id = subject.send(:get_container_id, nil, true) end expect(container_id).to eq('8c046cb0b72cd4c99f51b5591cd5b095967f58ee003710a45280c28ee1a9c7fa') @@ -103,7 +103,7 @@ def ino FileUtils.mkdir_p('/proc/self') File.write('/proc/self/cgroup', test_case[:input]) - result = subject.read_container_id('/proc/self/cgroup') + result = subject.send(:read_container_id, '/proc/self/cgroup') end expect(result).to eq(test_case[:expected]) end @@ -123,7 +123,7 @@ def ino File.write('/proc/self/cgroup', "") - container_id = subject.get_container_id(nil, true) + container_id = subject.send(:get_container_id, nil, true) end expect(container_id).to eq("fc7038bc73a8d3850c66ddbfb0b2901afa378bfcbb942cc384b051767e4ac6b0") @@ -305,7 +305,7 @@ def ino FakeFS.with_fresh do FileUtils.mkdir_p('/proc/self') File.write('/proc/self/mountinfo', test_case[:input]) - result = subject.read_mount_info('/proc/self/mountinfo') + result = subject.send(:read_mount_info, '/proc/self/mountinfo') end expect(result).to eq(test_case[:expected]) @@ -347,7 +347,7 @@ def ino File.write('/proc/self/mountinfo', mountinfo_contents) File.write('/proc/self/cgroup', '') # Needed if get_container_id uses it - container_id = subject.get_container_id(nil, true) + container_id = subject.send(:get_container_id, nil, true) end expect(container_id).to eq(cid) @@ -386,7 +386,7 @@ def ino test_cases.each do |test_case| it "parses correctly for: #{test_case[:name]}" do - result = subject.parse_cgroup_node_path(test_case[:content]) + result = subject.send(:parse_cgroup_node_path, test_case[:content]) expect(result).to eq(test_case[:expected]) end end @@ -463,7 +463,7 @@ def ino test_case[:expected_result] = test_case[:expected_result].gsub("{inode}", inode.to_s) end - result = subject.get_cgroup_inode('/sys/fs/cgroup', '/proc/self/cgroup') + result = subject.send(:get_cgroup_inode, '/sys/fs/cgroup', '/proc/self/cgroup') end expect(result).to eq(test_case[:expected_result]) @@ -518,7 +518,7 @@ def ino File.write('/proc/self/cgroup', test_case[:proc_self_cgroup_content].to_s) File.write('/proc/self/mountinfo', test_case[:mount_info_content].to_s) - allow(subject).to receive(:is_host_cgroup_namespace?) + allow(subject).to receive(:host_cgroup_namespace?) .and_return(test_case[:is_host_cgroup_ns] || false) if test_case[:cgroup_node_dir] @@ -532,7 +532,7 @@ def ino end end - container_id = subject.get_container_id(nil, true) + container_id = subject.send(:get_container_id, nil, true) end expect(container_id).to eq(test_case[:expected_result]) From 1c58afa0ca21bca546290b2727cd9e3301f1d2d8 Mon Sep 17 00:00:00 2001 From: Stephen Wakely Date: Wed, 9 Jul 2025 12:44:30 +0100 Subject: [PATCH 32/35] Remove unused instance vars --- lib/datadog/statsd/serialization/stat_serializer.rb | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/datadog/statsd/serialization/stat_serializer.rb b/lib/datadog/statsd/serialization/stat_serializer.rb index c6e3195e..d43bea4b 100644 --- a/lib/datadog/statsd/serialization/stat_serializer.rb +++ b/lib/datadog/statsd/serialization/stat_serializer.rb @@ -7,8 +7,6 @@ class StatSerializer def initialize(prefix, container_id, external_data, global_tags: []) @prefix = prefix @prefix_str = prefix.to_s - @container_id = container_id - @external_data = external_data @tag_serializer = TagSerializer.new(global_tags) @field_serializer = FieldSerializer.new(container_id, external_data) end From ecbbf3d9f812ff24f1c8824cd30ea3a246a80f8e Mon Sep 17 00:00:00 2001 From: Stephen Wakely Date: Thu, 10 Jul 2025 14:05:45 +0100 Subject: [PATCH 33/35] Add origin fields to telemetry metrics --- lib/datadog/statsd.rb | 6 +- lib/datadog/statsd/forwarder.rb | 6 ++ lib/datadog/statsd/message_buffer.rb | 4 +- lib/datadog/statsd/origin_detection.rb | 1 + .../statsd/serialization/field_serializer.rb | 3 +- lib/datadog/statsd/telemetry.rb | 10 +++- spec/statsd/telemetry_spec.rb | 59 ++++++++++++++++++- 7 files changed, 81 insertions(+), 8 deletions(-) diff --git a/lib/datadog/statsd.rb b/lib/datadog/statsd.rb index e1829cc8..d666fad2 100644 --- a/lib/datadog/statsd.rb +++ b/lib/datadog/statsd.rb @@ -157,6 +157,10 @@ def initialize( sender_queue_size: sender_queue_size, telemetry_flush_interval: telemetry_enable ? telemetry_flush_interval : nil, + container_id: container_id, + external_data: external_data, + cardinality: @cardinality, + serializer: serializer ) end @@ -464,7 +468,7 @@ def send_stats(stat, delta, type, opts = EMPTY_OPTIONS) if sample_rate == 1 || opts[:pre_sampled] || rand <= sample_rate full_stat = if @delay_serialization - [stat, delta, type, opts[:tags], sample_rate] + [stat, delta, type, opts[:tags], sample_rate, cardinality] else serializer.to_stat(stat, delta, type, tags: opts[:tags], sample_rate: sample_rate, cardinality: cardinality) end diff --git a/lib/datadog/statsd/forwarder.rb b/lib/datadog/statsd/forwarder.rb index 4e5f80e5..5cadeaac 100644 --- a/lib/datadog/statsd/forwarder.rb +++ b/lib/datadog/statsd/forwarder.rb @@ -22,6 +22,9 @@ def initialize( single_thread: false, logger: nil, + container_id: nil, + external_data: nil, + cardinality: nil, serializer: ) @@ -29,6 +32,9 @@ def initialize( @telemetry = if telemetry_flush_interval Telemetry.new(telemetry_flush_interval, + container_id, + external_data, + cardinality, global_tags: global_tags, transport_type: @transport_type ) diff --git a/lib/datadog/statsd/message_buffer.rb b/lib/datadog/statsd/message_buffer.rb index 2ea01d25..fc3bb7f7 100644 --- a/lib/datadog/statsd/message_buffer.rb +++ b/lib/datadog/statsd/message_buffer.rb @@ -28,8 +28,8 @@ def add(message) # Serializes the message if it hasn't been already. Part of the # delay_serialization feature. if message.is_a?(Array) - stat, delta, type, tags, sample_rate = message - message = @serializer.to_stat(stat, delta, type, tags: tags, sample_rate: sample_rate) + stat, delta, type, tags, sample_rate, cardinality = message + message = @serializer.to_stat(stat, delta, type, tags: tags, sample_rate: sample_rate, cardinality: cardinality) end message_size = message.bytesize diff --git a/lib/datadog/statsd/origin_detection.rb b/lib/datadog/statsd/origin_detection.rb index b025fe75..93d58d42 100644 --- a/lib/datadog/statsd/origin_detection.rb +++ b/lib/datadog/statsd/origin_detection.rb @@ -1,3 +1,4 @@ +# frozen_string_literal: true module Datadog class Statsd private diff --git a/lib/datadog/statsd/serialization/field_serializer.rb b/lib/datadog/statsd/serialization/field_serializer.rb index 9fab6ff7..f37ba42b 100644 --- a/lib/datadog/statsd/serialization/field_serializer.rb +++ b/lib/datadog/statsd/serialization/field_serializer.rb @@ -13,8 +13,7 @@ def initialize(container_id, external_data) def format(cardinality) if @container_id.nil? && @external_data.nil? && cardinality.nil? - # Avoid the allocation unless needed. - return nil + return "" end field = String.new diff --git a/lib/datadog/statsd/telemetry.rb b/lib/datadog/statsd/telemetry.rb index 89b19357..69a13418 100644 --- a/lib/datadog/statsd/telemetry.rb +++ b/lib/datadog/statsd/telemetry.rb @@ -19,7 +19,7 @@ class Telemetry # Rough estimation of maximum telemetry message size without tags MAX_TELEMETRY_MESSAGE_SIZE_WT_TAGS = 50 # bytes - def initialize(flush_interval, global_tags: [], transport_type: :udp) + def initialize(flush_interval, container_id, external_data, cardinality, global_tags: [], transport_type: :udp) @flush_interval = flush_interval @global_tags = global_tags @transport_type = transport_type @@ -32,6 +32,11 @@ def initialize(flush_interval, global_tags: [], transport_type: :udp) client_version: VERSION, client_transport: transport_type, ).format(global_tags) + + @serialized_fields = Serialization::FieldSerializer.new( + container_id, + external_data + ).format(cardinality) end def would_fit_in?(max_buffer_payload_size) @@ -98,9 +103,10 @@ def flush private attr_reader :serialized_tags + attr_reader :serialized_fields def pattern - @pattern ||= "datadog.dogstatsd.client.%s:%d|#{COUNTER_TYPE}|##{serialized_tags}" + @pattern ||= "datadog.dogstatsd.client.%s:%d|#{COUNTER_TYPE}|##{serialized_tags}#{serialized_fields}" end if Kernel.const_defined?('Process') && Process.respond_to?(:clock_gettime) diff --git a/spec/statsd/telemetry_spec.rb b/spec/statsd/telemetry_spec.rb index ae3ae392..02f5442c 100644 --- a/spec/statsd/telemetry_spec.rb +++ b/spec/statsd/telemetry_spec.rb @@ -2,7 +2,7 @@ describe Datadog::Statsd::Telemetry do subject do - described_class.new(2, + described_class.new(2, container_id, external_data, cardinality, global_tags: global_tags, transport_type: :doe ) @@ -12,6 +12,18 @@ [] end + let(:container_id) do + nil + end + + let(:external_data) do + nil + end + + let(:cardinality) do + nil + end + describe '#would_fit_in?' do # we will also check the size of telemetry automatic tags context 'with tags ["host:myhost", "network:ethernet"]' do @@ -106,6 +118,51 @@ end end + describe 'with origin fields' do + before do + subject.sent(metrics: 1, events: 2, service_checks: 3, bytes: 4, packets: 5) + subject.dropped_writer(bytes: 6, packets: 7) + subject.dropped_queue(bytes: 9, packets: 8) + subject.flush + end + + let(:container_id) do + "fc7038bc73a8d3850c66ddbfb0b2901afa378bfcbb942cc384b051767e4ac6b0" + end + + let(:external_data) do + "it-false,cn-comp-app,pu-abebb16c-c73e-41c9-ba37-4db4e75168ac" + end + + let(:cardinality) do + "low" + end + + it 'serializes the telemetry with origin fields' do + fields = "|c:fc7038bc73a8d3850c66ddbfb0b2901afa378bfcbb942cc384b051767e4ac6b0|e:it-false,cn-comp-app,pu-abebb16c-c73e-41c9-ba37-4db4e75168ac|card:low" + expect(subject.flush).to eq [ + "datadog.dogstatsd.client.metrics:1|c|#client:ruby,client_version:#{Datadog::Statsd::VERSION},client_transport:doe#{fields}", + "datadog.dogstatsd.client.events:2|c|#client:ruby,client_version:#{Datadog::Statsd::VERSION},client_transport:doe#{fields}", + "datadog.dogstatsd.client.service_checks:3|c|#client:ruby,client_version:#{Datadog::Statsd::VERSION},client_transport:doe#{fields}", + "datadog.dogstatsd.client.bytes_sent:4|c|#client:ruby,client_version:#{Datadog::Statsd::VERSION},client_transport:doe#{fields}", + "datadog.dogstatsd.client.bytes_dropped:15|c|#client:ruby,client_version:#{Datadog::Statsd::VERSION},client_transport:doe#{fields}", + "datadog.dogstatsd.client.bytes_dropped_queue:9|c|#client:ruby,client_version:#{Datadog::Statsd::VERSION},client_transport:doe#{fields}", + "datadog.dogstatsd.client.bytes_dropped_writer:6|c|#client:ruby,client_version:#{Datadog::Statsd::VERSION},client_transport:doe#{fields}", + "datadog.dogstatsd.client.packets_sent:5|c|#client:ruby,client_version:#{Datadog::Statsd::VERSION},client_transport:doe#{fields}", + "datadog.dogstatsd.client.packets_dropped:15|c|#client:ruby,client_version:#{Datadog::Statsd::VERSION},client_transport:doe#{fields}", + "datadog.dogstatsd.client.packets_dropped_queue:8|c|#client:ruby,client_version:#{Datadog::Statsd::VERSION},client_transport:doe#{fields}", + "datadog.dogstatsd.client.packets_dropped_writer:7|c|#client:ruby,client_version:#{Datadog::Statsd::VERSION},client_transport:doe#{fields}", + ] + end + + it do + skip 'Ruby too old' if RUBY_VERSION < '2.3.0' + expect do + subject.flush + end.to make_allocations(12) + end + end + describe '#reset' do before do Timecop.freeze(DateTime.new(2020, 2, 22, 12, 12, 12)) From 373be59be887c0b8eab075bffe331c47939f89f5 Mon Sep 17 00:00:00 2001 From: Stephen Wakely Date: Thu, 10 Jul 2025 16:31:19 +0100 Subject: [PATCH 34/35] Add delay serializer tests --- .../statsd/serialization/stat_serializer.rb | 2 +- spec/integrations/delay_serialization_spec.rb | 23 ++++++++++++++++++- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/lib/datadog/statsd/serialization/stat_serializer.rb b/lib/datadog/statsd/serialization/stat_serializer.rb index d43bea4b..1e37b7e2 100644 --- a/lib/datadog/statsd/serialization/stat_serializer.rb +++ b/lib/datadog/statsd/serialization/stat_serializer.rb @@ -17,7 +17,7 @@ def format(metric_name, delta, type, tags: [], sample_rate: 1, cardinality: nil) if sample_rate != 1 if tags_list = tag_serializer.format(tags) - "#{@prefix_str}#{metric_name}:#{delta}|#{type}|@#{sample_rate}#{fields}|##{tags_list}" + "#{@prefix_str}#{metric_name}:#{delta}|#{type}|@#{sample_rate}|##{tags_list}#{fields}" else "#{@prefix_str}#{metric_name}:#{delta}|#{type}|@#{sample_rate}#{fields}" end diff --git a/spec/integrations/delay_serialization_spec.rb b/spec/integrations/delay_serialization_spec.rb index 4a14a9f8..915061c6 100644 --- a/spec/integrations/delay_serialization_spec.rb +++ b/spec/integrations/delay_serialization_spec.rb @@ -6,7 +6,7 @@ # expects an Array is passed and not a String expect(buffer) .to receive(:add) - .with(["boo", 1, "c", nil, 1]) + .with(["boo", 1, "c", nil, 1, nil]) # and then expect no more adds! expect(buffer).to receive(:add).exactly(0).times expect(buffer) @@ -41,4 +41,25 @@ "pow:1|c|@2|#tag1:val1" ].join("\n")) end + + it "serializes container id normally" do + socket = FakeUDPSocket.new(copy_message: true) + allow(UDPSocket).to receive(:new).and_return(socket) + dogstats = Datadog::Statsd.new("localhost", 1234, + delay_serialization: true, + origin_detection: false, + container_id: "banana", + ) + + dogstats.increment("boo") + dogstats.increment("oob", tags: {tag1: "val1"}) + dogstats.increment("pow", tags: {tag1: "val1"}, sample_rate: 2) + dogstats.flush(sync: true) + + expect(socket.recv[0]).to eq([ + "boo:1|c|c:banana", + "oob:1|c|#tag1:val1|c:banana", + "pow:1|c|@2|#tag1:val1|c:banana" + ].join("\n")) + end end From cea749996aba5e864158684496f2697f591f5e69 Mon Sep 17 00:00:00 2001 From: Stephen Wakely Date: Thu, 10 Jul 2025 16:57:17 +0100 Subject: [PATCH 35/35] Fix field order --- spec/statsd/serialization/stat_serializer_spec.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/statsd/serialization/stat_serializer_spec.rb b/spec/statsd/serialization/stat_serializer_spec.rb index ecd6617b..2d97a2eb 100644 --- a/spec/statsd/serialization/stat_serializer_spec.rb +++ b/spec/statsd/serialization/stat_serializer_spec.rb @@ -132,7 +132,7 @@ end it 'adds the tags to the stat correctly' do - expect(subject.format('somecount', 42, 'c', tags: message_tags, sample_rate: 0.5)).to eq 'swag.somecount:42|c|@0.5|c:in-23|#globaltag1:value1,msgtag2:value2' + expect(subject.format('somecount', 42, 'c', tags: message_tags, sample_rate: 0.5)).to eq 'swag.somecount:42|c|@0.5|#globaltag1:value1,msgtag2:value2|c:in-23' end end