From 7148880bfeaa4f54ab804cfb90fd38ef0cd3d214 Mon Sep 17 00:00:00 2001 From: Kaise Cheng Date: Mon, 27 Oct 2025 19:30:27 +0000 Subject: [PATCH 1/7] fix `convert` from string to integer and float --- lib/logstash/filters/mutate.rb | 46 +++- spec/filters/mutate_spec.rb | 398 ++++++++++++++++++++++++++++++++- 2 files changed, 439 insertions(+), 5 deletions(-) diff --git a/lib/logstash/filters/mutate.rb b/lib/logstash/filters/mutate.rb index 076cce2..6fb9b29 100644 --- a/lib/logstash/filters/mutate.rb +++ b/lib/logstash/filters/mutate.rb @@ -1,6 +1,7 @@ # encoding: utf-8 require "logstash/filters/base" require "logstash/namespace" +require "bigdecimal" # The mutate filter allows you to perform general mutations on fields. You # can rename, replace, and modify fields in your events. @@ -344,15 +345,34 @@ def convert_boolean(value) def convert_integer(value) return 1 if value == true return 0 if value == false - return value.to_i if !value.is_a?(String) - value.tr(",", "").to_i + return value if value.is_a?(Integer) + + if value.is_a?(String) + value = value.strip.delete(',') + sign, unsigned_hex = parse_signed_hex_str(value) + # hex integer + return Integer(value) if unsigned_hex&.count('.') == 0 + # hex float + return Integer(sign * Float(unsigned_hex)) unless unsigned_hex.nil? + # floating point number + return Integer(BigDecimal(value)) if value.count('.') == 1 + end + + Integer(value) end def convert_float(value) return 1.0 if value == true return 0.0 if value == false - value = value.delete(",") if value.kind_of?(String) - value.to_f + + if value.is_a?(String) + value = value.strip.delete(',') + sign, unsigned_hex = parse_signed_hex_str(value) + # hex float + return sign * Float(unsigned_hex) unless unsigned_hex.nil? + end + + Float(value) end def convert_integer_eu(value) @@ -374,6 +394,24 @@ def cnv_replace_eu(value) value.tr(",.", ".,") end + # Parses a string to determine if it represents a signed hexadecimal number. + # If the string matches a signed hex format (eg "-0x1A"), returns an array + # containing the sign (-1 or 1) and the unsigned hex string (eg "0x1A"). + # BigDecimal() can't parse hex string. + # JRuby Float() can't parse signed hex string. + # + # @param value [String] the string to parse + # @return Array(Integer, String) the sign and unsigned hex string, or nil if not a hex string + def parse_signed_hex_str(value) + if value.match?(/^[+-]?0x/i) + sign = value.start_with?('-') ? -1 : 1 + unsigned = value.sub(/^[+-]/, '') + return [sign, unsigned] + end + + nil + end + def gsub(event) @gsub_parsed.each do |config| field = config[:field] diff --git a/spec/filters/mutate_spec.rb b/spec/filters/mutate_spec.rb index e352c98..439e1fc 100644 --- a/spec/filters/mutate_spec.rb +++ b/spec/filters/mutate_spec.rb @@ -318,7 +318,7 @@ def pattern_path(path) convert => [ "message", "int"] #should be integer } } - CONFIG + CONFIG sample "not_really_important" do expect {subject}.to raise_error(LogStash::ConfigurationError, /Invalid conversion type/) @@ -1122,4 +1122,400 @@ def pattern_path(path) end end + describe "convert makes types conversion" do + context "to string" do + config <<-CONFIG + filter { + mutate { convert => { "a" => "string" } } + } + CONFIG + + context "123" do + # Integer and Long + sample({ "a" => 123 }) do + expect(subject.get("a")).to be_a(String).and eq("123") + end + # Float and Double + sample({ "a" => Float(123.0) }) do + expect(subject.get("a")).to be_a(String).and eq("123.0") + end + # String + sample({ "a" => "123" }) do + expect(subject.get("a")).to be_a(String).and eq("123") + end + sample({ "a" => "0x7b" }) do + expect(subject.get("a")).to be_a(String).and eq("0x7b") + end + sample({ "a" => "123.0" }) do + expect(subject.get("a")).to be_a(String).and eq("123.0") + end + sample({ "a" => "1.230000e+02" }) do + expect(subject.get("a")).to be_a(String).and eq("1.230000e+02") + end + end + + context "123.45" do + # Float and Double + sample({ "a" => Float(123.45) }) do + expect(subject.get("a")).to be_a(String).and eq("123.45") + end + # String + sample({ "a" => "123.45" }) do + expect(subject.get("a")).to be_a(String).and eq("123.45") + end + sample({ "a" => "1.234500e+02" }) do + expect(subject.get("a")).to be_a(String).and eq("1.234500e+02") + end + sample({ "a" => "0x1.edcdp6" }) do + expect(subject.get("a")).to be_a(String).and eq( "0x1.edcdp6" ) + end + end + + context "16777217" do + # Integer and Long + sample({ "a" => 16777217 }) do + expect(subject.get("a")).to be_a(String).and eq("16777217") + end + # Float and Double + sample({ "a" => 1.6777217E7 }) do + expect(subject.get("a")).to be_a(String).and eq("16777217.0") + end + # String + sample({ "a" => "16777217" }) do + expect(subject.get("a")).to be_a(String).and eq("16777217") + end + sample({ "a" => "16777217.0" }) do + expect(subject.get("a")).to be_a(String).and eq("16777217.0") + end + end + + context "2147483648" do + # Long + sample({ "a" => 2147483648 }) do + expect(subject.get("a")).to be_a(String).and eq("2147483648") + end + # Double + sample({ "a" => 2.147483648E9 }) do + expect(subject.get("a")).to be_a(String).and eq("2147483648.0") + end + # String + sample({ "a" => "2147483648" }) do + expect(subject.get("a")).to be_a(String).and eq("2147483648") + end + sample({ "a" => "2147483648.0" }) do + expect(subject.get("a")).to be_a(String).and eq("2147483648.0") + end + end + + context "9007199254740993" do + # Long + sample({ "a" => 9007199254740993 }) do + expect(subject.get("a")).to be_a(String).and eq("9007199254740993") + end + # String + sample({ "a" => "9007199254740993" }) do + expect(subject.get("a")).to be_a(String).and eq("9007199254740993") + end + sample({ "a" => "9007199254740993.0" }) do + expect(subject.get("a")).to be_a(String).and eq("9007199254740993.0") + end + end + + + context "9223372036854775808" do + # String + sample({ "a" => "9223372036854775808" }) do + expect(subject.get("a")).to be_a(String).and eq("9223372036854775808") + end + sample({ "a" => "9223372036854775808.0" }) do + expect(subject.get("a")).to be_a(String).and eq("9223372036854775808.0") + end + end + + context "680564693277057720000000000000000000000" do + # String + sample({ "a" => "680564693277057720000000000000000000000" }) do + expect(subject.get("a")).to be_a(String).and eq("680564693277057720000000000000000000000") + end + sample({ "a" => "680564693277057720000000000000000000000.0" }) do + expect(subject.get("a")).to be_a(String).and eq("680564693277057720000000000000000000000.0") + end + end + end + + context "to integer" do + config <<-CONFIG + filter { + mutate { convert => { "a" => "integer" } } + } + CONFIG + + context "123" do + # Integer and Long + sample({ "a" => 123 }) do + expect(subject.get("a")).to be_a(Integer).and eq(123) + end + # Float and Double + sample({ "a" => Float(123.0) }) do + expect(subject.get("a")).to be_a(Integer).and eq(123) + end + # String + sample({ "a" => "123" }) do + expect(subject.get("a")).to be_a(Integer).and eq(123) + end + sample({ "a" => "0x7b" }) do + expect(subject.get("a")).to be_a(Integer).and eq(123) + end + sample({ "a" => "123.0" }) do + expect(subject.get("a")).to be_a(Integer).and eq(123) + end + sample({ "a" => "1.230000e+02" }) do + expect(subject.get("a")).to be_a(Integer).and eq(123) + end + end + + context "123.45" do + # Float and Double + sample({ "a" => Float(123.45) }) do + expect(subject.get("a")).to be_a(Integer).and eq(123) + end + # String + sample({ "a" => "123.45" }) do + expect(subject.get("a")).to be_a(Integer).and eq(123) + end + sample({ "a" => "1.234500e+02" }) do + expect(subject.get("a")).to be_a(Integer).and eq(123) + end + sample({ "a" => "0x1.edcdp6" }) do + expect(subject.get("a")).to be_a(Integer).and eq(123) + end + end + + context "-123.45" do + # Float and Double + sample({ "a" => Float(-123.45) }) do + expect(subject.get("a")).to be_a(Integer).and eq(-123) + end + # String + sample({ "a" => "-123.45" }) do + expect(subject.get("a")).to be_a(Integer).and eq(-123) + end + sample({ "a" => "-1.234500e+02" }) do + expect(subject.get("a")).to be_a(Integer).and eq(-123) + end + sample({ "a" => "-0x1.edcdp6" }) do + expect(subject.get("a")).to be_a(Integer).and eq(-123) + end + end + + context "16777217" do + # Integer and Long + sample({ "a" => 16777217 }) do + expect(subject.get("a")).to be_a(Integer).and eq(16777217) + end + # Double + sample({ "a" => 1.6777217E7 }) do + expect(subject.get("a")).to be_a(Integer).and eq(16777217) + end + # String + sample({ "a" => "16777217" }) do + expect(subject.get("a")).to be_a(Integer).and eq(16777217) + end + sample({ "a" => "16777217.0" }) do + expect(subject.get("a")).to be_a(Integer).and eq(16777217) + end + end + + context "2147483648" do + # Long + sample({ "a" => 2147483648 }) do + expect(subject.get("a")).to be_a(Integer).and eq(2147483648) + end + # Double + sample({ "a" => 2.147483648E9 }) do + expect(subject.get("a")).to be_a(Integer).and eq(2147483648) + end + # String + sample({ "a" => "2147483648" }) do + expect(subject.get("a")).to be_a(Integer).and eq(2147483648) + end + sample({ "a" => "2147483648.0" }) do + expect(subject.get("a")).to be_a(Integer).and eq(2147483648) + end + end + + context "9007199254740993" do + # Long + sample({ "a" => 9007199254740993 }) do + expect(subject.get("a")).to be_a(Integer).and eq(9007199254740993) + end + # String + sample({ "a" => "9007199254740993" }) do + expect(subject.get("a")).to be_a(Integer).and eq(9007199254740993) + end + sample({ "a" => "9007199254740993.0" }) do + expect(subject.get("a")).to be_a(Integer).and eq(9007199254740993) + end + end + + context "9223372036854775808" do + # String + sample({ "a" => "9223372036854775808" }) do + expect(subject.get("a")).to be_a(Integer).and eq(9223372036854775808) + end + sample({ "a" => "9223372036854775808.0" }) do + expect(subject.get("a")).to be_a(Integer).and eq(9223372036854775808) + end + end + + context "680564693277057720000000000000000000000" do + # String + sample({ "a" => "680564693277057720000000000000000000000" }) do + expect(subject.get("a")).to be_a(Integer).and eq(680564693277057720000000000000000000000) + end + sample({ "a" => "680564693277057720000000000000000000000.0" }) do + expect(subject.get("a")).to be_a(Integer).and eq(680564693277057720000000000000000000000) + end + end + + end + + context "to float" do + config <<-CONFIG + filter { + mutate { convert => { "a" => "float" } } + } + CONFIG + + context "123" do + # Integer and Long + sample({ "a" => 123 }) do + expect(subject.get("a")).to be_a(Float).and eq(123.0) + end + # Float and Double + sample({ "a" => Float(123) }) do + expect(subject.get("a")).to be_a(Float).and eq(123.0) + end + # String + sample({ "a" => "123" }) do + expect(subject.get("a")).to be_a(Float).and eq(123.0) + end + sample({ "a" => "0x7b" }) do + expect(subject.get("a")).to be_a(Float).and eq(123.0) + end + sample({ "a" => "123.0" }) do + expect(subject.get("a")).to be_a(Float).and eq(123.0) + end + sample({ "a" => "1.230000e+02" }) do + expect(subject.get("a")).to be_a(Float).and eq(123.0) + end + end + + context "123.45" do + # Float and Double + sample({ "a" => Float(123.45) }) do + expect(subject.get("a")).to be_a(Float).and eq(123.45) + end + # String + sample({ "a" => "123.45" }) do + expect(subject.get("a")).to be_a(Float).and eq(123.45) + end + sample({ "a" => "1.234500e+02" }) do + expect(subject.get("a")).to be_a(Float).and eq(123.45) + end + sample({ "a" => "0x1.edcdp6" }) do + expect(subject.get("a")).to be_a(Float).and eq(123.4501953125) + end + end + + context "-123.45" do + # Float and Double + sample({ "a" => Float(-123.45) }) do + expect(subject.get("a")).to be_a(Float).and eq(-123.45) + end + # String + sample({ "a" => "-123.45" }) do + expect(subject.get("a")).to be_a(Float).and eq(-123.45) + end + sample({ "a" => "-1.234500e+02" }) do + expect(subject.get("a")).to be_a(Float).and eq(-123.45) + end + sample({ "a" => "-0x1.edcdp6" }) do + expect(subject.get("a")).to be_a(Float).and eq(-123.4501953125) + end + end + + context "16777217" do + # Integer and Long + sample({ "a" => 16777217 }) do + expect(subject.get("a")).to be_a(Float).and eq(1.6777217E7) + end + # Float and Double + sample({ "a" => 1.6777217E7 }) do + expect(subject.get("a")).to be_a(Float).and eq(1.6777217E7) + end + # String + sample({ "a" => "16777217" }) do + expect(subject.get("a")).to be_a(Float).and eq(1.6777217E7) + end + sample({ "a" => "16777217.0" }) do + expect(subject.get("a")).to be_a(Float).and eq(1.6777217E7) + end + end + + + context "2147483648" do + # Long + sample({ "a" => 2147483648 }) do + expect(subject.get("a")).to be_a(Float).and eq(2.147483648E9) + end + # Double + sample({ "a" => 2.147483648E9 }) do + expect(subject.get("a")).to be_a(Float).and eq(2.147483648E9) + end + # String + sample({ "a" => "2147483648" }) do + expect(subject.get("a")).to be_a(Float).and eq(2.147483648E9) + end + sample({ "a" => "2147483648.0" }) do + expect(subject.get("a")).to be_a(Float).and eq(2.147483648E9) + end + end + + context "9007199254740993" do + # Long + sample({ "a" => 9007199254740993 }) do + expect(subject.get("a")).to be_a(Float).and eq(9.007199254740992E15) + end + # String + sample({ "a" => "9007199254740993" }) do + expect(subject.get("a")).to be_a(Float).and eq(9.007199254740992E15) + end + sample({ "a" => "9007199254740993.0" }) do + expect(subject.get("a")).to be_a(Float).and eq(9.007199254740992E15) + end + end + + context "9223372036854775808" do + # String + sample({ "a" => "9223372036854775808" }) do + expect(subject.get("a")).to be_a(Float).and eq(9.223372036854775808E18) + end + sample({ "a" => "9223372036854775808.0" }) do + expect(subject.get("a")).to be_a(Float).and eq(9.223372036854775808E18) + end + end + + context "680564693277057720000000000000000000000" do + # String + sample({ "a" => "680564693277057720000000000000000000000" }) do + expect(subject.get("a")).to be_a(Float).and eq(6.805646932770577e+38) + end + sample({ "a" => "680564693277057720000000000000000000000.0" }) do + expect(subject.get("a")).to be_a(Float).and eq(6.805646932770577e+38) + end + end + end + + end end From 7f4a9b0dd5a69d8d999f858599a930370a68ee86 Mon Sep 17 00:00:00 2001 From: Kaise Cheng Date: Tue, 28 Oct 2025 00:23:13 +0000 Subject: [PATCH 2/7] fallback to lenient type conversions for version < 8.14 --- lib/logstash/filters/mutate.rb | 42 +++++++++++++++++++++------------- spec/filters/mutate_spec.rb | 36 ++++++++++++++--------------- 2 files changed, 44 insertions(+), 34 deletions(-) diff --git a/lib/logstash/filters/mutate.rb b/lib/logstash/filters/mutate.rb index 6fb9b29..ca18f1f 100644 --- a/lib/logstash/filters/mutate.rb +++ b/lib/logstash/filters/mutate.rb @@ -228,6 +228,8 @@ def register ) end end + # fallback to old type conversion behavior prior to Logstash 8.14.0 + @lenient_conversion = self.class.is_lenient_version? @gsub_parsed = [] @gsub.nil? or @gsub.each_slice(3) do |field, needle, replacement| @@ -343,18 +345,22 @@ def convert_boolean(value) end def convert_integer(value) + value = value.strip.delete(',') if value.is_a?(String) + return 1 if value == true return 0 if value == false return value if value.is_a?(Integer) + return value.to_i if @lenient_conversion if value.is_a?(String) - value = value.strip.delete(',') - sign, unsigned_hex = parse_signed_hex_str(value) - # hex integer - return Integer(value) if unsigned_hex&.count('.') == 0 - # hex float - return Integer(sign * Float(unsigned_hex)) unless unsigned_hex.nil? - # floating point number + signed_float = parse_signed_hex_str(value) + # hex number + if signed_float + return Integer(value) if value.count(".").zero? + return Integer(signed_float) + end + + # floating point number. BigDecimal() can't parse hex string return Integer(BigDecimal(value)) if value.count('.') == 1 end @@ -362,14 +368,16 @@ def convert_integer(value) end def convert_float(value) + value = value.strip.delete(',') if value.is_a?(String) + return 1.0 if value == true return 0.0 if value == false + return value if value.is_a?(Float) + return value.to_f if @lenient_conversion if value.is_a?(String) - value = value.strip.delete(',') - sign, unsigned_hex = parse_signed_hex_str(value) - # hex float - return sign * Float(unsigned_hex) unless unsigned_hex.nil? + signed_float = parse_signed_hex_str(value) + return signed_float if signed_float end Float(value) @@ -395,18 +403,16 @@ def cnv_replace_eu(value) end # Parses a string to determine if it represents a signed hexadecimal number. - # If the string matches a signed hex format (eg "-0x1A"), returns an array - # containing the sign (-1 or 1) and the unsigned hex string (eg "0x1A"). - # BigDecimal() can't parse hex string. + # If the string matches a signed hex format (eg "-0x1A"), returns the signed float value. # JRuby Float() can't parse signed hex string. # # @param value [String] the string to parse - # @return Array(Integer, String) the sign and unsigned hex string, or nil if not a hex string + # @return [Float, nil] the signed float value if hex, or nil if not a hex string def parse_signed_hex_str(value) if value.match?(/^[+-]?0x/i) sign = value.start_with?('-') ? -1 : 1 unsigned = value.sub(/^[+-]/, '') - return [sign, unsigned] + return sign * Float(unsigned) end nil @@ -580,4 +586,8 @@ def copy(event) event.set(dest_field,LogStash::Util.deep_clone(original)) end end + + def self.is_lenient_version? + Gem::Version.new(LOGSTASH_VERSION) < Gem::Version.new("8.14.0") + end end diff --git a/spec/filters/mutate_spec.rb b/spec/filters/mutate_spec.rb index 439e1fc..70704e2 100644 --- a/spec/filters/mutate_spec.rb +++ b/spec/filters/mutate_spec.rb @@ -1122,7 +1122,7 @@ def pattern_path(path) end end - describe "convert makes types conversion" do + describe "convert makes type conversions" do context "to string" do config <<-CONFIG filter { @@ -1131,11 +1131,11 @@ def pattern_path(path) CONFIG context "123" do - # Integer and Long + # Integer sample({ "a" => 123 }) do expect(subject.get("a")).to be_a(String).and eq("123") end - # Float and Double + # Float sample({ "a" => Float(123.0) }) do expect(subject.get("a")).to be_a(String).and eq("123.0") end @@ -1155,7 +1155,7 @@ def pattern_path(path) end context "123.45" do - # Float and Double + # Float sample({ "a" => Float(123.45) }) do expect(subject.get("a")).to be_a(String).and eq("123.45") end @@ -1172,11 +1172,11 @@ def pattern_path(path) end context "16777217" do - # Integer and Long + # Integer sample({ "a" => 16777217 }) do expect(subject.get("a")).to be_a(String).and eq("16777217") end - # Float and Double + # Float sample({ "a" => 1.6777217E7 }) do expect(subject.get("a")).to be_a(String).and eq("16777217.0") end @@ -1251,11 +1251,11 @@ def pattern_path(path) CONFIG context "123" do - # Integer and Long + # Integer sample({ "a" => 123 }) do expect(subject.get("a")).to be_a(Integer).and eq(123) end - # Float and Double + # Float sample({ "a" => Float(123.0) }) do expect(subject.get("a")).to be_a(Integer).and eq(123) end @@ -1275,7 +1275,7 @@ def pattern_path(path) end context "123.45" do - # Float and Double + # Float sample({ "a" => Float(123.45) }) do expect(subject.get("a")).to be_a(Integer).and eq(123) end @@ -1292,7 +1292,7 @@ def pattern_path(path) end context "-123.45" do - # Float and Double + # Float sample({ "a" => Float(-123.45) }) do expect(subject.get("a")).to be_a(Integer).and eq(-123) end @@ -1309,7 +1309,7 @@ def pattern_path(path) end context "16777217" do - # Integer and Long + # Integer sample({ "a" => 16777217 }) do expect(subject.get("a")).to be_a(Integer).and eq(16777217) end @@ -1388,11 +1388,11 @@ def pattern_path(path) CONFIG context "123" do - # Integer and Long + # Integer sample({ "a" => 123 }) do expect(subject.get("a")).to be_a(Float).and eq(123.0) end - # Float and Double + # Float sample({ "a" => Float(123) }) do expect(subject.get("a")).to be_a(Float).and eq(123.0) end @@ -1412,7 +1412,7 @@ def pattern_path(path) end context "123.45" do - # Float and Double + # Float sample({ "a" => Float(123.45) }) do expect(subject.get("a")).to be_a(Float).and eq(123.45) end @@ -1429,7 +1429,7 @@ def pattern_path(path) end context "-123.45" do - # Float and Double + # Float sample({ "a" => Float(-123.45) }) do expect(subject.get("a")).to be_a(Float).and eq(-123.45) end @@ -1446,11 +1446,11 @@ def pattern_path(path) end context "16777217" do - # Integer and Long + # Integer sample({ "a" => 16777217 }) do expect(subject.get("a")).to be_a(Float).and eq(1.6777217E7) end - # Float and Double + # Float sample({ "a" => 1.6777217E7 }) do expect(subject.get("a")).to be_a(Float).and eq(1.6777217E7) end @@ -1517,5 +1517,5 @@ def pattern_path(path) end end - end + end unless LogStash::Filters::Mutate.is_lenient_version? # only test type conversions in v8.14+ end From f60229c6c8e88b344a00591830c101be2cba41a3 Mon Sep 17 00:00:00 2001 From: Kaise Cheng Date: Tue, 28 Oct 2025 13:24:11 +0000 Subject: [PATCH 3/7] changelog --- CHANGELOG.md | 3 +++ logstash-filter-mutate.gemspec | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8faf601..ce2a385 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +## 3.5.9 + - Fix `convert` to covert hexadecimal float notation and scientific notation string to float and integer [#175](https://github.com/logstash-plugins/logstash-filter-mutate/pull/175) + ## 3.5.8 - Fix "Can't modify frozen string" error when converting boolean to `string` [#171](https://github.com/logstash-plugins/logstash-filter-mutate/pull/171) diff --git a/logstash-filter-mutate.gemspec b/logstash-filter-mutate.gemspec index 3676ee8..8d7153d 100644 --- a/logstash-filter-mutate.gemspec +++ b/logstash-filter-mutate.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-filter-mutate' - s.version = '3.5.8' + s.version = '3.5.9' s.licenses = ['Apache License (2.0)'] s.summary = "Performs mutations on fields" s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" From dc60da9e6e8c2a752fc325733fea744ac5222cf3 Mon Sep 17 00:00:00 2001 From: Kaise Cheng Date: Tue, 28 Oct 2025 14:44:08 +0000 Subject: [PATCH 4/7] support uppercase hex str --- lib/logstash/filters/mutate.rb | 4 +-- spec/filters/mutate_spec.rb | 47 ++++++++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 2 deletions(-) diff --git a/lib/logstash/filters/mutate.rb b/lib/logstash/filters/mutate.rb index ca18f1f..c2d53a8 100644 --- a/lib/logstash/filters/mutate.rb +++ b/lib/logstash/filters/mutate.rb @@ -404,14 +404,14 @@ def cnv_replace_eu(value) # Parses a string to determine if it represents a signed hexadecimal number. # If the string matches a signed hex format (eg "-0x1A"), returns the signed float value. - # JRuby Float() can't parse signed hex string. + # JRuby Float() can't parse signed hex string and uppercase hex string. # # @param value [String] the string to parse # @return [Float, nil] the signed float value if hex, or nil if not a hex string def parse_signed_hex_str(value) if value.match?(/^[+-]?0x/i) sign = value.start_with?('-') ? -1 : 1 - unsigned = value.sub(/^[+-]/, '') + unsigned = value.sub(/^[+-]/, '').downcase return sign * Float(unsigned) end diff --git a/spec/filters/mutate_spec.rb b/spec/filters/mutate_spec.rb index 70704e2..5a7bc6c 100644 --- a/spec/filters/mutate_spec.rb +++ b/spec/filters/mutate_spec.rb @@ -1518,4 +1518,51 @@ def pattern_path(path) end end unless LogStash::Filters::Mutate.is_lenient_version? # only test type conversions in v8.14+ + + describe "parse_signed_hex_str" do + subject { LogStash::Filters::Mutate.new({ }) } + + context 'hexadecimal integers' do + it 'parses positive hex integer' do + expect(subject.send(:parse_signed_hex_str, '0x1A')).to eq(26.0) + expect(subject.send(:parse_signed_hex_str, '+0xFF')).to eq(255.0) + end + + it 'parses negative hex integer' do + expect(subject.send(:parse_signed_hex_str, '-0x1A')).to eq(-26.0) + expect(subject.send(:parse_signed_hex_str,'-0xFF')).to eq(-255.0) + end + + it 'ignores case in hex prefix' do + expect(subject.send(:parse_signed_hex_str,'0X1A')).to eq(26.0) + expect(subject.send(:parse_signed_hex_str,'-0X1A')).to eq(-26.0) + end + end + + context 'hexadecimal floats' do + it 'parses positive hex float' do + expect(subject.send(:parse_signed_hex_str,'0x1.8p+1')).to eq(3.0) + expect(subject.send(:parse_signed_hex_str,'+0x1.2p+2')).to eq(4.5) + end + + it 'parses negative hex float' do + expect(subject.send(:parse_signed_hex_str,'-0x1.8p+1')).to eq(-3.0) + expect(subject.send(:parse_signed_hex_str,'-0x1.2p+2')).to eq(-4.5) + end + end + + context 'non-hex strings' do + it 'returns nil for decimal numbers' do + expect(subject.send(:parse_signed_hex_str,'123')).to be_nil + expect(subject.send(:parse_signed_hex_str,'-456')).to be_nil + expect(subject.send(:parse_signed_hex_str,'1.23')).to be_nil + end + + it 'returns nil for random strings' do + expect(subject.send(:parse_signed_hex_str,'abc')).to be_nil + expect(subject.send(:parse_signed_hex_str,'0b1010')).to be_nil + end + end + end unless LogStash::Filters::Mutate.is_lenient_version? + end From f026a4763ab68c22d1ea5efcfcbb54a2fca4b618 Mon Sep 17 00:00:00 2001 From: Kaise Cheng Date: Thu, 30 Oct 2025 11:38:03 +0000 Subject: [PATCH 5/7] add support to jruby 10 for signed hex to float --- lib/logstash/filters/mutate.rb | 13 ++++++++++++- spec/filters/mutate_spec.rb | 2 -- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/lib/logstash/filters/mutate.rb b/lib/logstash/filters/mutate.rb index c2d53a8..bc50f51 100644 --- a/lib/logstash/filters/mutate.rb +++ b/lib/logstash/filters/mutate.rb @@ -230,6 +230,8 @@ def register end # fallback to old type conversion behavior prior to Logstash 8.14.0 @lenient_conversion = self.class.is_lenient_version? + # signed hex parsing support in jruby 10 + @support_signed_hex = self.class.can_parse_signed_hex? @gsub_parsed = [] @gsub.nil? or @gsub.each_slice(3) do |field, needle, replacement| @@ -404,11 +406,14 @@ def cnv_replace_eu(value) # Parses a string to determine if it represents a signed hexadecimal number. # If the string matches a signed hex format (eg "-0x1A"), returns the signed float value. - # JRuby Float() can't parse signed hex string and uppercase hex string. + # JRuby Float() can parse signed hex string and uppercase hex string in version 10+, + # but not in earlier versions. # # @param value [String] the string to parse # @return [Float, nil] the signed float value if hex, or nil if not a hex string def parse_signed_hex_str(value) + return Float(value) if @support_signed_hex + if value.match?(/^[+-]?0x/i) sign = value.start_with?('-') ? -1 : 1 unsigned = value.sub(/^[+-]/, '').downcase @@ -590,4 +595,10 @@ def copy(event) def self.is_lenient_version? Gem::Version.new(LOGSTASH_VERSION) < Gem::Version.new("8.14.0") end + + def self.can_parse_signed_hex? + Float("-0x1A") == -26.0 + rescue ArgumentError + false + end end diff --git a/spec/filters/mutate_spec.rb b/spec/filters/mutate_spec.rb index 5a7bc6c..b6364b7 100644 --- a/spec/filters/mutate_spec.rb +++ b/spec/filters/mutate_spec.rb @@ -1525,7 +1525,6 @@ def pattern_path(path) context 'hexadecimal integers' do it 'parses positive hex integer' do expect(subject.send(:parse_signed_hex_str, '0x1A')).to eq(26.0) - expect(subject.send(:parse_signed_hex_str, '+0xFF')).to eq(255.0) end it 'parses negative hex integer' do @@ -1542,7 +1541,6 @@ def pattern_path(path) context 'hexadecimal floats' do it 'parses positive hex float' do expect(subject.send(:parse_signed_hex_str,'0x1.8p+1')).to eq(3.0) - expect(subject.send(:parse_signed_hex_str,'+0x1.2p+2')).to eq(4.5) end it 'parses negative hex float' do From 03774e58a7be8ec80fafbae8ec457fa131adbb26 Mon Sep 17 00:00:00 2001 From: Kaise Cheng Date: Fri, 31 Oct 2025 14:28:12 +0000 Subject: [PATCH 6/7] only use BigDecimal when the string is scientific notation --- lib/logstash/filters/mutate.rb | 6 ++++-- spec/filters/mutate_spec.rb | 6 ++++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/lib/logstash/filters/mutate.rb b/lib/logstash/filters/mutate.rb index bc50f51..67c6f59 100644 --- a/lib/logstash/filters/mutate.rb +++ b/lib/logstash/filters/mutate.rb @@ -362,8 +362,10 @@ def convert_integer(value) return Integer(signed_float) end - # floating point number. BigDecimal() can't parse hex string - return Integer(BigDecimal(value)) if value.count('.') == 1 + # scientific notation. BigDecimal() can't parse hex string + return BigDecimal(value).to_i if value.include?("e") + # maybe a float string + return value.to_i end Integer(value) diff --git a/spec/filters/mutate_spec.rb b/spec/filters/mutate_spec.rb index b6364b7..eb6e47b 100644 --- a/spec/filters/mutate_spec.rb +++ b/spec/filters/mutate_spec.rb @@ -1356,6 +1356,9 @@ def pattern_path(path) sample({ "a" => "9007199254740993.0" }) do expect(subject.get("a")).to be_a(Integer).and eq(9007199254740993) end + sample({ "a" => "9.007199254740993e+15" }) do + expect(subject.get("a")).to be_a(Integer).and eq(9007199254740993) + end end context "9223372036854775808" do @@ -1494,6 +1497,9 @@ def pattern_path(path) sample({ "a" => "9007199254740993.0" }) do expect(subject.get("a")).to be_a(Float).and eq(9.007199254740992E15) end + sample({ "a" => "9.007199254740993e+15" }) do + expect(subject.get("a")).to be_a(Float).and eq(9007199254740992) + end end context "9223372036854775808" do From 2be6c3883b4f12828029a6f64c7c2ec4abfa1460 Mon Sep 17 00:00:00 2001 From: Kaise Cheng Date: Fri, 31 Oct 2025 14:51:44 +0000 Subject: [PATCH 7/7] notation case insensitive --- lib/logstash/filters/mutate.rb | 6 +++--- spec/filters/mutate_spec.rb | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/logstash/filters/mutate.rb b/lib/logstash/filters/mutate.rb index 67c6f59..d11ddc3 100644 --- a/lib/logstash/filters/mutate.rb +++ b/lib/logstash/filters/mutate.rb @@ -347,7 +347,7 @@ def convert_boolean(value) end def convert_integer(value) - value = value.strip.delete(',') if value.is_a?(String) + value = value.strip.delete(',').downcase if value.is_a?(String) return 1 if value == true return 0 if value == false @@ -372,7 +372,7 @@ def convert_integer(value) end def convert_float(value) - value = value.strip.delete(',') if value.is_a?(String) + value = value.strip.delete(',').downcase if value.is_a?(String) return 1.0 if value == true return 0.0 if value == false @@ -418,7 +418,7 @@ def parse_signed_hex_str(value) if value.match?(/^[+-]?0x/i) sign = value.start_with?('-') ? -1 : 1 - unsigned = value.sub(/^[+-]/, '').downcase + unsigned = value.sub(/^[+-]/, '') return sign * Float(unsigned) end diff --git a/spec/filters/mutate_spec.rb b/spec/filters/mutate_spec.rb index eb6e47b..591c6a3 100644 --- a/spec/filters/mutate_spec.rb +++ b/spec/filters/mutate_spec.rb @@ -1539,8 +1539,8 @@ def pattern_path(path) end it 'ignores case in hex prefix' do - expect(subject.send(:parse_signed_hex_str,'0X1A')).to eq(26.0) - expect(subject.send(:parse_signed_hex_str,'-0X1A')).to eq(-26.0) + expect(subject.send(:parse_signed_hex_str,'0x1A')).to eq(26.0) + expect(subject.send(:parse_signed_hex_str,'-0x1A')).to eq(-26.0) end end