diff --git a/CHANGELOG.md b/CHANGELOG.md index 8faf601..ce2a385 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +## 3.5.9 + - Fix `convert` to covert hexadecimal float notation and scientific notation string to float and integer [#175](https://github.com/logstash-plugins/logstash-filter-mutate/pull/175) + ## 3.5.8 - Fix "Can't modify frozen string" error when converting boolean to `string` [#171](https://github.com/logstash-plugins/logstash-filter-mutate/pull/171) diff --git a/lib/logstash/filters/mutate.rb b/lib/logstash/filters/mutate.rb index 076cce2..d11ddc3 100644 --- a/lib/logstash/filters/mutate.rb +++ b/lib/logstash/filters/mutate.rb @@ -1,6 +1,7 @@ # encoding: utf-8 require "logstash/filters/base" require "logstash/namespace" +require "bigdecimal" # The mutate filter allows you to perform general mutations on fields. You # can rename, replace, and modify fields in your events. @@ -227,6 +228,10 @@ def register ) end end + # fallback to old type conversion behavior prior to Logstash 8.14.0 + @lenient_conversion = self.class.is_lenient_version? + # signed hex parsing support in jruby 10 + @support_signed_hex = self.class.can_parse_signed_hex? @gsub_parsed = [] @gsub.nil? or @gsub.each_slice(3) do |field, needle, replacement| @@ -342,17 +347,44 @@ def convert_boolean(value) end def convert_integer(value) + value = value.strip.delete(',').downcase if value.is_a?(String) + return 1 if value == true return 0 if value == false - return value.to_i if !value.is_a?(String) - value.tr(",", "").to_i + return value if value.is_a?(Integer) + return value.to_i if @lenient_conversion + + if value.is_a?(String) + signed_float = parse_signed_hex_str(value) + # hex number + if signed_float + return Integer(value) if value.count(".").zero? + return Integer(signed_float) + end + + # scientific notation. BigDecimal() can't parse hex string + return BigDecimal(value).to_i if value.include?("e") + # maybe a float string + return value.to_i + end + + Integer(value) end def convert_float(value) + value = value.strip.delete(',').downcase if value.is_a?(String) + return 1.0 if value == true return 0.0 if value == false - value = value.delete(",") if value.kind_of?(String) - value.to_f + return value if value.is_a?(Float) + return value.to_f if @lenient_conversion + + if value.is_a?(String) + signed_float = parse_signed_hex_str(value) + return signed_float if signed_float + end + + Float(value) end def convert_integer_eu(value) @@ -374,6 +406,25 @@ def cnv_replace_eu(value) value.tr(",.", ".,") end + # Parses a string to determine if it represents a signed hexadecimal number. + # If the string matches a signed hex format (eg "-0x1A"), returns the signed float value. + # JRuby Float() can parse signed hex string and uppercase hex string in version 10+, + # but not in earlier versions. + # + # @param value [String] the string to parse + # @return [Float, nil] the signed float value if hex, or nil if not a hex string + def parse_signed_hex_str(value) + return Float(value) if @support_signed_hex + + if value.match?(/^[+-]?0x/i) + sign = value.start_with?('-') ? -1 : 1 + unsigned = value.sub(/^[+-]/, '') + return sign * Float(unsigned) + end + + nil + end + def gsub(event) @gsub_parsed.each do |config| field = config[:field] @@ -542,4 +593,14 @@ def copy(event) event.set(dest_field,LogStash::Util.deep_clone(original)) end end + + def self.is_lenient_version? + Gem::Version.new(LOGSTASH_VERSION) < Gem::Version.new("8.14.0") + end + + def self.can_parse_signed_hex? + Float("-0x1A") == -26.0 + rescue ArgumentError + false + end end diff --git a/logstash-filter-mutate.gemspec b/logstash-filter-mutate.gemspec index 3676ee8..8d7153d 100644 --- a/logstash-filter-mutate.gemspec +++ b/logstash-filter-mutate.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-filter-mutate' - s.version = '3.5.8' + s.version = '3.5.9' s.licenses = ['Apache License (2.0)'] s.summary = "Performs mutations on fields" s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" diff --git a/spec/filters/mutate_spec.rb b/spec/filters/mutate_spec.rb index e352c98..591c6a3 100644 --- a/spec/filters/mutate_spec.rb +++ b/spec/filters/mutate_spec.rb @@ -318,7 +318,7 @@ def pattern_path(path) convert => [ "message", "int"] #should be integer } } - CONFIG + CONFIG sample "not_really_important" do expect {subject}.to raise_error(LogStash::ConfigurationError, /Invalid conversion type/) @@ -1122,4 +1122,451 @@ def pattern_path(path) end end + describe "convert makes type conversions" do + context "to string" do + config <<-CONFIG + filter { + mutate { convert => { "a" => "string" } } + } + CONFIG + + context "123" do + # Integer + sample({ "a" => 123 }) do + expect(subject.get("a")).to be_a(String).and eq("123") + end + # Float + sample({ "a" => Float(123.0) }) do + expect(subject.get("a")).to be_a(String).and eq("123.0") + end + # String + sample({ "a" => "123" }) do + expect(subject.get("a")).to be_a(String).and eq("123") + end + sample({ "a" => "0x7b" }) do + expect(subject.get("a")).to be_a(String).and eq("0x7b") + end + sample({ "a" => "123.0" }) do + expect(subject.get("a")).to be_a(String).and eq("123.0") + end + sample({ "a" => "1.230000e+02" }) do + expect(subject.get("a")).to be_a(String).and eq("1.230000e+02") + end + end + + context "123.45" do + # Float + sample({ "a" => Float(123.45) }) do + expect(subject.get("a")).to be_a(String).and eq("123.45") + end + # String + sample({ "a" => "123.45" }) do + expect(subject.get("a")).to be_a(String).and eq("123.45") + end + sample({ "a" => "1.234500e+02" }) do + expect(subject.get("a")).to be_a(String).and eq("1.234500e+02") + end + sample({ "a" => "0x1.edcdp6" }) do + expect(subject.get("a")).to be_a(String).and eq( "0x1.edcdp6" ) + end + end + + context "16777217" do + # Integer + sample({ "a" => 16777217 }) do + expect(subject.get("a")).to be_a(String).and eq("16777217") + end + # Float + sample({ "a" => 1.6777217E7 }) do + expect(subject.get("a")).to be_a(String).and eq("16777217.0") + end + # String + sample({ "a" => "16777217" }) do + expect(subject.get("a")).to be_a(String).and eq("16777217") + end + sample({ "a" => "16777217.0" }) do + expect(subject.get("a")).to be_a(String).and eq("16777217.0") + end + end + + context "2147483648" do + # Long + sample({ "a" => 2147483648 }) do + expect(subject.get("a")).to be_a(String).and eq("2147483648") + end + # Double + sample({ "a" => 2.147483648E9 }) do + expect(subject.get("a")).to be_a(String).and eq("2147483648.0") + end + # String + sample({ "a" => "2147483648" }) do + expect(subject.get("a")).to be_a(String).and eq("2147483648") + end + sample({ "a" => "2147483648.0" }) do + expect(subject.get("a")).to be_a(String).and eq("2147483648.0") + end + end + + context "9007199254740993" do + # Long + sample({ "a" => 9007199254740993 }) do + expect(subject.get("a")).to be_a(String).and eq("9007199254740993") + end + # String + sample({ "a" => "9007199254740993" }) do + expect(subject.get("a")).to be_a(String).and eq("9007199254740993") + end + sample({ "a" => "9007199254740993.0" }) do + expect(subject.get("a")).to be_a(String).and eq("9007199254740993.0") + end + end + + + context "9223372036854775808" do + # String + sample({ "a" => "9223372036854775808" }) do + expect(subject.get("a")).to be_a(String).and eq("9223372036854775808") + end + sample({ "a" => "9223372036854775808.0" }) do + expect(subject.get("a")).to be_a(String).and eq("9223372036854775808.0") + end + end + + context "680564693277057720000000000000000000000" do + # String + sample({ "a" => "680564693277057720000000000000000000000" }) do + expect(subject.get("a")).to be_a(String).and eq("680564693277057720000000000000000000000") + end + sample({ "a" => "680564693277057720000000000000000000000.0" }) do + expect(subject.get("a")).to be_a(String).and eq("680564693277057720000000000000000000000.0") + end + end + end + + context "to integer" do + config <<-CONFIG + filter { + mutate { convert => { "a" => "integer" } } + } + CONFIG + + context "123" do + # Integer + sample({ "a" => 123 }) do + expect(subject.get("a")).to be_a(Integer).and eq(123) + end + # Float + sample({ "a" => Float(123.0) }) do + expect(subject.get("a")).to be_a(Integer).and eq(123) + end + # String + sample({ "a" => "123" }) do + expect(subject.get("a")).to be_a(Integer).and eq(123) + end + sample({ "a" => "0x7b" }) do + expect(subject.get("a")).to be_a(Integer).and eq(123) + end + sample({ "a" => "123.0" }) do + expect(subject.get("a")).to be_a(Integer).and eq(123) + end + sample({ "a" => "1.230000e+02" }) do + expect(subject.get("a")).to be_a(Integer).and eq(123) + end + end + + context "123.45" do + # Float + sample({ "a" => Float(123.45) }) do + expect(subject.get("a")).to be_a(Integer).and eq(123) + end + # String + sample({ "a" => "123.45" }) do + expect(subject.get("a")).to be_a(Integer).and eq(123) + end + sample({ "a" => "1.234500e+02" }) do + expect(subject.get("a")).to be_a(Integer).and eq(123) + end + sample({ "a" => "0x1.edcdp6" }) do + expect(subject.get("a")).to be_a(Integer).and eq(123) + end + end + + context "-123.45" do + # Float + sample({ "a" => Float(-123.45) }) do + expect(subject.get("a")).to be_a(Integer).and eq(-123) + end + # String + sample({ "a" => "-123.45" }) do + expect(subject.get("a")).to be_a(Integer).and eq(-123) + end + sample({ "a" => "-1.234500e+02" }) do + expect(subject.get("a")).to be_a(Integer).and eq(-123) + end + sample({ "a" => "-0x1.edcdp6" }) do + expect(subject.get("a")).to be_a(Integer).and eq(-123) + end + end + + context "16777217" do + # Integer + sample({ "a" => 16777217 }) do + expect(subject.get("a")).to be_a(Integer).and eq(16777217) + end + # Double + sample({ "a" => 1.6777217E7 }) do + expect(subject.get("a")).to be_a(Integer).and eq(16777217) + end + # String + sample({ "a" => "16777217" }) do + expect(subject.get("a")).to be_a(Integer).and eq(16777217) + end + sample({ "a" => "16777217.0" }) do + expect(subject.get("a")).to be_a(Integer).and eq(16777217) + end + end + + context "2147483648" do + # Long + sample({ "a" => 2147483648 }) do + expect(subject.get("a")).to be_a(Integer).and eq(2147483648) + end + # Double + sample({ "a" => 2.147483648E9 }) do + expect(subject.get("a")).to be_a(Integer).and eq(2147483648) + end + # String + sample({ "a" => "2147483648" }) do + expect(subject.get("a")).to be_a(Integer).and eq(2147483648) + end + sample({ "a" => "2147483648.0" }) do + expect(subject.get("a")).to be_a(Integer).and eq(2147483648) + end + end + + context "9007199254740993" do + # Long + sample({ "a" => 9007199254740993 }) do + expect(subject.get("a")).to be_a(Integer).and eq(9007199254740993) + end + # String + sample({ "a" => "9007199254740993" }) do + expect(subject.get("a")).to be_a(Integer).and eq(9007199254740993) + end + sample({ "a" => "9007199254740993.0" }) do + expect(subject.get("a")).to be_a(Integer).and eq(9007199254740993) + end + sample({ "a" => "9.007199254740993e+15" }) do + expect(subject.get("a")).to be_a(Integer).and eq(9007199254740993) + end + end + + context "9223372036854775808" do + # String + sample({ "a" => "9223372036854775808" }) do + expect(subject.get("a")).to be_a(Integer).and eq(9223372036854775808) + end + sample({ "a" => "9223372036854775808.0" }) do + expect(subject.get("a")).to be_a(Integer).and eq(9223372036854775808) + end + end + + context "680564693277057720000000000000000000000" do + # String + sample({ "a" => "680564693277057720000000000000000000000" }) do + expect(subject.get("a")).to be_a(Integer).and eq(680564693277057720000000000000000000000) + end + sample({ "a" => "680564693277057720000000000000000000000.0" }) do + expect(subject.get("a")).to be_a(Integer).and eq(680564693277057720000000000000000000000) + end + end + + end + + context "to float" do + config <<-CONFIG + filter { + mutate { convert => { "a" => "float" } } + } + CONFIG + + context "123" do + # Integer + sample({ "a" => 123 }) do + expect(subject.get("a")).to be_a(Float).and eq(123.0) + end + # Float + sample({ "a" => Float(123) }) do + expect(subject.get("a")).to be_a(Float).and eq(123.0) + end + # String + sample({ "a" => "123" }) do + expect(subject.get("a")).to be_a(Float).and eq(123.0) + end + sample({ "a" => "0x7b" }) do + expect(subject.get("a")).to be_a(Float).and eq(123.0) + end + sample({ "a" => "123.0" }) do + expect(subject.get("a")).to be_a(Float).and eq(123.0) + end + sample({ "a" => "1.230000e+02" }) do + expect(subject.get("a")).to be_a(Float).and eq(123.0) + end + end + + context "123.45" do + # Float + sample({ "a" => Float(123.45) }) do + expect(subject.get("a")).to be_a(Float).and eq(123.45) + end + # String + sample({ "a" => "123.45" }) do + expect(subject.get("a")).to be_a(Float).and eq(123.45) + end + sample({ "a" => "1.234500e+02" }) do + expect(subject.get("a")).to be_a(Float).and eq(123.45) + end + sample({ "a" => "0x1.edcdp6" }) do + expect(subject.get("a")).to be_a(Float).and eq(123.4501953125) + end + end + + context "-123.45" do + # Float + sample({ "a" => Float(-123.45) }) do + expect(subject.get("a")).to be_a(Float).and eq(-123.45) + end + # String + sample({ "a" => "-123.45" }) do + expect(subject.get("a")).to be_a(Float).and eq(-123.45) + end + sample({ "a" => "-1.234500e+02" }) do + expect(subject.get("a")).to be_a(Float).and eq(-123.45) + end + sample({ "a" => "-0x1.edcdp6" }) do + expect(subject.get("a")).to be_a(Float).and eq(-123.4501953125) + end + end + + context "16777217" do + # Integer + sample({ "a" => 16777217 }) do + expect(subject.get("a")).to be_a(Float).and eq(1.6777217E7) + end + # Float + sample({ "a" => 1.6777217E7 }) do + expect(subject.get("a")).to be_a(Float).and eq(1.6777217E7) + end + # String + sample({ "a" => "16777217" }) do + expect(subject.get("a")).to be_a(Float).and eq(1.6777217E7) + end + sample({ "a" => "16777217.0" }) do + expect(subject.get("a")).to be_a(Float).and eq(1.6777217E7) + end + end + + + context "2147483648" do + # Long + sample({ "a" => 2147483648 }) do + expect(subject.get("a")).to be_a(Float).and eq(2.147483648E9) + end + # Double + sample({ "a" => 2.147483648E9 }) do + expect(subject.get("a")).to be_a(Float).and eq(2.147483648E9) + end + # String + sample({ "a" => "2147483648" }) do + expect(subject.get("a")).to be_a(Float).and eq(2.147483648E9) + end + sample({ "a" => "2147483648.0" }) do + expect(subject.get("a")).to be_a(Float).and eq(2.147483648E9) + end + end + + context "9007199254740993" do + # Long + sample({ "a" => 9007199254740993 }) do + expect(subject.get("a")).to be_a(Float).and eq(9.007199254740992E15) + end + # String + sample({ "a" => "9007199254740993" }) do + expect(subject.get("a")).to be_a(Float).and eq(9.007199254740992E15) + end + sample({ "a" => "9007199254740993.0" }) do + expect(subject.get("a")).to be_a(Float).and eq(9.007199254740992E15) + end + sample({ "a" => "9.007199254740993e+15" }) do + expect(subject.get("a")).to be_a(Float).and eq(9007199254740992) + end + end + + context "9223372036854775808" do + # String + sample({ "a" => "9223372036854775808" }) do + expect(subject.get("a")).to be_a(Float).and eq(9.223372036854775808E18) + end + sample({ "a" => "9223372036854775808.0" }) do + expect(subject.get("a")).to be_a(Float).and eq(9.223372036854775808E18) + end + end + + context "680564693277057720000000000000000000000" do + # String + sample({ "a" => "680564693277057720000000000000000000000" }) do + expect(subject.get("a")).to be_a(Float).and eq(6.805646932770577e+38) + end + sample({ "a" => "680564693277057720000000000000000000000.0" }) do + expect(subject.get("a")).to be_a(Float).and eq(6.805646932770577e+38) + end + end + end + + end unless LogStash::Filters::Mutate.is_lenient_version? # only test type conversions in v8.14+ + + describe "parse_signed_hex_str" do + subject { LogStash::Filters::Mutate.new({ }) } + + context 'hexadecimal integers' do + it 'parses positive hex integer' do + expect(subject.send(:parse_signed_hex_str, '0x1A')).to eq(26.0) + end + + it 'parses negative hex integer' do + expect(subject.send(:parse_signed_hex_str, '-0x1A')).to eq(-26.0) + expect(subject.send(:parse_signed_hex_str,'-0xFF')).to eq(-255.0) + end + + it 'ignores case in hex prefix' do + expect(subject.send(:parse_signed_hex_str,'0x1A')).to eq(26.0) + expect(subject.send(:parse_signed_hex_str,'-0x1A')).to eq(-26.0) + end + end + + context 'hexadecimal floats' do + it 'parses positive hex float' do + expect(subject.send(:parse_signed_hex_str,'0x1.8p+1')).to eq(3.0) + end + + it 'parses negative hex float' do + expect(subject.send(:parse_signed_hex_str,'-0x1.8p+1')).to eq(-3.0) + expect(subject.send(:parse_signed_hex_str,'-0x1.2p+2')).to eq(-4.5) + end + end + + context 'non-hex strings' do + it 'returns nil for decimal numbers' do + expect(subject.send(:parse_signed_hex_str,'123')).to be_nil + expect(subject.send(:parse_signed_hex_str,'-456')).to be_nil + expect(subject.send(:parse_signed_hex_str,'1.23')).to be_nil + end + + it 'returns nil for random strings' do + expect(subject.send(:parse_signed_hex_str,'abc')).to be_nil + expect(subject.send(:parse_signed_hex_str,'0b1010')).to be_nil + end + end + end unless LogStash::Filters::Mutate.is_lenient_version? + end