From d68937d58c14b6eaa52557ef82d41d5908e600e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Helmut=20H=C3=A4nsel?= Date: Sun, 17 Nov 2024 01:28:08 +0100 Subject: [PATCH 1/7] support user-defined mapping for Inf and NaN via keyword arg --- src/write.jl | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/src/write.jl b/src/write.jl index bfff8a1..101f3f6 100644 --- a/src/write.jl +++ b/src/write.jl @@ -279,19 +279,23 @@ function write(::NumberType, buf, pos, len, x::AbstractFloat; allow_inf::Bool=fa return buf, pos, len end -@inline function write(::NumberType, buf, pos, len, x::T; allow_inf::Bool=false, kw...) where {T <: Base.IEEEFloat} - isfinite(x) || allow_inf || error("$x not allowed to be written in JSON spec") - if isinf(x) +_std_mapping(x) = x == Inf ? "Infinity" : x == -Inf ? "-Infinity" : "NaN" + +@inline function write(::NumberType, buf, pos, len, x::T; inf_mapping::Function = _std_mapping, allow_inf::Bool = inf_mapping !== _std_mapping, kw...) where {T <: Base.IEEEFloat} + if isfinite(x) + @check Ryu.neededdigits(T) + pos = Ryu.writeshortest(buf, pos, x) + else + allow_inf || error("$x not allowed to be written in JSON spec") # Although this is non-standard JSON, "Infinity" is commonly used. # See https://docs.python.org/3/library/json.html#infinite-and-nan-number-values. - if sign(x) == -1 - @writechar '-' + bytes = codeunits(inf_mapping(x)) + @check length(bytes) + for b in bytes + @inbounds buf[pos] = b + pos += 1 end - @writechar 'I' 'n' 'f' 'i' 'n' 'i' 't' 'y' - return buf, pos, len end - @check Ryu.neededdigits(T) - pos = Ryu.writeshortest(buf, pos, x) return buf, pos, len end From 1ce1b286b83f5ffd76cf189e5b48e8be25299c6a Mon Sep 17 00:00:00 2001 From: hhaensel Date: Wed, 15 Jan 2025 10:44:40 +0100 Subject: [PATCH 2/7] allow_inf: fix lower performance for default_mapping, add `underscore_inf_mapping` and `quoted_inf_mapping` plus docstrings and tests --- src/write.jl | 50 ++++++++++++++++++++++++++++++++++++++++++-------- test/json.jl | 8 ++++++++ 2 files changed, 50 insertions(+), 8 deletions(-) diff --git a/src/write.jl b/src/write.jl index 101f3f6..9a2e566 100644 --- a/src/write.jl +++ b/src/write.jl @@ -279,21 +279,55 @@ function write(::NumberType, buf, pos, len, x::AbstractFloat; allow_inf::Bool=fa return buf, pos, len end -_std_mapping(x) = x == Inf ? "Infinity" : x == -Inf ? "-Infinity" : "NaN" +""" + underscore_inf_mapping(x) + +This function provides alternative string mappings for `Inf` and `NaN` values. +### Example + +``` +inf_mapping = JSON3.underscore_inf_mapping +JSON3.write(NaN; inf_mapping) # "\\"__nan__\\"" +``` + +Alternative mappings can easily be defined by the user, e.g. a quoted version of the default mapping: + +``` +inf_mapping(x) = x == Inf ? "\\"my_infinity\\"" : x == -Inf ? "\\"-my_infinity\\"" : "\\"my_nan\\"" +JSON3.write(NaN; inf_mapping) # "\\"my_nan\\"" +``` + +See also [`quoted_inf_mapping`](@ref). +""" +underscore_inf_mapping(x) = x == Inf ? "\"__inf__\"" : x == -Inf ? "\"__neginf__\"" : "\"__nan__\"" + +""" + quoted_inf_mapping(x) + +Provides a quoted version of the default mappings for `Inf` and `NaN` values. + +See also [`underscore_inf_mapping`](@ref). +""" +quoted_inf_mapping(x) = x == Inf ? "\"Infinity\"" : x == -Inf ? "\"-Infinity\"" : "\"NaN\"" -@inline function write(::NumberType, buf, pos, len, x::T; inf_mapping::Function = _std_mapping, allow_inf::Bool = inf_mapping !== _std_mapping, kw...) where {T <: Base.IEEEFloat} - if isfinite(x) +@inline function write(::NumberType, buf, pos, len, x::T; inf_mapping::Union{Function, Nothing} = nothing, allow_inf::Bool = inf_mapping !== nothing, kw...) where {T <: Base.IEEEFloat} + if isfinite(x) || allow_inf && inf_mapping === nothing && isnan(x) @check Ryu.neededdigits(T) pos = Ryu.writeshortest(buf, pos, x) else allow_inf || error("$x not allowed to be written in JSON spec") # Although this is non-standard JSON, "Infinity" is commonly used. # See https://docs.python.org/3/library/json.html#infinite-and-nan-number-values. - bytes = codeunits(inf_mapping(x)) - @check length(bytes) - for b in bytes - @inbounds buf[pos] = b - pos += 1 + if inf_mapping === nothing + sign(x) == -1 && @writechar '-' + @writechar 'I' 'n' 'f' 'i' 'n' 'i' 't' 'y' + else + bytes = codeunits(inf_mapping(x)) + @check length(bytes) + for b in bytes + @inbounds buf[pos] = b + pos += 1 + end end end return buf, pos, len diff --git a/test/json.jl b/test/json.jl index 98b1e2c..13361cf 100644 --- a/test/json.jl +++ b/test/json.jl @@ -46,6 +46,14 @@ end @test JSON3.read("Inf"; allow_inf=true) === Inf @test JSON3.read("Infinity"; allow_inf=true) === Inf @test JSON3.read("-Infinity"; allow_inf=true) === -Inf + + @test JSON3.write(NaN, inf_mapping = JSON3.underscore_inf_mapping) == "\"__nan__\"" + @test JSON3.write(Inf, inf_mapping = JSON3.underscore_inf_mapping) == "\"__inf__\"" + @test JSON3.write(-Inf, inf_mapping = JSON3.underscore_inf_mapping) == "\"__neginf__\"" + + @test JSON3.write(NaN, inf_mapping = JSON3.quoted_inf_mapping) == "\"NaN\"" + @test JSON3.write(Inf, inf_mapping = JSON3.quoted_inf_mapping) == "\"Infinity\"" + @test JSON3.write(-Inf, inf_mapping = JSON3.quoted_inf_mapping) == "\"-Infinity\"" end @testset "Char" begin From f6568137c6ce93a45fbf5e4bcbe4781f579280bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Helmut=20H=C3=A4nsel?= Date: Thu, 16 Jan 2025 00:42:04 +0100 Subject: [PATCH 3/7] remove custom inf_mappings and add kw inf_mapping to docstring --- src/write.jl | 37 ++++++------------------------------- test/json.jl | 11 ++++------- 2 files changed, 10 insertions(+), 38 deletions(-) diff --git a/src/write.jl b/src/write.jl index 9a2e566..a1c245d 100644 --- a/src/write.jl +++ b/src/write.jl @@ -25,6 +25,12 @@ Write JSON. ## Keyword Args * `allow_inf`: Allow writing of `Inf` and `NaN` values (not part of the JSON standard). [default `false`] +* `inf_mapping`: A function to map `Inf`, `-Inf` and `NaN` values to a custom representation. [default `nothing`] + + e.g. a quoted version of the default mapping + + `quoted_inf_mapping(x) = x == Inf ? "\\"Infinity\\"" : x == -Inf ? "\\"-Infinity\\"" : "\\"NaN\\""` + * `dateformat`: A [`DateFormat`](https://docs.julialang.org/en/v1/stdlib/Dates/#Dates.DateFormat) describing how to format `Date`s in the object. [default `Dates.default_format(T)`] """ function write(io::IO, obj::T; kw...) where {T} @@ -279,37 +285,6 @@ function write(::NumberType, buf, pos, len, x::AbstractFloat; allow_inf::Bool=fa return buf, pos, len end -""" - underscore_inf_mapping(x) - -This function provides alternative string mappings for `Inf` and `NaN` values. -### Example - -``` -inf_mapping = JSON3.underscore_inf_mapping -JSON3.write(NaN; inf_mapping) # "\\"__nan__\\"" -``` - -Alternative mappings can easily be defined by the user, e.g. a quoted version of the default mapping: - -``` -inf_mapping(x) = x == Inf ? "\\"my_infinity\\"" : x == -Inf ? "\\"-my_infinity\\"" : "\\"my_nan\\"" -JSON3.write(NaN; inf_mapping) # "\\"my_nan\\"" -``` - -See also [`quoted_inf_mapping`](@ref). -""" -underscore_inf_mapping(x) = x == Inf ? "\"__inf__\"" : x == -Inf ? "\"__neginf__\"" : "\"__nan__\"" - -""" - quoted_inf_mapping(x) - -Provides a quoted version of the default mappings for `Inf` and `NaN` values. - -See also [`underscore_inf_mapping`](@ref). -""" -quoted_inf_mapping(x) = x == Inf ? "\"Infinity\"" : x == -Inf ? "\"-Infinity\"" : "\"NaN\"" - @inline function write(::NumberType, buf, pos, len, x::T; inf_mapping::Union{Function, Nothing} = nothing, allow_inf::Bool = inf_mapping !== nothing, kw...) where {T <: Base.IEEEFloat} if isfinite(x) || allow_inf && inf_mapping === nothing && isnan(x) @check Ryu.neededdigits(T) diff --git a/test/json.jl b/test/json.jl index 13361cf..40f8ecd 100644 --- a/test/json.jl +++ b/test/json.jl @@ -47,13 +47,10 @@ end @test JSON3.read("Infinity"; allow_inf=true) === Inf @test JSON3.read("-Infinity"; allow_inf=true) === -Inf - @test JSON3.write(NaN, inf_mapping = JSON3.underscore_inf_mapping) == "\"__nan__\"" - @test JSON3.write(Inf, inf_mapping = JSON3.underscore_inf_mapping) == "\"__inf__\"" - @test JSON3.write(-Inf, inf_mapping = JSON3.underscore_inf_mapping) == "\"__neginf__\"" - - @test JSON3.write(NaN, inf_mapping = JSON3.quoted_inf_mapping) == "\"NaN\"" - @test JSON3.write(Inf, inf_mapping = JSON3.quoted_inf_mapping) == "\"Infinity\"" - @test JSON3.write(-Inf, inf_mapping = JSON3.quoted_inf_mapping) == "\"-Infinity\"" + quoted_inf_mapping(x) = x == Inf ? "\"Infinity\"" : x == -Inf ? "\"-Infinity\"" : "\"NaN\"" + @test JSON3.write(NaN, inf_mapping = quoted_inf_mapping) == "\"NaN\"" + @test JSON3.write(Inf, inf_mapping = quoted_inf_mapping) == "\"Infinity\"" + @test JSON3.write(-Inf, inf_mapping = quoted_inf_mapping) == "\"-Infinity\"" end @testset "Char" begin From 62f8efbcc41e4ba0370df9b3e647abab4a18ecfa Mon Sep 17 00:00:00 2001 From: hhaensel Date: Thu, 16 Jan 2025 15:14:16 +0100 Subject: [PATCH 4/7] modify docstring for inf_mapping --- src/write.jl | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/write.jl b/src/write.jl index a1c245d..eafcfc6 100644 --- a/src/write.jl +++ b/src/write.jl @@ -27,10 +27,9 @@ Write JSON. * `allow_inf`: Allow writing of `Inf` and `NaN` values (not part of the JSON standard). [default `false`] * `inf_mapping`: A function to map `Inf`, `-Inf` and `NaN` values to a custom representation. [default `nothing`] - e.g. a quoted version of the default mapping + if `inf_mapping` is `nothing` the mapping is equivalent to - `quoted_inf_mapping(x) = x == Inf ? "\\"Infinity\\"" : x == -Inf ? "\\"-Infinity\\"" : "\\"NaN\\""` - + `inf_mapping = x -> x == Inf ? "Infinity" : x == -Inf ? "-Infinity" : "NaN"`` * `dateformat`: A [`DateFormat`](https://docs.julialang.org/en/v1/stdlib/Dates/#Dates.DateFormat) describing how to format `Date`s in the object. [default `Dates.default_format(T)`] """ function write(io::IO, obj::T; kw...) where {T} From 0dc027712c8af05b7eb8714b12cf43031e13effb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Helmut=20H=C3=A4nsel?= Date: Wed, 29 Jan 2025 07:02:21 +0100 Subject: [PATCH 5/7] revise docstring of write, add brackets for clarity --- src/write.jl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/write.jl b/src/write.jl index eafcfc6..cc07cfa 100644 --- a/src/write.jl +++ b/src/write.jl @@ -28,8 +28,8 @@ Write JSON. * `inf_mapping`: A function to map `Inf`, `-Inf` and `NaN` values to a custom representation. [default `nothing`] if `inf_mapping` is `nothing` the mapping is equivalent to - - `inf_mapping = x -> x == Inf ? "Infinity" : x == -Inf ? "-Infinity" : "NaN"`` + `inf_mapping = x -> x == Inf ? "Infinity" : x == -Inf ? "-Infinity" : "NaN"`. + Specifying `inf_mapping` will automatically set the default value of `allow_inf` to `true`. * `dateformat`: A [`DateFormat`](https://docs.julialang.org/en/v1/stdlib/Dates/#Dates.DateFormat) describing how to format `Date`s in the object. [default `Dates.default_format(T)`] """ function write(io::IO, obj::T; kw...) where {T} @@ -285,7 +285,7 @@ function write(::NumberType, buf, pos, len, x::AbstractFloat; allow_inf::Bool=fa end @inline function write(::NumberType, buf, pos, len, x::T; inf_mapping::Union{Function, Nothing} = nothing, allow_inf::Bool = inf_mapping !== nothing, kw...) where {T <: Base.IEEEFloat} - if isfinite(x) || allow_inf && inf_mapping === nothing && isnan(x) + if isfinite(x) || (allow_inf && inf_mapping === nothing && isnan(x)) @check Ryu.neededdigits(T) pos = Ryu.writeshortest(buf, pos, x) else From 4fc6b4399d59fb96be144c86bc6ba15d11b56ce1 Mon Sep 17 00:00:00 2001 From: hhaensel Date: Fri, 31 Jan 2025 12:35:03 +0100 Subject: [PATCH 6/7] support `read()` with inf_mapping --- src/read.jl | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/src/read.jl b/src/read.jl index 8fcc60d..11fb230 100644 --- a/src/read.jl +++ b/src/read.jl @@ -93,13 +93,13 @@ end const FLOAT_INT_BOUND = 2.0^53 -function read!(buf, pos, len, b, tape, tapeidx, ::Type{Any}, checkint=true; allow_inf::Bool=false) +function read!(buf, pos, len, b, tape, tapeidx, ::Type{Any}, checkint=true; inf_mapping::Union{Function,Nothing}=nothing, allow_inf::Bool=(inf_mapping !== nothing)) if b == UInt8('{') - return read!(buf, pos, len, b, tape, tapeidx, Object, checkint; allow_inf=allow_inf) + return read!(buf, pos, len, b, tape, tapeidx, Object, checkint; allow_inf=allow_inf, inf_mapping=inf_mapping) elseif b == UInt8('[') - return read!(buf, pos, len, b, tape, tapeidx, Array, checkint; allow_inf=allow_inf) + return read!(buf, pos, len, b, tape, tapeidx, Array, checkint; allow_inf=allow_inf, inf_mapping=inf_mapping) elseif b == UInt8('"') - return read!(buf, pos, len, b, tape, tapeidx, String) + return read!(buf, pos, len, b, tape, tapeidx, String; inf_mapping=inf_mapping) elseif b == UInt8('n') return read!(buf, pos, len, b, tape, tapeidx, Nothing) elseif b == UInt8('t') @@ -148,7 +148,7 @@ function read!(buf, pos, len, b, tape, tapeidx, ::Type{Any}, checkint=true; allo invalid(InvalidChar, buf, pos, Any) end -function read!(buf, pos, len, b, tape, tapeidx, ::Type{String}) +function read!(buf, pos, len, b, tape, tapeidx, ::Type{String}; inf_mapping::Union{Function,Nothing}=nothing) pos += 1 @eof strpos = pos @@ -171,6 +171,23 @@ function read!(buf, pos, len, b, tape, tapeidx, ::Type{String}) b = getbyte(buf, pos) end @check + if inf_mapping !== nothing + val = view(buf, strpos:pos-1) + float = if val == codeunits(inf_mapping(Inf))[2:end-1] + Inf + elseif val == codeunits(inf_mapping(-Inf))[2:end-1] + -Inf + elseif val == codeunits(inf_mapping(NaN))[2:end-1] + NaN + else + 0.0 + end + if float != 0.0 + @inbounds tape[tapeidx] = FLOAT + @inbounds tape[tapeidx+1] = Core.bitcast(UInt64, float) + return pos + 1, tapeidx + 2 + end + end @inbounds tape[tapeidx] = string(strlen) @inbounds tape[tapeidx+1] = ifelse(escaped, ESCAPE_BIT | strpos, strpos) return pos + 1, tapeidx + 2 From 7f0831826c41fbf1fd59e973d2652b7d70967f41 Mon Sep 17 00:00:00 2001 From: hhaensel Date: Fri, 31 Jan 2025 14:57:22 +0100 Subject: [PATCH 7/7] include quotes in test when reading inf and nan strings --- src/read.jl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/read.jl b/src/read.jl index 11fb230..cd4a8da 100644 --- a/src/read.jl +++ b/src/read.jl @@ -172,12 +172,12 @@ function read!(buf, pos, len, b, tape, tapeidx, ::Type{String}; inf_mapping::Uni end @check if inf_mapping !== nothing - val = view(buf, strpos:pos-1) - float = if val == codeunits(inf_mapping(Inf))[2:end-1] + val = view(buf, strpos-1:pos) + float = if val == codeunits(inf_mapping(Inf)) Inf - elseif val == codeunits(inf_mapping(-Inf))[2:end-1] + elseif val == codeunits(inf_mapping(-Inf)) -Inf - elseif val == codeunits(inf_mapping(NaN))[2:end-1] + elseif val == codeunits(inf_mapping(NaN)) NaN else 0.0