From 7a1c83f9c1127cb3e53cfeb75b557299cfc1913e Mon Sep 17 00:00:00 2001 From: kanishka-linux Date: Mon, 21 Jan 2019 18:18:08 +0530 Subject: [PATCH 01/38] add empty parameter to gen --- lib/jake/boolean.ex | 2 +- lib/jake/integer.ex | 2 +- lib/jake/null.ex | 2 +- lib/jake/number.ex | 2 +- lib/jake/string.ex | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/jake/boolean.ex b/lib/jake/boolean.ex index 3f6fba8..0d25962 100644 --- a/lib/jake/boolean.ex +++ b/lib/jake/boolean.ex @@ -1,5 +1,5 @@ defmodule Jake.Boolean do - def gen(_) do + def gen(_, _) do StreamData.boolean() end end diff --git a/lib/jake/integer.ex b/lib/jake/integer.ex index 4d0e073..4e72b3d 100644 --- a/lib/jake/integer.ex +++ b/lib/jake/integer.ex @@ -1,7 +1,7 @@ defmodule Jake.Integer do alias Jake.StreamUtil - def gen(spec) do + def gen(spec, _) do min = Map.get(spec, "minimum") max = Map.get(spec, "maximum") exclusive_min = Map.get(spec, "exclusiveMinimum", false) diff --git a/lib/jake/null.ex b/lib/jake/null.ex index 9c9ff2c..505cebc 100644 --- a/lib/jake/null.ex +++ b/lib/jake/null.ex @@ -1,5 +1,5 @@ defmodule Jake.Null do - def gen(_) do + def gen(_, _) do StreamData.constant(nil) end end diff --git a/lib/jake/number.ex b/lib/jake/number.ex index 35295ff..f0ca695 100644 --- a/lib/jake/number.ex +++ b/lib/jake/number.ex @@ -1,7 +1,7 @@ defmodule Jake.Number do alias Jake.StreamUtil - def gen(spec) do + def gen(spec, _) do min = Map.get(spec, "minimum") max = Map.get(spec, "maximum") exclusive_min = Map.get(spec, "exclusiveMinimum", false) diff --git a/lib/jake/string.ex b/lib/jake/string.ex index 648bf24..672aec6 100644 --- a/lib/jake/string.ex +++ b/lib/jake/string.ex @@ -1,5 +1,5 @@ defmodule Jake.String do - def gen(spec) do + def gen(spec, _) do options = [] min_length = Map.get(spec, "minLength") max_length = Map.get(spec, "maxLength") From f961ecbfbe54cf06c5da7d9e3fec98fdc55bbb3e Mon Sep 17 00:00:00 2001 From: kanishka-linux Date: Mon, 21 Jan 2019 18:23:31 +0530 Subject: [PATCH 02/38] handle case when $ref is absent --- lib/jake/ref.ex | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 lib/jake/ref.ex diff --git a/lib/jake/ref.ex b/lib/jake/ref.ex new file mode 100644 index 0000000..d786bc9 --- /dev/null +++ b/lib/jake/ref.ex @@ -0,0 +1,6 @@ +defmodule Jake.Ref do + def expand_ref(ref, map, _omap) + when is_nil(ref) or is_map(ref) do + {map, false} + end +end From c9c698a79864ef26f1e2c20786ca1e9c4d146bb9 Mon Sep 17 00:00:00 2001 From: kanishka-linux Date: Mon, 21 Jan 2019 18:30:41 +0530 Subject: [PATCH 03/38] use generator instead of gen in verify --- test/jake_test.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/jake_test.exs b/test/jake_test.exs index c67dc61..2a7f8ad 100644 --- a/test/jake_test.exs +++ b/test/jake_test.exs @@ -95,7 +95,7 @@ defmodule JakeTest do end def verify(schema) do - Jake.gen(schema) + Jake.generator(schema) |> Enum.take(100) |> Enum.each(fn value -> result = Validator.validate(schema, value) From 5114855af289a93979d9b0fc389d609dd3f4320d Mon Sep 17 00:00:00 2001 From: kanishka-linux Date: Mon, 21 Jan 2019 18:34:17 +0530 Subject: [PATCH 04/38] pass context to gen_init and handle generator resizing --- lib/jake.ex | 79 +++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 61 insertions(+), 18 deletions(-) diff --git a/lib/jake.ex b/lib/jake.ex index 40f91d4..b053c5b 100644 --- a/lib/jake.ex +++ b/lib/jake.ex @@ -11,46 +11,89 @@ defmodule Jake do "string" ] - def gen(%{"anyOf" => options} = spec) when is_list(options) do + def generator(map) do + StreamData.sized(fn size -> + Map.put(%{}, "map", map) |> Map.put("omap", map) |> Map.put("size", size) |> gen_init() + end) + end + + def gen_init(schema) do + StreamData.bind( + get_lazy_streamkey(schema), + fn {nmap, nsize} -> + nschema = Map.put(schema, "map", nmap) |> Map.put("size", nsize) + + if nmap["enum"] do + gen_enum(nschema, nmap["enum"]) + else + gen(nmap, nschema) + end + |> StreamData.resize(nsize) + end + ) + end + + def get_lazy_streamkey(schema) do + {map, ref} = + get_in(schema, ["map", "$ref"]) |> Jake.Ref.expand_ref(schema["map"], schema["omap"]) + + if ref do + StreamData.constant({map, trunc(schema["size"] / 2)}) + else + StreamData.constant({map, schema["size"]}) + end + end + + def gen_enum(schema, enum) when is_list(enum) do + map = schema["map"] + + StreamData.member_of(enum) + |> StreamData.filter(fn x -> ExJsonSchema.Validator.valid?(map, x) end) + end + + def gen(%{"anyOf" => options} = spec, schema) when is_list(options) do Enum.map(options, fn option -> - gen(Map.merge(Map.drop(spec, ["anyOf"]), option)) + map = Map.merge(Map.drop(spec, ["anyOf"]), option) + Map.put(schema, "map", map) |> gen_init() end) |> StreamData.one_of() end - def gen(%{"allOf" => options} = spec) when is_list(options) do + def gen(%{"allOf" => options} = spec, schema) when is_list(options) do properties = options |> Enum.reduce(%{}, fn x, acc -> MapUtil.deep_merge(x, acc) end) - spec - |> Map.drop(["allOf"]) - |> MapUtil.deep_merge(properties) - |> gen + map = + spec + |> Map.drop(["allOf"]) + |> MapUtil.deep_merge(properties) + + Map.put(schema, "map", map) |> gen_init() end - def gen(%{"type" => type} = spec) when is_binary(type) do + def gen(%{"type" => type} = spec, schema) when is_binary(type) do module = String.to_existing_atom("Elixir.Jake.#{String.capitalize(type)}") - apply(module, :gen, [spec]) + apply(module, :gen, [spec, schema]) end - def gen(%{"type" => types} = spec) when is_list(types) do + def gen(%{"type" => types} = spec, schema) when is_list(types) do Enum.map(types, fn type -> - gen(%{spec | "type" => type}) + map = %{spec | "type" => type} + + Map.put(schema, "map", map) + |> gen_init() end) |> StreamData.one_of() end - def gen(%{"enum" => enum}) when is_list(enum) do - StreamData.member_of(enum) - end - # type not present - def gen(spec) do + def gen(spec, schema) do StreamData.member_of(@types) |> StreamData.bind(fn type -> - Map.put(spec, "type", type) - |> gen() + map = Map.put(spec, "type", type) + size = trunc(schema["size"] / 2) + Map.put(schema, "map", map) |> Map.put("size", size) |> gen_init() end) end end From 73f588d641255e98a6db577ff15a1ff742dd0d76 Mon Sep 17 00:00:00 2001 From: kanishka-linux Date: Mon, 21 Jan 2019 18:36:55 +0530 Subject: [PATCH 05/38] pass context to gen_init --- lib/jake/array.ex | 12 ++++++++---- lib/jake/object.ex | 18 +++++++++--------- 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/lib/jake/array.ex b/lib/jake/array.ex index b77d090..c1dc772 100644 --- a/lib/jake/array.ex +++ b/lib/jake/array.ex @@ -1,5 +1,5 @@ defmodule Jake.Array do - def gen(spec) do + def gen(spec, schema) do items = Map.get(spec, "items", %{}) uniq = Map.get(spec, "uniqueItems", false) additional_items = Map.get(spec, "additionalItems", %{}) @@ -19,15 +19,19 @@ defmodule Jake.Array do items = Stream.concat([items, additional_items]) StreamData.bind(StreamData.integer(min_items..max_items), fn count -> - Enum.take(items, count) - |> Enum.map(&Jake.gen(&1)) + list = Enum.take(items, count) + + for(n <- list, do: Map.put(schema, "map", n)) + |> Enum.map(&Jake.gen_init(&1)) |> StreamData.fixed_list() end) |> StreamData.filter(fn x -> !uniq || length(Enum.uniq(x)) == length(x) end) else - list_of.(Jake.gen(items), min_length: min_items, max_length: max_items) + Map.put(schema, "map", items) + |> Jake.gen_init() + |> list_of.(min_length: min_items, max_length: max_items) end end end diff --git a/lib/jake/object.ex b/lib/jake/object.ex index 2df08ce..a33ad89 100644 --- a/lib/jake/object.ex +++ b/lib/jake/object.ex @@ -1,7 +1,7 @@ defmodule Jake.Object do alias Jake.StreamUtil - def gen(spec) do + def gen(spec, schema) do properties = Map.get(spec, "properties", %{}) required = Map.get(spec, "required", []) all_properties = Map.keys(properties) @@ -15,27 +15,27 @@ defmodule Jake.Object do additional_min = optional_min - length(optional) additional_max = max - length(required) - length(optional) - additional(additional_properties, all_properties, additional_min..additional_max) + additional(additional_properties, all_properties, additional_min..additional_max, schema) |> StreamUtil.merge( StreamUtil.optional_map( - as_map(properties, optional), + as_map(properties, optional, schema), optional_min..optional_max ) ) - |> StreamUtil.merge(StreamData.fixed_map(as_map(properties, required))) + |> StreamUtil.merge(StreamData.fixed_map(as_map(properties, required, schema))) end - defp additional(properties, _all, min.._max) when min < 0 or not is_map(properties) do + defp additional(properties, _all, min.._max, schema) when min < 0 or not is_map(properties) do StreamData.constant(%{}) end - defp additional(properties, all, min..max) do + defp additional(properties, all, min..max, schema) do Randex.stream(~r/[a-zA-Z_]\w{0,5}/, mod: Randex.Generator.StreamData) |> StreamData.filter(fn x -> !Enum.member?(all, x) end) |> StreamData.uniq_list_of(min_length: min, max_length: max) |> StreamData.bind(fn keys -> Enum.map(keys, fn key -> - {key, Jake.gen(properties)} + {key, Jake.gen_init(Map.put(schema, "map", properties))} end) |> Enum.into(%{}) |> StreamData.fixed_map() @@ -43,10 +43,10 @@ defmodule Jake.Object do |> StreamData.scale(fn size -> trunc(size / 10) end) end - defp as_map(properties, keys) do + defp as_map(properties, keys, schema) do Map.take(properties, keys) |> Enum.map(fn {name, spec} -> - {name, Jake.gen(spec)} + {name, Jake.gen_init(Map.put(schema, "map", spec))} end) |> Enum.into(%{}) end From 60c9a4a341edbbfe30ea92519c413e2c7354f8c3 Mon Sep 17 00:00:00 2001 From: kanishka-linux Date: Tue, 22 Jan 2019 23:04:44 +0530 Subject: [PATCH 06/38] remove gen_enum from gen_init --- lib/jake.ex | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/lib/jake.ex b/lib/jake.ex index b053c5b..63a3169 100644 --- a/lib/jake.ex +++ b/lib/jake.ex @@ -22,13 +22,7 @@ defmodule Jake do get_lazy_streamkey(schema), fn {nmap, nsize} -> nschema = Map.put(schema, "map", nmap) |> Map.put("size", nsize) - - if nmap["enum"] do - gen_enum(nschema, nmap["enum"]) - else - gen(nmap, nschema) - end - |> StreamData.resize(nsize) + gen(nmap, nschema) |> StreamData.resize(nsize) end ) end @@ -44,11 +38,9 @@ defmodule Jake do end end - def gen_enum(schema, enum) when is_list(enum) do - map = schema["map"] - + def gen(%{"enum" => enum} = spec, schema) when is_list(enum) do StreamData.member_of(enum) - |> StreamData.filter(fn x -> ExJsonSchema.Validator.valid?(map, x) end) + |> StreamData.filter(fn x -> ExJsonSchema.Validator.valid?(spec, x) end) end def gen(%{"anyOf" => options} = spec, schema) when is_list(options) do From 4cc799043c6ba4324ef1db05d57d1be52e036318 Mon Sep 17 00:00:00 2001 From: kanishka-linux Date: Tue, 22 Jan 2019 23:12:26 +0530 Subject: [PATCH 07/38] rename gen_init to gen_lazy --- lib/jake.ex | 12 ++++++------ lib/jake/array.ex | 4 ++-- lib/jake/object.ex | 4 ++-- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/lib/jake.ex b/lib/jake.ex index 63a3169..5e587c9 100644 --- a/lib/jake.ex +++ b/lib/jake.ex @@ -13,11 +13,11 @@ defmodule Jake do def generator(map) do StreamData.sized(fn size -> - Map.put(%{}, "map", map) |> Map.put("omap", map) |> Map.put("size", size) |> gen_init() + Map.put(%{}, "map", map) |> Map.put("omap", map) |> Map.put("size", size) |> gen_lazy() end) end - def gen_init(schema) do + def gen_lazy(schema) do StreamData.bind( get_lazy_streamkey(schema), fn {nmap, nsize} -> @@ -46,7 +46,7 @@ defmodule Jake do def gen(%{"anyOf" => options} = spec, schema) when is_list(options) do Enum.map(options, fn option -> map = Map.merge(Map.drop(spec, ["anyOf"]), option) - Map.put(schema, "map", map) |> gen_init() + Map.put(schema, "map", map) |> gen_lazy() end) |> StreamData.one_of() end @@ -61,7 +61,7 @@ defmodule Jake do |> Map.drop(["allOf"]) |> MapUtil.deep_merge(properties) - Map.put(schema, "map", map) |> gen_init() + Map.put(schema, "map", map) |> gen_lazy() end def gen(%{"type" => type} = spec, schema) when is_binary(type) do @@ -74,7 +74,7 @@ defmodule Jake do map = %{spec | "type" => type} Map.put(schema, "map", map) - |> gen_init() + |> gen_lazy() end) |> StreamData.one_of() end @@ -85,7 +85,7 @@ defmodule Jake do |> StreamData.bind(fn type -> map = Map.put(spec, "type", type) size = trunc(schema["size"] / 2) - Map.put(schema, "map", map) |> Map.put("size", size) |> gen_init() + Map.put(schema, "map", map) |> Map.put("size", size) |> gen_lazy() end) end end diff --git a/lib/jake/array.ex b/lib/jake/array.ex index c1dc772..c515442 100644 --- a/lib/jake/array.ex +++ b/lib/jake/array.ex @@ -22,7 +22,7 @@ defmodule Jake.Array do list = Enum.take(items, count) for(n <- list, do: Map.put(schema, "map", n)) - |> Enum.map(&Jake.gen_init(&1)) + |> Enum.map(&Jake.gen_lazy(&1)) |> StreamData.fixed_list() end) |> StreamData.filter(fn x -> @@ -30,7 +30,7 @@ defmodule Jake.Array do end) else Map.put(schema, "map", items) - |> Jake.gen_init() + |> Jake.gen_lazy() |> list_of.(min_length: min_items, max_length: max_items) end end diff --git a/lib/jake/object.ex b/lib/jake/object.ex index a33ad89..6c52f05 100644 --- a/lib/jake/object.ex +++ b/lib/jake/object.ex @@ -35,7 +35,7 @@ defmodule Jake.Object do |> StreamData.uniq_list_of(min_length: min, max_length: max) |> StreamData.bind(fn keys -> Enum.map(keys, fn key -> - {key, Jake.gen_init(Map.put(schema, "map", properties))} + {key, Jake.gen_lazy(Map.put(schema, "map", properties))} end) |> Enum.into(%{}) |> StreamData.fixed_map() @@ -46,7 +46,7 @@ defmodule Jake.Object do defp as_map(properties, keys, schema) do Map.take(properties, keys) |> Enum.map(fn {name, spec} -> - {name, Jake.gen_init(Map.put(schema, "map", spec))} + {name, Jake.gen_lazy(Map.put(schema, "map", spec))} end) |> Enum.into(%{}) end From 179a5ad61fd1b21e550b72d757f4a457bff2d8f4 Mon Sep 17 00:00:00 2001 From: kanishka-linux Date: Wed, 23 Jan 2019 13:27:01 +0530 Subject: [PATCH 08/38] add separate Context module for holding root schema, child schema and generator size as Struct --- lib/jake/context.ex | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 lib/jake/context.ex diff --git a/lib/jake/context.ex b/lib/jake/context.ex new file mode 100644 index 0000000..ea1efa5 --- /dev/null +++ b/lib/jake/context.ex @@ -0,0 +1,3 @@ +defmodule Jake.Context do + defstruct root: %{}, child: %{}, size: 0 +end From 565f36abf6f28b21513e3855dab773c4a8e63f6f Mon Sep 17 00:00:00 2001 From: kanishka-linux Date: Wed, 23 Jan 2019 13:30:38 +0530 Subject: [PATCH 09/38] pass context as struct instead of map to gen_lazy --- lib/jake.ex | 56 +++++++++++++++++++++++----------------------- lib/jake/array.ex | 6 ++--- lib/jake/object.ex | 16 ++++++------- 3 files changed, 39 insertions(+), 39 deletions(-) diff --git a/lib/jake.ex b/lib/jake.ex index 5e587c9..1763e1f 100644 --- a/lib/jake.ex +++ b/lib/jake.ex @@ -1,5 +1,6 @@ defmodule Jake do alias Jake.MapUtil + alias Jake.Context @types [ "array", @@ -11,81 +12,80 @@ defmodule Jake do "string" ] - def generator(map) do + def generator(schema) do StreamData.sized(fn size -> - Map.put(%{}, "map", map) |> Map.put("omap", map) |> Map.put("size", size) |> gen_lazy() + %Context{root: schema, child: schema, size: size} |> gen_lazy() end) end - def gen_lazy(schema) do + def gen_lazy(context) do StreamData.bind( - get_lazy_streamkey(schema), - fn {nmap, nsize} -> - nschema = Map.put(schema, "map", nmap) |> Map.put("size", nsize) - gen(nmap, nschema) |> StreamData.resize(nsize) + get_lazy_streamkey(context), + fn {child, size} -> + gen(child, %{context | child: child, size: size}) |> StreamData.resize(size) end ) end - def get_lazy_streamkey(schema) do - {map, ref} = - get_in(schema, ["map", "$ref"]) |> Jake.Ref.expand_ref(schema["map"], schema["omap"]) + def get_lazy_streamkey(context) do + {child, ref} = + get_in(context.child, ["$ref"]) |> Jake.Ref.expand_ref(context.child, context.root) if ref do - StreamData.constant({map, trunc(schema["size"] / 2)}) + StreamData.constant({child, trunc(context.size / 2)}) else - StreamData.constant({map, schema["size"]}) + StreamData.constant({child, context.size}) end end - def gen(%{"enum" => enum} = spec, schema) when is_list(enum) do + def gen(%{"enum" => enum} = spec, _context) when is_list(enum) do StreamData.member_of(enum) |> StreamData.filter(fn x -> ExJsonSchema.Validator.valid?(spec, x) end) end - def gen(%{"anyOf" => options} = spec, schema) when is_list(options) do + def gen(%{"anyOf" => options} = spec, context) when is_list(options) do Enum.map(options, fn option -> - map = Map.merge(Map.drop(spec, ["anyOf"]), option) - Map.put(schema, "map", map) |> gen_lazy() + child = Map.merge(Map.drop(spec, ["anyOf"]), option) + %{context | child: child} |> gen_lazy() end) |> StreamData.one_of() end - def gen(%{"allOf" => options} = spec, schema) when is_list(options) do + def gen(%{"allOf" => options} = spec, context) when is_list(options) do properties = options |> Enum.reduce(%{}, fn x, acc -> MapUtil.deep_merge(x, acc) end) - map = + child = spec |> Map.drop(["allOf"]) |> MapUtil.deep_merge(properties) - Map.put(schema, "map", map) |> gen_lazy() + %{context | child: child} |> gen_lazy() end - def gen(%{"type" => type} = spec, schema) when is_binary(type) do + def gen(%{"type" => type} = spec, context) when is_binary(type) do module = String.to_existing_atom("Elixir.Jake.#{String.capitalize(type)}") - apply(module, :gen, [spec, schema]) + apply(module, :gen, [spec, context]) end - def gen(%{"type" => types} = spec, schema) when is_list(types) do + def gen(%{"type" => types} = spec, context) when is_list(types) do Enum.map(types, fn type -> - map = %{spec | "type" => type} + child = %{spec | "type" => type} - Map.put(schema, "map", map) + %{context | child: child} |> gen_lazy() end) |> StreamData.one_of() end # type not present - def gen(spec, schema) do + def gen(spec, context) do StreamData.member_of(@types) |> StreamData.bind(fn type -> - map = Map.put(spec, "type", type) - size = trunc(schema["size"] / 2) - Map.put(schema, "map", map) |> Map.put("size", size) |> gen_lazy() + child = Map.put(spec, "type", type) + size = trunc(context.size / 2) + %{context | child: child, size: size} |> gen_lazy() end) end end diff --git a/lib/jake/array.ex b/lib/jake/array.ex index c515442..cdc0de7 100644 --- a/lib/jake/array.ex +++ b/lib/jake/array.ex @@ -1,5 +1,5 @@ defmodule Jake.Array do - def gen(spec, schema) do + def gen(spec, context) do items = Map.get(spec, "items", %{}) uniq = Map.get(spec, "uniqueItems", false) additional_items = Map.get(spec, "additionalItems", %{}) @@ -21,7 +21,7 @@ defmodule Jake.Array do StreamData.bind(StreamData.integer(min_items..max_items), fn count -> list = Enum.take(items, count) - for(n <- list, do: Map.put(schema, "map", n)) + for(n <- list, do: %{context | child: n}) |> Enum.map(&Jake.gen_lazy(&1)) |> StreamData.fixed_list() end) @@ -29,7 +29,7 @@ defmodule Jake.Array do !uniq || length(Enum.uniq(x)) == length(x) end) else - Map.put(schema, "map", items) + %{context | child: items} |> Jake.gen_lazy() |> list_of.(min_length: min_items, max_length: max_items) end diff --git a/lib/jake/object.ex b/lib/jake/object.ex index 6c52f05..b4bb37f 100644 --- a/lib/jake/object.ex +++ b/lib/jake/object.ex @@ -1,7 +1,7 @@ defmodule Jake.Object do alias Jake.StreamUtil - def gen(spec, schema) do + def gen(spec, context) do properties = Map.get(spec, "properties", %{}) required = Map.get(spec, "required", []) all_properties = Map.keys(properties) @@ -15,27 +15,27 @@ defmodule Jake.Object do additional_min = optional_min - length(optional) additional_max = max - length(required) - length(optional) - additional(additional_properties, all_properties, additional_min..additional_max, schema) + additional(additional_properties, all_properties, additional_min..additional_max, context) |> StreamUtil.merge( StreamUtil.optional_map( - as_map(properties, optional, schema), + as_map(properties, optional, context), optional_min..optional_max ) ) - |> StreamUtil.merge(StreamData.fixed_map(as_map(properties, required, schema))) + |> StreamUtil.merge(StreamData.fixed_map(as_map(properties, required, context))) end defp additional(properties, _all, min.._max, schema) when min < 0 or not is_map(properties) do StreamData.constant(%{}) end - defp additional(properties, all, min..max, schema) do + defp additional(properties, all, min..max, context) do Randex.stream(~r/[a-zA-Z_]\w{0,5}/, mod: Randex.Generator.StreamData) |> StreamData.filter(fn x -> !Enum.member?(all, x) end) |> StreamData.uniq_list_of(min_length: min, max_length: max) |> StreamData.bind(fn keys -> Enum.map(keys, fn key -> - {key, Jake.gen_lazy(Map.put(schema, "map", properties))} + {key, Jake.gen_lazy(%{context | child: properties})} end) |> Enum.into(%{}) |> StreamData.fixed_map() @@ -43,10 +43,10 @@ defmodule Jake.Object do |> StreamData.scale(fn size -> trunc(size / 10) end) end - defp as_map(properties, keys, schema) do + defp as_map(properties, keys, context) do Map.take(properties, keys) |> Enum.map(fn {name, spec} -> - {name, Jake.gen_lazy(Map.put(schema, "map", spec))} + {name, Jake.gen_lazy(%{context | child: spec})} end) |> Enum.into(%{}) end From 468e47d43cebd1d0506d207d8ed0e8197e3f1b39 Mon Sep 17 00:00:00 2001 From: kanishka-linux Date: Wed, 23 Jan 2019 15:26:25 +0530 Subject: [PATCH 10/38] use single empty parameter in gen --- lib/jake/boolean.ex | 2 +- lib/jake/null.ex | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/jake/boolean.ex b/lib/jake/boolean.ex index 0d25962..3f6fba8 100644 --- a/lib/jake/boolean.ex +++ b/lib/jake/boolean.ex @@ -1,5 +1,5 @@ defmodule Jake.Boolean do - def gen(_, _) do + def gen(_) do StreamData.boolean() end end diff --git a/lib/jake/null.ex b/lib/jake/null.ex index 505cebc..9c9ff2c 100644 --- a/lib/jake/null.ex +++ b/lib/jake/null.ex @@ -1,5 +1,5 @@ defmodule Jake.Null do - def gen(_, _) do + def gen(_) do StreamData.constant(nil) end end From 160eeec69f056809a50bfbe7b89eedbaa2440e9d Mon Sep 17 00:00:00 2001 From: kanishka-linux Date: Wed, 23 Jan 2019 15:28:40 +0530 Subject: [PATCH 11/38] use Context with pattern matching in gen --- lib/jake.ex | 16 ++++++++-------- lib/jake/array.ex | 4 +++- lib/jake/number.ex | 5 +++-- lib/jake/object.ex | 5 +++-- lib/jake/string.ex | 4 +++- 5 files changed, 20 insertions(+), 14 deletions(-) diff --git a/lib/jake.ex b/lib/jake.ex index 1763e1f..e160819 100644 --- a/lib/jake.ex +++ b/lib/jake.ex @@ -22,7 +22,7 @@ defmodule Jake do StreamData.bind( get_lazy_streamkey(context), fn {child, size} -> - gen(child, %{context | child: child, size: size}) |> StreamData.resize(size) + gen(%{context | child: child, size: size}) |> StreamData.resize(size) end ) end @@ -38,12 +38,12 @@ defmodule Jake do end end - def gen(%{"enum" => enum} = spec, _context) when is_list(enum) do + def gen(%Context{child: %{"enum" => enum} = spec} = context) when is_list(enum) do StreamData.member_of(enum) |> StreamData.filter(fn x -> ExJsonSchema.Validator.valid?(spec, x) end) end - def gen(%{"anyOf" => options} = spec, context) when is_list(options) do + def gen(%Context{child: %{"anyOf" => options} = spec} = context) when is_list(options) do Enum.map(options, fn option -> child = Map.merge(Map.drop(spec, ["anyOf"]), option) %{context | child: child} |> gen_lazy() @@ -51,7 +51,7 @@ defmodule Jake do |> StreamData.one_of() end - def gen(%{"allOf" => options} = spec, context) when is_list(options) do + def gen(%Context{child: %{"allOf" => options} = spec} = context) when is_list(options) do properties = options |> Enum.reduce(%{}, fn x, acc -> MapUtil.deep_merge(x, acc) end) @@ -64,12 +64,12 @@ defmodule Jake do %{context | child: child} |> gen_lazy() end - def gen(%{"type" => type} = spec, context) when is_binary(type) do + def gen(%Context{child: %{"type" => type} = spec} = context) when is_binary(type) do module = String.to_existing_atom("Elixir.Jake.#{String.capitalize(type)}") - apply(module, :gen, [spec, context]) + apply(module, :gen, [context]) end - def gen(%{"type" => types} = spec, context) when is_list(types) do + def gen(%Context{child: %{"type" => types} = spec} = context) when is_list(types) do Enum.map(types, fn type -> child = %{spec | "type" => type} @@ -80,7 +80,7 @@ defmodule Jake do end # type not present - def gen(spec, context) do + def gen(%Context{child: spec} = context) do StreamData.member_of(@types) |> StreamData.bind(fn type -> child = Map.put(spec, "type", type) diff --git a/lib/jake/array.ex b/lib/jake/array.ex index cdc0de7..5419387 100644 --- a/lib/jake/array.ex +++ b/lib/jake/array.ex @@ -1,5 +1,7 @@ defmodule Jake.Array do - def gen(spec, context) do + alias Jake.Context + + def gen(%Context{child: spec} = context) do items = Map.get(spec, "items", %{}) uniq = Map.get(spec, "uniqueItems", false) additional_items = Map.get(spec, "additionalItems", %{}) diff --git a/lib/jake/number.ex b/lib/jake/number.ex index f0ca695..df88aae 100644 --- a/lib/jake/number.ex +++ b/lib/jake/number.ex @@ -1,7 +1,8 @@ defmodule Jake.Number do alias Jake.StreamUtil - - def gen(spec, _) do + alias Jake.Context + + def gen(%Context{child: spec} = context) do min = Map.get(spec, "minimum") max = Map.get(spec, "maximum") exclusive_min = Map.get(spec, "exclusiveMinimum", false) diff --git a/lib/jake/object.ex b/lib/jake/object.ex index b4bb37f..d73593d 100644 --- a/lib/jake/object.ex +++ b/lib/jake/object.ex @@ -1,7 +1,8 @@ defmodule Jake.Object do alias Jake.StreamUtil - - def gen(spec, context) do + alias Jake.Context + + def gen(%Context{child: spec} = context) do properties = Map.get(spec, "properties", %{}) required = Map.get(spec, "required", []) all_properties = Map.keys(properties) diff --git a/lib/jake/string.ex b/lib/jake/string.ex index 672aec6..bdc9372 100644 --- a/lib/jake/string.ex +++ b/lib/jake/string.ex @@ -1,5 +1,7 @@ defmodule Jake.String do - def gen(spec, _) do + alias Jake.Context + + def gen(%Context{child: spec} = context) do options = [] min_length = Map.get(spec, "minLength") max_length = Map.get(spec, "maxLength") From 437fabce3c921b3c7904e9f07dbc752008cb57a3 Mon Sep 17 00:00:00 2001 From: kanishka-linux Date: Wed, 23 Jan 2019 15:29:35 +0530 Subject: [PATCH 12/38] use Context with pattern matching in gen --- lib/jake/integer.ex | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/jake/integer.ex b/lib/jake/integer.ex index 4e72b3d..ce7d1d8 100644 --- a/lib/jake/integer.ex +++ b/lib/jake/integer.ex @@ -1,7 +1,8 @@ defmodule Jake.Integer do alias Jake.StreamUtil - - def gen(spec, _) do + alias Jake.Context + + def gen(%Context{child: spec} = context) do min = Map.get(spec, "minimum") max = Map.get(spec, "maximum") exclusive_min = Map.get(spec, "exclusiveMinimum", false) From fb2117033356afa3c63734c428fa5079ccc23a6a Mon Sep 17 00:00:00 2001 From: kanishka-linux Date: Wed, 23 Jan 2019 15:54:49 +0530 Subject: [PATCH 13/38] pass only context to expand_ref --- lib/jake.ex | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/lib/jake.ex b/lib/jake.ex index e160819..e859d2c 100644 --- a/lib/jake.ex +++ b/lib/jake.ex @@ -21,21 +21,23 @@ defmodule Jake do def gen_lazy(context) do StreamData.bind( get_lazy_streamkey(context), - fn {child, size} -> + fn %Context{child: child, size: size} -> gen(%{context | child: child, size: size}) |> StreamData.resize(size) end ) end def get_lazy_streamkey(context) do - {child, ref} = - get_in(context.child, ["$ref"]) |> Jake.Ref.expand_ref(context.child, context.root) + {new_context, ref} = Jake.Ref.expand_ref(context) - if ref do - StreamData.constant({child, trunc(context.size / 2)}) - else - StreamData.constant({child, context.size}) - end + size = + if ref do + trunc(new_context.size / 2) + else + new_context.size + end + + StreamData.constant(%{new_context | size: size}) end def gen(%Context{child: %{"enum" => enum} = spec} = context) when is_list(enum) do From 006254efb0a0b27bc9cdf3c14f5274c8e8b283b9 Mon Sep 17 00:00:00 2001 From: kanishka-linux Date: Wed, 23 Jan 2019 15:56:34 +0530 Subject: [PATCH 14/38] pass only context parameter to expand_ref --- lib/jake/ref.ex | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/jake/ref.ex b/lib/jake/ref.ex index d786bc9..88944ca 100644 --- a/lib/jake/ref.ex +++ b/lib/jake/ref.ex @@ -1,6 +1,7 @@ defmodule Jake.Ref do - def expand_ref(ref, map, _omap) - when is_nil(ref) or is_map(ref) do - {map, false} + alias Jake.Context + + def expand_ref(context) do + {context, false} end end From 2ba758002f66c02ffe1067f2c08a0e32b5214c98 Mon Sep 17 00:00:00 2001 From: kanishka-linux Date: Wed, 23 Jan 2019 15:57:47 +0530 Subject: [PATCH 15/38] mix format --- lib/jake/array.ex | 2 +- lib/jake/integer.ex | 2 +- lib/jake/number.ex | 2 +- lib/jake/object.ex | 2 +- lib/jake/string.ex | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/jake/array.ex b/lib/jake/array.ex index 5419387..ef66cdd 100644 --- a/lib/jake/array.ex +++ b/lib/jake/array.ex @@ -1,6 +1,6 @@ defmodule Jake.Array do alias Jake.Context - + def gen(%Context{child: spec} = context) do items = Map.get(spec, "items", %{}) uniq = Map.get(spec, "uniqueItems", false) diff --git a/lib/jake/integer.ex b/lib/jake/integer.ex index ce7d1d8..53b7b91 100644 --- a/lib/jake/integer.ex +++ b/lib/jake/integer.ex @@ -1,7 +1,7 @@ defmodule Jake.Integer do alias Jake.StreamUtil alias Jake.Context - + def gen(%Context{child: spec} = context) do min = Map.get(spec, "minimum") max = Map.get(spec, "maximum") diff --git a/lib/jake/number.ex b/lib/jake/number.ex index df88aae..c8fcb89 100644 --- a/lib/jake/number.ex +++ b/lib/jake/number.ex @@ -1,7 +1,7 @@ defmodule Jake.Number do alias Jake.StreamUtil alias Jake.Context - + def gen(%Context{child: spec} = context) do min = Map.get(spec, "minimum") max = Map.get(spec, "maximum") diff --git a/lib/jake/object.ex b/lib/jake/object.ex index d73593d..68fbcc9 100644 --- a/lib/jake/object.ex +++ b/lib/jake/object.ex @@ -1,7 +1,7 @@ defmodule Jake.Object do alias Jake.StreamUtil alias Jake.Context - + def gen(%Context{child: spec} = context) do properties = Map.get(spec, "properties", %{}) required = Map.get(spec, "required", []) diff --git a/lib/jake/string.ex b/lib/jake/string.ex index bdc9372..3f4cff8 100644 --- a/lib/jake/string.ex +++ b/lib/jake/string.ex @@ -1,6 +1,6 @@ defmodule Jake.String do alias Jake.Context - + def gen(%Context{child: spec} = context) do options = [] min_length = Map.get(spec, "minLength") From 0379a9ef1a2d11e0d555d952e75a2e0bf90e4c45 Mon Sep 17 00:00:00 2001 From: kanishka-linux Date: Wed, 23 Jan 2019 16:12:16 +0530 Subject: [PATCH 16/38] modify arguments to gen_lazy and remove for loop --- lib/jake/array.ex | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/lib/jake/array.ex b/lib/jake/array.ex index ef66cdd..1677905 100644 --- a/lib/jake/array.ex +++ b/lib/jake/array.ex @@ -21,10 +21,8 @@ defmodule Jake.Array do items = Stream.concat([items, additional_items]) StreamData.bind(StreamData.integer(min_items..max_items), fn count -> - list = Enum.take(items, count) - - for(n <- list, do: %{context | child: n}) - |> Enum.map(&Jake.gen_lazy(&1)) + Enum.take(items, count) + |> Enum.map(&Jake.gen_lazy(%{context | child: &1})) |> StreamData.fixed_list() end) |> StreamData.filter(fn x -> From ccb46b1f4104ab9ef75968529ce0b797f10cc3c1 Mon Sep 17 00:00:00 2001 From: kanishka-linux Date: Wed, 23 Jan 2019 16:32:46 +0530 Subject: [PATCH 17/38] fix unused variable warning --- lib/jake.ex | 4 ++-- lib/jake/integer.ex | 2 +- lib/jake/number.ex | 2 +- lib/jake/object.ex | 2 +- lib/jake/ref.ex | 2 +- lib/jake/string.ex | 2 +- 6 files changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/jake.ex b/lib/jake.ex index e859d2c..1f82201 100644 --- a/lib/jake.ex +++ b/lib/jake.ex @@ -40,7 +40,7 @@ defmodule Jake do StreamData.constant(%{new_context | size: size}) end - def gen(%Context{child: %{"enum" => enum} = spec} = context) when is_list(enum) do + def gen(%Context{child: %{"enum" => enum} = spec} = _context) when is_list(enum) do StreamData.member_of(enum) |> StreamData.filter(fn x -> ExJsonSchema.Validator.valid?(spec, x) end) end @@ -66,7 +66,7 @@ defmodule Jake do %{context | child: child} |> gen_lazy() end - def gen(%Context{child: %{"type" => type} = spec} = context) when is_binary(type) do + def gen(%Context{child: %{"type" => type} = _spec} = context) when is_binary(type) do module = String.to_existing_atom("Elixir.Jake.#{String.capitalize(type)}") apply(module, :gen, [context]) end diff --git a/lib/jake/integer.ex b/lib/jake/integer.ex index 53b7b91..95ad4e7 100644 --- a/lib/jake/integer.ex +++ b/lib/jake/integer.ex @@ -2,7 +2,7 @@ defmodule Jake.Integer do alias Jake.StreamUtil alias Jake.Context - def gen(%Context{child: spec} = context) do + def gen(%Context{child: spec} = _context) do min = Map.get(spec, "minimum") max = Map.get(spec, "maximum") exclusive_min = Map.get(spec, "exclusiveMinimum", false) diff --git a/lib/jake/number.ex b/lib/jake/number.ex index c8fcb89..b776ac8 100644 --- a/lib/jake/number.ex +++ b/lib/jake/number.ex @@ -2,7 +2,7 @@ defmodule Jake.Number do alias Jake.StreamUtil alias Jake.Context - def gen(%Context{child: spec} = context) do + def gen(%Context{child: spec} = _context) do min = Map.get(spec, "minimum") max = Map.get(spec, "maximum") exclusive_min = Map.get(spec, "exclusiveMinimum", false) diff --git a/lib/jake/object.ex b/lib/jake/object.ex index 68fbcc9..dee02a7 100644 --- a/lib/jake/object.ex +++ b/lib/jake/object.ex @@ -26,7 +26,7 @@ defmodule Jake.Object do |> StreamUtil.merge(StreamData.fixed_map(as_map(properties, required, context))) end - defp additional(properties, _all, min.._max, schema) when min < 0 or not is_map(properties) do + defp additional(properties, _all, min.._max, _context) when min < 0 or not is_map(properties) do StreamData.constant(%{}) end diff --git a/lib/jake/ref.ex b/lib/jake/ref.ex index 88944ca..6f95c08 100644 --- a/lib/jake/ref.ex +++ b/lib/jake/ref.ex @@ -1,7 +1,7 @@ defmodule Jake.Ref do alias Jake.Context - def expand_ref(context) do + def expand_ref(%Context{child: _spec} = context) do {context, false} end end diff --git a/lib/jake/string.ex b/lib/jake/string.ex index 3f4cff8..4c29ac7 100644 --- a/lib/jake/string.ex +++ b/lib/jake/string.ex @@ -1,7 +1,7 @@ defmodule Jake.String do alias Jake.Context - def gen(%Context{child: spec} = context) do + def gen(%Context{child: spec} = _context) do options = [] min_length = Map.get(spec, "minLength") max_length = Map.get(spec, "maxLength") From c87f87885001f2c2fec66e5eff1455d71c3e6fb8 Mon Sep 17 00:00:00 2001 From: kanishka-linux Date: Thu, 24 Jan 2019 00:32:29 +0530 Subject: [PATCH 18/38] add separate tests for $ref --- test/jake_test_ref.exs | 160 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 160 insertions(+) create mode 100644 test/jake_test_ref.exs diff --git a/test/jake_test_ref.exs b/test/jake_test_ref.exs new file mode 100644 index 0000000..003c912 --- /dev/null +++ b/test/jake_test_ref.exs @@ -0,0 +1,160 @@ +defmodule JakeTestRef do + use ExUnitProperties + use ExUnit.Case + doctest Jake + + def test_generator_property(jschema) do + schema = Jason.decode!(jschema) + gen = Jake.generator(schema) + + check all a <- gen do + assert ExJsonSchema.Validator.valid?(schema, a) + end + end + + property "test ref simple" do + jschema = ~s({"properties": { + "foo": {"type": "integer"}, + "bar": {"$ref": "#/properties/foo"} + }}) + test_generator_property(jschema) + end + + property "test ref escape pointer" do + jschema = ~s({"tilda~field": {"type": "integer"}, + "slash/field": {"type": "integer"}, + "percent%field": {"type": "integer"}, + "properties": { + "tilda": {"$ref": "#/tilda~0field"}, + "slash": {"$ref": "#/slash~1field"}, + "percent": {"$ref": "#/percent%25field"} + }}) + test_generator_property(jschema) + end + + property "test ref nested schema" do + jschema = ~s({"definitions": { + "a": {"type": "integer"}, + "b": {"$ref": "#/definitions/a"}, + "c": {"$ref": "#/definitions/b"} + }, + "$ref": "#/definitions/c"}) + test_generator_property(jschema) + end + + property "test ref overrides any sibling keywords" do + jschema = ~s({"definitions": { + "reffed": { + "type": "array" + } + }, + "properties": { + "foo": { + "$ref": "#/definitions/reffed", + "maxItems": 2 + } + }}) + test_generator_property(jschema) + end + + property "test ref not reference" do + jschema = ~s({"properties": { + "$ref": {"type": "string"} + }}) + test_generator_property(jschema) + end + + property "test ref array index" do + jschema = ~s({"items": [ + {"type": "integer"}, + {"$ref": "#/items/0"} + ]}) + test_generator_property(jschema) + end + + property "test ref root" do + jschema = ~s({"properties": { + "bar" : {"type":"integer"}, + "foo": {"$ref": "#"} + }, + "additionalProperties": false}) + test_generator_property(jschema) + end + + property "test ref simple recursive" do + jschema = ~s({"properties": { + "foo_bar" : {"type":"integer"}, + "bar" : {"$ref": "#/properties"}, + "foo": {"$ref": "#/properties/bar"} + }, + "additionalProperties": false}) + test_generator_property(jschema) + end + + property "test ref complex recursive" do + jschema = ~s({"definitions": { + "person": { + "type": "object", + "properties": { + "name": { "type": "string" }, + "children": { + "type": "array", + "items": { "$ref": "#/definitions/person" } + + } + }, "required": ["name"], "additionalProperties": false + } + }, + "type": "object", + "properties": { + "person": { "$ref": "#/definitions/person" } + }, "required": ["person"], "additionalProperties": false + }) + test_generator_property(jschema) + end + + property "test ref complex recursive no required" do + jschema = ~s({"definitions": { + "person": { + "type": "object", + "properties": { + "name": { "type": "string", "minLength": 5 }, + "children": { + "type": "array", + "items": { "$ref": "#/definitions/person" } + + } + }, "additionalProperties": false + } + }, + "type": "object", + "properties": { + "person": { "$ref": "#/definitions/person" } + }, "additionalProperties": false + }) + test_generator_property(jschema) + end + + property "test complex ref" do + jschema = ~s({ "definitions": { + "address": { + "type": "object", + "additionalProperties": false, + "properties": { + "street_address": { "type": "string" }, + "city": { "type": "string" }, + "state": { "type": "string" } + }, + "required": ["street_address", "city", "state"] + } + }, + "type": "object", + "properties": { + "billing_address": { "$ref": "#/definitions/address" }, + "shipping_address": { "$ref": "#/definitions/address" } + }, + "additionalProperties": false + }) + test_generator_property(jschema) + end +end From 736234f045bd7cdcb171374c805a2d03cbd975fe Mon Sep 17 00:00:00 2001 From: kanishka-linux Date: Thu, 24 Jan 2019 00:33:49 +0530 Subject: [PATCH 19/38] handle nested ref in gen_lazy --- lib/jake.ex | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/lib/jake.ex b/lib/jake.ex index 1f82201..ba0362f 100644 --- a/lib/jake.ex +++ b/lib/jake.ex @@ -22,7 +22,14 @@ defmodule Jake do StreamData.bind( get_lazy_streamkey(context), fn %Context{child: child, size: size} -> - gen(%{context | child: child, size: size}) |> StreamData.resize(size) + new_context = %{context | child: child, size: size} + + if child["$ref"] do + gen_lazy(new_context) + else + gen(new_context) + end + |> StreamData.resize(size) end ) end From 5f579d457acd24d519b9dc2b667627f3a2027334 Mon Sep 17 00:00:00 2001 From: kanishka-linux Date: Thu, 24 Jan 2019 00:34:37 +0530 Subject: [PATCH 20/38] handle $ref when not nil --- lib/jake/ref.ex | 85 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) diff --git a/lib/jake/ref.ex b/lib/jake/ref.ex index 6f95c08..b74b78a 100644 --- a/lib/jake/ref.ex +++ b/lib/jake/ref.ex @@ -1,7 +1,92 @@ defmodule Jake.Ref do alias Jake.Context + def expand_ref(%Context{child: %{"$ref" => ref} = spec} = context) + when ref == "#" do + new_child = Map.drop(spec, ["$ref"]) |> Map.merge(context.root) + {%{context | child: new_child}, true} + end + + def expand_ref(%Context{child: %{"$ref" => ref} = spec} = context) when is_binary(ref) do + uri = URI.decode(ref) + + ref_map = + if String.starts_with?(uri, "http") do + process_http_path(uri) + else + process_local_path(uri) |> get_head_list_path(context.root) + end + + new_child = Map.drop(spec, ["$ref"]) |> Map.merge(ref_map) + + {%{context | child: new_child}, true} + end + def expand_ref(%Context{child: _spec} = context) do {context, false} end + + def get_head_list_path(path_list, root_schema) do + {head, tail} = Enum.split(path_list, -1) + + head_path = + if length(head) > 0 do + get_in(root_schema, head) + else + get_in(root_schema, path_list) + end + + tail = + if is_list(head_path) do + Enum.fetch!(tail, 0) + else + nil + end + + if tail != nil and is_numeric(tail) do + {index, ""} = Integer.parse(tail) + Enum.fetch!(head_path, index) + else + get_in(root_schema, path_list) + end + end + + def process_http_path(url) do + [url, local] = + if String.contains?(url, "#/") do + String.split(url, "#/") + else + [url, nil] + end + + IO.inspect({url, local}) + {:ok, {{_, 200, _}, _, schema}} = :httpc.request(:get, {to_charlist(url), []}, [], []) + jschema = Poison.decode!(schema) + + if is_nil(local) do + jschema + else + process_local_path(local) |> get_head_list_path(jschema) + end + end + + def process_local_path(path) do + str = + String.replace(path, "~0", "~") + |> String.replace("#/", "", global: false) + + if String.contains?(str, "~1") do + strlist = String.split(str, "/") + for n <- strlist, do: String.replace(n, "~1", "/") + else + String.split(str, "/") + end + end + + def is_numeric(str) do + case Integer.parse(str) do + {_num, ""} -> true + _ -> false + end + end end From 42e52dccff58df2c89196bc3be5e9e447648f1e5 Mon Sep 17 00:00:00 2001 From: kanishka-linux Date: Thu, 24 Jan 2019 01:07:52 +0530 Subject: [PATCH 21/38] replace Poison with Jason --- lib/jake/ref.ex | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/jake/ref.ex b/lib/jake/ref.ex index b74b78a..de4a7f0 100644 --- a/lib/jake/ref.ex +++ b/lib/jake/ref.ex @@ -59,9 +59,8 @@ defmodule Jake.Ref do [url, nil] end - IO.inspect({url, local}) {:ok, {{_, 200, _}, _, schema}} = :httpc.request(:get, {to_charlist(url), []}, [], []) - jschema = Poison.decode!(schema) + jschema = Jason.decode!(schema) if is_nil(local) do jschema From 040af6381de12942e8cd3bfb3f8ab7b138226a65 Mon Sep 17 00:00:00 2001 From: kanishka-linux Date: Thu, 24 Jan 2019 13:25:47 +0530 Subject: [PATCH 22/38] add refExtra.json --- test_suite/tests/draft4/refExtra.json | 150 ++++++++++++++++++++++++++ 1 file changed, 150 insertions(+) create mode 100644 test_suite/tests/draft4/refExtra.json diff --git a/test_suite/tests/draft4/refExtra.json b/test_suite/tests/draft4/refExtra.json new file mode 100644 index 0000000..2358566 --- /dev/null +++ b/test_suite/tests/draft4/refExtra.json @@ -0,0 +1,150 @@ +[ + { + "description": "root pointer ref", + "schema": { + "properties": { + "foo": {"$ref": "#"} + }, + "additionalProperties": false + } + }, + { + "description": "relative pointer ref to object", + "schema": { + "properties": { + "foo": {"type": "integer"}, + "bar": {"$ref": "#/properties/foo"} + } + } + }, + { + "description": "relative pointer ref to array", + "schema": { + "items": [ + {"type": "integer"}, + {"$ref": "#/items/0"} + ] + } + }, + { + "description": "escaped pointer ref", + "schema": { + "tilda~field": {"type": "integer"}, + "slash/field": {"type": "integer"}, + "percent%field": {"type": "integer"}, + "properties": { + "tilda": {"$ref": "#/tilda~0field"}, + "slash": {"$ref": "#/slash~1field"}, + "percent": {"$ref": "#/percent%25field"} + } + } + }, + { + "description": "nested refs", + "schema": { + "definitions": { + "a": {"type": "integer"}, + "b": {"$ref": "#/definitions/a"}, + "c": {"$ref": "#/definitions/b"} + }, + "$ref": "#/definitions/c" + } + }, + { + "description": "ref overrides any sibling keywords", + "schema": { + "definitions": { + "reffed": { + "type": "array" + } + }, + "properties": { + "foo": { + "$ref": "#/definitions/reffed", + "maxItems": 2 + } + } + } + }, + { + "description": "property named $ref that is not a reference", + "schema": { + "properties": { + "$ref": {"type": "string"} + } + } + }, + { + "description": "test ref simple recursive", + "schema": {"properties": { + "foo_bar" : {"type":"integer"}, + "bar" : {"$ref": "#/properties"}, + "foo": {"$ref": "#/properties/bar"} + }, + "additionalProperties": false} + }, + { + "description": "test ref complex recursive", + "schema": {"definitions": { + "person": { + "type": "object", + "properties": { + "name": { "type": "string" }, + "children": { + "type": "array", + "items": { "$ref": "#/definitions/person" } + + } + }, "required": ["name"], "additionalProperties": false + } + }, + "type": "object", + "properties": { + "person": { "$ref": "#/definitions/person" } + }, "required": ["person"], "additionalProperties": false + } + }, + { + "description": "test ref complex recursive no required", + "schema": {"definitions": { + "person": { + "type": "object", + "properties": { + "name": { "type": "string", "minLength": 5 }, + "children": { + "type": "array", + "items": { "$ref": "#/definitions/person" } + + } + }, "additionalProperties": false + } + }, + "type": "object", + "properties": { + "person": { "$ref": "#/definitions/person" } + }, "additionalProperties": false + } + }, + { + "description": "test complex ref", + "schema": { "definitions": { + "address": { + "type": "object", + "additionalProperties": false, + "properties": { + "street_address": { "type": "string" }, + "city": { "type": "string" }, + "state": { "type": "string" } + }, + "required": ["street_address", "city", "state"] + } + }, + "type": "object", + "properties": { + "billing_address": { "$ref": "#/definitions/address" }, + "shipping_address": { "$ref": "#/definitions/address" } + }, + "additionalProperties": false + } + } +] From 87ae1f5af928e301f6b9db84889939fabb681ce0 Mon Sep 17 00:00:00 2001 From: kanishka-linux Date: Thu, 24 Jan 2019 13:26:56 +0530 Subject: [PATCH 23/38] read test schema from refExtra.json --- test/jake_test_ref.exs | 156 +++-------------------------------------- 1 file changed, 10 insertions(+), 146 deletions(-) diff --git a/test/jake_test_ref.exs b/test/jake_test_ref.exs index 003c912..84fdec6 100644 --- a/test/jake_test_ref.exs +++ b/test/jake_test_ref.exs @@ -3,8 +3,7 @@ defmodule JakeTestRef do use ExUnit.Case doctest Jake - def test_generator_property(jschema) do - schema = Jason.decode!(jschema) + def test_generator_property(schema) do gen = Jake.generator(schema) check all a <- gen do @@ -12,149 +11,14 @@ defmodule JakeTestRef do end end - property "test ref simple" do - jschema = ~s({"properties": { - "foo": {"type": "integer"}, - "bar": {"$ref": "#/properties/foo"} - }}) - test_generator_property(jschema) - end - - property "test ref escape pointer" do - jschema = ~s({"tilda~field": {"type": "integer"}, - "slash/field": {"type": "integer"}, - "percent%field": {"type": "integer"}, - "properties": { - "tilda": {"$ref": "#/tilda~0field"}, - "slash": {"$ref": "#/slash~1field"}, - "percent": {"$ref": "#/percent%25field"} - }}) - test_generator_property(jschema) - end - - property "test ref nested schema" do - jschema = ~s({"definitions": { - "a": {"type": "integer"}, - "b": {"$ref": "#/definitions/a"}, - "c": {"$ref": "#/definitions/b"} - }, - "$ref": "#/definitions/c"}) - test_generator_property(jschema) - end - - property "test ref overrides any sibling keywords" do - jschema = ~s({"definitions": { - "reffed": { - "type": "array" - } - }, - "properties": { - "foo": { - "$ref": "#/definitions/reffed", - "maxItems": 2 - } - }}) - test_generator_property(jschema) - end - - property "test ref not reference" do - jschema = ~s({"properties": { - "$ref": {"type": "string"} - }}) - test_generator_property(jschema) - end - - property "test ref array index" do - jschema = ~s({"items": [ - {"type": "integer"}, - {"$ref": "#/items/0"} - ]}) - test_generator_property(jschema) - end - - property "test ref root" do - jschema = ~s({"properties": { - "bar" : {"type":"integer"}, - "foo": {"$ref": "#"} - }, - "additionalProperties": false}) - test_generator_property(jschema) - end - - property "test ref simple recursive" do - jschema = ~s({"properties": { - "foo_bar" : {"type":"integer"}, - "bar" : {"$ref": "#/properties"}, - "foo": {"$ref": "#/properties/bar"} - }, - "additionalProperties": false}) - test_generator_property(jschema) - end - - property "test ref complex recursive" do - jschema = ~s({"definitions": { - "person": { - "type": "object", - "properties": { - "name": { "type": "string" }, - "children": { - "type": "array", - "items": { "$ref": "#/definitions/person" } - - } - }, "required": ["name"], "additionalProperties": false - } - }, - "type": "object", - "properties": { - "person": { "$ref": "#/definitions/person" } - }, "required": ["person"], "additionalProperties": false - }) - test_generator_property(jschema) - end - - property "test ref complex recursive no required" do - jschema = ~s({"definitions": { - "person": { - "type": "object", - "properties": { - "name": { "type": "string", "minLength": 5 }, - "children": { - "type": "array", - "items": { "$ref": "#/definitions/person" } - - } - }, "additionalProperties": false - } - }, - "type": "object", - "properties": { - "person": { "$ref": "#/definitions/person" } - }, "additionalProperties": false - }) - test_generator_property(jschema) - end - - property "test complex ref" do - jschema = ~s({ "definitions": { - "address": { - "type": "object", - "additionalProperties": false, - "properties": { - "street_address": { "type": "string" }, - "city": { "type": "string" }, - "state": { "type": "string" } - }, - "required": ["street_address", "city", "state"] - } - }, - "type": "object", - "properties": { - "billing_address": { "$ref": "#/definitions/address" }, - "shipping_address": { "$ref": "#/definitions/address" } - }, - "additionalProperties": false - }) - test_generator_property(jschema) + property "test suite ref" do + for path <- [ + "draft4/refExtra.json" + ] do + Path.wildcard("test_suite/tests/#{path}") + |> Enum.map(fn path -> File.read!(path) |> Jason.decode!() end) + |> Enum.concat() + |> Enum.map(fn %{"schema" => schema} -> test_generator_property(schema) end) + end end end From 2c25999190a6f9fab380e9c1e3aa44482e961902 Mon Sep 17 00:00:00 2001 From: kanishka-linux Date: Fri, 25 Jan 2019 12:49:01 +0530 Subject: [PATCH 24/38] add odgn_json_pointer as dependency --- mix.exs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mix.exs b/mix.exs index c990bc4..0ebbd25 100644 --- a/mix.exs +++ b/mix.exs @@ -22,7 +22,8 @@ defmodule Jake.MixProject do {:stream_data, "~> 0.4"}, {:jason, "~> 1.1"}, {:randex, "~> 0.4"}, - {:ex_json_schema, "~> 0.5.4"} + {:ex_json_schema, "~> 0.5.4"}, + {:odgn_json_pointer, "~> 2.4"} ] end end From 8d8ca2ef148c880642ed208b48c35a1320759717 Mon Sep 17 00:00:00 2001 From: kanishka-linux Date: Fri, 25 Jan 2019 12:54:02 +0530 Subject: [PATCH 25/38] use JSONPointer to resolve json pointer --- lib/jake/ref.ex | 51 ++++--------------------------------------------- 1 file changed, 4 insertions(+), 47 deletions(-) diff --git a/lib/jake/ref.ex b/lib/jake/ref.ex index de4a7f0..f4e7d43 100644 --- a/lib/jake/ref.ex +++ b/lib/jake/ref.ex @@ -14,7 +14,8 @@ defmodule Jake.Ref do if String.starts_with?(uri, "http") do process_http_path(uri) else - process_local_path(uri) |> get_head_list_path(context.root) + {:ok, value} = JSONPointer.get(context.root, uri) + value end new_child = Map.drop(spec, ["$ref"]) |> Map.merge(ref_map) @@ -26,31 +27,6 @@ defmodule Jake.Ref do {context, false} end - def get_head_list_path(path_list, root_schema) do - {head, tail} = Enum.split(path_list, -1) - - head_path = - if length(head) > 0 do - get_in(root_schema, head) - else - get_in(root_schema, path_list) - end - - tail = - if is_list(head_path) do - Enum.fetch!(tail, 0) - else - nil - end - - if tail != nil and is_numeric(tail) do - {index, ""} = Integer.parse(tail) - Enum.fetch!(head_path, index) - else - get_in(root_schema, path_list) - end - end - def process_http_path(url) do [url, local] = if String.contains?(url, "#/") do @@ -65,27 +41,8 @@ defmodule Jake.Ref do if is_nil(local) do jschema else - process_local_path(local) |> get_head_list_path(jschema) - end - end - - def process_local_path(path) do - str = - String.replace(path, "~0", "~") - |> String.replace("#/", "", global: false) - - if String.contains?(str, "~1") do - strlist = String.split(str, "/") - for n <- strlist, do: String.replace(n, "~1", "/") - else - String.split(str, "/") - end - end - - def is_numeric(str) do - case Integer.parse(str) do - {_num, ""} -> true - _ -> false + {:ok, value} = JSONPointer.get(jschema, local) + value end end end From f8a2fee43630ac5eaf68b88a6ca263de3c9510a2 Mon Sep 17 00:00:00 2001 From: kanishka-linux Date: Fri, 25 Jan 2019 13:13:48 +0530 Subject: [PATCH 26/38] use URI.parse instead of manually splitting --- lib/jake/ref.ex | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/jake/ref.ex b/lib/jake/ref.ex index f4e7d43..7dd29e2 100644 --- a/lib/jake/ref.ex +++ b/lib/jake/ref.ex @@ -29,8 +29,9 @@ defmodule Jake.Ref do def process_http_path(url) do [url, local] = - if String.contains?(url, "#/") do - String.split(url, "#/") + if String.contains?(url, "#") do + u = URI.parse(url) + ["#{u.scheme}//#{u.authority}:#{u.port}#{u.path}", u.fragment] else [url, nil] end From 80ce5d7357bd280c92ab4ce5dd880f26da362c20 Mon Sep 17 00:00:00 2001 From: kanishka-linux Date: Fri, 25 Jan 2019 17:32:44 +0530 Subject: [PATCH 27/38] add default_path and cache to context --- lib/jake/context.ex | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/jake/context.ex b/lib/jake/context.ex index ea1efa5..55f5156 100644 --- a/lib/jake/context.ex +++ b/lib/jake/context.ex @@ -1,3 +1,7 @@ defmodule Jake.Context do - defstruct root: %{}, child: %{}, size: 0 + defstruct root: %{}, + child: %{}, + size: 0, + default_path: File.cwd!(), + cache: %{} end From 98ebb9f50738c4526cf3630d5053ebac26d57888 Mon Sep 17 00:00:00 2001 From: kanishka-linux Date: Fri, 25 Jan 2019 17:33:39 +0530 Subject: [PATCH 28/38] add test for http $ref --- test_suite/tests/draft4/refExtra.json | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test_suite/tests/draft4/refExtra.json b/test_suite/tests/draft4/refExtra.json index 2358566..ccedba4 100644 --- a/test_suite/tests/draft4/refExtra.json +++ b/test_suite/tests/draft4/refExtra.json @@ -146,5 +146,11 @@ }, "additionalProperties": false } + }, + { + "description": "test http json resource", + "schema": { + "$ref": "http://json-schema.org/draft-04/schema#/definitions/positiveInteger" + } } ] From 2f260d46e937ea89ae13217a311077453c2d8ab0 Mon Sep 17 00:00:00 2001 From: kanishka-linux Date: Fri, 25 Jan 2019 17:37:32 +0530 Subject: [PATCH 29/38] modify expand_ref to include local file schema and caching of http linked schema --- lib/jake/ref.ex | 47 +++++++++++++++++++++++++++++------------------ 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/lib/jake/ref.ex b/lib/jake/ref.ex index 7dd29e2..a70453c 100644 --- a/lib/jake/ref.ex +++ b/lib/jake/ref.ex @@ -8,14 +8,24 @@ defmodule Jake.Ref do end def expand_ref(%Context{child: %{"$ref" => ref} = spec} = context) when is_binary(ref) do - uri = URI.decode(ref) + uri_parse = URI.decode(ref) |> URI.parse() - ref_map = - if String.starts_with?(uri, "http") do - process_http_path(uri) - else - {:ok, value} = JSONPointer.get(context.root, uri) - value + {context, ref_map} = + cond do + uri_parse.scheme in ["http", "https"] -> + process_http_path(uri_parse, context) + + uri_parse.path == nil and is_binary(uri_parse.fragment) -> + {context, JSONPointer.get!(context.root, uri_parse.fragment)} + + is_binary(uri_parse.path) and is_binary(uri_parse.fragment) -> + ref_map = + Path.join(context.default_path, uri_parse.path) + |> File.read!() + |> Jason.decode!() + |> JSONPointer.get!(uri_parse.fragment) + + {context, ref_map} end new_child = Map.drop(spec, ["$ref"]) |> Map.merge(ref_map) @@ -27,23 +37,24 @@ defmodule Jake.Ref do {context, false} end - def process_http_path(url) do - [url, local] = - if String.contains?(url, "#") do - u = URI.parse(url) - ["#{u.scheme}//#{u.authority}:#{u.port}#{u.path}", u.fragment] + def process_http_path(uri_parse, context) do + url = "#{uri_parse.scheme}://#{uri_parse.authority}:#{uri_parse.port}#{uri_parse.path}" + + {context, schema} = + if context.cache[url] == nil do + {:ok, {{_, 200, _}, _, schema}} = :httpc.request(:get, {to_charlist(url), []}, [], []) + new_cache = Map.put(context.cache, url, schema) + {%{context | cache: new_cache}, schema} else - [url, nil] + {context, context.cache[url]} end - {:ok, {{_, 200, _}, _, schema}} = :httpc.request(:get, {to_charlist(url), []}, [], []) jschema = Jason.decode!(schema) - if is_nil(local) do - jschema + if uri_parse.fragment do + {context, JSONPointer.get!(jschema, uri_parse.fragment)} else - {:ok, value} = JSONPointer.get(jschema, local) - value + {context, jschema} end end end From 7231c94b05fc18d1b8e889af12ca6b8a90861bec Mon Sep 17 00:00:00 2001 From: kanishka-linux Date: Sat, 26 Jan 2019 14:28:00 +0530 Subject: [PATCH 30/38] rename default_path to root_dir --- lib/jake/context.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/jake/context.ex b/lib/jake/context.ex index 55f5156..daba8d5 100644 --- a/lib/jake/context.ex +++ b/lib/jake/context.ex @@ -2,6 +2,6 @@ defmodule Jake.Context do defstruct root: %{}, child: %{}, size: 0, - default_path: File.cwd!(), + root_dir: "/", cache: %{} end From 52729b2e89a00a7da7fe35c18462d4334744d1a0 Mon Sep 17 00:00:00 2001 From: kanishka-linux Date: Sat, 26 Jan 2019 14:28:43 +0530 Subject: [PATCH 31/38] rename uri_parse to parsed_uri --- lib/jake/ref.ex | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/lib/jake/ref.ex b/lib/jake/ref.ex index a70453c..c4ed68c 100644 --- a/lib/jake/ref.ex +++ b/lib/jake/ref.ex @@ -8,22 +8,23 @@ defmodule Jake.Ref do end def expand_ref(%Context{child: %{"$ref" => ref} = spec} = context) when is_binary(ref) do - uri_parse = URI.decode(ref) |> URI.parse() + parsed_uri = URI.decode(ref) |> URI.parse() {context, ref_map} = cond do - uri_parse.scheme in ["http", "https"] -> - process_http_path(uri_parse, context) + parsed_uri.scheme in ["http", "https"] -> + process_http_path(parsed_uri, context) - uri_parse.path == nil and is_binary(uri_parse.fragment) -> - {context, JSONPointer.get!(context.root, uri_parse.fragment)} + parsed_uri.path == nil and is_binary(parsed_uri.fragment) -> + {context, JSONPointer.get!(context.root, parsed_uri.fragment)} - is_binary(uri_parse.path) and is_binary(uri_parse.fragment) -> + is_binary(parsed_uri.path) and is_binary(parsed_uri.fragment) -> ref_map = - Path.join(context.default_path, uri_parse.path) + File.cwd!() + |> Path.join(parsed_uri.path) |> File.read!() |> Jason.decode!() - |> JSONPointer.get!(uri_parse.fragment) + |> JSONPointer.get!(parsed_uri.fragment) {context, ref_map} end @@ -37,8 +38,8 @@ defmodule Jake.Ref do {context, false} end - def process_http_path(uri_parse, context) do - url = "#{uri_parse.scheme}://#{uri_parse.authority}:#{uri_parse.port}#{uri_parse.path}" + def process_http_path(parsed_uri, context) do + url = "#{parsed_uri.scheme}://#{parsed_uri.authority}:#{parsed_uri.port}#{parsed_uri.path}" {context, schema} = if context.cache[url] == nil do @@ -51,8 +52,8 @@ defmodule Jake.Ref do jschema = Jason.decode!(schema) - if uri_parse.fragment do - {context, JSONPointer.get!(jschema, uri_parse.fragment)} + if parsed_uri.fragment do + {context, JSONPointer.get!(jschema, parsed_uri.fragment)} else {context, jschema} end From 3d7f020e9f87d0874060882b0ab546aef6740ef0 Mon Sep 17 00:00:00 2001 From: kanishka-linux Date: Sat, 26 Jan 2019 14:29:42 +0530 Subject: [PATCH 32/38] format json file --- test_suite/tests/draft4/refExtra.json | 210 ++++++++++++++++++-------- 1 file changed, 143 insertions(+), 67 deletions(-) diff --git a/test_suite/tests/draft4/refExtra.json b/test_suite/tests/draft4/refExtra.json index ccedba4..74866a2 100644 --- a/test_suite/tests/draft4/refExtra.json +++ b/test_suite/tests/draft4/refExtra.json @@ -3,7 +3,9 @@ "description": "root pointer ref", "schema": { "properties": { - "foo": {"$ref": "#"} + "foo": { + "$ref": "#" + } }, "additionalProperties": false } @@ -12,8 +14,12 @@ "description": "relative pointer ref to object", "schema": { "properties": { - "foo": {"type": "integer"}, - "bar": {"$ref": "#/properties/foo"} + "foo": { + "type": "integer" + }, + "bar": { + "$ref": "#/properties/foo" + } } } }, @@ -21,21 +27,37 @@ "description": "relative pointer ref to array", "schema": { "items": [ - {"type": "integer"}, - {"$ref": "#/items/0"} + { + "type": "integer" + }, + { + "$ref": "#/items/0" + } ] } }, { "description": "escaped pointer ref", "schema": { - "tilda~field": {"type": "integer"}, - "slash/field": {"type": "integer"}, - "percent%field": {"type": "integer"}, + "tilda~field": { + "type": "integer" + }, + "slash/field": { + "type": "integer" + }, + "percent%field": { + "type": "integer" + }, "properties": { - "tilda": {"$ref": "#/tilda~0field"}, - "slash": {"$ref": "#/slash~1field"}, - "percent": {"$ref": "#/percent%25field"} + "tilda": { + "$ref": "#/tilda~0field" + }, + "slash": { + "$ref": "#/slash~1field" + }, + "percent": { + "$ref": "#/percent%25field" + } } } }, @@ -43,9 +65,15 @@ "description": "nested refs", "schema": { "definitions": { - "a": {"type": "integer"}, - "b": {"$ref": "#/definitions/a"}, - "c": {"$ref": "#/definitions/b"} + "a": { + "type": "integer" + }, + "b": { + "$ref": "#/definitions/a" + }, + "c": { + "$ref": "#/definitions/b" + } }, "$ref": "#/definitions/c" } @@ -70,82 +98,130 @@ "description": "property named $ref that is not a reference", "schema": { "properties": { - "$ref": {"type": "string"} + "$ref": { + "type": "string" + } } } }, { "description": "test ref simple recursive", - "schema": {"properties": { - "foo_bar" : {"type":"integer"}, - "bar" : {"$ref": "#/properties"}, - "foo": {"$ref": "#/properties/bar"} + "schema": { + "properties": { + "foo_bar": { + "type": "integer" + }, + "bar": { + "$ref": "#/properties" + }, + "foo": { + "$ref": "#/properties/bar" + } }, - "additionalProperties": false} + "additionalProperties": false + } }, { "description": "test ref complex recursive", - "schema": {"definitions": { - "person": { - "type": "object", - "properties": { - "name": { "type": "string" }, + "schema": { + "definitions": { + "person": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, "children": { - "type": "array", - "items": { "$ref": "#/definitions/person" } - + "type": "array", + "items": { + "$ref": "#/definitions/person" + } } - }, "required": ["name"], "additionalProperties": false - } - }, - "type": "object", - "properties": { - "person": { "$ref": "#/definitions/person" } - }, "required": ["person"], "additionalProperties": false + }, + "required": [ + "name" + ], + "additionalProperties": false } + }, + "type": "object", + "properties": { + "person": { + "$ref": "#/definitions/person" + } + }, + "required": [ + "person" + ], + "additionalProperties": false + } }, { "description": "test ref complex recursive no required", - "schema": {"definitions": { - "person": { - "type": "object", - "properties": { - "name": { "type": "string", "minLength": 5 }, + "schema": { + "definitions": { + "person": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 5 + }, "children": { - "type": "array", - "items": { "$ref": "#/definitions/person" } - + "type": "array", + "items": { + "$ref": "#/definitions/person" + } } - }, "additionalProperties": false - } - }, - "type": "object", - "properties": { - "person": { "$ref": "#/definitions/person" } - }, "additionalProperties": false + }, + "additionalProperties": false } + }, + "type": "object", + "properties": { + "person": { + "$ref": "#/definitions/person" + } + }, + "additionalProperties": false + } }, { "description": "test complex ref", - "schema": { "definitions": { - "address": { - "type": "object", - "additionalProperties": false, - "properties": { - "street_address": { "type": "string" }, - "city": { "type": "string" }, - "state": { "type": "string" } - }, - "required": ["street_address", "city", "state"] + "schema": { + "definitions": { + "address": { + "type": "object", + "additionalProperties": false, + "properties": { + "street_address": { + "type": "string" + }, + "city": { + "type": "string" + }, + "state": { + "type": "string" } - }, - "type": "object", - "properties": { - "billing_address": { "$ref": "#/definitions/address" }, - "shipping_address": { "$ref": "#/definitions/address" } - }, - "additionalProperties": false + }, + "required": [ + "street_address", + "city", + "state" + ] + } + }, + "type": "object", + "properties": { + "billing_address": { + "$ref": "#/definitions/address" + }, + "shipping_address": { + "$ref": "#/definitions/address" } + }, + "additionalProperties": false + } }, { "description": "test http json resource", From 375fda5e75548bf5f3af928981d9e19c3f2d9221 Mon Sep 17 00:00:00 2001 From: kanishka-linux Date: Tue, 29 Jan 2019 11:50:24 +0530 Subject: [PATCH 33/38] add support for patternProperties --- lib/jake/object.ex | 71 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/lib/jake/object.ex b/lib/jake/object.ex index dee02a7..fa7be93 100644 --- a/lib/jake/object.ex +++ b/lib/jake/object.ex @@ -2,8 +2,41 @@ defmodule Jake.Object do alias Jake.StreamUtil alias Jake.Context + defp gen_pattern_properties( + %Context{child: %{"patternProperties" => patternProperties} = _spec} = context + ) do + nlist = + for {k, v} <- patternProperties, + do: build_and_verify_patterns(k, v, patternProperties, context) + + merge_patterns(nlist) + end + def gen(%Context{child: spec} = context) do properties = Map.get(spec, "properties", %{}) + + if map_size(properties) == 0 and spec["patternProperties"] do + gen_pattern_properties(context) + else + new_child = Map.put(spec, "properties", properties) + gen_regular_object(%{context | child: new_child}) + end + end + + defp gen_regular_object(%Context{child: %{"properties" => properties} = spec} = context) do + nproperties = check_pattern_properties(spec, properties, spec["patternProperties"]) + + pattern_prop = + if nproperties != nil and is_list(nproperties) do + nlist = for n <- nproperties, length(n) > 0, do: Enum.fetch!(n, 0) + Enum.reduce(nlist, %{}, fn x, acc -> Map.merge(x, acc) end) + else + properties + end + + properties = pattern_prop + spec = Map.put(spec, "properties", properties) + context = %{context | child: spec} required = Map.get(spec, "required", []) all_properties = Map.keys(properties) optional = Enum.filter(all_properties, &(!Enum.member?(required, &1))) @@ -26,6 +59,44 @@ defmodule Jake.Object do |> StreamUtil.merge(StreamData.fixed_map(as_map(properties, required, context))) end + defp check_pattern_properties(_spec, properties, pprop) do + if pprop do + for {k, v} <- properties do + for {key, value} <- pprop, + Regex.match?(~r/#{key}/, k), + do: Map.put(properties, k, Map.merge(v, value)) + end + else + properties + end + end + + defp merge_patterns(nlist) do + merge_maps = fn list -> Enum.reduce(list, %{}, fn x, acc -> Map.merge(acc, x) end) end + + StreamData.bind(StreamData.fixed_list(nlist), fn list -> + StreamData.constant(merge_maps.(list)) + end) + end + + defp build_and_verify_patterns(key, value, pprop, context) do + pprop_schema = %{"patternProperties" => pprop} + # IO.inspect(pprop_schema) + nkey = Randex.stream(~r/#{key}/, mod: Randex.Generator.StreamData) + nval = %{context | child: value} |> Jake.gen_lazy() + + StreamData.bind(nkey, fn k -> + StreamData.bind_filter( + nval, + fn v -> + result = ExJsonSchema.Validator.valid?(pprop_schema, %{k => v}) + if result, do: {:cont, StreamData.constant(%{k => v})}, else: :skip + end, + 100 + ) + end) + end + defp additional(properties, _all, min.._max, _context) when min < 0 or not is_map(properties) do StreamData.constant(%{}) end From 229533bfe340ee3c763716871b82cc36c0637852 Mon Sep 17 00:00:00 2001 From: kanishka-linux Date: Tue, 29 Jan 2019 11:51:03 +0530 Subject: [PATCH 34/38] add test for patternProperties --- test/jake_test.exs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/jake_test.exs b/test/jake_test.exs index 2a7f8ad..93eeddf 100644 --- a/test/jake_test.exs +++ b/test/jake_test.exs @@ -85,7 +85,8 @@ defmodule JakeTest do "draft4/additionalItems.json", "draft4/maxProperties.json", "draft4/minProperties.json", - "draft4/additionalProperties.json" + "draft4/additionalProperties.json", + "draft4/patternProperties.json" ] do Path.wildcard("test_suite/tests/#{path}") |> Enum.map(fn path -> File.read!(path) |> Jason.decode!() end) From 1d52d8482a718ab0ee577b0f08ccd5bdab0e048a Mon Sep 17 00:00:00 2001 From: kanishka-linux Date: Tue, 29 Jan 2019 11:51:59 +0530 Subject: [PATCH 35/38] add one more test --- test_suite/tests/draft4/patternProperties.json | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/test_suite/tests/draft4/patternProperties.json b/test_suite/tests/draft4/patternProperties.json index 5f741df..7360a4b 100644 --- a/test_suite/tests/draft4/patternProperties.json +++ b/test_suite/tests/draft4/patternProperties.json @@ -116,5 +116,15 @@ "valid": false } ] + }, + { + "description": "patternProperties with properties", + "schema": { + "type": "object", + "properties": {"foo": {"maximum": 100}}, + "patternProperties": { + "f.*o": {"type": "integer"} + } + } } ] From d5bc5f6b102be3d9bdbf9ab58e0ca84e893a3e8d Mon Sep 17 00:00:00 2001 From: kanishka-linux Date: Tue, 29 Jan 2019 22:21:44 +0530 Subject: [PATCH 36/38] modify check_pattern_properties --- lib/jake/object.ex | 45 ++++++++++++++++++++++++--------------------- 1 file changed, 24 insertions(+), 21 deletions(-) diff --git a/lib/jake/object.ex b/lib/jake/object.ex index fa7be93..8c6f6d3 100644 --- a/lib/jake/object.ex +++ b/lib/jake/object.ex @@ -1,16 +1,7 @@ defmodule Jake.Object do alias Jake.StreamUtil alias Jake.Context - - defp gen_pattern_properties( - %Context{child: %{"patternProperties" => patternProperties} = _spec} = context - ) do - nlist = - for {k, v} <- patternProperties, - do: build_and_verify_patterns(k, v, patternProperties, context) - - merge_patterns(nlist) - end + alias Jake.MapUtil def gen(%Context{child: spec} = context) do properties = Map.get(spec, "properties", %{}) @@ -23,18 +14,25 @@ defmodule Jake.Object do end end + defp gen_pattern_properties( + %Context{child: %{"patternProperties" => patternProperties} = _spec} = context + ) do + nlist = + for {k, v} <- patternProperties, + do: build_and_verify_patterns(k, v, patternProperties, context) + + merge_patterns(nlist) + end + defp gen_regular_object(%Context{child: %{"properties" => properties} = spec} = context) do nproperties = check_pattern_properties(spec, properties, spec["patternProperties"]) - pattern_prop = - if nproperties != nil and is_list(nproperties) do - nlist = for n <- nproperties, length(n) > 0, do: Enum.fetch!(n, 0) - Enum.reduce(nlist, %{}, fn x, acc -> Map.merge(x, acc) end) + properties = + if is_list(nproperties) and length(nproperties) > 0 do + Enum.reduce(nproperties, %{}, fn x, acc -> MapUtil.deep_merge(x, acc) end) else properties end - - properties = pattern_prop spec = Map.put(spec, "properties", properties) context = %{context | child: spec} required = Map.get(spec, "required", []) @@ -61,11 +59,16 @@ defmodule Jake.Object do defp check_pattern_properties(_spec, properties, pprop) do if pprop do - for {k, v} <- properties do - for {key, value} <- pprop, - Regex.match?(~r/#{key}/, k), - do: Map.put(properties, k, Map.merge(v, value)) - end + pprop_list = Map.to_list(pprop) + + Map.to_list(properties) + |> Enum.map(fn {k, v} -> + Enum.map(pprop_list, fn {key, value} -> + if Regex.match?(~r/#{key}/, k) do + Map.put(properties, k, Map.merge(v, value)) + end + end) + end) |> List.flatten() |> Enum.uniq() |> List.delete(nil) else properties end From 86f1c9252d1b250180f0109f897ea34d582c3bb5 Mon Sep 17 00:00:00 2001 From: kanishka-linux Date: Tue, 29 Jan 2019 22:25:17 +0530 Subject: [PATCH 37/38] modify test --- test_suite/tests/draft4/patternProperties.json | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test_suite/tests/draft4/patternProperties.json b/test_suite/tests/draft4/patternProperties.json index 7360a4b..d3ee422 100644 --- a/test_suite/tests/draft4/patternProperties.json +++ b/test_suite/tests/draft4/patternProperties.json @@ -121,9 +121,10 @@ "description": "patternProperties with properties", "schema": { "type": "object", - "properties": {"foo": {"maximum": 100}}, + "properties": {"foo": {"maximum": 100}, "bar": {"type":"integer"}}, "patternProperties": { - "f.*o": {"type": "integer"} + "f.*o": {"type": "integer"}, + "ba*r": {"minimum": 20} } } } From ab37258a72a8f352fa4d3e7e44d1750773c8e164 Mon Sep 17 00:00:00 2001 From: kanishka-linux Date: Wed, 30 Jan 2019 00:40:53 +0530 Subject: [PATCH 38/38] mix format --- lib/jake/object.ex | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/jake/object.ex b/lib/jake/object.ex index 8c6f6d3..2b6e1f7 100644 --- a/lib/jake/object.ex +++ b/lib/jake/object.ex @@ -33,6 +33,7 @@ defmodule Jake.Object do else properties end + spec = Map.put(spec, "properties", properties) context = %{context | child: spec} required = Map.get(spec, "required", []) @@ -68,7 +69,10 @@ defmodule Jake.Object do Map.put(properties, k, Map.merge(v, value)) end end) - end) |> List.flatten() |> Enum.uniq() |> List.delete(nil) + end) + |> List.flatten() + |> Enum.uniq() + |> List.delete(nil) else properties end